[Refactor] refactors the cache module by reorganizing cache-related files (#63759)

Signed-off-by: trueeyu <lxhhust350@qq.com>
This commit is contained in:
trueeyu 2025-10-01 19:05:00 +08:00 committed by GitHub
parent 0094eeaa6d
commit d23e5cee79
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
56 changed files with 93 additions and 97 deletions

View File

@ -24,7 +24,7 @@
#include <filesystem>
#include <memory>
#include "cache/starcache_engine.h"
#include "cache/disk_cache/starcache_engine.h"
#include "starcache/common/types.h"
#include "util/logging.h"
#include "util/random.h"

View File

@ -18,9 +18,9 @@
#include <random>
#include "cache/cache_options.h"
#include "cache/lrucache_engine.h"
#include "cache/object_cache/page_cache.h"
#include "cache/starcache_engine.h"
#include "cache/disk_cache/starcache_engine.h"
#include "cache/mem_cache/lrucache_engine.h"
#include "cache/mem_cache/page_cache.h"
#include "common/config.h"
#include "runtime/current_thread.h"
#include "runtime/exec_env.h"
@ -105,7 +105,7 @@ std::string ObjectCacheBench::get_cache_type_str(CacheType type) {
}
void ObjectCacheBench::init_cache(CacheType cache_type) {
CacheOptions opt;
DiskCacheOptions opt;
opt.mem_space_size = _capacity;
opt.block_size = config::datacache_block_size;
opt.max_flying_memory_mb = config::datacache_max_flying_memory_mb;

View File

@ -23,15 +23,15 @@ set(CACHE_FILES
mem_space_monitor.cpp
datacache.cpp
datacache_utils.cpp
lrucache_engine.cpp
block_cache/block_cache.cpp
block_cache/io_buffer.cpp
block_cache/block_cache_hit_rate_counter.hpp
object_cache/page_cache.cpp
mem_cache/lrucache_engine.cpp
mem_cache/page_cache.cpp
disk_cache/block_cache.cpp
disk_cache/io_buffer.cpp
disk_cache/block_cache_hit_rate_counter.hpp
)
if (${WITH_STARCACHE} STREQUAL "ON")
list(APPEND CACHE_FILES starcache_engine.cpp)
list(APPEND CACHE_FILES disk_cache/starcache_engine.cpp)
list(APPEND CACHE_FILES peer_cache_engine.cpp)
endif()

View File

@ -16,9 +16,9 @@
#include "cache/datacache_utils.h"
#include "cache/disk_space_monitor.h"
#include "cache/lrucache_engine.h"
#include "cache/mem_cache/lrucache_engine.h"
#include "cache/mem_cache/page_cache.h"
#include "cache/mem_space_monitor.h"
#include "cache/object_cache/page_cache.h"
#include "common/status.h"
#include "gutil/strings/split.h"
#include "gutil/strings/strip.h"
@ -27,8 +27,8 @@
#include "util/parse_util.h"
#ifdef WITH_STARCACHE
#include "cache/disk_cache/starcache_engine.h"
#include "cache/peer_cache_engine.h"
#include "cache/starcache_engine.h"
#endif
namespace starrocks {

View File

@ -14,9 +14,9 @@
#pragma once
#include "cache/block_cache/block_cache.h"
#include "cache/local_disk_cache_engine.h"
#include "cache/local_mem_cache_engine.h"
#include "cache/disk_cache/block_cache.h"
#include "cache/disk_cache/local_disk_cache_engine.h"
#include "cache/mem_cache/local_mem_cache_engine.h"
#include "common/status.h"
namespace starrocks {

View File

@ -15,7 +15,7 @@
#pragma once
#include "cache/cache_metrics.h"
#include "cache/local_disk_cache_engine.h"
#include "cache/disk_cache/local_disk_cache_engine.h"
#include "gen_cpp/DataCache_types.h"
#include "storage/options.h"

View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cache/block_cache/block_cache.h"
#include "cache/disk_cache/block_cache.h"
#include <fmt/format.h>

View File

@ -16,7 +16,7 @@
#include <atomic>
#include "cache/local_disk_cache_engine.h"
#include "cache/disk_cache/local_disk_cache_engine.h"
#include "cache/remote_cache_engine.h"
#include "common/status.h"

View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cache/block_cache/io_buffer.h"
#include "cache/disk_cache/io_buffer.h"
#include "gutil/strings/fastmem.h"

View File

@ -14,9 +14,9 @@
#pragma once
#include "cache/block_cache/io_buffer.h"
#include "cache/cache_options.h"
#include "cache/object_cache/cache_types.h"
#include "cache/disk_cache/io_buffer.h"
#include "cache/mem_cache/cache_types.h"
#include "common/status.h"
namespace starrocks {

View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cache/starcache_engine.h"
#include "cache/disk_cache/starcache_engine.h"
#include <filesystem>

View File

@ -14,7 +14,7 @@
#pragma once
#include "cache/local_disk_cache_engine.h"
#include "cache/disk_cache/local_disk_cache_engine.h"
#include "common/status.h"
#include "starcache/star_cache.h"
#include "starcache/time_based_cache_adaptor.h"

View File

@ -20,7 +20,7 @@
#include <unordered_map>
#include "cache/cache_options.h"
#include "cache/local_disk_cache_engine.h"
#include "cache/disk_cache/local_disk_cache_engine.h"
#include "common/status.h"
#include "fs/fs.h"
#include "util/disk_info.h"

View File

@ -14,9 +14,9 @@
#pragma once
#include "cache/block_cache/io_buffer.h"
#include "cache/cache_options.h"
#include "cache/object_cache/cache_types.h"
#include "cache/disk_cache/io_buffer.h"
#include "cache/mem_cache/cache_types.h"
#include "common/status.h"
namespace starrocks {

View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cache/lrucache_engine.h"
#include "cache/mem_cache/lrucache_engine.h"
#include <butil/fast_rand.h>

View File

@ -16,7 +16,7 @@
#include <atomic>
#include "cache/local_mem_cache_engine.h"
#include "cache/mem_cache/local_mem_cache_engine.h"
#include "util/lru_cache.h"
namespace starrocks {

View File

@ -32,7 +32,7 @@
// specific language governing permissions and limitations
// under the License.
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include <malloc.h>

View File

@ -14,7 +14,7 @@
#include "cache/mem_space_monitor.h"
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include "common/config.h"
#include "runtime/exec_env.h"
#include "runtime/mem_tracker.h"

View File

@ -14,8 +14,8 @@
#pragma once
#include "cache/block_cache/io_buffer.h"
#include "cache/cache_options.h"
#include "cache/disk_cache/io_buffer.h"
#include "common/status.h"
namespace starrocks {

View File

@ -27,7 +27,7 @@
#include <mutex>
#include "cache/datacache.h"
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include "common/config.h"
#include "gutil/endian.h"
#include "gutil/stringprintf.h"

View File

@ -14,7 +14,7 @@
#include "exec/hdfs_scanner/hdfs_scanner.h"
#include "cache/block_cache/block_cache_hit_rate_counter.hpp"
#include "cache/disk_cache/block_cache_hit_rate_counter.hpp"
#include "column/column_helper.h"
#include "column/type_traits.h"
#include "connector/deletion_vector/deletion_vector.h"

View File

@ -21,7 +21,7 @@
#include "runtime/exec_env.h"
#ifdef WITH_STARCACHE
#include "cache/starcache_engine.h"
#include "cache/disk_cache/starcache_engine.h"
#endif
namespace starrocks {

View File

@ -20,7 +20,7 @@
#include <string>
#include <vector>
#include "cache/block_cache/block_cache.h"
#include "cache/disk_cache/block_cache.h"
#include "column/vectorized_fwd.h"
#include "common/status.h"
#include "common/statusor.h"

View File

@ -21,15 +21,15 @@
#include <string>
#include "cache/block_cache/block_cache_hit_rate_counter.hpp"
#include "cache/local_disk_cache_engine.h"
#include "cache/disk_cache/block_cache_hit_rate_counter.hpp"
#include "cache/disk_cache/local_disk_cache_engine.h"
#include "http/http_channel.h"
#include "http/http_headers.h"
#include "http/http_request.h"
#include "http/http_status.h"
#ifdef WITH_STARCACHE
#include "cache/starcache_engine.h"
#include "cache/disk_cache/starcache_engine.h"
#endif
namespace starrocks {

View File

@ -44,7 +44,7 @@
#include "agent/agent_common.h"
#include "agent/agent_server.h"
#include "cache/datacache.h"
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include "common/configbase.h"
#include "common/status.h"
#include "exec/workgroup/scan_executor.h"

View File

@ -17,8 +17,8 @@
#include <memory>
#include <string>
#include "cache/block_cache/block_cache.h"
#include "cache/block_cache/io_buffer.h"
#include "cache/disk_cache/block_cache.h"
#include "cache/disk_cache/io_buffer.h"
#include "io/shared_buffered_input_stream.h"
namespace starrocks::io {

View File

@ -44,8 +44,8 @@
#include <unordered_map>
#include <vector>
#include "cache/block_cache/block_cache.h"
#include "cache/datacache_utils.h"
#include "cache/disk_cache/block_cache.h"
#include "cctz/time_zone.h"
#include "common/global_types.h"
#include "common/object_pool.h"

View File

@ -21,8 +21,8 @@
#include "agent/heartbeat_server.h"
#include "backend_service.h"
#include "cache/block_cache/block_cache.h"
#include "cache/datacache.h"
#include "cache/disk_cache/block_cache.h"
#include "common/config.h"
#include "common/daemon.h"
#include "common/process_exit.h"
@ -49,7 +49,7 @@
#include "util/thrift_server.h"
#ifdef WITH_STARCACHE
#include "cache/starcache_engine.h"
#include "cache/disk_cache/starcache_engine.h"
#endif
namespace brpc {

View File

@ -34,7 +34,7 @@
#pragma once
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include "storage/rowset/page_handle_fwd.h"
namespace starrocks {

View File

@ -38,7 +38,7 @@
#include <string>
#include <string_view>
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include "column/column.h"
#include "common/logging.h"
#include "fs/fs.h"

View File

@ -46,7 +46,7 @@
#ifdef USE_STAROS
#include "fslib/star_cache_handler.h"
#endif
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include "gutil/strings/split.h" // for string split
#include "gutil/strtoint.h" // for atoi64
#include "io/io_profiler.h"

View File

@ -408,7 +408,8 @@ set(EXEC_FILES
./service/service_be/internal_service_test.cpp
./service/staros_worker_test.cpp
./cache/datacache_utils_test.cpp
./cache/block_cache/block_cache_hit_rate_counter_test.cpp
./cache/disk_cache/block_cache_hit_rate_counter_test.cpp
./cache/mem_cache/page_cache_test.cpp
./cache/peer_cache_test.cpp
./util/thrift_rpc_helper_test.cpp
./util/thrift_client_test.cpp
@ -602,9 +603,9 @@ SET(DW_TEST_FILES
if ("${WITH_STARCACHE}" STREQUAL "ON")
list(APPEND DW_TEST_FILES ./cache/disk_space_monitor_test.cpp)
list(APPEND DW_TEST_FILES ./cache/lrucache_engine_test.cpp)
list(APPEND DW_TEST_FILES ./cache/starcache_engine_test.cpp)
list(APPEND DW_TEST_FILES ./cache/block_cache/block_cache_test.cpp)
list(APPEND DW_TEST_FILES ./cache/disk_cache/starcache_engine_test.cpp)
list(APPEND DW_TEST_FILES ./cache/disk_cache/block_cache_test.cpp)
list(APPEND DW_TEST_FILES ./cache/mem_cache/lrucache_engine_test.cpp)
list(APPEND DW_TEST_FILES ./io/cache_input_stream_test.cpp)
endif ()

View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cache/block_cache/block_cache_hit_rate_counter.hpp"
#include "cache/disk_cache/block_cache_hit_rate_counter.hpp"
#include <gtest/gtest.h>

View File

@ -12,16 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cache/block_cache/block_cache.h"
#include "cache/disk_cache/block_cache.h"
#include <gtest/gtest.h>
#include <cstring>
#include <filesystem>
#include "cache/block_cache/test_cache_utils.h"
#include "cache/datacache_utils.h"
#include "common/logging.h"
#include "cache/disk_cache/test_cache_utils.h"
#include "common/statusor.h"
#include "fs/fs_util.h"
#include "testutil/assert.h"
@ -30,10 +29,6 @@ namespace starrocks {
class BlockCacheTest : public ::testing::Test {
protected:
static void SetUpTestCase() {}
static void TearDownTestCase() {}
void SetUp() override {
_saved_enable_auto_adjust = config::enable_datacache_disk_auto_adjust;
config::enable_datacache_disk_auto_adjust = false;

View File

@ -12,11 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cache/starcache_engine.h"
#include "cache/disk_cache/starcache_engine.h"
#include <gtest/gtest.h>
#include "cache/block_cache/test_cache_utils.h"
#include "cache/disk_cache/test_cache_utils.h"
#include "fs/fs_util.h"
#include "testutil/assert.h"

View File

@ -17,9 +17,9 @@
#include <fmt/format.h>
#include <gtest/gtest.h>
#include "cache/block_cache/block_cache.h"
#include "cache/disk_cache/block_cache.h"
#include "cache/disk_cache/starcache_engine.h"
#include "cache/peer_cache_engine.h"
#include "cache/starcache_engine.h"
#include "common/logging.h"
#include "testutil/assert.h"

View File

@ -20,8 +20,8 @@
#include <cstring>
#include <filesystem>
#include "cache/block_cache/block_cache.h"
#include "cache/block_cache/test_cache_utils.h"
#include "cache/disk_cache/block_cache.h"
#include "cache/disk_cache/test_cache_utils.h"
#include "common/logging.h"
#include "common/statusor.h"
#include "fs/fs_util.h"

View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cache/lrucache_engine.h"
#include "cache/mem_cache/lrucache_engine.h"
#include <gtest/gtest.h>

View File

@ -32,11 +32,11 @@
// specific language governing permissions and limitations
// under the License.
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include <gtest/gtest.h>
#include "cache/lrucache_engine.h"
#include "cache/mem_cache/lrucache_engine.h"
#include "testutil/assert.h"
namespace starrocks {
@ -54,7 +54,7 @@ protected:
};
void StoragePageCacheTest::SetUp() {
CacheOptions opts{.mem_space_size = _capacity};
MemCacheOptions opts{.mem_space_size = _capacity};
_lru_cache = std::make_shared<LRUCacheEngine>();
ASSERT_OK(_lru_cache->init(opts));
_page_cache = std::make_shared<StoragePageCache>(_lru_cache.get());
@ -96,7 +96,7 @@ TEST_F(StoragePageCacheTest, normal) {
PageCacheHandle handle;
ObjectCacheWriteOptions opts;
ASSERT_OK(_page_cache->insert(key, data.get(), &opts, &handle));
ASSERT_OK(_page_cache->insert(key, data.get(), opts, &handle));
ASSERT_EQ(handle.data(), data.get());
auto* check_data = data.release();

View File

@ -15,8 +15,8 @@
#include <fmt/format.h>
#include <gtest/gtest.h>
#include "cache/block_cache/io_buffer.h"
#include "cache/cache_options.h"
#include "cache/disk_cache/io_buffer.h"
#include "cache/peer_cache_engine.h"
#include "common/statusor.h"

View File

@ -18,7 +18,7 @@
#include <memory>
#include "cache/block_cache/block_cache.h"
#include "cache/disk_cache/block_cache.h"
#include "column/column_helper.h"
#include "exec/hdfs_scanner/hdfs_scanner_orc.h"
#include "exec/hdfs_scanner/hdfs_scanner_parquet.h"

View File

@ -18,10 +18,10 @@
#include <memory>
#include "cache/block_cache/block_cache.h"
#include "cache/block_cache/test_cache_utils.h"
#include "cache/datacache.h"
#include "cache/starcache_engine.h"
#include "cache/disk_cache/block_cache.h"
#include "cache/disk_cache/starcache_engine.h"
#include "cache/disk_cache/test_cache_utils.h"
#include "column/column_helper.h"
#include "exec/hdfs_scanner/hdfs_scanner_orc.h"
#include "exec/hdfs_scanner/hdfs_scanner_parquet.h"

View File

@ -20,10 +20,10 @@
#include <random>
#include <set>
#include "cache/block_cache/block_cache.h"
#include "cache/block_cache/test_cache_utils.h"
#include "cache/lrucache_engine.h"
#include "cache/starcache_engine.h"
#include "cache/disk_cache/block_cache.h"
#include "cache/disk_cache/starcache_engine.h"
#include "cache/disk_cache/test_cache_utils.h"
#include "cache/mem_cache/lrucache_engine.h"
#include "column/column_helper.h"
#include "column/fixed_length_column.h"
#include "common/logging.h"

View File

@ -16,9 +16,9 @@
#include <gtest/gtest.h>
#include "cache/block_cache/test_cache_utils.h"
#include "cache/datacache.h"
#include "cache/starcache_engine.h"
#include "cache/disk_cache/starcache_engine.h"
#include "cache/disk_cache/test_cache_utils.h"
#include "fs/fs_util.h"
#include "runtime/exec_env.h"
#include "storage/persistent_index_load_executor.h"

View File

@ -19,9 +19,9 @@
#include <gtest/gtest.h>
#include <rapidjson/document.h>
#include "cache/block_cache/block_cache_hit_rate_counter.hpp"
#include "cache/block_cache/test_cache_utils.h"
#include "cache/starcache_engine.h"
#include "cache/disk_cache/block_cache_hit_rate_counter.hpp"
#include "cache/disk_cache/starcache_engine.h"
#include "cache/disk_cache/test_cache_utils.h"
#include "gen_cpp/HeartbeatService_types.h"
#include "http/http_channel.h"
#include "http/http_request.h"

View File

@ -16,9 +16,9 @@
#include <gtest/gtest.h>
#include "cache/block_cache/test_cache_utils.h"
#include "cache/datacache.h"
#include "cache/starcache_engine.h"
#include "cache/disk_cache/starcache_engine.h"
#include "cache/disk_cache/test_cache_utils.h"
#include "fs/fs_util.h"
#include "runtime/exec_env.h"
#include "testutil/assert.h"

View File

@ -19,8 +19,8 @@
#include <memory>
#include "cache/block_cache/test_cache_utils.h"
#include "cache/datacache.h"
#include "cache/disk_cache/test_cache_utils.h"
#include "common/utils.h"
#include "exec/tablet_sink_index_channel.h"
#include "runtime/exec_env.h"

View File

@ -40,7 +40,7 @@
#include <memory>
#include <string>
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include "common/logging.h"
#include "fs/fs_memory.h"
#include "runtime/mem_tracker.h"

View File

@ -16,8 +16,8 @@
#include <gtest/gtest.h>
#include "cache/lrucache_engine.h"
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/lrucache_engine.h"
#include "cache/mem_cache/page_cache.h"
#include "testutil/assert.h"
namespace starrocks {

View File

@ -16,8 +16,8 @@
#include <gtest/gtest.h>
#include "cache/lrucache_engine.h"
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/lrucache_engine.h"
#include "cache/mem_cache/page_cache.h"
#include "fs/fs_memory.h"
#include "storage/rowset/binary_plain_page.h"
#include "storage/rowset/bitshuffle_page.h"

View File

@ -39,7 +39,7 @@
#include <memory>
#include <string>
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/page_cache.h"
#include "column/binary_column.h"
#include "common/config.h"
#include "fs/fs_memory.h"

View File

@ -36,8 +36,8 @@
#include <gtest/gtest.h>
#include "cache/lrucache_engine.h"
#include "cache/object_cache/page_cache.h"
#include "cache/mem_cache/lrucache_engine.h"
#include "cache/mem_cache/page_cache.h"
#include "common/config.h"
#include "testutil/assert.h"