[UT] Fix broken BE UT caused of page cache (#34604)

Some test cases will delete the page cache, causing other test cases that use the page cache to fail.

Fixes #34602

Signed-off-by: Alex Zhu <zhuming9011@gmail.com>
This commit is contained in:
Alex Zhu 2023-11-09 00:00:29 +08:00 committed by GitHub
parent e66a35e082
commit dd2f7ac989
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 15 additions and 31 deletions

View File

@ -63,6 +63,10 @@ void StoragePageCache::release_global_cache() {
}
}
void StoragePageCache::prune() {
_cache->prune();
}
static void init_metrics() {
StarRocksMetrics::instance()->metrics()->register_metric("page_cache_lookup_count", &page_cache_lookup_count);
StarRocksMetrics::instance()->metrics()->register_hook("page_cache_lookup_count", []() {

View File

@ -116,6 +116,8 @@ public:
bool adjust_capacity(int64_t delta, size_t min_capacity = 0);
void prune();
private:
static StoragePageCache* _s_instance;

View File

@ -57,7 +57,6 @@ public:
protected:
void SetUp() override {
StoragePageCache::create_global_cache(&_tracker, 1000000000);
_fs = std::make_shared<MemoryFileSystem>();
ASSERT_TRUE(_fs->create_dir(kTestDir).ok());
@ -66,7 +65,7 @@ protected:
_opts.skip_fill_data_cache = false;
_opts.stats = &_stats;
}
void TearDown() override { StoragePageCache::release_global_cache(); }
void TearDown() override { StoragePageCache::instance()->prune(); }
void get_bitmap_reader_iter(RandomAccessFile* rfile, const ColumnIndexMetaPB& meta, BitmapIndexReader** reader,
BitmapIndexIterator** iter) {
@ -95,7 +94,6 @@ protected:
}
std::shared_ptr<MemoryFileSystem> _fs = nullptr;
MemTracker _tracker;
MemPool _pool;
IndexReadOptions _opts;
OlapReaderStatistics _stats;

View File

@ -52,8 +52,6 @@ const std::string kTestDir = "/bloom_filter_index_reader_writer_test";
class BloomFilterIndexReaderWriterTest : public testing::Test {
protected:
void SetUp() override {
_mem_tracker = std::make_unique<MemTracker>();
StoragePageCache::create_global_cache(_mem_tracker.get(), 1000000000);
_fs = std::make_shared<MemoryFileSystem>();
ASSERT_TRUE(_fs->create_dir(kTestDir).ok());
@ -62,7 +60,7 @@ protected:
_opts.skip_fill_data_cache = false;
_opts.stats = &_stats;
}
void TearDown() override { StoragePageCache::release_global_cache(); }
void TearDown() override { StoragePageCache::instance()->prune(); }
template <LogicalType type>
void write_bloom_filter_index_file(const std::string& file_name, const void* values, size_t value_count,
@ -165,7 +163,6 @@ protected:
}
}
std::unique_ptr<MemTracker> _mem_tracker = nullptr;
std::shared_ptr<MemoryFileSystem> _fs = nullptr;
IndexReadOptions _opts;
OlapReaderStatistics _stats;

View File

@ -160,9 +160,6 @@ protected:
}
// read and check
{
// create page cache
std::unique_ptr<MemTracker> page_cache_mem_tracker = std::make_unique<MemTracker>();
StoragePageCache::create_global_cache(page_cache_mem_tracker.get(), 1000000000);
// read and check
auto res = ColumnReader::create(&meta, segment.get());
ASSERT_TRUE(res.ok());

View File

@ -53,16 +53,13 @@ public:
const std::string kTestDir = "/ordinal_page_index_test";
void SetUp() override {
_mem_tracker = std::make_unique<MemTracker>();
StoragePageCache::create_global_cache(_mem_tracker.get(), 1000000000);
_fs = std::make_shared<MemoryFileSystem>();
ASSERT_TRUE(_fs->create_dir(kTestDir).ok());
}
void TearDown() override { StoragePageCache::release_global_cache(); }
void TearDown() override { StoragePageCache::instance()->prune(); }
protected:
std::unique_ptr<MemTracker> _mem_tracker = nullptr;
std::shared_ptr<MemoryFileSystem> _fs = nullptr;
};

View File

@ -77,7 +77,6 @@ protected:
OlapReaderStatistics _stats;
void SetUp() override {
_page_cache_mem_tracker = std::make_unique<MemTracker>();
config::tablet_map_shard_size = 1;
config::txn_map_shard_size = 1;
config::txn_shard_size = 1;
@ -101,7 +100,6 @@ protected:
ASSERT_TRUE(fs::create_directories(rowset_dir).ok());
ASSERT_TRUE(fs::create_directories(config::storage_root_path + "/data/rowset_test_seg").ok());
ASSERT_TRUE(fs::create_directories(config::storage_root_path + "/data/rowset_test_delete").ok());
StoragePageCache::create_global_cache(_page_cache_mem_tracker.get(), 1000000000);
i++;
}
@ -112,7 +110,7 @@ protected:
if (fs::path_exist(config::storage_root_path)) {
ASSERT_TRUE(fs::remove_all(config::storage_root_path).ok());
}
StoragePageCache::release_global_cache();
StoragePageCache::instance()->prune();
config::storage_root_path = _default_storage_root_path;
}
@ -242,7 +240,6 @@ protected:
void test_final_merge(bool has_merge_condition);
private:
std::unique_ptr<MemTracker> _page_cache_mem_tracker = nullptr;
std::string _default_storage_root_path;
};

View File

@ -42,15 +42,12 @@ public:
void SetUp() override {
_fs = std::make_shared<MemoryFileSystem>();
ASSERT_TRUE(_fs->create_dir(kSegmentDir).ok());
_page_cache_mem_tracker = std::make_unique<MemTracker>();
StoragePageCache::create_global_cache(_page_cache_mem_tracker.get(), 1000000000);
}
void TearDown() override { StoragePageCache::release_global_cache(); }
void TearDown() override { StoragePageCache::instance()->prune(); }
const std::string kSegmentDir = "/segment_test";
std::shared_ptr<MemoryFileSystem> _fs = nullptr;
std::unique_ptr<MemTracker> _page_cache_mem_tracker = nullptr;
};
namespace test {

View File

@ -49,20 +49,16 @@ protected:
void SetUp() override {
_fs = FileSystem::CreateSharedFromString("posix://").value();
ASSERT_OK(_fs->create_dir_recursive(kSegmentDir));
_page_cache_mem_tracker = std::make_unique<MemTracker>();
StoragePageCache::create_global_cache(_page_cache_mem_tracker.get(), 1000000000);
}
void TearDown() override {
ASSERT_TRUE(fs::remove_all(kSegmentDir).ok());
StoragePageCache::release_global_cache();
StoragePageCache::instance()->prune();
}
const std::string kSegmentDir = "./ut_dir/segment_rewriter_test";
std::shared_ptr<FileSystem> _fs;
std::unique_ptr<MemTracker> _page_cache_mem_tracker;
};
TEST_F(SegmentRewriterTest, rewrite_test) {

View File

@ -77,11 +77,9 @@ protected:
void SetUp() override {
_fs = std::make_shared<MemoryFileSystem>();
ASSERT_TRUE(_fs->create_dir(kSegmentDir).ok());
_page_cache_mem_tracker = std::make_unique<MemTracker>();
StoragePageCache::create_global_cache(_page_cache_mem_tracker.get(), 1000000000);
}
void TearDown() override { StoragePageCache::release_global_cache(); }
void TearDown() override { StoragePageCache::instance()->prune(); }
void build_segment(const SegmentWriterOptions& opts, const TabletSchemaCSPtr& build_schema,
const TabletSchemaCSPtr& query_schema, size_t nrows, const ValueGenerator& generator,

View File

@ -53,10 +53,11 @@ protected:
auto _page_cache_mem_tracker = std::make_unique<MemTracker>();
static const int kNumShardBits = 5;
static const int kNumShards = 1 << kNumShardBits;
StoragePageCache::release_global_cache();
StoragePageCache::create_global_cache(_page_cache_mem_tracker.get(), kNumShards * 100000);
}
void TearDown() override { StoragePageCache::release_global_cache(); }
void TearDown() override { StoragePageCache::instance()->prune(); }
};
class TestMetricsVisitor : public MetricsVisitor {