[Feature](function) support sec_to_time (#62797)

Signed-off-by: Murphy <mofei@starrocks.com>

.gitignore

@@ -33,6 +33,7 @@ fe_plugins/output
 fe/mocked
 fe/ut_ports
 fe/*/target
+fe/*/bin
 fe/plugin/*/target/
 fe/fe-core/gen
 fe/fe-grammar/gen
@@ -97,3 +98,15 @@ CMakeLists.txt
 .claude
 CLAUDE.md
 .clangd
+
+
+!build-mac/CMakeLists.txt
+# build-mac generated files
+build-mac/.ninja_deps
+build-mac/.ninja_lock
+build-mac/.ninja_log
+build-mac/CMakeCache.txt
+build-mac/CMakeFiles/
+build-mac/build.ninja
+build-mac/build_version.cc
+build-mac/cmake_install.cmake

README.md

@@ -130,6 +130,8 @@ This project is used by the following companies. Learn more about their use case
 
 - [Airbnb](https://www.youtube.com/watch?v=AzDxEZuMBwM&ab_channel=StarRocks_labs)
 - [Airtable](https://medium.com/airtable-eng/live-shard-data-archive-export-and-ingestion-to-starrocks-for-validation-6af555e8b3fe)
+- [Alibaba](https://www.youtube.com/watch?v=7N34q65mthk)
+- [Celonis](https://www.youtube.com/watch?v=Jm-D0xvOtek)
 - [Coinbase](https://www.youtube.com/watch?v=3Z9jSCaHnYg&list=PL0eWwaesODdhBhKSnvpfIEAB9sgk8rKmy)
 - [Demandbase](https://starrocks.medium.com/demandbase-ditches-denormalization-by-switching-off-clickhouse-44195d795a83)
 - [Didi](https://www.starrocks.io/blog/reduced-80-cost-didis-journey-from-multiple-olap-engines-to-starrocks)
@@ -137,6 +139,7 @@ This project is used by the following companies. Learn more about their use case
 - [Fanatics](https://www.youtube.com/watch?v=hbXovqR6tOc)
 - [Fresha](https://medium.com/fresha-data-engineering/how-we-accidentally-became-one-of-uks-first-starrocks-production-pioneers-7db249f10010)
 - [Grab](https://engineering.grab.com/building-a-spark-observability)
+- [Haezoom](https://www.starrocks.io/blog/haezoom-and-cloudshift-overcome-apache-druids-limits-with-starrocks)
 - [HerdWatch](https://medium.com/p/a7916a7e87bf)
 - [Intuit](https://www.youtube.com/watch?v=tUC3FS3ki10)
 - [iQiyi](https://medium.com/starrocks-engineering/real-time-analytics-at-scale-why-we-use-starrocks-0aa3c859cbeb)

be/CMakeLists.txt

@@ -650,6 +650,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror")
     endif()
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unused-parameter -Wno-documentation -Wno-weak-vtables")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option")
     # Turn on following warning as error explicitly
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror=string-plus-int")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror=pessimizing-move")
@@ -670,6 +671,10 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
         # ignore warning from apache-orc
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-switch-default")
     endif ()
+    # Add -rtlib=compiler-rt for ARM architecture to fix LLVM bug: https://bugs.llvm.org/show_bug.cgi?id=16404
+    if ("${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "aarch64")
+        set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -rtlib=compiler-rt")
+    endif()
 else ()
     if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0.0")
         # ignore error from apache-orc

be/src/agent/task_worker_pool.cpp

@@ -848,15 +848,22 @@ void* ReportDataCacheMetricsTaskWorkerPool::_worker_thread_callback(void* arg_th
         request.__set_report_version(g_report_version.load(std::memory_order_relaxed));
 
         TDataCacheMetrics t_metrics{};
-        // TODO: mem_metrics + disk_metrics
-        const LocalCacheEngine* cache = DataCache::GetInstance()->local_disk_cache();
-        if (cache != nullptr && cache->is_initialized()) {
-            const auto metrics = cache->cache_metrics();
-            DataCacheUtils::set_metrics_from_thrift(t_metrics, metrics);
-        } else {
-            t_metrics.__set_status(TDataCacheStatus::DISABLED);
-        }
+        const LocalDiskCacheEngine* disk_cache = DataCache::GetInstance()->local_disk_cache();
+        const LocalMemCacheEngine* mem_cache = DataCache::GetInstance()->local_mem_cache();
+        bool disk_cache_inited = disk_cache != nullptr && disk_cache->is_initialized();
+        bool mem_cache_inited = mem_cache != nullptr && mem_cache->is_initialized();
 
+        if (!disk_cache_inited && !mem_cache_inited) {
+            t_metrics.__set_status(TDataCacheStatus::DISABLED);
+        } else {
+            if (mem_cache_inited) {
+                t_metrics.__set_status(TDataCacheStatus::NORMAL);
+                DataCacheUtils::set_metrics_to_thrift(t_metrics, mem_cache->cache_metrics());
+            }
+            if (disk_cache_inited) {
+                DataCacheUtils::set_metrics_to_thrift(t_metrics, disk_cache->cache_metrics());
+            }
+        }
         request.__set_datacache_metrics(t_metrics);
 
         TMasterResult result;

be/src/bench/block_cache_bench.cpp

@@ -24,7 +24,7 @@
 #include <filesystem>
 #include <memory>
 
-#include "cache/starcache_engine.h"
+#include "cache/disk_cache/starcache_engine.h"
 #include "starcache/common/types.h"
 #include "util/logging.h"
 #include "util/random.h"

be/src/bench/object_cache_bench.cpp

@@ -18,9 +18,9 @@
 #include <random>
 
 #include "cache/cache_options.h"
-#include "cache/lrucache_engine.h"
-#include "cache/object_cache/page_cache.h"
-#include "cache/starcache_engine.h"
+#include "cache/disk_cache/starcache_engine.h"
+#include "cache/mem_cache/lrucache_engine.h"
+#include "cache/mem_cache/page_cache.h"
 #include "common/config.h"
 #include "runtime/current_thread.h"
 #include "runtime/exec_env.h"
@@ -105,14 +105,13 @@ std::string ObjectCacheBench::get_cache_type_str(CacheType type) {
 }
 
 void ObjectCacheBench::init_cache(CacheType cache_type) {
-    CacheOptions opt;
+    DiskCacheOptions opt;
     opt.mem_space_size = _capacity;
     opt.block_size = config::datacache_block_size;
     opt.max_flying_memory_mb = config::datacache_max_flying_memory_mb;
     opt.max_concurrent_inserts = config::datacache_max_concurrent_inserts;
     opt.enable_checksum = config::datacache_checksum_enable;
     opt.enable_direct_io = config::datacache_direct_io_enable;
-    opt.enable_tiered_cache = config::datacache_tiered_cache_enable;
     opt.skip_read_factor = config::datacache_skip_read_factor;
     opt.scheduler_threads_per_cpu = config::datacache_scheduler_threads_per_cpu;
     opt.enable_datacache_persistence = false;
@@ -146,7 +145,7 @@ void ObjectCacheBench::prepare_sequence_data(StoragePageCache* cache, int64_t co
         auto* ptr = new std::vector<uint8_t>(_page_size);
         (*ptr)[0] = 1;
         PageCacheHandle handle;
-        ObjectCacheWriteOptions options;
+        MemCacheWriteOptions options;
         Status st = cache->insert(key, ptr, options, &handle);
         if (!st.ok()) {
             if (!st.is_already_exist()) {
@@ -162,7 +161,7 @@ void ObjectCacheBench::prepare_data(StoragePageCache* cache, int64_t count) {
         auto* ptr = new std::vector<uint8_t>(_page_size);
         (*ptr)[0] = 1;
         PageCacheHandle handle;
-        ObjectCacheWriteOptions options;
+        MemCacheWriteOptions options;
         Status st = cache->insert(key, ptr, options, &handle);
         if (!st.ok()) {
             if (!st.is_already_exist()) {
@@ -211,7 +210,7 @@ void ObjectCacheBench::random_insert_multi_threads(benchmark::State* state, Stor
         auto* ptr = new std::vector<uint8_t>(page_size);
         (*ptr)[0] = 1;
         PageCacheHandle handle;
-        ObjectCacheWriteOptions options;
+        MemCacheWriteOptions options;
         Status st = cache->insert(key, ptr, options, &handle);
         if (!st.ok()) {
             if (!st.is_already_exist()) {

be/src/cache/CMakeLists.txt

@@ -23,15 +23,15 @@ set(CACHE_FILES
   mem_space_monitor.cpp
   datacache.cpp
   datacache_utils.cpp
-  lrucache_engine.cpp
-  block_cache/block_cache.cpp
-  block_cache/io_buffer.cpp
-  block_cache/block_cache_hit_rate_counter.hpp
-  object_cache/page_cache.cpp
+  mem_cache/lrucache_engine.cpp
+  mem_cache/page_cache.cpp
+  disk_cache/block_cache.cpp
+  disk_cache/io_buffer.cpp
+  disk_cache/block_cache_hit_rate_counter.hpp
 )
 
 if (${WITH_STARCACHE} STREQUAL "ON")
-    list(APPEND CACHE_FILES starcache_engine.cpp)
+    list(APPEND CACHE_FILES disk_cache/starcache_engine.cpp)
     list(APPEND CACHE_FILES peer_cache_engine.cpp)
 endif()
 

be/src/cache/cache_metrics.h

@@ -55,19 +55,4 @@ struct DataCacheStatusUtils {
     }
 };
 
-struct DataCacheMetrics {
-    DataCacheStatus status;
-
-    size_t mem_quota_bytes;
-    size_t mem_used_bytes;
-    size_t disk_quota_bytes;
-    size_t disk_used_bytes;
-    size_t meta_used_bytes = 0;
-};
-
-#ifdef WITH_STARCACHE
-using StarCacheMetrics = starcache::CacheMetrics;
-using StarCacheStatus = starcache::CacheStatus;
-#endif
-
 } // namespace starrocks

be/src/cache/cache_options.h

@@ -14,17 +14,8 @@
 
 #pragma once
 
-#include <cstdint>
-#include <functional>
-#include <string>
-#include <vector>
-
-#include "cache/cache_metrics.h"
-#include "common/status.h"
-
 namespace starrocks {
 
-// Options to control how to create DataCache instance
 struct DataCacheOptions {
     bool enable_datacache = false;
     bool enable_cache_select = false;
@@ -37,82 +28,4 @@ struct DataCacheOptions {
     int64_t datacache_ttl_seconds = 0;
 };
 
-struct DirSpace {
-    std::string path;
-    size_t size;
-};
-
-struct RemoteCacheOptions {
-    double skip_read_factor = 0;
-};
-
-struct MemCacheOptions {
-    size_t mem_space_size = 0;
-};
-
-struct DiskCacheOptions {
-    // basic
-    size_t mem_space_size = 0;
-    std::vector<DirSpace> dir_spaces;
-    std::string meta_path;
-
-    // advanced
-    size_t block_size = 0;
-    bool enable_checksum = false;
-    bool enable_direct_io = false;
-    bool enable_tiered_cache = true;
-    bool enable_datacache_persistence = false;
-    size_t max_concurrent_inserts = 0;
-    size_t max_flying_memory_mb = 0;
-    double scheduler_threads_per_cpu = 0;
-    double skip_read_factor = 0;
-    uint32_t inline_item_count_limit = 0;
-    std::string eviction_policy;
-};
-
-struct BlockCacheOptions {
-    size_t block_size = 0;
-};
-
-struct WriteCacheOptions {
-    int8_t priority = 0;
-    // If ttl_seconds=0 (default), no ttl restriction will be set. If an old one exists, remove it.
-    uint64_t ttl_seconds = 0;
-    // If overwrite=true, the cache value will be replaced if it already exists.
-    bool overwrite = false;
-    bool async = false;
-    // When allow_zero_copy=true, it means the caller can ensure the target buffer not be released before
-    // to write finish. So the cache library can use the buffer directly without copying it to another buffer.
-    bool allow_zero_copy = false;
-    std::function<void(int, const std::string&)> callback = nullptr;
-
-    // The probability to evict other items if the cache space is full, which can help avoid frequent cache replacement
-    // and improve cache hit rate sometimes.
-    // It is expressed as a percentage. If evict_probability is 10, it means the probability to evict other data is 10%.
-    int32_t evict_probability = 100;
-
-    // The base frequency for target cache.
-    // When using multiple segment lru, a higher frequency may cause the cache is written to warm segment directly.
-    // For the default cache options, that `lru_segment_freq_bits` is 0:
-    // * The default `frequency=0` indicates the cache will be written to cold segment.
-    // * A frequency value greater than 0 indicates writing this cache directly to the warm segment.
-    int8_t frequency = 0;
-
-    struct Stats {
-        int64_t write_mem_bytes = 0;
-        int64_t write_disk_bytes = 0;
-    } stats;
-};
-
-struct ReadCacheOptions {
-    bool use_adaptor = false;
-    std::string remote_host;
-    int32_t remote_port;
-
-    struct Stats {
-        int64_t read_mem_bytes = 0;
-        int64_t read_disk_bytes = 0;
-    } stats;
-};
-
 } // namespace starrocks

be/src/cache/datacache.cpp

@@ -16,9 +16,9 @@
 
 #include "cache/datacache_utils.h"
 #include "cache/disk_space_monitor.h"
-#include "cache/lrucache_engine.h"
+#include "cache/mem_cache/lrucache_engine.h"
+#include "cache/mem_cache/page_cache.h"
 #include "cache/mem_space_monitor.h"
-#include "cache/object_cache/page_cache.h"
 #include "common/status.h"
 #include "gutil/strings/split.h"
 #include "gutil/strings/strip.h"
@@ -27,8 +27,8 @@
 #include "util/parse_util.h"
 
 #ifdef WITH_STARCACHE
+#include "cache/disk_cache/starcache_engine.h"
 #include "cache/peer_cache_engine.h"
-#include "cache/starcache_engine.h"
 #endif
 
 namespace starrocks {
@@ -44,11 +44,6 @@ Status DataCache::init(const std::vector<StorePath>& store_paths) {
     _block_cache = std::make_shared<BlockCache>();
     _page_cache = std::make_shared<StoragePageCache>();
 
-#if defined(WITH_STARCACHE)
-    _local_disk_cache_engine = "starcache";
-#endif
-    _local_mem_cache_engine = "lrucache";
-
     if (!config::datacache_enable) {
         config::disable_storage_page_cache = true;
         config::block_cache_enable = false;
@@ -173,89 +168,88 @@ BlockCacheOptions DataCache::_init_block_cache_options() {
     return cache_options;
 }
 
+#if defined(WITH_STARCACHE)
 StatusOr<DiskCacheOptions> DataCache::_init_disk_cache_options() {
     DiskCacheOptions cache_options;
 
-    if (_local_disk_cache_engine == "starcache") {
 #ifdef USE_STAROS
-        std::vector<string> corresponding_starlet_dirs;
-        if (config::datacache_unified_instance_enable && !config::starlet_cache_dir.empty()) {
-            // in older versions, users might set `starlet_cache_dir` instead of `storage_root_path` for starlet cache,
-            // we need to move starlet cache into storage_root_path/datacache
-            auto s = DataCacheUtils::get_corresponding_starlet_cache_dir(_store_paths, config::starlet_cache_dir);
-            if (!s.ok()) {
-                LOG(WARNING) << s.status().message() << ", change config::datacache_unified_instance_enable to false";
-                config::datacache_unified_instance_enable = false;
-            } else {
-                corresponding_starlet_dirs = *s;
-            }
+    std::vector<string> corresponding_starlet_dirs;
+    if (config::datacache_unified_instance_enable && !config::starlet_cache_dir.empty()) {
+        // in older versions, users might set `starlet_cache_dir` instead of `storage_root_path` for starlet cache,
+        // we need to move starlet cache into storage_root_path/datacache
+        auto s = DataCacheUtils::get_corresponding_starlet_cache_dir(_store_paths, config::starlet_cache_dir);
+        if (!s.ok()) {
+            LOG(WARNING) << s.status().message() << ", change config::datacache_unified_instance_enable to false";
+            config::datacache_unified_instance_enable = false;
+        } else {
+            corresponding_starlet_dirs = *s;
         }
-        int idx = 0;
-#endif
-
-        for (auto& root_path : _store_paths) {
-            // Because we have unified the datacache between datalake and starlet, we also need to unify the
-            // cache path and quota.
-            // To reuse the old cache data in `starlet_cache` directory, we try to rename it to the new `datacache`
-            // directory if it exists. To avoid the risk of cross disk renaming of a large amount of cached data,
-            // we do not automatically rename it when the source and destination directories are on different disks.
-            // In this case, users should manually remount the directories and restart them.
-            std::string datacache_path = root_path.path + "/datacache";
-#ifdef USE_STAROS
-            if (config::datacache_unified_instance_enable) {
-                std::string starlet_cache_path;
-                if (idx < corresponding_starlet_dirs.size()) {
-                    starlet_cache_path = corresponding_starlet_dirs[idx++];
-                } else {
-                    starlet_cache_path = root_path.path + "/starlet_cache/star_cache";
-                }
-                RETURN_IF_ERROR(DataCacheUtils::change_disk_path(starlet_cache_path, datacache_path));
-            }
-#endif
-            // Create it if not exist
-            Status st = FileSystem::Default()->create_dir_if_missing(datacache_path);
-            if (!st.ok()) {
-                LOG(ERROR) << "Fail to create datacache directory: " << datacache_path << ", reason: " << st.message();
-                return Status::InternalError("Fail to create datacache directory");
-            }
-
-            ASSIGN_OR_RETURN(int64_t disk_size, DataCacheUtils::parse_conf_datacache_disk_size(
-                                                        datacache_path, config::datacache_disk_size, -1));
-#ifdef USE_STAROS
-            // If the `datacache_disk_size` is manually set a positive value, we will use the maximum cache quota between
-            // dataleke and starlet cache as the quota of the unified cache. Otherwise, the cache quota will remain zero
-            // and then automatically adjusted based on the current avalible disk space.
-            if (config::datacache_unified_instance_enable &&
-                (!config::enable_datacache_disk_auto_adjust || disk_size > 0)) {
-                ASSIGN_OR_RETURN(
-                        int64_t starlet_cache_size,
-                        DataCacheUtils::parse_conf_datacache_disk_size(
-                                datacache_path, fmt::format("{}%", config::starlet_star_cache_disk_size_percent), -1));
-                disk_size = std::max(disk_size, starlet_cache_size);
-            }
-#endif
-            cache_options.dir_spaces.push_back({.path = datacache_path, .size = static_cast<size_t>(disk_size)});
-        }
-
-        if (cache_options.dir_spaces.empty()) {
-            config::enable_datacache_disk_auto_adjust = false;
-        }
-
-        cache_options.block_size = config::datacache_block_size;
-        cache_options.max_flying_memory_mb = config::datacache_max_flying_memory_mb;
-        cache_options.max_concurrent_inserts = config::datacache_max_concurrent_inserts;
-        cache_options.enable_checksum = config::datacache_checksum_enable;
-        cache_options.enable_direct_io = config::datacache_direct_io_enable;
-        cache_options.enable_tiered_cache = config::datacache_tiered_cache_enable;
-        cache_options.skip_read_factor = config::datacache_skip_read_factor;
-        cache_options.scheduler_threads_per_cpu = config::datacache_scheduler_threads_per_cpu;
-        cache_options.enable_datacache_persistence = config::datacache_persistence_enable;
-        cache_options.inline_item_count_limit = config::datacache_inline_item_count_limit;
-        cache_options.eviction_policy = config::datacache_eviction_policy;
     }
+    int idx = 0;
+#endif
+
+    for (auto& root_path : _store_paths) {
+        // Because we have unified the datacache between datalake and starlet, we also need to unify the
+        // cache path and quota.
+        // To reuse the old cache data in `starlet_cache` directory, we try to rename it to the new `datacache`
+        // directory if it exists. To avoid the risk of cross disk renaming of a large amount of cached data,
+        // we do not automatically rename it when the source and destination directories are on different disks.
+        // In this case, users should manually remount the directories and restart them.
+        std::string datacache_path = root_path.path + "/datacache";
+#ifdef USE_STAROS
+        if (config::datacache_unified_instance_enable) {
+            std::string starlet_cache_path;
+            if (idx < corresponding_starlet_dirs.size()) {
+                starlet_cache_path = corresponding_starlet_dirs[idx++];
+            } else {
+                starlet_cache_path = root_path.path + "/starlet_cache/star_cache";
+            }
+            RETURN_IF_ERROR(DataCacheUtils::change_disk_path(starlet_cache_path, datacache_path));
+        }
+#endif
+        // Create it if not exist
+        Status st = FileSystem::Default()->create_dir_if_missing(datacache_path);
+        if (!st.ok()) {
+            LOG(ERROR) << "Fail to create datacache directory: " << datacache_path << ", reason: " << st.message();
+            return Status::InternalError("Fail to create datacache directory");
+        }
+
+        ASSIGN_OR_RETURN(int64_t disk_size, DataCacheUtils::parse_conf_datacache_disk_size(
+                                                    datacache_path, config::datacache_disk_size, -1));
+#ifdef USE_STAROS
+        // If the `datacache_disk_size` is manually set a positive value, we will use the maximum cache quota between
+        // dataleke and starlet cache as the quota of the unified cache. Otherwise, the cache quota will remain zero
+        // and then automatically adjusted based on the current avalible disk space.
+        if (config::datacache_unified_instance_enable &&
+            (!config::enable_datacache_disk_auto_adjust || disk_size > 0)) {
+            ASSIGN_OR_RETURN(
+                    int64_t starlet_cache_size,
+                    DataCacheUtils::parse_conf_datacache_disk_size(
+                            datacache_path, fmt::format("{}%", config::starlet_star_cache_disk_size_percent), -1));
+            disk_size = std::max(disk_size, starlet_cache_size);
+        }
+#endif
+        cache_options.dir_spaces.push_back({.path = datacache_path, .size = static_cast<size_t>(disk_size)});
+    }
+
+    if (cache_options.dir_spaces.empty()) {
+        config::enable_datacache_disk_auto_adjust = false;
+    }
+
+    cache_options.block_size = config::datacache_block_size;
+    cache_options.max_flying_memory_mb = config::datacache_max_flying_memory_mb;
+    cache_options.max_concurrent_inserts = config::datacache_max_concurrent_inserts;
+    cache_options.enable_checksum = config::datacache_checksum_enable;
+    cache_options.enable_direct_io = config::datacache_direct_io_enable;
+    cache_options.skip_read_factor = config::datacache_skip_read_factor;
+    cache_options.scheduler_threads_per_cpu = config::datacache_scheduler_threads_per_cpu;
+    cache_options.enable_datacache_persistence = config::datacache_persistence_enable;
+    cache_options.inline_item_count_limit = config::datacache_inline_item_count_limit;
+    cache_options.eviction_policy = config::datacache_eviction_policy;
 
     return cache_options;
 }
+#endif
 
 static bool parse_resource_str(const string& str, string* value) {
     if (!str.empty()) {
@@ -289,7 +283,7 @@ void DataCache::try_release_resource_before_core_dump() {
     };
 
     if (_local_mem_cache != nullptr && need_release("data_cache")) {
-        (void)_local_mem_cache->update_mem_quota(0, false);
+        (void)_local_mem_cache->update_mem_quota(0);
     }
 }
 

be/src/cache/datacache.h

@@ -14,8 +14,9 @@
 
 #pragma once
 
-#include "cache/block_cache/block_cache.h"
-#include "cache/local_cache_engine.h"
+#include "cache/disk_cache/block_cache.h"
+#include "cache/disk_cache/local_disk_cache_engine.h"
+#include "cache/mem_cache/local_mem_cache_engine.h"
 #include "common/status.h"
 
 namespace starrocks {
@@ -39,16 +40,13 @@ public:
 
     void try_release_resource_before_core_dump();
 
-    void set_local_mem_cache(std::shared_ptr<LocalCacheEngine> local_mem_cache) {
-        _local_mem_cache = std::move(local_mem_cache);
-    }
-    void set_local_disk_cache(std::shared_ptr<LocalCacheEngine> local_disk_cache) {
+    void set_local_disk_cache(std::shared_ptr<LocalDiskCacheEngine> local_disk_cache) {
         _local_disk_cache = std::move(local_disk_cache);
     }
     void set_page_cache(std::shared_ptr<StoragePageCache> page_cache) { _page_cache = std::move(page_cache); }
 
-    LocalCacheEngine* local_mem_cache() { return _local_mem_cache.get(); }
-    LocalCacheEngine* local_disk_cache() { return _local_disk_cache.get(); }
+    LocalMemCacheEngine* local_mem_cache() { return _local_mem_cache.get(); }
+    LocalDiskCacheEngine* local_disk_cache() { return _local_disk_cache.get(); }
     BlockCache* block_cache() const { return _block_cache.get(); }
     void set_block_cache(std::shared_ptr<BlockCache> block_cache) { _block_cache = std::move(block_cache); }
     StoragePageCache* page_cache() const { return _page_cache.get(); }
@@ -63,11 +61,11 @@ public:
 
 private:
     StatusOr<MemCacheOptions> _init_mem_cache_options();
-    StatusOr<DiskCacheOptions> _init_disk_cache_options();
     RemoteCacheOptions _init_remote_cache_options();
     BlockCacheOptions _init_block_cache_options();
 
 #if defined(WITH_STARCACHE)
+    StatusOr<DiskCacheOptions> _init_disk_cache_options();
     Status _init_starcache_engine(DiskCacheOptions* cache_options);
     Status _init_peer_cache(const RemoteCacheOptions& cache_options);
 #endif
@@ -78,10 +76,8 @@ private:
     std::vector<StorePath> _store_paths;
 
     // cache engine
-    std::string _local_mem_cache_engine;
-    std::string _local_disk_cache_engine;
-    std::shared_ptr<LocalCacheEngine> _local_mem_cache;
-    std::shared_ptr<LocalCacheEngine> _local_disk_cache;
+    std::shared_ptr<LocalMemCacheEngine> _local_mem_cache;
+    std::shared_ptr<LocalDiskCacheEngine> _local_disk_cache;
     std::shared_ptr<RemoteCacheEngine> _remote_cache;
 
     std::shared_ptr<BlockCache> _block_cache;

be/src/cache/datacache_utils.cpp

@@ -21,26 +21,29 @@
 
 #include "absl/status/statusor.h"
 #include "absl/strings/str_split.h"
+#include "cache/mem_cache/local_mem_cache_engine.h"
 #include "fs/fs.h"
 #include "gutil/strings/split.h"
 #include "util/parse_util.h"
 
 namespace starrocks {
-void DataCacheUtils::set_metrics_from_thrift(TDataCacheMetrics& t_metrics, const DataCacheMetrics& metrics) {
+
+void DataCacheUtils::set_metrics_to_thrift(TDataCacheMetrics& t_metrics, const DataCacheDiskMetrics& metrics) {
     t_metrics.__set_status(DataCacheStatusUtils::to_thrift(metrics.status));
     t_metrics.__set_disk_quota_bytes(metrics.disk_quota_bytes);
     t_metrics.__set_disk_used_bytes(metrics.disk_used_bytes);
+}
+
+void DataCacheUtils::set_metrics_to_thrift(TDataCacheMetrics& t_metrics, const DataCacheMemMetrics& metrics) {
     t_metrics.__set_mem_quota_bytes(metrics.mem_quota_bytes);
     t_metrics.__set_mem_used_bytes(metrics.mem_used_bytes);
 }
 
 #ifdef WITH_STARCACHE
-void DataCacheUtils::set_metrics_from_thrift(TDataCacheMetrics& t_metrics, const StarCacheMetrics& metrics) {
+void DataCacheUtils::set_disk_metrics_to_thrift(TDataCacheMetrics& t_metrics, const StarCacheMetrics& metrics) {
     t_metrics.__set_status(DataCacheStatusUtils::to_thrift(static_cast<DataCacheStatus>(metrics.status)));
     t_metrics.__set_disk_quota_bytes(metrics.disk_quota_bytes);
     t_metrics.__set_disk_used_bytes(metrics.disk_used_bytes);
-    t_metrics.__set_mem_quota_bytes(metrics.mem_quota_bytes);
-    t_metrics.__set_mem_used_bytes(metrics.mem_used_bytes);
 }
 #endif
 

be/src/cache/datacache_utils.h

@@ -15,18 +15,21 @@
 #pragma once
 
 #include "cache/cache_metrics.h"
-#include "cache/local_cache_engine.h"
+#include "cache/disk_cache/starcache_engine.h"
 #include "gen_cpp/DataCache_types.h"
 #include "storage/options.h"
 
 namespace starrocks {
 
+class DataCacheMemMetrics;
+
 class DataCacheUtils {
 public:
-    static void set_metrics_from_thrift(TDataCacheMetrics& t_metrics, const DataCacheMetrics& metrics);
+    static void set_metrics_to_thrift(TDataCacheMetrics& t_metrics, const DataCacheDiskMetrics& metrics);
+    static void set_metrics_to_thrift(TDataCacheMetrics& t_metrics, const DataCacheMemMetrics& metrics);
 
 #ifdef WITH_STARCACHE
-    static void set_metrics_from_thrift(TDataCacheMetrics& t_metrics, const StarCacheMetrics& metrics);
+    static void set_disk_metrics_to_thrift(TDataCacheMetrics& t_metrics, const StarCacheMetrics& metrics);
 #endif
 
     static Status parse_conf_datacache_mem_size(const std::string& conf_mem_size_str, int64_t mem_limit,

be/src/cache/disk_cache/block_cache.cpp

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "cache/block_cache/block_cache.h"
+#include "cache/disk_cache/block_cache.h"
 
 #include <fmt/format.h>
 
@@ -36,7 +36,7 @@ BlockCache::~BlockCache() {
     (void)shutdown();
 }
 
-Status BlockCache::init(const BlockCacheOptions& options, std::shared_ptr<LocalCacheEngine> local_cache,
+Status BlockCache::init(const BlockCacheOptions& options, std::shared_ptr<LocalDiskCacheEngine> local_cache,
                         std::shared_ptr<RemoteCacheEngine> remote_cache) {
     _block_size = std::min(options.block_size, MAX_BLOCK_SIZE);
     _local_cache = std::move(local_cache);
@@ -45,7 +45,8 @@ Status BlockCache::init(const BlockCacheOptions& options, std::shared_ptr<LocalC
     return Status::OK();
 }
 
-Status BlockCache::write(const CacheKey& cache_key, off_t offset, const IOBuffer& buffer, WriteCacheOptions* options) {
+Status BlockCache::write(const CacheKey& cache_key, off_t offset, const IOBuffer& buffer,
+                         DiskCacheWriteOptions* options) {
     if (offset % _block_size != 0) {
         LOG(WARNING) << "write block key: " << cache_key << " with invalid args, offset: " << offset;
         return Status::InvalidArgument(strings::Substitute("offset must be aligned by block size $0", _block_size));
@@ -62,7 +63,7 @@ Status BlockCache::write(const CacheKey& cache_key, off_t offset, const IOBuffer
 static void empty_deleter(void*) {}
 
 Status BlockCache::write(const CacheKey& cache_key, off_t offset, size_t size, const char* data,
-                         WriteCacheOptions* options) {
+                         DiskCacheWriteOptions* options) {
     if (!data) {
         return Status::InvalidArgument("invalid data buffer");
     }
@@ -73,7 +74,7 @@ Status BlockCache::write(const CacheKey& cache_key, off_t offset, size_t size, c
 }
 
 Status BlockCache::read(const CacheKey& cache_key, off_t offset, size_t size, IOBuffer* buffer,
-                        ReadCacheOptions* options) {
+                        DiskCacheReadOptions* options) {
     if (size == 0) {
         return Status::OK();
     }
@@ -84,7 +85,7 @@ Status BlockCache::read(const CacheKey& cache_key, off_t offset, size_t size, IO
 }
 
 StatusOr<size_t> BlockCache::read(const CacheKey& cache_key, off_t offset, size_t size, char* data,
-                                  ReadCacheOptions* options) {
+                                  DiskCacheReadOptions* options) {
     IOBuffer buffer;
     RETURN_IF_ERROR(read(cache_key, offset, size, &buffer, options));
     buffer.copy_to(data);
@@ -117,7 +118,7 @@ Status BlockCache::remove(const CacheKey& cache_key, off_t offset, size_t size)
 }
 
 Status BlockCache::read_buffer_from_remote_cache(const std::string& cache_key, size_t offset, size_t size,
-                                                 IOBuffer* buffer, ReadCacheOptions* options) {
+                                                 IOBuffer* buffer, DiskCacheReadOptions* options) {
     if (size == 0) {
         return Status::OK();
     }

be/src/cache/disk_cache/block_cache.h

@@ -16,12 +16,16 @@
 
 #include <atomic>
 
-#include "cache/local_cache_engine.h"
+#include "cache/disk_cache/local_disk_cache_engine.h"
 #include "cache/remote_cache_engine.h"
 #include "common/status.h"
 
 namespace starrocks {
 
+struct BlockCacheOptions {
+    size_t block_size = 0;
+};
+
 class BlockCache {
 public:
     using CacheKey = std::string;
@@ -33,22 +37,23 @@ public:
     ~BlockCache();
 
     // Init the block cache instance
-    Status init(const BlockCacheOptions& options, std::shared_ptr<LocalCacheEngine> local_cache,
+    Status init(const BlockCacheOptions& options, std::shared_ptr<LocalDiskCacheEngine> local_cache,
                 std::shared_ptr<RemoteCacheEngine> remote_cache);
 
     // Write data buffer to cache, the `offset` must be aligned by block size
-    Status write(const CacheKey& cache_key, off_t offset, const IOBuffer& buffer, WriteCacheOptions* options = nullptr);
+    Status write(const CacheKey& cache_key, off_t offset, const IOBuffer& buffer,
+                 DiskCacheWriteOptions* options = nullptr);
 
     Status write(const CacheKey& cache_key, off_t offset, size_t size, const char* data,
-                 WriteCacheOptions* options = nullptr);
+                 DiskCacheWriteOptions* options = nullptr);
 
     // Read data from cache, it returns the data size if successful; otherwise the error status
     // will be returned. The offset and size must be aligned by block size.
     Status read(const CacheKey& cache_key, off_t offset, size_t size, IOBuffer* buffer,
-                ReadCacheOptions* options = nullptr);
+                DiskCacheReadOptions* options = nullptr);
 
     StatusOr<size_t> read(const CacheKey& cache_key, off_t offset, size_t size, char* data,
-                          ReadCacheOptions* options = nullptr);
+                          DiskCacheReadOptions* options = nullptr);
 
     bool exist(const CacheKey& cache_key, off_t offset, size_t size) const;
 
@@ -57,7 +62,7 @@ public:
 
     // Read data from remote cache
     Status read_buffer_from_remote_cache(const std::string& cache_key, size_t offset, size_t size, IOBuffer* buffer,
-                                         ReadCacheOptions* options);
+                                         DiskCacheReadOptions* options);
 
     void record_read_local_cache(size_t size, int64_t latency_us);
 
@@ -73,15 +78,14 @@ public:
     bool is_initialized() const { return _initialized.load(std::memory_order_relaxed); }
 
     bool available() const { return is_initialized() && _local_cache->available(); }
-    bool mem_cache_available() const { return is_initialized() && _local_cache->mem_cache_available(); }
 
-    std::shared_ptr<LocalCacheEngine> local_cache() { return _local_cache; }
+    std::shared_ptr<LocalDiskCacheEngine> local_cache() { return _local_cache; }
 
     static const size_t MAX_BLOCK_SIZE;
 
 private:
     size_t _block_size = 0;
-    std::shared_ptr<LocalCacheEngine> _local_cache;
+    std::shared_ptr<LocalDiskCacheEngine> _local_cache;
     std::shared_ptr<RemoteCacheEngine> _remote_cache;
     std::atomic<bool> _initialized = false;
 };

be/src/cache/disk_cache/io_buffer.cpp

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "cache/block_cache/io_buffer.h"
+#include "cache/disk_cache/io_buffer.h"
 
 #include "gutil/strings/fastmem.h"
 

be/src/cache/disk_cache/local_disk_cache_engine.h

@@ -0,0 +1,137 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "cache/cache_metrics.h"
+#include "cache/disk_cache/io_buffer.h"
+#include "common/status.h"
+
+namespace starrocks {
+
+struct DirSpace {
+    std::string path;
+    size_t size;
+};
+
+struct DiskCacheOptions {
+    // basic
+    size_t mem_space_size = 0;
+    std::vector<DirSpace> dir_spaces;
+    std::string meta_path;
+
+    // advanced
+    size_t block_size = 0;
+    bool enable_checksum = false;
+    bool enable_direct_io = false;
+    bool enable_datacache_persistence = false;
+    size_t max_concurrent_inserts = 0;
+    size_t max_flying_memory_mb = 0;
+    double scheduler_threads_per_cpu = 0;
+    double skip_read_factor = 0;
+    uint32_t inline_item_count_limit = 0;
+    std::string eviction_policy;
+};
+
+struct DiskCacheWriteOptions {
+    int8_t priority = 0;
+    // If ttl_seconds=0 (default), no ttl restriction will be set. If an old one exists, remove it.
+    uint64_t ttl_seconds = 0;
+    // If overwrite=true, the cache value will be replaced if it already exists.
+    bool overwrite = false;
+    bool async = false;
+    // When allow_zero_copy=true, it means the caller can ensure the target buffer not be released before
+    // to write finish. So the cache library can use the buffer directly without copying it to another buffer.
+    bool allow_zero_copy = false;
+    std::function<void(int, const std::string&)> callback = nullptr;
+
+    // The base frequency for target cache.
+    // When using multiple segment lru, a higher frequency may cause the cache is written to warm segment directly.
+    // For the default cache options, that `lru_segment_freq_bits` is 0:
+    // * The default `frequency=0` indicates the cache will be written to cold segment.
+    // * A frequency value greater than 0 indicates writing this cache directly to the warm segment.
+    int8_t frequency = 0;
+
+    struct Stats {
+        int64_t write_mem_bytes = 0;
+        int64_t write_disk_bytes = 0;
+    } stats;
+};
+
+struct DiskCacheReadOptions {
+    bool use_adaptor = false;
+    std::string remote_host;
+    int32_t remote_port;
+
+    struct Stats {
+        int64_t read_mem_bytes = 0;
+        int64_t read_disk_bytes = 0;
+    } stats;
+};
+
+struct DataCacheDiskMetrics {
+    DataCacheStatus status;
+
+    size_t disk_quota_bytes;
+    size_t disk_used_bytes;
+};
+
+class LocalDiskCacheEngine {
+public:
+    virtual ~LocalDiskCacheEngine() = default;
+
+    virtual bool is_initialized() const = 0;
+
+    // Write data to cache
+    virtual Status write(const std::string& key, const IOBuffer& buffer, DiskCacheWriteOptions* options) = 0;
+
+    // Read data from cache, it returns the data size if successful; otherwise the error status
+    // will be returned.
+    virtual Status read(const std::string& key, size_t off, size_t size, IOBuffer* buffer,
+                        DiskCacheReadOptions* options) = 0;
+
+    virtual bool exist(const std::string& key) const = 0;
+
+    // Remove data from cache.
+    virtual Status remove(const std::string& key) = 0;
+
+    // Update the datacache disk space information, such as disk quota or disk path.
+    virtual Status update_disk_spaces(const std::vector<DirSpace>& spaces) = 0;
+
+    // Update the datacache inline cache count limit
+    virtual Status update_inline_cache_count_limit(int32_t limit) = 0;
+
+    virtual const DataCacheDiskMetrics cache_metrics() const = 0;
+
+    virtual void record_read_remote(size_t size, int64_t latency_us) = 0;
+
+    virtual void record_read_cache(size_t size, int64_t latency_us) = 0;
+
+    virtual Status shutdown() = 0;
+
+    virtual bool has_disk_cache() const = 0;
+    virtual bool available() const = 0;
+    virtual void disk_spaces(std::vector<DirSpace>* spaces) const = 0;
+
+    // Get the lookup count, including cache hit count and cache miss count.
+    virtual size_t lookup_count() const = 0;
+
+    // Get the cache hit count.
+    virtual size_t hit_count() const = 0;
+
+    // Remove all cache entries that are not actively in use.
+    virtual Status prune() = 0;
+};
+
+} // namespace starrocks

be/src/cache/disk_cache/starcache_engine.cpp

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "cache/starcache_engine.h"
+#include "cache/disk_cache/starcache_engine.h"
 
 #include <filesystem>
 
@@ -55,8 +55,6 @@ Status StarCacheEngine::init(const DiskCacheOptions& options) {
     }
     opt.lru_segment_freq_bits = 0;
 
-    _enable_tiered_cache = options.enable_tiered_cache;
-    _enable_datacache_persistence = options.enable_datacache_persistence;
     _cache = std::make_shared<starcache::StarCache>();
     RETURN_IF_ERROR(to_status(_cache->init(opt)));
 
@@ -68,7 +66,7 @@ Status StarCacheEngine::init(const DiskCacheOptions& options) {
     return Status::OK();
 }
 
-Status StarCacheEngine::write(const std::string& key, const IOBuffer& buffer, WriteCacheOptions* options) {
+Status StarCacheEngine::write(const std::string& key, const IOBuffer& buffer, DiskCacheWriteOptions* options) {
     if (!options) {
         return to_status(_cache->set(key, buffer.const_raw_buf(), nullptr));
     }
@@ -80,12 +78,8 @@ Status StarCacheEngine::write(const std::string& key, const IOBuffer& buffer, Wr
     opts.async = options->async;
     opts.keep_alive = options->allow_zero_copy;
     opts.callback = options->callback;
-    if (!_enable_datacache_persistence && _enable_tiered_cache) {
-        opts.mode = starcache::WriteOptions::WriteMode::WRITE_BACK;
-    } else {
-        opts.mode = starcache::WriteOptions::WriteMode::WRITE_THROUGH;
-    }
-    opts.evict_probability = options->evict_probability;
+    opts.mode = starcache::WriteOptions::WriteMode::WRITE_THROUGH;
+    opts.evict_probability = 100;
     opts.ignore_inline = true;
     Status st;
     {
@@ -104,14 +98,13 @@ Status StarCacheEngine::write(const std::string& key, const IOBuffer& buffer, Wr
 }
 
 Status StarCacheEngine::read(const std::string& key, size_t off, size_t size, IOBuffer* buffer,
-                             ReadCacheOptions* options) {
+                             DiskCacheReadOptions* options) {
     if (!options) {
         return to_status(_cache->read(key, off, size, &buffer->raw_buf(), nullptr));
     }
     starcache::ReadOptions opts;
     opts.use_adaptor = options->use_adaptor;
-    opts.mode = _enable_tiered_cache ? starcache::ReadOptions::ReadMode::READ_BACK
-                                     : starcache::ReadOptions::ReadMode::READ_THROUGH;
+    opts.mode = starcache::ReadOptions::ReadMode::READ_THROUGH;
     auto st = to_status(_cache->read(key, off, size, &buffer->raw_buf(), &opts));
     if (st.ok()) {
         options->stats.read_mem_bytes = opts.stats.read_mem_bytes;
@@ -120,52 +113,6 @@ Status StarCacheEngine::read(const std::string& key, size_t off, size_t size, IO
     return st;
 }
 
-Status StarCacheEngine::insert(const std::string& key, void* value, size_t size, ObjectCacheDeleter deleter,
-                               ObjectCacheHandlePtr* handle, const ObjectCacheWriteOptions& options) {
-    starcache::ObjectHandle* obj_hdl = new starcache::ObjectHandle;
-    auto obj_deleter = [deleter, key, value] {
-        // For temporary compatibility with old deleters.
-        CacheKey cache_key(key);
-        deleter(cache_key, value);
-    };
-    starcache::WriteOptions opts;
-    opts.priority = options.priority;
-    opts.ttl_seconds = options.ttl_seconds;
-    opts.overwrite = options.overwrite;
-    opts.evict_probability = options.evict_probability;
-    Status st = to_status(_cache->set_object(key, value, size, obj_deleter, obj_hdl, &opts));
-    if (!st.ok()) {
-        delete obj_hdl;
-    } else if (handle) {
-        // Try release the old handle before fill it with a new one.
-        _try_release_obj_handle(*handle);
-        *handle = reinterpret_cast<ObjectCacheHandlePtr>(obj_hdl);
-    }
-    return st;
-}
-
-Status StarCacheEngine::lookup(const std::string& key, ObjectCacheHandlePtr* handle, ObjectCacheReadOptions* options) {
-    starcache::ObjectHandle* obj_hdl = new starcache::ObjectHandle;
-    // Skip checking options temporarily because there is no valid members in `ObjectCacheReadOptions` now.
-    Status st = to_status(_cache->get_object(key, obj_hdl, nullptr));
-    if (!st.ok()) {
-        delete obj_hdl;
-    } else if (handle) {
-        _try_release_obj_handle(*handle);
-        *handle = reinterpret_cast<ObjectCacheHandlePtr>(obj_hdl);
-    }
-    return st;
-}
-
-void StarCacheEngine::release(ObjectCacheHandlePtr handle) {
-    _try_release_obj_handle(handle);
-}
-
-const void* StarCacheEngine::value(ObjectCacheHandlePtr handle) {
-    auto obj_hdl = reinterpret_cast<starcache::ObjectHandle*>(handle);
-    return obj_hdl->ptr();
-}
-
 bool StarCacheEngine::exist(const std::string& key) const {
     return _cache->exist(key);
 }
@@ -174,22 +121,6 @@ Status StarCacheEngine::remove(const std::string& key) {
     return to_status(_cache->remove(key));
 }
 
-Status StarCacheEngine::update_mem_quota(size_t quota_bytes, bool flush_to_disk) {
-    Status st = to_status(_cache->update_mem_quota(quota_bytes, flush_to_disk));
-    _refresh_quota();
-    return st;
-}
-
-Status StarCacheEngine::adjust_mem_quota(int64_t delta, size_t min_capacity) {
-    auto starcache_metrics = _cache->metrics();
-    size_t capacity = starcache_metrics.mem_quota_bytes;
-    int64_t new_capacity = capacity + delta;
-    if (new_capacity < (int64_t)min_capacity) {
-        return Status::InvalidArgument("target capacity is less than the minimum capacity");
-    }
-    return to_status(_cache->update_mem_quota(new_capacity, false));
-}
-
 Status StarCacheEngine::update_disk_spaces(const std::vector<DirSpace>& spaces) {
     std::vector<starcache::DirSpace> disk_spaces;
     disk_spaces.reserve(spaces.size());
@@ -209,14 +140,11 @@ const StarCacheMetrics StarCacheEngine::starcache_metrics(int level) const {
     return _cache->metrics(level);
 }
 
-const DataCacheMetrics StarCacheEngine::cache_metrics() const {
+const DataCacheDiskMetrics StarCacheEngine::cache_metrics() const {
     auto starcache_metrics = _cache->metrics(0);
-    DataCacheMetrics metrics = {.status = static_cast<DataCacheStatus>(starcache_metrics.status),
-                                .mem_quota_bytes = starcache_metrics.mem_quota_bytes,
-                                .mem_used_bytes = starcache_metrics.mem_used_bytes,
-                                .disk_quota_bytes = starcache_metrics.disk_quota_bytes,
-                                .disk_used_bytes = starcache_metrics.disk_used_bytes,
-                                .meta_used_bytes = starcache_metrics.meta_used_bytes};
+    DataCacheDiskMetrics metrics = {.status = static_cast<DataCacheStatus>(starcache_metrics.status),
+                                    .disk_quota_bytes = starcache_metrics.disk_quota_bytes,
+                                    .disk_used_bytes = starcache_metrics.disk_used_bytes};
     return metrics;
 }
 
@@ -239,18 +167,9 @@ Status StarCacheEngine::shutdown() {
 
 void StarCacheEngine::_refresh_quota() {
     auto metrics = starcache_metrics(0);
-    _mem_quota.store(metrics.mem_quota_bytes, std::memory_order_relaxed);
     _disk_quota.store(metrics.disk_quota_bytes, std::memory_order_relaxed);
 }
 
-void StarCacheEngine::_try_release_obj_handle(ObjectCacheHandlePtr handle) {
-    if (handle) {
-        auto obj_hdl = reinterpret_cast<starcache::ObjectHandle*>(handle);
-        obj_hdl->release();
-        delete obj_hdl;
-    }
-}
-
 void StarCacheEngine::disk_spaces(std::vector<DirSpace>* spaces) const {
     spaces->clear();
     auto metrics = starcache_metrics(0);
@@ -259,18 +178,6 @@ void StarCacheEngine::disk_spaces(std::vector<DirSpace>* spaces) const {
     }
 }
 
-size_t StarCacheEngine::mem_quota() const {
-    starcache::CacheMetrics metrics = _cache->metrics(0);
-    // TODO: optimizer later
-    return metrics.mem_quota_bytes;
-}
-
-size_t StarCacheEngine::mem_usage() const {
-    // TODO: add meta size?
-    starcache::CacheMetrics metrics = _cache->metrics(0);
-    return metrics.mem_used_bytes;
-}
-
 size_t StarCacheEngine::lookup_count() const {
     starcache::CacheMetrics metrics = _cache->metrics(1);
     return metrics.detail_l1->hit_count + metrics.detail_l1->miss_count;
@@ -281,17 +188,6 @@ size_t StarCacheEngine::hit_count() const {
     return metrics.detail_l1->hit_count;
 }
 
-const ObjectCacheMetrics StarCacheEngine::metrics() const {
-    auto starcache_metrics = _cache->metrics(2);
-    ObjectCacheMetrics m;
-    m.capacity = starcache_metrics.mem_quota_bytes;
-    m.usage = starcache_metrics.mem_used_bytes;
-    m.lookup_count = starcache_metrics.detail_l1->hit_count + starcache_metrics.detail_l1->miss_count;
-    m.hit_count = starcache_metrics.detail_l1->hit_count;
-    m.object_item_count = starcache_metrics.detail_l2->object_item_count;
-    return m;
-}
-
 Status StarCacheEngine::prune() {
     return to_status(_cache->update_mem_quota(0, false));
 }

be/src/cache/disk_cache/starcache_engine.h

@@ -14,48 +14,51 @@
 
 #pragma once
 
-#include "cache/local_cache_engine.h"
+#include "cache/disk_cache/local_disk_cache_engine.h"
 #include "common/status.h"
+
+#ifdef WITH_STARCACHE
 #include "starcache/star_cache.h"
 #include "starcache/time_based_cache_adaptor.h"
+#else
+namespace starcache {
+class StarCache;
+class TimeBasedCacheAdaptor;
+struct CacheMetrics;
+} // namespace starcache
+#endif
 
 namespace starrocks {
 
-class StarCacheEngine : public LocalCacheEngine {
+#ifdef WITH_STARCACHE
+using StarCacheMetrics = starcache::CacheMetrics;
+#endif
+
+class StarCacheEngine : public LocalDiskCacheEngine {
 public:
     StarCacheEngine() = default;
-    virtual ~StarCacheEngine() override = default;
+    ~StarCacheEngine() override = default;
 
     Status init(const DiskCacheOptions& options);
     bool is_initialized() const override { return _initialized.load(std::memory_order_relaxed); }
 
-    Status write(const std::string& key, const IOBuffer& buffer, WriteCacheOptions* options) override;
-    Status read(const std::string& key, size_t off, size_t size, IOBuffer* buffer, ReadCacheOptions* options) override;
-
-    Status insert(const std::string& key, void* value, size_t size, ObjectCacheDeleter deleter,
-                  ObjectCacheHandlePtr* handle, const ObjectCacheWriteOptions& options) override;
-
-    Status lookup(const std::string& key, ObjectCacheHandlePtr* handle, ObjectCacheReadOptions* options) override;
-
-    void release(ObjectCacheHandlePtr handle) override;
-
-    const void* value(ObjectCacheHandlePtr handle) override;
+    Status write(const std::string& key, const IOBuffer& buffer, DiskCacheWriteOptions* options) override;
+    Status read(const std::string& key, size_t off, size_t size, IOBuffer* buffer,
+                DiskCacheReadOptions* options) override;
 
     bool exist(const std::string& key) const override;
 
     Status remove(const std::string& key) override;
 
-    Status adjust_mem_quota(int64_t delta, size_t min_capacity) override;
-
-    Status update_mem_quota(size_t quota_bytes, bool flush_to_disk) override;
-
     Status update_disk_spaces(const std::vector<DirSpace>& spaces) override;
 
     Status update_inline_cache_count_limit(int32_t limit) override;
 
+#ifdef WITH_STARCACHE
     const StarCacheMetrics starcache_metrics(int level) const;
+#endif
 
-    const DataCacheMetrics cache_metrics() const override;
+    const DataCacheDiskMetrics cache_metrics() const override;
 
     void record_read_remote(size_t size, int64_t latency_us) override;
 
@@ -63,38 +66,25 @@ public:
 
     Status shutdown() override;
 
-    LocalCacheEngineType engine_type() override { return LocalCacheEngineType::STARCACHE; }
-
     std::shared_ptr<starcache::StarCache> starcache_instance() { return _cache; }
-    bool has_mem_cache() const override { return _mem_quota.load(std::memory_order_relaxed) > 0; }
     bool has_disk_cache() const override { return _disk_quota.load(std::memory_order_relaxed) > 0; }
-    bool available() const override { return is_initialized() && (has_mem_cache() || has_disk_cache()); }
-    bool mem_cache_available() const override { return is_initialized() && has_mem_cache(); }
+    bool available() const override { return is_initialized() && has_disk_cache(); }
 
     void disk_spaces(std::vector<DirSpace>* spaces) const override;
 
-    size_t mem_quota() const override;
-    size_t mem_usage() const override;
-
     size_t lookup_count() const override;
 
     size_t hit_count() const override;
 
-    const ObjectCacheMetrics metrics() const override;
-
     Status prune() override;
 
 private:
     void _refresh_quota();
-    void _try_release_obj_handle(ObjectCacheHandlePtr handle);
 
     std::shared_ptr<starcache::StarCache> _cache;
     std::unique_ptr<starcache::TimeBasedCacheAdaptor> _cache_adaptor;
-    bool _enable_tiered_cache = false;
-    bool _enable_datacache_persistence = false;
     std::atomic<bool> _initialized = false;
 
-    std::atomic<size_t> _mem_quota = 0;
     std::atomic<size_t> _disk_quota = 0;
 };
 } // namespace starrocks

be/src/cache/disk_space_monitor.cpp

@@ -224,10 +224,10 @@ dev_t DiskSpace::FileSystemWrapper::device_id(const std::string& path) {
     return DataCacheUtils::disk_device_id(path);
 }
 
-DiskSpaceMonitor::DiskSpaceMonitor(LocalCacheEngine* cache)
+DiskSpaceMonitor::DiskSpaceMonitor(LocalDiskCacheEngine* cache)
         : _cache(cache), _fs(std::make_shared<DiskSpace::FileSystemWrapper>()) {}
 
-DiskSpaceMonitor::DiskSpaceMonitor(LocalCacheEngine* cache, std::shared_ptr<DiskSpace::FileSystemWrapper> fs)
+DiskSpaceMonitor::DiskSpaceMonitor(LocalDiskCacheEngine* cache, std::shared_ptr<DiskSpace::FileSystemWrapper> fs)
         : _cache(cache), _fs(std::move(fs)) {}
 
 DiskSpaceMonitor::~DiskSpaceMonitor() {

be/src/cache/disk_space_monitor.h

@@ -20,7 +20,7 @@
 #include <unordered_map>
 
 #include "cache/cache_options.h"
-#include "cache/local_cache_engine.h"
+#include "cache/disk_cache/local_disk_cache_engine.h"
 #include "common/status.h"
 #include "fs/fs.h"
 #include "util/disk_info.h"
@@ -118,8 +118,8 @@ private:
 
 class DiskSpaceMonitor {
 public:
-    DiskSpaceMonitor(LocalCacheEngine* cache);
-    DiskSpaceMonitor(LocalCacheEngine* cache, std::shared_ptr<DiskSpace::FileSystemWrapper> fs);
+    DiskSpaceMonitor(LocalDiskCacheEngine* cache);
+    DiskSpaceMonitor(LocalDiskCacheEngine* cache, std::shared_ptr<DiskSpace::FileSystemWrapper> fs);
     ~DiskSpaceMonitor();
 
     Status init(std::vector<DirSpace>* dir_spaces);
@@ -152,7 +152,7 @@ private:
 
     size_t _total_cache_usage = 0;
     size_t _total_cache_quota = 0;
-    LocalCacheEngine* _cache = nullptr;
+    LocalDiskCacheEngine* _cache = nullptr;
     std::shared_ptr<DiskSpace::FileSystemWrapper> _fs = nullptr;
 };
 

be/src/cache/mem_cache/local_mem_cache_engine.h

@@ -14,46 +14,69 @@
 
 #pragma once
 
-#include "cache/block_cache/io_buffer.h"
-#include "cache/cache_options.h"
-#include "cache/object_cache/cache_types.h"
+#include "cache/cache_metrics.h"
+#include "cache/disk_cache/io_buffer.h"
 #include "common/status.h"
 
 namespace starrocks {
+class CacheKey;
 
-enum class LocalCacheEngineType { STARCACHE, LRUCACHE };
+struct MemCacheOptions {
+    size_t mem_space_size = 0;
+};
 
-class LocalCacheEngine {
+struct MemCacheWriteOptions {
+    // The priority of the cache object, only support 0 and 1 now.
+    int8_t priority = 0;
+
+    // The probability to evict other items if the cache space is full, which can help avoid frequent cache replacement
+    // and improve cache hit rate sometimes.
+    // It is expressed as a percentage. If evict_probability is 10, it means the probability to evict other data is 10%.
+    int32_t evict_probability = 100;
+};
+
+struct MemCacheReadOptions {};
+
+struct MemCacheHandle {};
+
+using MemCacheHandlePtr = MemCacheHandle*;
+
+// using CacheDeleter = std::function<void(const std::string&, void*)>;
+//
+// We only use the deleter function of the lru cache temporarily.
+// Maybe a std::function object or a function pointer like `void (*)(std::string&, void*)` which
+// independent on lru cache is more appropriate, but it is not easy to convert them to the lru
+// cache deleter when using a lru cache module.
+using MemCacheDeleter = void (*)(const CacheKey&, void*);
+
+struct DataCacheMemMetrics {
+    size_t mem_quota_bytes = 0;
+    size_t mem_used_bytes = 0;
+};
+
+class LocalMemCacheEngine {
 public:
-    virtual ~LocalCacheEngine() = default;
+    virtual ~LocalMemCacheEngine() = default;
 
     virtual bool is_initialized() const = 0;
 
-    // Write data to cache
-    virtual Status write(const std::string& key, const IOBuffer& buffer, WriteCacheOptions* options) = 0;
-
-    // Read data from cache, it returns the data size if successful; otherwise the error status
-    // will be returned.
-    virtual Status read(const std::string& key, size_t off, size_t size, IOBuffer* buffer,
-                        ReadCacheOptions* options) = 0;
-
     // Insert object to cache
-    virtual Status insert(const std::string& key, void* value, size_t size, ObjectCacheDeleter deleter,
-                          ObjectCacheHandlePtr* handle, const ObjectCacheWriteOptions& options) = 0;
+    virtual Status insert(const std::string& key, void* value, size_t size, MemCacheDeleter deleter,
+                          MemCacheHandlePtr* handle, const MemCacheWriteOptions& options) = 0;
 
     // Lookup object from cache, the `handle` wraps the object pointer.
     // As long as the handle object is not destroyed and the user does not manually call the `handle->release()`
     // function, the corresponding pointer will never be freed by the cache system.
-    virtual Status lookup(const std::string& key, ObjectCacheHandlePtr* handle,
-                          ObjectCacheReadOptions* options = nullptr) = 0;
+    virtual Status lookup(const std::string& key, MemCacheHandlePtr* handle,
+                          MemCacheReadOptions* options = nullptr) = 0;
 
     // Release a handle returned by a previous insert() or lookup().
     // The handle must have not been released yet.
-    virtual void release(ObjectCacheHandlePtr handle) = 0;
+    virtual void release(MemCacheHandlePtr handle) = 0;
 
     // Return the value in the given handle returned by a previous insert() or lookup().
     // The handle must have not been released yet.
-    virtual const void* value(ObjectCacheHandlePtr handle) = 0;
+    virtual const void* value(MemCacheHandlePtr handle) = 0;
 
     virtual bool exist(const std::string& key) const = 0;
 
@@ -65,29 +88,15 @@ public:
     virtual Status adjust_mem_quota(int64_t delta, size_t min_capacity) = 0;
 
     // Update the datacache memory quota.
-    virtual Status update_mem_quota(size_t quota_bytes, bool flush_to_disk) = 0;
+    virtual Status update_mem_quota(size_t quota_bytes) = 0;
 
-    // Update the datacache disk space information, such as disk quota or disk path.
-    virtual Status update_disk_spaces(const std::vector<DirSpace>& spaces) = 0;
-
-    // Update the datacache inline cache count limit
-    virtual Status update_inline_cache_count_limit(int32_t limit) = 0;
-
-    virtual const DataCacheMetrics cache_metrics() const = 0;
-
-    virtual void record_read_remote(size_t size, int64_t latency_us) = 0;
-
-    virtual void record_read_cache(size_t size, int64_t latency_us) = 0;
+    virtual const DataCacheMemMetrics cache_metrics() const = 0;
 
     virtual Status shutdown() = 0;
 
-    virtual LocalCacheEngineType engine_type() = 0;
-
     virtual bool has_mem_cache() const = 0;
-    virtual bool has_disk_cache() const = 0;
     virtual bool available() const = 0;
     virtual bool mem_cache_available() const = 0;
-    virtual void disk_spaces(std::vector<DirSpace>* spaces) const = 0;
 
     virtual size_t mem_quota() const = 0;
     virtual size_t mem_usage() const = 0;
@@ -98,11 +107,17 @@ public:
     // Get the cache hit count.
     virtual size_t hit_count() const = 0;
 
-    // Get all cache metrics together.
-    virtual const ObjectCacheMetrics metrics() const = 0;
+    // Get the insert count.
+    virtual size_t insert_count() const = 0;
+
+    // Get the insert evict count.
+    virtual size_t insert_evict_count() const = 0;
+
+    // Get the release evict count.
+    virtual size_t release_evict_count() const = 0;
 
     // Remove all cache entries that are not actively in use.
     virtual Status prune() = 0;
 };
 
-} // namespace starrocks
+} // namespace starrocks

be/src/cache/mem_cache/lrucache_engine.cpp

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "cache/lrucache_engine.h"
+#include "cache/mem_cache/lrucache_engine.h"
 
 #include <butil/fast_rand.h>
 
@@ -23,33 +23,24 @@ Status LRUCacheEngine::init(const MemCacheOptions& options) {
     return Status::OK();
 }
 
-Status LRUCacheEngine::write(const std::string& key, const IOBuffer& buffer, WriteCacheOptions* options) {
-    return Status::NotSupported("LRUCache engine don't support write block");
-}
-
-Status LRUCacheEngine::read(const std::string& key, size_t off, size_t size, IOBuffer* buffer,
-                            ReadCacheOptions* options) {
-    return Status::NotSupported("LRUCache engine don't support read block");
-}
-
-Status LRUCacheEngine::insert(const std::string& key, void* value, size_t size, ObjectCacheDeleter deleter,
-                              ObjectCacheHandlePtr* handle, const ObjectCacheWriteOptions& options) {
+Status LRUCacheEngine::insert(const std::string& key, void* value, size_t size, MemCacheDeleter deleter,
+                              MemCacheHandlePtr* handle, const MemCacheWriteOptions& options) {
     if (!_check_write(size, options)) {
         return Status::InternalError("cache insertion is rejected");
     }
     auto* lru_handle = _cache->insert(key, value, size, deleter, static_cast<CachePriority>(options.priority));
     if (handle) {
-        *handle = reinterpret_cast<ObjectCacheHandlePtr>(lru_handle);
+        *handle = reinterpret_cast<MemCacheHandlePtr>(lru_handle);
     }
     return Status::OK();
 }
 
-Status LRUCacheEngine::lookup(const std::string& key, ObjectCacheHandlePtr* handle, ObjectCacheReadOptions* options) {
+Status LRUCacheEngine::lookup(const std::string& key, MemCacheHandlePtr* handle, MemCacheReadOptions* options) {
     auto* lru_handle = _cache->lookup(CacheKey(key));
     if (!lru_handle) {
         return Status::NotFound("no such entry");
     }
-    *handle = reinterpret_cast<ObjectCacheHandlePtr>(lru_handle);
+    *handle = reinterpret_cast<MemCacheHandlePtr>(lru_handle);
     return Status::OK();
 }
 
@@ -68,26 +59,13 @@ Status LRUCacheEngine::remove(const std::string& key) {
     return Status::OK();
 }
 
-Status LRUCacheEngine::update_mem_quota(size_t quota_bytes, bool flush_to_disk) {
+Status LRUCacheEngine::update_mem_quota(size_t quota_bytes) {
     _cache->set_capacity(quota_bytes);
     return Status::OK();
 }
 
-Status LRUCacheEngine::update_disk_spaces(const std::vector<DirSpace>& spaces) {
-    return Status::NotSupported("LRUCache engine don't support update disk spaces");
-}
-
-Status LRUCacheEngine::update_inline_cache_count_limit(int32_t limit) {
-    return Status::NotSupported("LRUCache engine don't support update inline cache count limit");
-}
-
-const DataCacheMetrics LRUCacheEngine::cache_metrics() const {
-    return DataCacheMetrics{.status = DataCacheStatus::NORMAL,
-                            .mem_quota_bytes = _cache->get_capacity(),
-                            .mem_used_bytes = _cache->get_memory_usage(),
-                            .disk_quota_bytes = 0,
-                            .disk_used_bytes = 0,
-                            .meta_used_bytes = 0};
+const DataCacheMemMetrics LRUCacheEngine::cache_metrics() const {
+    return DataCacheMemMetrics{.mem_quota_bytes = _cache->get_capacity(), .mem_used_bytes = _cache->get_memory_usage()};
 }
 
 Status LRUCacheEngine::shutdown() {
@@ -100,12 +78,12 @@ Status LRUCacheEngine::prune() {
     return Status::OK();
 }
 
-void LRUCacheEngine::release(ObjectCacheHandlePtr handle) {
+void LRUCacheEngine::release(MemCacheHandlePtr handle) {
     auto lru_handle = reinterpret_cast<Cache::Handle*>(handle);
     _cache->release(lru_handle);
 }
 
-const void* LRUCacheEngine::value(ObjectCacheHandlePtr handle) {
+const void* LRUCacheEngine::value(MemCacheHandlePtr handle) {
     auto lru_handle = reinterpret_cast<Cache::Handle*>(handle);
     return _cache->value(lru_handle);
 }
@@ -133,18 +111,19 @@ size_t LRUCacheEngine::hit_count() const {
     return _cache->get_hit_count();
 }
 
-const ObjectCacheMetrics LRUCacheEngine::metrics() const {
-    ObjectCacheMetrics m;
-    m.capacity = _cache->get_capacity();
-    m.usage = _cache->get_memory_usage();
-    m.lookup_count = _cache->get_lookup_count();
-    m.hit_count = _cache->get_hit_count();
-    // Unsupported
-    m.object_item_count = 0;
-    return m;
+size_t LRUCacheEngine::insert_count() const {
+    return _cache->get_insert_count();
 }
 
-bool LRUCacheEngine::_check_write(size_t charge, const ObjectCacheWriteOptions& options) const {
+size_t LRUCacheEngine::insert_evict_count() const {
+    return _cache->get_insert_evict_count();
+}
+
+size_t LRUCacheEngine::release_evict_count() const {
+    return _cache->get_release_evict_count();
+}
+
+bool LRUCacheEngine::_check_write(size_t charge, const MemCacheWriteOptions& options) const {
     if (options.evict_probability >= 100) {
         return true;
     }
@@ -164,5 +143,4 @@ bool LRUCacheEngine::_check_write(size_t charge, const ObjectCacheWriteOptions&
     }
     return false;
 }
-
 } // namespace starrocks

be/src/cache/mem_cache/lrucache_engine.h

@@ -16,11 +16,11 @@
 
 #include <atomic>
 
-#include "cache/local_cache_engine.h"
+#include "cache/mem_cache/local_mem_cache_engine.h"
 #include "util/lru_cache.h"
 
 namespace starrocks {
-class LRUCacheEngine final : public LocalCacheEngine {
+class LRUCacheEngine final : public LocalMemCacheEngine {
 public:
     LRUCacheEngine() = default;
     ~LRUCacheEngine() override = default;
@@ -28,36 +28,25 @@ public:
     Status init(const MemCacheOptions& options);
     bool is_initialized() const override { return _initialized.load(std::memory_order_relaxed); }
 
-    Status write(const std::string& key, const IOBuffer& buffer, WriteCacheOptions* options) override;
-    Status read(const std::string& key, size_t off, size_t size, IOBuffer* buffer, ReadCacheOptions* options) override;
-
-    Status insert(const std::string& key, void* value, size_t size, ObjectCacheDeleter deleter,
-                  ObjectCacheHandlePtr* handle, const ObjectCacheWriteOptions& options) override;
-    Status lookup(const std::string& key, ObjectCacheHandlePtr* handle, ObjectCacheReadOptions* options) override;
+    Status insert(const std::string& key, void* value, size_t size, MemCacheDeleter deleter, MemCacheHandlePtr* handle,
+                  const MemCacheWriteOptions& options) override;
+    Status lookup(const std::string& key, MemCacheHandlePtr* handle, MemCacheReadOptions* options) override;
 
     bool exist(const std::string& key) const override;
     Status remove(const std::string& key) override;
 
-    Status update_mem_quota(size_t quota_bytes, bool flush_to_disk) override;
-    Status update_disk_spaces(const std::vector<DirSpace>& spaces) override;
-    Status update_inline_cache_count_limit(int32_t limit) override;
+    Status update_mem_quota(size_t quota_bytes) override;
 
-    const DataCacheMetrics cache_metrics() const override;
-    void record_read_remote(size_t size, int64_t latency_us) override {}
-    void record_read_cache(size_t size, int64_t latency_us) override {}
+    const DataCacheMemMetrics cache_metrics() const override;
 
     Status shutdown() override;
-    LocalCacheEngineType engine_type() override { return LocalCacheEngineType::LRUCACHE; }
     bool has_mem_cache() const override { return _cache->get_capacity() > 0; }
-    bool has_disk_cache() const override { return false; }
 
     bool available() const override { return is_initialized() && has_mem_cache(); }
     bool mem_cache_available() const override { return is_initialized() && has_mem_cache(); }
 
-    void disk_spaces(std::vector<DirSpace>* spaces) const override {}
-
-    void release(ObjectCacheHandlePtr handle) override;
-    const void* value(ObjectCacheHandlePtr handle) override;
+    void release(MemCacheHandlePtr handle) override;
+    const void* value(MemCacheHandlePtr handle) override;
 
     Status adjust_mem_quota(int64_t delta, size_t min_capacity) override;
 
@@ -68,14 +57,18 @@ public:
 
     size_t hit_count() const override;
 
-    const ObjectCacheMetrics metrics() const override;
+    size_t insert_count() const override;
+
+    size_t insert_evict_count() const override;
+
+    size_t release_evict_count() const override;
 
     Status prune() override;
 
 private:
-    bool _check_write(size_t charge, const ObjectCacheWriteOptions& options) const;
+    bool _check_write(size_t charge, const MemCacheWriteOptions& options) const;
 
     std::atomic<bool> _initialized = false;
     std::unique_ptr<ShardedLRUCache> _cache;
 };
-} // namespace starrocks
+} // namespace starrocks

be/src/cache/mem_cache/page_cache.cpp

@@ -32,7 +32,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "cache/object_cache/page_cache.h"
+#include "cache/mem_cache/page_cache.h"
 
 #include <malloc.h>
 
@@ -49,6 +49,9 @@ std::atomic<size_t> StoragePageCacheMetrics::released_page_handle_count{};
 
 METRIC_DEFINE_UINT_GAUGE(page_cache_lookup_count, MetricUnit::OPERATIONS);
 METRIC_DEFINE_UINT_GAUGE(page_cache_hit_count, MetricUnit::OPERATIONS);
+METRIC_DEFINE_UINT_GAUGE(page_cache_insert_count, MetricUnit::OPERATIONS);
+METRIC_DEFINE_UINT_GAUGE(page_cache_insert_evict_count, MetricUnit::OPERATIONS);
+METRIC_DEFINE_UINT_GAUGE(page_cache_release_evict_count, MetricUnit::OPERATIONS);
 METRIC_DEFINE_UINT_GAUGE(page_cache_capacity, MetricUnit::BYTES);
 METRIC_DEFINE_UINT_GAUGE(page_cache_pinned_count, MetricUnit::BYTES);
 
@@ -61,6 +64,22 @@ void StoragePageCache::init_metrics() {
     StarRocksMetrics::instance()->metrics()->register_hook(
             "page_cache_hit_count", [this]() { page_cache_hit_count.set_value(get_hit_count()); });
 
+    StarRocksMetrics::instance()->metrics()->register_metric("page_cache_insert_count", &page_cache_insert_count);
+    StarRocksMetrics::instance()->metrics()->register_hook(
+            "page_cache_insert_count", [this]() { page_cache_insert_count.set_value(get_insert_count()); });
+
+    StarRocksMetrics::instance()->metrics()->register_metric("page_cache_insert_evict_count",
+                                                             &page_cache_insert_evict_count);
+    StarRocksMetrics::instance()->metrics()->register_hook("page_cache_insert_evict_count", [this]() {
+        page_cache_insert_evict_count.set_value(get_insert_evict_count());
+    });
+
+    StarRocksMetrics::instance()->metrics()->register_metric("page_cache_release_evict_count",
+                                                             &page_cache_release_evict_count);
+    StarRocksMetrics::instance()->metrics()->register_hook("page_cache_release_evict_count", [this]() {
+        page_cache_release_evict_count.set_value(get_release_evict_count());
+    });
+
     StarRocksMetrics::instance()->metrics()->register_metric("page_cache_capacity", &page_cache_capacity);
     StarRocksMetrics::instance()->metrics()->register_hook("page_cache_capacity",
                                                            [this]() { page_cache_capacity.set_value(get_capacity()); });
@@ -75,7 +94,7 @@ void StoragePageCache::prune() {
 }
 
 void StoragePageCache::set_capacity(size_t capacity) {
-    Status st = _cache->update_mem_quota(capacity, false);
+    Status st = _cache->update_mem_quota(capacity);
     LOG_IF(INFO, !st.ok()) << "Fail to set cache capacity to " << capacity << ", reason: " << st.message();
 }
 
@@ -91,6 +110,18 @@ uint64_t StoragePageCache::get_hit_count() const {
     return _cache->hit_count();
 }
 
+uint64_t StoragePageCache::get_insert_count() const {
+    return _cache->insert_count();
+}
+
+uint64_t StoragePageCache::get_insert_evict_count() const {
+    return _cache->insert_evict_count();
+}
+
+uint64_t StoragePageCache::get_release_evict_count() const {
+    return _cache->release_evict_count();
+}
+
 bool StoragePageCache::adjust_capacity(int64_t delta, size_t min_capacity) {
     Status st = _cache->adjust_mem_quota(delta, min_capacity);
     if (!st.ok()) {
@@ -105,7 +136,7 @@ size_t StoragePageCache::get_pinned_count() const {
 }
 
 bool StoragePageCache::lookup(const std::string& key, PageCacheHandle* handle) {
-    ObjectCacheHandle* obj_handle = nullptr;
+    MemCacheHandle* obj_handle = nullptr;
     Status st = _cache->lookup(key, &obj_handle);
     if (!st.ok()) {
         return false;
@@ -115,7 +146,7 @@ bool StoragePageCache::lookup(const std::string& key, PageCacheHandle* handle) {
     return true;
 }
 
-Status StoragePageCache::insert(const std::string& key, std::vector<uint8_t>* data, const ObjectCacheWriteOptions& opts,
+Status StoragePageCache::insert(const std::string& key, std::vector<uint8_t>* data, const MemCacheWriteOptions& opts,
                                 PageCacheHandle* handle) {
 #ifndef BE_TEST
     int64_t mem_size = malloc_usable_size(data->data()) + sizeof(*data);
@@ -131,7 +162,7 @@ Status StoragePageCache::insert(const std::string& key, std::vector<uint8_t>* da
         delete cache_item;
     };
 
-    ObjectCacheHandle* obj_handle = nullptr;
+    MemCacheHandle* obj_handle = nullptr;
     // Use mem size managed by memory allocator as this record charge size.
     // At the same time, we should record this record size for data fetching when lookup.
     Status st = _cache->insert(key, (void*)data, mem_size, deleter, &obj_handle, opts);
@@ -142,9 +173,9 @@ Status StoragePageCache::insert(const std::string& key, std::vector<uint8_t>* da
     return st;
 }
 
-Status StoragePageCache::insert(const std::string& key, void* data, int64_t size, ObjectCacheDeleter deleter,
-                                const ObjectCacheWriteOptions& opts, PageCacheHandle* handle) {
-    ObjectCacheHandle* obj_handle = nullptr;
+Status StoragePageCache::insert(const std::string& key, void* data, int64_t size, MemCacheDeleter deleter,
+                                const MemCacheWriteOptions& opts, PageCacheHandle* handle) {
+    MemCacheHandle* obj_handle = nullptr;
     Status st = _cache->insert(key, data, size, deleter, &obj_handle, opts);
     if (st.ok()) {
         *handle = PageCacheHandle(_cache, obj_handle);

be/src/cache/mem_cache/page_cache.h

@@ -43,7 +43,7 @@ namespace starrocks {
 
 class PageCacheHandle;
 class MemTracker;
-struct ObjectCacheWriteOptions;
+struct MemCacheWriteOptions;
 
 // Page cache min size is 256MB
 static constexpr int64_t kcacheMinSize = 268435456;
@@ -66,9 +66,9 @@ public:
     // Client should call create_global_cache before.
     static StoragePageCache* instance() { return DataCache::GetInstance()->page_cache(); }
 
-    StoragePageCache(LocalCacheEngine* cache_engine) : _cache(cache_engine), _initialized(true) {}
+    StoragePageCache(LocalMemCacheEngine* cache_engine) : _cache(cache_engine), _initialized(true) {}
 
-    void init(LocalCacheEngine* cache_engine) {
+    void init(LocalMemCacheEngine* cache_engine) {
         _cache = cache_engine;
         _initialized.store(true, std::memory_order_relaxed);
     }
@@ -87,11 +87,11 @@ public:
     // This function is thread-safe, and when two clients insert two same key
     // concurrently, this function can assure that only one page is cached.
     // The in_memory page will have higher priority.
-    Status insert(const std::string& key, std::vector<uint8_t>* data, const ObjectCacheWriteOptions& opts,
+    Status insert(const std::string& key, std::vector<uint8_t>* data, const MemCacheWriteOptions& opts,
                   PageCacheHandle* handle);
 
-    Status insert(const std::string& key, void* data, int64_t size, ObjectCacheDeleter deleter,
-                  const ObjectCacheWriteOptions& opts, PageCacheHandle* handle);
+    Status insert(const std::string& key, void* data, int64_t size, MemCacheDeleter deleter,
+                  const MemCacheWriteOptions& opts, PageCacheHandle* handle);
 
     size_t memory_usage() const { return _cache->mem_usage(); }
 
@@ -103,6 +103,12 @@ public:
 
     uint64_t get_hit_count() const;
 
+    uint64_t get_insert_count() const;
+
+    uint64_t get_insert_evict_count() const;
+
+    uint64_t get_release_evict_count() const;
+
     bool adjust_capacity(int64_t delta, size_t min_capacity = 0);
 
     void prune();
@@ -115,7 +121,7 @@ public:
     size_t get_pinned_count() const;
 
 private:
-    LocalCacheEngine* _cache = nullptr;
+    LocalMemCacheEngine* _cache = nullptr;
     std::atomic<bool> _initialized = false;
 };
 
@@ -125,7 +131,12 @@ private:
 class PageCacheHandle {
 public:
     PageCacheHandle() = default;
-    PageCacheHandle(LocalCacheEngine* cache, ObjectCacheHandle* handle) : _cache(cache), _handle(handle) {}
+    PageCacheHandle(LocalMemCacheEngine* cache, MemCacheHandle* handle) : _cache(cache), _handle(handle) {}
+
+    // Don't allow copy and assign
+    PageCacheHandle(const PageCacheHandle&) = delete;
+    const PageCacheHandle& operator=(const PageCacheHandle&) = delete;
+
     ~PageCacheHandle() {
         if (_handle != nullptr) {
             StoragePageCacheMetrics::released_page_handle_count++;
@@ -145,16 +156,12 @@ public:
         return *this;
     }
 
-    LocalCacheEngine* cache() const { return _cache; }
+    LocalMemCacheEngine* cache() const { return _cache; }
     const void* data() const { return _cache->value(_handle); }
 
 private:
-    LocalCacheEngine* _cache = nullptr;
-    ObjectCacheHandle* _handle = nullptr;
-
-    // Don't allow copy and assign
-    PageCacheHandle(const PageCacheHandle&) = delete;
-    const PageCacheHandle& operator=(const PageCacheHandle&) = delete;
+    LocalMemCacheEngine* _cache = nullptr;
+    MemCacheHandle* _handle = nullptr;
 };
 
 } // namespace starrocks

be/src/cache/mem_space_monitor.cpp

@@ -14,7 +14,7 @@
 
 #include "cache/mem_space_monitor.h"
 
-#include "cache/object_cache/page_cache.h"
+#include "cache/mem_cache/page_cache.h"
 #include "common/config.h"
 #include "runtime/exec_env.h"
 #include "runtime/mem_tracker.h"

be/src/cache/object_cache/cache_types.h

@@ -1,75 +0,0 @@
-// Copyright 2021-present StarRocks, Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-#include <functional>
-#include <ostream>
-#include <string>
-
-// Not a good way to import lru cache header here, just for temporary compatibility with old deleters.
-#include "util/lru_cache.h"
-
-namespace starrocks {
-
-enum class ObjectCacheModuleType { LRUCACHE, STARCACHE };
-
-struct ObjectCacheWriteOptions {
-    // The priority of the cache object, only support 0 and 1 now.
-    int8_t priority = 0;
-    // If ttl_seconds=0 (default), no ttl restriction will be set. If an old one exists, remove it.
-    uint64_t ttl_seconds = 0;
-    // If overwrite=true, the cache value will be replaced if it already exists.
-    bool overwrite = false;
-    // The probability to evict other items if the cache space is full, which can help avoid frequent cache replacement
-    // and improve cache hit rate sometimes.
-    // It is expressed as a percentage. If evict_probability is 10, it means the probability to evict other data is 10%.
-    int32_t evict_probability = 100;
-};
-
-struct ObjectCacheReadOptions {};
-
-struct ObjectCacheHandle {};
-
-struct ObjectCacheMetrics {
-    size_t capacity = 0;
-    size_t usage = 0;
-    size_t lookup_count = 0;
-    size_t hit_count = 0;
-    size_t object_item_count = 0;
-};
-
-using ObjectCacheHandlePtr = ObjectCacheHandle*;
-
-// using CacheDeleter = std::function<void(const std::string&, void*)>;
-//
-// We only use the deleter function of the lru cache temporarily.
-// Maybe a std::function object or a function pointer like `void (*)(std::string&, void*)` which
-// independent on lru cache is more appropriate, but it is not easy to convert them to the lru
-// cache deleter when using a lru cache module.
-using ObjectCacheDeleter = void (*)(const CacheKey&, void*);
-
-inline std::ostream& operator<<(std::ostream& os, const ObjectCacheModuleType& module) {
-    switch (module) {
-    case ObjectCacheModuleType::LRUCACHE:
-        os << "lrucache";
-        break;
-    case ObjectCacheModuleType::STARCACHE:
-        os << "starcache";
-        break;
-    }
-    return os;
-}
-
-} // namespace starrocks

be/src/cache/peer_cache_engine.cpp

@@ -29,7 +29,7 @@ Status PeerCacheEngine::init(const RemoteCacheOptions& options) {
 }
 
 Status PeerCacheEngine::read(const std::string& key, size_t off, size_t size, IOBuffer* buffer,
-                             ReadCacheOptions* options) {
+                             DiskCacheReadOptions* options) {
     if (options->use_adaptor && !_cache_adaptor->check_read_cache()) {
         return Status::ResourceBusy("resource is busy");
     }

be/src/cache/peer_cache_engine.h

@@ -26,9 +26,10 @@ public:
 
     Status init(const RemoteCacheOptions& options) override;
 
-    Status read(const std::string& key, size_t off, size_t size, IOBuffer* buffer, ReadCacheOptions* options) override;
+    Status read(const std::string& key, size_t off, size_t size, IOBuffer* buffer,
+                DiskCacheReadOptions* options) override;
 
-    Status write(const std::string& key, const IOBuffer& buffer, WriteCacheOptions* options) override {
+    Status write(const std::string& key, const IOBuffer& buffer, DiskCacheWriteOptions* options) override {
         return Status::NotSupported("write data to peer cache is unsupported");
     }
 

be/src/cache/remote_cache_engine.h

@@ -14,12 +14,16 @@
 
 #pragma once
 
-#include "cache/block_cache/io_buffer.h"
-#include "cache/cache_options.h"
+#include "cache/disk_cache/io_buffer.h"
+#include "cache/disk_cache/local_disk_cache_engine.h"
 #include "common/status.h"
 
 namespace starrocks {
 
+struct RemoteCacheOptions {
+    double skip_read_factor = 0;
+};
+
 class RemoteCacheEngine {
 public:
     virtual ~RemoteCacheEngine() = default;
@@ -28,12 +32,12 @@ public:
     virtual Status init(const RemoteCacheOptions& options) = 0;
 
     // Write data to remote cache
-    virtual Status write(const std::string& key, const IOBuffer& buffer, WriteCacheOptions* options) = 0;
+    virtual Status write(const std::string& key, const IOBuffer& buffer, DiskCacheWriteOptions* options) = 0;
 
     // Read data from remote cache, it returns the data size if successful; otherwise the error status
     // will be returned.
     virtual Status read(const std::string& key, size_t off, size_t size, IOBuffer* buffer,
-                        ReadCacheOptions* options) = 0;
+                        DiskCacheReadOptions* options) = 0;
 
     // Remove data from cache.
     virtual Status remove(const std::string& key) = 0;

be/src/column/nullable_column.cpp

@@ -529,11 +529,11 @@ void NullableColumn::put_mysql_row_buffer(MysqlRowBuffer* buf, size_t idx, bool
 }
 
 void NullableColumn::check_or_die() const {
-    CHECK_EQ(_null_column->size(), _data_column->size());
+    DCHECK_EQ(_null_column->size(), _data_column->size());
     // when _has_null=true, the column may have no null value, so don't check.
     if (!_has_null) {
         auto null_data = _null_column->immutable_data();
-        CHECK(!SIMD::contain_nonzero(null_data, 0));
+        DCHECK(!SIMD::contain_nonzero(null_data, 0));
     }
     _data_column->check_or_die();
     _null_column->check_or_die();

be/src/common/config.h

@@ -412,7 +412,7 @@ CONF_Bool(enable_event_based_compaction_framework, "true");
 CONF_Bool(enable_size_tiered_compaction_strategy, "true");
 CONF_mBool(enable_pk_size_tiered_compaction_strategy, "true");
 // Enable parallel execution within tablet for primary key tables.
-CONF_mBool(enable_pk_parallel_execution, "false");
+CONF_mBool(enable_pk_parallel_execution, "true");
 // The minimum threshold of data size for enabling pk parallel execution.
 // Default is 300MB.
 CONF_mInt64(pk_parallel_execution_threshold_bytes, "314572800");
@@ -1299,19 +1299,8 @@ CONF_Bool(datacache_block_buffer_enable, "true");
 // To control how many threads will be created for datacache synchronous tasks.
 // For the default value, it means for every 8 cpu, one thread will be created.
 CONF_Double(datacache_scheduler_threads_per_cpu, "0.125");
-// To control whether cache raw data both in memory and disk.
-// If true, the raw data will be written to the tiered cache composed of memory cache and disk cache,
-// and the memory cache hotter data than disk.
-// If false, the raw data will be written to disk directly and read from disk without promotion.
-// For object data, such as parquet footer object, which can only be cached in memory are not affected
-// by this configuration.
-CONF_Bool(datacache_tiered_cache_enable, "false");
 // Whether to persist cached data
 CONF_Bool(datacache_persistence_enable, "true");
-// DataCache engines, alternatives: starcache, lrucache
-// Set the default value empty to indicate whether it is manually configured by users.
-// If not, we need to adjust the default engine based on build switches like "WITH_STARCACHE".
-CONF_String_enum(datacache_engine, "", ",starcache,lrucache");
 // The interval time (millisecond) for agent report datacache metrics to FE.
 CONF_mInt32(report_datacache_metrics_interval_ms, "60000");
 
@@ -1365,7 +1354,6 @@ CONF_Alias(datacache_block_size, block_cache_block_size);
 CONF_Alias(datacache_max_concurrent_inserts, block_cache_max_concurrent_inserts);
 CONF_Alias(datacache_checksum_enable, block_cache_checksum_enable);
 CONF_Alias(datacache_direct_io_enable, block_cache_direct_io_enable);
-CONF_Alias(datacache_engine, block_cache_engine);
 
 CONF_mInt64(l0_l1_merge_ratio, "10");
 // max wal file size in l0
@@ -1616,7 +1604,7 @@ CONF_mBool(apply_del_vec_after_all_index_filter, "true");
 // connector sink memory watermark
 CONF_mDouble(connector_sink_mem_high_watermark_ratio, "0.3");
 CONF_mDouble(connector_sink_mem_low_watermark_ratio, "0.1");
-CONF_mDouble(connector_sink_mem_urgent_space_ratio, "0.1");
+CONF_mDouble(connector_sink_mem_urgent_space_ratio, "0.05");
 // Whether enable spill intermediate data for connector sink.
 CONF_mBool(enable_connector_sink_spill, "true");
 

be/src/common/configbase.h

@@ -340,7 +340,7 @@ void TEST_clear_configs();
 
 template <>
 struct fmt::formatter<starrocks::config::MutableString> : formatter<std::string> {
-    auto format(const starrocks::config::MutableString& s, format_context& ctx) {
+    auto format(const starrocks::config::MutableString& s, format_context& ctx) const {
         return formatter<std::string>::format(s.value(), ctx);
     }
 };

be/src/common/daemon.cpp

@@ -44,6 +44,13 @@
 #ifdef USE_STAROS
 #include "fslib/star_cache_handler.h"
 #endif
+#include <fmt/ranges.h>
+
+#include <csignal>
+// Need POSIX signal APIs like sigaction/siginfo_t.
+// NOLINTNEXTLINE(modernize-deprecated-headers)
+#include <signal.h>
+
 #include "fs/encrypt_file.h"
 #include "gutil/cpu.h"
 #include "jemalloc/jemalloc.h"
@@ -164,6 +171,12 @@ struct JemallocStats {
 };
 
 static void retrieve_jemalloc_stats(JemallocStats* stats) {
+    // On macOS, jemalloc may define je_mallctl as mallctl via macro in jemalloc.h
+#ifdef __APPLE__
+#ifndef je_mallctl
+#define je_mallctl mallctl
+#endif
+#endif
     uint64_t epoch = 1;
     size_t sz = sizeof(epoch);
     je_mallctl("epoch", &epoch, &sz, &epoch, sz);

be/src/common/logconfig.cpp

@@ -18,6 +18,12 @@
 #include <glog/logging.h>
 #include <glog/vlog_is_on.h>
 #include <jemalloc/jemalloc.h>
+#ifdef __APPLE__
+#include <mach/mach_init.h>
+#include <mach/mach_port.h>
+#include <mach/thread_act.h>
+#include <pthread.h>
+#endif
 
 #include <cerrno>
 #include <cstdio>
@@ -27,7 +33,7 @@
 #include <mutex>
 
 #include "cache/datacache.h"
-#include "cache/object_cache/page_cache.h"
+#include "cache/mem_cache/page_cache.h"
 #include "common/config.h"
 #include "gutil/endian.h"
 #include "gutil/stringprintf.h"
@@ -128,7 +134,12 @@ static void dontdump_unused_pages() {
     static bool start_dump = false;
     struct timeval tv;
     gettimeofday(&tv, nullptr);
+    // On macOS, pthread_t is an opaque pointer; convert to a numeric id for fmt
+#ifdef __APPLE__
+    uint64_t tid = static_cast<uint64_t>(pthread_mach_thread_np(pthread_self()));
+#else
     pthread_t tid = pthread_self();
+#endif
     const uint32_t MAX_BUFFER_SIZE = 1024;
     char buffer[MAX_BUFFER_SIZE] = {};
     // memory_buffer allocate 500 bytes from stack
@@ -136,7 +147,13 @@ static void dontdump_unused_pages() {
     if (!start_dump) {
         int res = snprintf(buffer, MAX_BUFFER_SIZE, "arena.%d.purge", MALLCTL_ARENAS_ALL);
         buffer[res] = '\0';
-        int ret = je_mallctl(buffer, nullptr, nullptr, nullptr, 0);
+        int ret =
+#ifdef __APPLE__
+                mallctl
+#else
+                je_mallctl
+#endif
+                (buffer, nullptr, nullptr, nullptr, 0);
 
         if (ret != 0) {
             FMT_LOG("je_mallctl execute purge failed, errno:{}", ret);
@@ -146,7 +163,13 @@ static void dontdump_unused_pages() {
 
         res = snprintf(buffer, MAX_BUFFER_SIZE, "arena.%d.dontdump", MALLCTL_ARENAS_ALL);
         buffer[res] = '\0';
-        ret = je_mallctl(buffer, nullptr, nullptr, nullptr, 0);
+        ret =
+#ifdef __APPLE__
+                mallctl
+#else
+                je_mallctl
+#endif
+                (buffer, nullptr, nullptr, nullptr, 0);
 
         if (ret != 0) {
             FMT_LOG("je_mallctl execute dontdump failed, errno:{}", ret);
@@ -199,8 +222,10 @@ bool init_glog(const char* basename, bool install_signal_handler) {
     FLAGS_logbuflevel = 0;
     // Buffer log messages for at most this many seconds.
     FLAGS_logbufsecs = 30;
-    // Set roll num.
+    // Set roll num. Not available with Homebrew glog on macOS.
+#ifndef __APPLE__
     FLAGS_log_filenum_quota = config::sys_log_roll_num;
+#endif
 
     // Set log level.
     std::string loglevel = config::sys_log_level;
@@ -230,13 +255,19 @@ bool init_glog(const char* basename, bool install_signal_handler) {
     std::string sizeflag = "SIZE-MB-";
     bool ok = false;
     if (rollmode.compare("TIME-DAY") == 0) {
+#ifndef __APPLE__
         FLAGS_log_split_method = "day";
+#endif
         ok = true;
     } else if (rollmode.compare("TIME-HOUR") == 0) {
+#ifndef __APPLE__
         FLAGS_log_split_method = "hour";
+#endif
         ok = true;
     } else if (rollmode.substr(0, sizeflag.length()).compare(sizeflag) == 0) {
+#ifndef __APPLE__
         FLAGS_log_split_method = "size";
+#endif
         std::string sizestr = rollmode.substr(sizeflag.size(), rollmode.size() - sizeflag.size());
         if (sizestr.size() != 0) {
             char* end = nullptr;
@@ -278,7 +309,10 @@ bool init_glog(const char* basename, bool install_signal_handler) {
     if (config::dump_trace_info) {
         google::InstallFailureWriter(failure_writer);
         google::InstallFailureFunction((google::logging_fail_func_t)failure_function);
+#ifndef MACOS_DISABLE_GLOG_STACKTRACE
+        // This symbol may be unavailable on macOS builds using system glog.
         google::InstallFailureHandlerAfterOutputLog(failure_handler_after_output_log);
+#endif
     }
 
     logging_initialized = true;

be/src/common/prof/heap_prof.cpp

@@ -27,21 +27,45 @@
 namespace starrocks {
 // detail implements for allocator
 static int set_jemalloc_profiling(bool enable) {
-    int ret = je_mallctl("prof.active", nullptr, nullptr, &enable, 1);
-    ret |= je_mallctl("prof.thread_active_init", nullptr, nullptr, &enable, 1);
+    int ret =
+#ifdef __APPLE__
+            mallctl
+#else
+            je_mallctl
+#endif
+            ("prof.active", nullptr, nullptr, &enable, 1);
+    ret |=
+#ifdef __APPLE__
+            mallctl
+#else
+            je_mallctl
+#endif
+            ("prof.thread_active_init", nullptr, nullptr, &enable, 1);
     return ret;
 }
 
 static int has_enable_heap_profile() {
     int value = 0;
     size_t size = sizeof(value);
-    je_mallctl("prof.active", &value, &size, nullptr, 0);
+
+#ifdef __APPLE__
+    mallctl
+#else
+    je_mallctl
+#endif
+            ("prof.active", &value, &size, nullptr, 0);
     return value;
 }
 
 bool dump_snapshot(const std::string& filename) {
     const char* fname = filename.c_str();
-    return je_mallctl("prof.dump", nullptr, nullptr, &fname, sizeof(const char*)) == 0;
+    return (
+#ifdef __APPLE__
+                   mallctl
+#else
+                   je_mallctl
+#endif
+                   ("prof.dump", nullptr, nullptr, &fname, sizeof(const char*))) == 0;
 }
 
 // declare exec from script
@@ -85,4 +109,4 @@ std::string HeapProf::to_dot_format(const std::string& heapdump_filename) {
     return exec(fmt::format("{} --dot {} {}", jeprof, binary, heapdump_filename));
 }
 
-} // namespace starrocks
+} // namespace starrocks

be/src/connector/connector_chunk_sink.cpp

@@ -41,7 +41,8 @@ Status ConnectorChunkSink::init() {
 }
 
 Status ConnectorChunkSink::write_partition_chunk(const std::string& partition,
-                                                 const std::vector<int8_t>& partition_field_null_list, Chunk* chunk) {
+                                                 const std::vector<int8_t>& partition_field_null_list,
+                                                 const ChunkPtr& chunk) {
     // partition_field_null_list is used to distinguish with the secenario like NULL and string "null"
     // They are under the same dir path, but should not in the same data file.
     // We should record them in different files so that each data file could has its own meta info.
@@ -64,13 +65,13 @@ Status ConnectorChunkSink::write_partition_chunk(const std::string& partition,
     return Status::OK();
 }
 
-Status ConnectorChunkSink::add(Chunk* chunk) {
+Status ConnectorChunkSink::add(const ChunkPtr& chunk) {
     std::string partition = DEFAULT_PARTITION;
     bool partitioned = !_partition_column_names.empty();
     if (partitioned) {
         ASSIGN_OR_RETURN(partition,
-                         HiveUtils::make_partition_name(_partition_column_names, _partition_column_evaluators, chunk,
-                                                        _support_null_partition));
+                         HiveUtils::make_partition_name(_partition_column_names, _partition_column_evaluators,
+                                                        chunk.get(), _support_null_partition));
     }
 
     RETURN_IF_ERROR(
@@ -79,6 +80,13 @@ Status ConnectorChunkSink::add(Chunk* chunk) {
 }
 
 Status ConnectorChunkSink::finish() {
+    // Flushing data to disk to make more memory space for subsequent merge operations.
+    for (auto& [partition_key, writer] : _partition_chunk_writers) {
+        RETURN_IF_ERROR(writer->flush());
+    }
+    for (auto& [partition_key, writer] : _partition_chunk_writers) {
+        RETURN_IF_ERROR(writer->wait_flush());
+    }
     for (auto& [partition_key, writer] : _partition_chunk_writers) {
         RETURN_IF_ERROR(writer->finish());
     }

be/src/connector/connector_chunk_sink.h

@@ -47,7 +47,7 @@ public:
 
     Status init();
 
-    virtual Status add(Chunk* chunk);
+    virtual Status add(const ChunkPtr& chunk);
 
     Status finish();
 
@@ -58,7 +58,7 @@ public:
     virtual void callback_on_commit(const CommitResult& result) = 0;
 
     Status write_partition_chunk(const std::string& partition, const vector<int8_t>& partition_field_null_list,
-                                 Chunk* chunk);
+                                 const ChunkPtr& chunk);
 
     Status status();
 

be/src/connector/connector_sink_executor.cpp

@@ -50,13 +50,16 @@ int ConnectorSinkSpillExecutor::calc_max_thread_num() {
 }
 
 void ChunkSpillTask::run() {
+    SCOPED_THREAD_LOCAL_MEM_TRACKER_SETTER(_mem_tracker);
     auto res = _load_chunk_spiller->spill(*_chunk);
     if (_cb) {
         _cb(_chunk, res);
     }
+    _chunk.reset();
 }
 
 void MergeBlockTask::run() {
+    SCOPED_THREAD_LOCAL_MEM_TRACKER_SETTER(_mem_tracker);
     auto st = _writer->merge_blocks();
     if (_cb) {
         _cb(st);

be/src/connector/connector_sink_executor.h

@@ -71,9 +71,12 @@ protected:
 
 class ChunkSpillTask final : public Runnable {
 public:
-    ChunkSpillTask(LoadChunkSpiller* load_chunk_spiller, ChunkPtr chunk,
+    ChunkSpillTask(LoadChunkSpiller* load_chunk_spiller, ChunkPtr chunk, MemTracker* mem_tracker,
                    std::function<void(ChunkPtr chunk, const StatusOr<size_t>&)> cb)
-            : _load_chunk_spiller(load_chunk_spiller), _chunk(chunk), _cb(std::move(cb)) {}
+            : _load_chunk_spiller(load_chunk_spiller),
+              _chunk(std::move(chunk)),
+              _mem_tracker(mem_tracker),
+              _cb(std::move(cb)) {}
 
     ~ChunkSpillTask() override = default;
 
@@ -82,18 +85,20 @@ public:
 private:
     LoadChunkSpiller* _load_chunk_spiller;
     ChunkPtr _chunk;
+    MemTracker* _mem_tracker;
     std::function<void(ChunkPtr, const StatusOr<size_t>&)> _cb;
 };
 
 class MergeBlockTask : public Runnable {
 public:
-    MergeBlockTask(SpillPartitionChunkWriter* writer, std::function<void(const Status&)> cb)
-            : _writer(writer), _cb(std::move(cb)) {}
+    MergeBlockTask(SpillPartitionChunkWriter* writer, MemTracker* mem_tracker, std::function<void(const Status&)> cb)
+            : _writer(writer), _mem_tracker(mem_tracker), _cb(std::move(cb)) {}
 
     void run() override;
 
 private:
     SpillPartitionChunkWriter* _writer;
+    MemTracker* _mem_tracker;
     std::function<void(const Status&)> _cb;
 };
 

be/src/connector/file_chunk_sink.cpp

@@ -56,8 +56,9 @@ StatusOr<std::unique_ptr<ConnectorChunkSink>> FileChunkSinkProvider::create_chun
     std::shared_ptr<formats::FileWriterFactory> file_writer_factory;
     if (boost::iequals(ctx->format, formats::PARQUET)) {
         file_writer_factory = std::make_shared<formats::ParquetFileWriterFactory>(
-                fs, ctx->compression_type, ctx->options, ctx->column_names, std::move(column_evaluators), std::nullopt,
-                ctx->executor, runtime_state);
+                fs, ctx->compression_type, ctx->options, ctx->column_names,
+                std::make_shared<std::vector<std::unique_ptr<ColumnEvaluator>>>(std::move(column_evaluators)),
+                std::nullopt, ctx->executor, runtime_state);
     } else if (boost::iequals(ctx->format, formats::ORC)) {
         file_writer_factory = std::make_shared<formats::ORCFileWriterFactory>(
                 fs, ctx->compression_type, ctx->options, ctx->column_names, std::move(column_evaluators), ctx->executor,

be/src/connector/hive_chunk_sink.cpp

@@ -66,7 +66,8 @@ StatusOr<std::unique_ptr<ConnectorChunkSink>> HiveChunkSinkProvider::create_chun
         ctx->options[formats::ParquetWriterOptions::USE_LEGACY_DECIMAL_ENCODING] = "true";
         ctx->options[formats::ParquetWriterOptions::USE_INT96_TIMESTAMP_ENCODING] = "true";
         file_writer_factory = std::make_shared<formats::ParquetFileWriterFactory>(
-                fs, ctx->compression_type, ctx->options, ctx->data_column_names, std::move(data_column_evaluators),
+                fs, ctx->compression_type, ctx->options, ctx->data_column_names,
+                std::make_shared<std::vector<std::unique_ptr<ColumnEvaluator>>>(std::move(data_column_evaluators)),
                 std::nullopt, ctx->executor, runtime_state);
     } else if (boost::iequals(ctx->format, formats::ORC)) {
         file_writer_factory = std::make_shared<formats::ORCFileWriterFactory>(

be/src/connector/iceberg_chunk_sink.cpp

@@ -82,7 +82,8 @@ StatusOr<std::unique_ptr<ConnectorChunkSink>> IcebergChunkSinkProvider::create_c
     auto ctx = std::dynamic_pointer_cast<IcebergChunkSinkContext>(context);
     auto runtime_state = ctx->fragment_context->runtime_state();
     std::shared_ptr<FileSystem> fs = FileSystem::CreateUniqueFromString(ctx->path, FSOptions(&ctx->cloud_conf)).value();
-    auto column_evaluators = ColumnEvaluator::clone(ctx->column_evaluators);
+    auto column_evaluators = std::make_shared<std::vector<std::unique_ptr<ColumnEvaluator>>>(
+            ColumnEvaluator::clone(ctx->column_evaluators));
     auto location_provider = std::make_shared<connector::LocationProvider>(
             ctx->path, print_id(ctx->fragment_context->query_id()), runtime_state->be_number(), driver_id,
             boost::to_lower_copy(ctx->format));
@@ -93,8 +94,8 @@ StatusOr<std::unique_ptr<ConnectorChunkSink>> IcebergChunkSinkProvider::create_c
     std::shared_ptr<formats::FileWriterFactory> file_writer_factory;
     if (boost::iequals(ctx->format, formats::PARQUET)) {
         file_writer_factory = std::make_shared<formats::ParquetFileWriterFactory>(
-                fs, ctx->compression_type, ctx->options, ctx->column_names, std::move(column_evaluators),
-                ctx->parquet_field_ids, ctx->executor, runtime_state);
+                fs, ctx->compression_type, ctx->options, ctx->column_names, column_evaluators, ctx->parquet_field_ids,
+                ctx->executor, runtime_state);
     } else {
         file_writer_factory = std::make_shared<formats::UnknownFileWriterFactory>(ctx->format);
     }
@@ -107,7 +108,7 @@ StatusOr<std::unique_ptr<ConnectorChunkSink>> IcebergChunkSinkProvider::create_c
                         fs,
                         ctx->fragment_context,
                         runtime_state->desc_tbl().get_tuple_descriptor(ctx->tuple_desc_id),
-                        &ctx->column_evaluators,
+                        column_evaluators,
                         ctx->sort_ordering});
         partition_chunk_writer_factory = std::make_unique<SpillPartitionChunkWriterFactory>(partition_chunk_writer_ctx);
     } else {
@@ -123,14 +124,14 @@ StatusOr<std::unique_ptr<ConnectorChunkSink>> IcebergChunkSinkProvider::create_c
                                                          std::move(partition_chunk_writer_factory), runtime_state);
 }
 
-Status IcebergChunkSink::add(Chunk* chunk) {
+Status IcebergChunkSink::add(const ChunkPtr& chunk) {
     std::string partition = DEFAULT_PARTITION;
     bool partitioned = !_partition_column_names.empty();
     std::vector<int8_t> partition_field_null_list;
     if (partitioned) {
         ASSIGN_OR_RETURN(partition, HiveUtils::iceberg_make_partition_name(
                                             _partition_column_names, _partition_column_evaluators,
-                                            dynamic_cast<IcebergChunkSink*>(this)->transform_expr(), chunk,
+                                            dynamic_cast<IcebergChunkSink*>(this)->transform_expr(), chunk.get(),
                                             _support_null_partition, partition_field_null_list));
     }
 

be/src/connector/iceberg_chunk_sink.h

@@ -45,7 +45,7 @@ public:
 
     const std::vector<std::string>& transform_expr() const { return _transform_exprs; }
 
-    Status add(Chunk* chunk) override;
+    Status add(const ChunkPtr& chunk) override;
 
 private:
     std::vector<std::string> _transform_exprs;

be/src/connector/lake_connector.cpp

@@ -625,7 +625,7 @@ void LakeDataSource::init_counter(RuntimeState* state) {
             ADD_CHILD_COUNTER(_runtime_profile, "ShortKeyRangeNumber", TUnit::UNIT, segment_init_name);
     _column_iterator_init_timer = ADD_CHILD_TIMER(_runtime_profile, "ColumnIteratorInit", segment_init_name);
     _bitmap_index_iterator_init_timer = ADD_CHILD_TIMER(_runtime_profile, "BitmapIndexIteratorInit", segment_init_name);
-    _zone_map_filter_timer = ADD_CHILD_TIMER(_runtime_profile, "ZoneMapIndexFiter", segment_init_name);
+    _zone_map_filter_timer = ADD_CHILD_TIMER(_runtime_profile, "ZoneMapIndexFilter", segment_init_name);
     _rows_key_range_filter_timer = ADD_CHILD_TIMER(_runtime_profile, "ShortKeyFilter", segment_init_name);
     _bf_filter_timer = ADD_CHILD_TIMER(_runtime_profile, "BloomFilterFilter", segment_init_name);
 

be/src/connector/partition_chunk_writer.cpp

@@ -72,12 +72,12 @@ Status BufferPartitionChunkWriter::init() {
     return Status::OK();
 }
 
-Status BufferPartitionChunkWriter::write(Chunk* chunk) {
-    RETURN_IF_ERROR(create_file_writer_if_needed());
-    if (_file_writer->get_written_bytes() >= _max_file_size) {
+Status BufferPartitionChunkWriter::write(const ChunkPtr& chunk) {
+    if (_file_writer && _file_writer->get_written_bytes() >= _max_file_size) {
         commit_file();
     }
-    return _file_writer->write(chunk);
+    RETURN_IF_ERROR(create_file_writer_if_needed());
+    return _file_writer->write(chunk.get());
 }
 
 Status BufferPartitionChunkWriter::flush() {
@@ -85,6 +85,10 @@ Status BufferPartitionChunkWriter::flush() {
     return Status::OK();
 }
 
+Status BufferPartitionChunkWriter::wait_flush() {
+    return Status::OK();
+}
+
 Status BufferPartitionChunkWriter::finish() {
     commit_file();
     return Status::OK();
@@ -102,6 +106,7 @@ SpillPartitionChunkWriter::SpillPartitionChunkWriter(std::string partition,
     _block_merge_token = StorageEngine::instance()->load_spill_block_merge_executor()->create_token();
     _tuple_desc = ctx->tuple_desc;
     _writer_id = generate_uuid();
+    _spill_mode = _sort_ordering != nullptr;
 }
 
 SpillPartitionChunkWriter::~SpillPartitionChunkWriter() {
@@ -120,16 +125,16 @@ Status SpillPartitionChunkWriter::init() {
     RETURN_IF_ERROR(_load_spill_block_mgr->init());
     _load_chunk_spiller = std::make_unique<LoadChunkSpiller>(_load_spill_block_mgr.get(),
                                                              _fragment_context->runtime_state()->runtime_profile());
-    if (_column_evaluators) {
-        RETURN_IF_ERROR(ColumnEvaluator::init(*_column_evaluators));
-    }
     return Status::OK();
 }
 
-Status SpillPartitionChunkWriter::write(Chunk* chunk) {
+Status SpillPartitionChunkWriter::write(const ChunkPtr& chunk) {
     RETURN_IF_ERROR(create_file_writer_if_needed());
+    if (!_spill_mode) {
+        return _write_chunk(chunk.get());
+    }
 
-    _chunks.push_back(chunk->clone_unique());
+    _chunks.push_back(chunk);
     _chunk_bytes_usage += chunk->bytes_usage();
     if (!_base_chunk) {
         _base_chunk = _chunks.back();
@@ -149,11 +154,21 @@ Status SpillPartitionChunkWriter::write(Chunk* chunk) {
 
 Status SpillPartitionChunkWriter::flush() {
     RETURN_IF(!_file_writer, Status::OK());
+    // Change to spill mode if memory is insufficent.
+    if (!_spill_mode) {
+        _spill_mode = true;
+        commit_file();
+        return Status::OK();
+    }
     return _spill();
 }
 
-Status SpillPartitionChunkWriter::finish() {
+Status SpillPartitionChunkWriter::wait_flush() {
     _chunk_spill_token->wait();
+    return Status::OK();
+}
+
+Status SpillPartitionChunkWriter::finish() {
     // If no chunks have been spilled, flush data to remote file directly.
     if (_load_chunk_spiller->empty()) {
         VLOG(2) << "flush to remote directly when finish, query_id: " << print_id(_fragment_context->query_id())
@@ -169,7 +184,8 @@ Status SpillPartitionChunkWriter::finish() {
         _handle_err(st);
         commit_file();
     };
-    auto merge_task = std::make_shared<MergeBlockTask>(this, cb);
+    auto merge_task = std::make_shared<MergeBlockTask>(this, _fragment_context->runtime_state()->instance_mem_tracker(),
+                                                       std::move(cb));
     return _block_merge_token->submit(merge_task);
 }
 
@@ -182,9 +198,7 @@ bool SpillPartitionChunkWriter::is_finished() {
 }
 
 Status SpillPartitionChunkWriter::merge_blocks() {
-    RETURN_IF_ERROR(flush());
     _chunk_spill_token->wait();
-
     auto write_func = [this](Chunk* chunk) { return _flush_chunk(chunk, false); };
     auto flush_func = [this]() {
         // Commit file after each merge function to ensure the data written to one file is ordered,
@@ -239,10 +253,13 @@ Status SpillPartitionChunkWriter::_spill() {
         }
         _spilling_bytes_usage.fetch_sub(chunk->bytes_usage(), std::memory_order_relaxed);
     };
-    auto spill_task = std::make_shared<ChunkSpillTask>(_load_chunk_spiller.get(), _result_chunk, callback);
+    auto spill_task = std::make_shared<ChunkSpillTask>(_load_chunk_spiller.get(), _result_chunk,
+                                                       _fragment_context->runtime_state()->instance_mem_tracker(),
+                                                       std::move(callback));
     RETURN_IF_ERROR(_chunk_spill_token->submit(spill_task));
     _spilling_bytes_usage.fetch_add(_result_chunk->bytes_usage(), std::memory_order_relaxed);
     _chunk_bytes_usage = 0;
+    _result_chunk.reset();
     return Status::OK();
 }
 
@@ -330,7 +347,6 @@ Status SpillPartitionChunkWriter::_merge_chunks() {
                 }
             }
         }
-
         chunk.reset();
     }
 

be/src/connector/partition_chunk_writer.h

@@ -50,7 +50,7 @@ struct SpillPartitionChunkWriterContext : public PartitionChunkWriterContext {
     std::shared_ptr<FileSystem> fs;
     pipeline::FragmentContext* fragment_context = nullptr;
     TupleDescriptor* tuple_desc = nullptr;
-    std::vector<std::unique_ptr<ColumnEvaluator>>* column_evaluators;
+    std::shared_ptr<std::vector<std::unique_ptr<ColumnEvaluator>>> column_evaluators;
     std::shared_ptr<SortOrdering> sort_ordering;
 };
 
@@ -63,10 +63,12 @@ public:
 
     virtual Status init() = 0;
 
-    virtual Status write(Chunk* chunk) = 0;
+    virtual Status write(const ChunkPtr& chunk) = 0;
 
     virtual Status flush() = 0;
 
+    virtual Status wait_flush() = 0;
+
     virtual Status finish() = 0;
 
     virtual bool is_finished() = 0;
@@ -118,10 +120,12 @@ public:
 
     Status init() override;
 
-    Status write(Chunk* chunk) override;
+    Status write(const ChunkPtr& chunk) override;
 
     Status flush() override;
 
+    Status wait_flush() override;
+
     Status finish() override;
 
     bool is_finished() override { return true; }
@@ -140,10 +144,12 @@ public:
 
     Status init() override;
 
-    Status write(Chunk* chunk) override;
+    Status write(const ChunkPtr& chunk) override;
 
     Status flush() override;
 
+    Status wait_flush() override;
+
     Status finish() override;
 
     bool is_finished() override;
@@ -156,7 +162,12 @@ public:
                _file_writer->get_written_bytes();
     }
 
-    int64_t get_flushable_bytes() override { return _chunk_bytes_usage; }
+    int64_t get_flushable_bytes() override {
+        if (!_spill_mode) {
+            return _file_writer ? _file_writer->get_written_bytes() : 0;
+        }
+        return _chunk_bytes_usage;
+    }
 
     Status merge_blocks();
 
@@ -185,13 +196,12 @@ private:
     std::shared_ptr<FileSystem> _fs = nullptr;
     pipeline::FragmentContext* _fragment_context = nullptr;
     TupleDescriptor* _tuple_desc = nullptr;
-    std::vector<std::unique_ptr<ColumnEvaluator>>* _column_evaluators;
+    std::shared_ptr<std::vector<std::unique_ptr<ColumnEvaluator>>> _column_evaluators;
     std::shared_ptr<SortOrdering> _sort_ordering;
     std::unique_ptr<ThreadPoolToken> _chunk_spill_token;
     std::unique_ptr<ThreadPoolToken> _block_merge_token;
     std::unique_ptr<LoadSpillBlockManager> _load_spill_block_mgr;
     std::shared_ptr<LoadChunkSpiller> _load_chunk_spiller;
-    //std::function<StatusOr<ColumnPtr>(Chunk*, size_t)> _column_eval_func;
     TUniqueId _writer_id;
 
     std::list<ChunkPtr> _chunks;
@@ -201,6 +211,7 @@ private:
     ChunkPtr _base_chunk;
     SchemaPtr _schema;
     std::unordered_map<int, int> _col_index_map; // result chunk index -> chunk index
+    bool _spill_mode = false;
 
     static const int64_t kWaitMilliseconds;
 };

be/src/connector/sink_memory_manager.cpp

@@ -51,8 +51,9 @@ bool SinkOperatorMemoryManager::kill_victim() {
     // The flush will decrease the writer flushable memory bytes, so it usually
     // will not be choosed in a short time.
     const auto filename = victim->out_stream()->filename();
+    size_t flush_bytes = victim->get_flushable_bytes();
     const auto result = victim->flush();
-    LOG(INFO) << "kill victim: " << filename << ", result: " << result;
+    LOG(INFO) << "kill victim: " << filename << ", result: " << result << ", flushable_bytes: " << flush_bytes;
     return true;
 }
 
@@ -118,7 +119,7 @@ bool SinkMemoryManager::_apply_on_mem_tracker(SinkOperatorMemoryManager* child_m
 
     auto available_memory = [&]() { return mem_tracker->limit() - mem_tracker->consumption(); };
     auto low_watermark = static_cast<int64_t>(mem_tracker->limit() * _low_watermark_ratio);
-    int64_t flush_watermark = _query_tracker->limit() * _urgent_space_ratio;
+    int64_t flush_watermark = mem_tracker->limit() * _urgent_space_ratio;
     while (available_memory() <= low_watermark) {
         child_manager->update_writer_occupied_memory();
         int64_t total_occupied_memory = _total_writer_occupied_memory();

be/src/exec/hash_joiner.cpp

@@ -556,7 +556,13 @@ Status HashJoiner::_create_runtime_in_filters(RuntimeState* state) {
     SCOPED_TIMER(build_metrics().build_runtime_filter_timer);
     size_t ht_row_count = get_ht_row_count();
 
-    if (ht_row_count > config::max_pushdown_conditions_per_column) {
+    // Use FE session variable if set, otherwise fall back to BE config
+    size_t max_conditions = config::max_pushdown_conditions_per_column;
+    if (state->query_options().__isset.max_pushdown_conditions_per_column) {
+        max_conditions = state->query_options().max_pushdown_conditions_per_column;
+    }
+
+    if (ht_row_count > max_conditions) {
         return Status::OK();
     }
 

be/src/exec/hdfs_scanner/hdfs_scanner.cpp

@@ -14,7 +14,7 @@
 
 #include "exec/hdfs_scanner/hdfs_scanner.h"
 
-#include "cache/block_cache/block_cache_hit_rate_counter.hpp"
+#include "cache/disk_cache/block_cache_hit_rate_counter.hpp"
 #include "column/column_helper.h"
 #include "column/type_traits.h"
 #include "connector/deletion_vector/deletion_vector.h"

be/src/exec/hdfs_scanner/hdfs_scanner.h

@@ -17,6 +17,7 @@
 #include <atomic>
 #include <boost/algorithm/string.hpp>
 
+#include "cache/cache_options.h"
 #include "connector/deletion_vector/deletion_bitmap.h"
 #include "exec/olap_scan_prepare.h"
 #include "exec/pipeline/scan/morsel.h"

be/src/exec/pipeline/scan/chunk_source.cpp

@@ -80,12 +80,19 @@ Status ChunkSource::buffer_next_batch_chunks_blocking(RuntimeState* state, size_
                 if (_status.is_end_of_file()) {
                     chunk->owner_info().set_owner_id(owner_id, true);
                     _chunk_buffer.put(_scan_operator_seq, std::move(chunk), std::move(_chunk_token));
+                    break;
                 } else if (_status.is_time_out()) {
                     chunk->owner_info().set_owner_id(owner_id, false);
                     _chunk_buffer.put(_scan_operator_seq, std::move(chunk), std::move(_chunk_token));
                     _status = Status::OK();
+                    break;
+                } else if (_status.is_eagain()) {
+                    // EAGAIN is normal case, but sleep a while to avoid busy loop
+                    SleepFor(MonoDelta::FromNanoseconds(workgroup::WorkGroup::YIELD_PREEMPT_MAX_TIME_SPENT));
+                    _status = Status::OK();
+                } else {
+                    break;
                 }
-                break;
             }
 
             // schema won't be used by the computing layer, here we just reset it.

be/src/exec/pipeline/scan/connector_scan_operator.cpp

@@ -17,7 +17,6 @@
 #include "exec/connector_scan_node.h"
 #include "exec/pipeline/pipeline_driver.h"
 #include "exec/pipeline/scan/balanced_chunk_buffer.h"
-#include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
 
 namespace starrocks::pipeline {
@@ -763,7 +762,7 @@ Status ConnectorChunkSource::_read_chunk(RuntimeState* state, ChunkPtr* chunk) {
         RETURN_IF_ERROR(_open_data_source(state, &mem_alloc_failed));
         if (mem_alloc_failed) {
             _mem_alloc_failed_count += 1;
-            return Status::TimedOut("");
+            return Status::EAgain("");
         }
         if (state->is_cancelled()) {
             return Status::Cancelled("canceled state");

be/src/exec/pipeline/scan/olap_chunk_source.cpp

@@ -178,7 +178,7 @@ void OlapChunkSource::_init_counter(RuntimeState* state) {
             ADD_CHILD_COUNTER(_runtime_profile, "RemainingRowsAfterShortKeyFilter", TUnit::UNIT, segment_init_name);
     _column_iterator_init_timer = ADD_CHILD_TIMER(_runtime_profile, "ColumnIteratorInit", segment_init_name);
     _bitmap_index_iterator_init_timer = ADD_CHILD_TIMER(_runtime_profile, "BitmapIndexIteratorInit", segment_init_name);
-    _zone_map_filter_timer = ADD_CHILD_TIMER(_runtime_profile, "ZoneMapIndexFiter", segment_init_name);
+    _zone_map_filter_timer = ADD_CHILD_TIMER(_runtime_profile, "ZoneMapIndexFilter", segment_init_name);
     _rows_key_range_filter_timer = ADD_CHILD_TIMER(_runtime_profile, "ShortKeyFilter", segment_init_name);
     _rows_key_range_counter =
             ADD_CHILD_COUNTER(_runtime_profile, "ShortKeyRangeNumber", TUnit::UNIT, segment_init_name);

be/src/exec/pipeline/scan/olap_meta_scan_operator.cpp

@@ -1,110 +0,0 @@
-// Copyright 2021-present StarRocks, Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "exec/pipeline/scan/olap_meta_scan_operator.h"
-
-#include <utility>
-
-#include "exec/olap_meta_scanner.h"
-#include "exec/pipeline/scan/olap_meta_chunk_source.h"
-#include "exec/pipeline/scan/olap_meta_scan_context.h"
-
-namespace starrocks::pipeline {
-
-OlapMetaScanOperatorFactory::OlapMetaScanOperatorFactory(int32_t id, ScanNode* meta_scan_node, size_t dop,
-                                                         std::shared_ptr<OlapMetaScanContextFactory> ctx_factory)
-        : ScanOperatorFactory(id, meta_scan_node), _ctx_factory(std::move(ctx_factory)) {}
-
-Status OlapMetaScanOperatorFactory::do_prepare(RuntimeState* state) {
-    return Status::OK();
-}
-
-void OlapMetaScanOperatorFactory::do_close(RuntimeState* state) {}
-
-OperatorPtr OlapMetaScanOperatorFactory::do_create(int32_t dop, int32_t driver_sequence) {
-    return std::make_shared<OlapMetaScanOperator>(this, _id, driver_sequence, dop, _scan_node,
-                                                  _ctx_factory->get_or_create(driver_sequence));
-}
-
-OlapMetaScanOperator::OlapMetaScanOperator(OperatorFactory* factory, int32_t id, int32_t driver_sequence, int32_t dop,
-                                           ScanNode* meta_scan_node, OlapMetaScanContextPtr ctx)
-        : ScanOperator(factory, id, driver_sequence, dop, meta_scan_node), _ctx(std::move(ctx)) {}
-
-OlapMetaScanOperator::~OlapMetaScanOperator() = default;
-
-bool OlapMetaScanOperator::has_output() const {
-    if (!_ctx->is_prepare_finished()) {
-        return false;
-    }
-    return ScanOperator::has_output();
-}
-
-bool OlapMetaScanOperator::is_finished() const {
-    if (!_ctx->is_prepare_finished()) {
-        return false;
-    }
-    return ScanOperator::is_finished();
-}
-
-Status OlapMetaScanOperator::do_prepare(RuntimeState* state) {
-    return Status::OK();
-}
-
-void OlapMetaScanOperator::do_close(RuntimeState* state) {}
-
-ChunkSourcePtr OlapMetaScanOperator::create_chunk_source(MorselPtr morsel, int32_t chunk_source_index) {
-    return std::make_shared<OlapMetaChunkSource>(this, _runtime_profile.get(), std::move(morsel), _ctx);
-}
-
-ChunkPtr OlapMetaScanOperator::get_chunk_from_buffer() {
-    ChunkPtr chunk = nullptr;
-    if (_ctx->get_chunk_buffer().try_get(_driver_sequence, &chunk)) {
-        return chunk;
-    }
-    return nullptr;
-}
-
-size_t OlapMetaScanOperator::num_buffered_chunks() const {
-    return _ctx->get_chunk_buffer().size(_driver_sequence);
-}
-
-size_t OlapMetaScanOperator::buffer_size() const {
-    return _ctx->get_chunk_buffer().limiter()->size();
-}
-
-size_t OlapMetaScanOperator::buffer_capacity() const {
-    return _ctx->get_chunk_buffer().limiter()->capacity();
-}
-
-size_t OlapMetaScanOperator::buffer_memory_usage() const {
-    return _ctx->get_chunk_buffer().memory_usage();
-}
-
-size_t OlapMetaScanOperator::default_buffer_capacity() const {
-    return _ctx->get_chunk_buffer().limiter()->default_capacity();
-}
-
-ChunkBufferTokenPtr OlapMetaScanOperator::pin_chunk(int num_chunks) {
-    return _ctx->get_chunk_buffer().limiter()->pin(num_chunks);
-}
-
-bool OlapMetaScanOperator::is_buffer_full() const {
-    return _ctx->get_chunk_buffer().limiter()->is_full();
-}
-
-void OlapMetaScanOperator::set_buffer_finished() {
-    _ctx->get_chunk_buffer().set_finished(_driver_sequence);
-}
-
-} // namespace starrocks::pipeline

be/src/exec/pipeline/scan/olap_meta_scan_operator.h

@@ -1,73 +0,0 @@
-// Copyright 2021-present StarRocks, Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-#include "exec/olap_meta_scan_node.h"
-#include "exec/pipeline/pipeline_builder.h"
-#include "exec/pipeline/scan/balanced_chunk_buffer.h"
-#include "exec/pipeline/scan/olap_meta_scan_context.h"
-#include "exec/pipeline/scan/scan_operator.h"
-#include "gen_cpp/Types_types.h"
-
-namespace starrocks::pipeline {
-
-class OlapMetaScanOperatorFactory final : public ScanOperatorFactory {
-public:
-    OlapMetaScanOperatorFactory(int32_t id, ScanNode* meta_scan_node, size_t dop,
-                                std::shared_ptr<OlapMetaScanContextFactory> ctx_factory);
-
-    ~OlapMetaScanOperatorFactory() override = default;
-
-    bool with_morsels() const override { return true; }
-
-    Status do_prepare(RuntimeState* state) override;
-    void do_close(RuntimeState* state) override;
-    OperatorPtr do_create(int32_t dop, int32_t driver_sequence) override;
-
-private:
-    std::shared_ptr<OlapMetaScanContextFactory> _ctx_factory;
-};
-
-class OlapMetaScanOperator final : public ScanOperator {
-public:
-    OlapMetaScanOperator(OperatorFactory* factory, int32_t id, int32_t driver_sequence, int32_t dop,
-                         ScanNode* scan_node, OlapMetaScanContextPtr scan_ctx);
-
-    ~OlapMetaScanOperator() override;
-
-    bool has_output() const override;
-    bool is_finished() const override;
-
-    Status do_prepare(RuntimeState* state) override;
-    void do_close(RuntimeState* state) override;
-    ChunkSourcePtr create_chunk_source(MorselPtr morsel, int32_t chunk_source_index) override;
-
-private:
-    void attach_chunk_source(int32_t source_index) override {}
-    void detach_chunk_source(int32_t source_index) override {}
-    bool has_shared_chunk_source() const override { return false; }
-    ChunkPtr get_chunk_from_buffer() override;
-    size_t num_buffered_chunks() const override;
-    size_t buffer_size() const override;
-    size_t buffer_capacity() const override;
-    size_t buffer_memory_usage() const override;
-    size_t default_buffer_capacity() const override;
-    ChunkBufferTokenPtr pin_chunk(int num_chunks) override;
-    bool is_buffer_full() const override;
-    void set_buffer_finished() override;
-
-    OlapMetaScanContextPtr _ctx;
-};
-} // namespace starrocks::pipeline

be/src/exec/pipeline/sink/connector_sink_operator.cpp

@@ -109,7 +109,7 @@ StatusOr<ChunkPtr> ConnectorSinkOperator::pull_chunk(RuntimeState* state) {
 }
 
 Status ConnectorSinkOperator::push_chunk(RuntimeState* state, const ChunkPtr& chunk) {
-    RETURN_IF_ERROR(_connector_chunk_sink->add(chunk.get()));
+    RETURN_IF_ERROR(_connector_chunk_sink->add(chunk));
     return Status::OK();
 }
 

be/src/exec/pipeline/spill_process_channel.h

@@ -18,6 +18,14 @@
 #include <memory>
 #include <utility>
 
+// On macOS, system headers may define a macro named current_task(),
+// which conflicts with the method name below. Undefine to avoid collisions.
+#ifdef __APPLE__
+#ifdef current_task
+#undef current_task
+#endif
+#endif
+
 #include "column/vectorized_fwd.h"
 #include "common/statusor.h"
 #include "exec/spill/spiller.h"

be/src/exec/scan_node.cpp

@@ -94,17 +94,19 @@ Status ScanNode::prepare(RuntimeState* state) {
 }
 
 // Distribute morsels from a single queue to multiple queues
-static std::map<int, pipeline::MorselQueuePtr> uniform_distribute_morsels(pipeline::MorselQueuePtr morsel_queue,
-                                                                          int dop) {
+static StatusOr<std::map<int, pipeline::MorselQueuePtr>> uniform_distribute_morsels(
+        pipeline::MorselQueuePtr morsel_queue, int dop) {
+    std::map<int, pipeline::MorselQueuePtr> queue_per_driver;
     std::map<int, pipeline::Morsels> morsels_per_driver;
     int driver_seq = 0;
     while (!morsel_queue->empty()) {
-        auto maybe_morsel = morsel_queue->try_get();
-        DCHECK(maybe_morsel.ok());
-        morsels_per_driver[driver_seq].push_back(std::move(maybe_morsel.value()));
+        auto maybe_morsel_status_or = morsel_queue->try_get();
+        if (UNLIKELY(!maybe_morsel_status_or.ok())) {
+            return maybe_morsel_status_or.status();
+        }
+        morsels_per_driver[driver_seq].push_back(std::move(maybe_morsel_status_or.value()));
         driver_seq = (driver_seq + 1) % dop;
     }
-    std::map<int, pipeline::MorselQueuePtr> queue_per_driver;
 
     auto morsel_queue_type = morsel_queue->type();
     DCHECK(morsel_queue_type == pipeline::MorselQueue::Type::FIXED ||
@@ -144,7 +146,7 @@ StatusOr<pipeline::MorselQueueFactoryPtr> ScanNode::convert_scan_range_to_morsel
         // If not so much morsels, try to assign morsel uniformly among operators to avoid data skew
         if (!always_shared_scan() && scan_dop > 1 && is_fixed_or_dynamic_morsel_queue &&
             morsel_queue->num_original_morsels() <= io_parallelism) {
-            auto morsel_queue_map = uniform_distribute_morsels(std::move(morsel_queue), scan_dop);
+            ASSIGN_OR_RETURN(auto morsel_queue_map, uniform_distribute_morsels(std::move(morsel_queue), scan_dop));
             return std::make_unique<pipeline::IndividualMorselQueueFactory>(std::move(morsel_queue_map),
                                                                             /*could_local_shuffle*/ true);
         } else {

be/src/exec/schema_scanner/schema_be_datacache_metrics_scanner.cpp

@@ -21,7 +21,7 @@
 #include "runtime/exec_env.h"
 
 #ifdef WITH_STARCACHE
-#include "cache/starcache_engine.h"
+#include "cache/disk_cache/starcache_engine.h"
 #endif
 
 namespace starrocks {
@@ -68,9 +68,14 @@ Status SchemaBeDataCacheMetricsScanner::get_next(ChunkPtr* chunk, bool* eos) {
     row.emplace_back(_be_id);
 
     // TODO: Support LRUCacheEngine
-    auto* cache = DataCache::GetInstance()->local_disk_cache();
-    if (cache != nullptr && cache->is_initialized() && cache->engine_type() == LocalCacheEngineType::STARCACHE) {
-        auto* starcache = reinterpret_cast<StarCacheEngine*>(cache);
+    auto* mem_cache = DataCache::GetInstance()->local_mem_cache();
+    DataCacheMemMetrics mem_metrics;
+    if (mem_cache != nullptr && mem_cache->is_initialized()) {
+        mem_metrics = mem_cache->cache_metrics();
+    }
+    auto* disk_cache = DataCache::GetInstance()->local_disk_cache();
+    if (disk_cache != nullptr && disk_cache->is_initialized()) {
+        auto* starcache = reinterpret_cast<StarCacheEngine*>(disk_cache);
         // retrieve different priority's used bytes from level = 2 metrics
         metrics = starcache->starcache_metrics(2);
 
@@ -79,8 +84,8 @@ Status SchemaBeDataCacheMetricsScanner::get_next(ChunkPtr* chunk, bool* eos) {
         row.emplace_back(Slice(status));
         row.emplace_back(metrics.disk_quota_bytes);
         row.emplace_back(metrics.disk_used_bytes);
-        row.emplace_back(metrics.mem_quota_bytes);
-        row.emplace_back(metrics.mem_used_bytes);
+        row.emplace_back(mem_metrics.mem_quota_bytes);
+        row.emplace_back(mem_metrics.mem_used_bytes);
         row.emplace_back(metrics.meta_used_bytes);
 
         const auto& dir_spaces = metrics.disk_dir_spaces;

be/src/exec/spill/dir_manager.h

@@ -14,7 +14,11 @@
 
 #pragma once
 
+#ifdef __APPLE__
+#include <sys/mount.h>
+#else
 #include <sys/statfs.h>
+#endif
 
 #include <atomic>
 #include <memory>
@@ -101,7 +105,11 @@ private:
         struct statfs stat1, stat2;
         statfs(path1.c_str(), &stat1);
         statfs(path2.c_str(), &stat2);
+#ifdef __APPLE__
+        return stat1.f_fsid.val[0] == stat2.f_fsid.val[0] && stat1.f_fsid.val[1] == stat2.f_fsid.val[1];
+#else
         return stat1.f_fsid.__val[0] == stat2.f_fsid.__val[0] && stat1.f_fsid.__val[1] == stat2.f_fsid.__val[1];
+#endif
     }
 
     std::vector<DirPtr> _dirs;
@@ -117,4 +125,4 @@ private:
     Status::RuntimeError(fmt::format("acquire size error: dir {} try acquire:{} usage:{} capacity:{}", dir->dir(), \
                                      acquire_size, dir->get_current_size(), dir->get_max_size()))
 
-} // namespace starrocks::spill
+} // namespace starrocks::spill

be/src/exec/spill/spill_components.cpp

@@ -325,8 +325,6 @@ void PartitionedSpillerWriter::_add_partition(SpilledPartitionPtr&& partition_pt
 }
 
 void PartitionedSpillerWriter::_remove_partition(const SpilledPartition* partition) {
-    auto affinity_group = partition->block_group->get_affinity_group();
-    DCHECK(affinity_group != kDefaultBlockAffinityGroup);
     _id_to_partitions.erase(partition->partition_id);
     size_t level = partition->level;
     auto& partitions = _level_to_partitions[level];
@@ -334,6 +332,12 @@ void PartitionedSpillerWriter::_remove_partition(const SpilledPartition* partiti
     auto iter = std::find_if(partitions.begin(), partitions.end(),
                              [partition](auto& val) { return val->partition_id == partition->partition_id; });
     _total_partition_num -= (iter != partitions.end());
+    if (partition->block_group != nullptr) {
+        auto affinity_group = partition->block_group->get_affinity_group();
+        DCHECK(affinity_group != kDefaultBlockAffinityGroup);
+        WARN_IF_ERROR(_spiller->block_manager()->release_affinity_group(affinity_group),
+                      fmt::format("release affinity group {} error", affinity_group));
+    }
     partitions.erase(iter);
     if (partitions.empty()) {
         _level_to_partitions.erase(level);
@@ -341,8 +345,6 @@ void PartitionedSpillerWriter::_remove_partition(const SpilledPartition* partiti
             _min_level = level + 1;
         }
     }
-    WARN_IF_ERROR(_spiller->block_manager()->release_affinity_group(affinity_group),
-                  fmt::format("release affinity group {} error", affinity_group));
 }
 
 Status PartitionedSpillerWriter::_choose_partitions_to_flush(bool is_final_flush,

be/src/exec/tablet_sink.cpp

@@ -46,9 +46,9 @@
 #include "column/column_helper.h"
 #include "column/map_column.h"
 #include "column/nullable_column.h"
+#include "common/config.h"
 #include "common/statusor.h"
 #include "common/tracer.h"
-#include "config.h"
 #include "exec/pipeline/query_context.h"
 #include "exec/tablet_sink_colocate_sender.h"
 #include "exprs/expr.h"

be/src/exec/tablet_sink_index_channel.cpp

@@ -19,10 +19,10 @@
 #include "column/chunk.h"
 #include "column/column_viewer.h"
 #include "column/nullable_column.h"
+#include "common/config.h"
 #include "common/statusor.h"
 #include "common/tracer.h"
 #include "common/utils.h"
-#include "config.h"
 #include "exec/tablet_sink.h"
 #include "exprs/expr_context.h"
 #include "gutil/strings/fastmem.h"

be/src/exprs/CMakeLists.txt

@@ -27,6 +27,7 @@ set(EXPR_FILES
   agg/factory/aggregate_resolver_minmaxany.cpp
   agg/factory/aggregate_resolver_others.cpp
   agg/factory/aggregate_resolver_sumcount.cpp
+  agg/factory/aggregate_resolver_distinct.cpp
   agg/factory/aggregate_resolver_stream.cpp
   agg/factory/aggregate_resolver_utility.cpp
   agg/factory/aggregate_resolver_variance.cpp

be/src/exprs/agg/aggregate_traits.h

@@ -16,12 +16,12 @@
 
 #include <type_traits>
 
+#include "column/array_column.h"
 #include "column/type_traits.h"
 #include "gutil/strings/fastmem.h"
 #include "types/logical_type.h"
 
 namespace starrocks {
-
 // Type traits from aggregate functions
 template <LogicalType lt, typename = guard::Guard>
 struct AggDataTypeTraits {};
@@ -41,6 +41,7 @@ struct AggDataTypeTraits<lt, FixedLengthLTGuard<lt>> {
     static RefType get_ref(const ValueType& value) { return value; }
 
     static void update_max(ValueType& current, const RefType& input) { current = std::max<ValueType>(current, input); }
+
     static void update_min(ValueType& current, const RefType& input) { current = std::min<ValueType>(current, input); }
 
     static bool is_equal(const RefType& lhs, const RefType& rhs) { return lhs == rhs; }
@@ -56,7 +57,9 @@ struct AggDataTypeTraits<lt, ObjectFamilyLTGuard<lt>> {
     using RefType = RunTimeCppType<lt>;
 
     static void assign_value(ValueType& value, RefType ref) { value = *ref; }
+
     static void assign_value(ColumnType* column, size_t row, const RefType& ref) { *column->get_object(row) = *ref; }
+
     static void assign_value(ColumnType* column, size_t row, const ValueType& ref) { *column->get_object(row) = ref; }
 
     static void append_value(ColumnType* column, const ValueType& value) { column->append(&value); }
@@ -65,12 +68,47 @@ struct AggDataTypeTraits<lt, ObjectFamilyLTGuard<lt>> {
     static const RefType get_row_ref(const ColumnType& column, size_t row) { return column.get_object(row); }
 
     static void update_max(ValueType& current, const RefType& input) { current = std::max<ValueType>(current, *input); }
+
     static void update_min(ValueType& current, const RefType& input) { current = std::min<ValueType>(current, *input); }
 
     static bool is_equal(const RefType& lhs, const RefType& rhs) { return *lhs == *rhs; }
     static bool equals(const ValueType& lhs, const RefType& rhs) { return lhs == *rhs; }
 };
 
+// For pointer ref types
+template <LogicalType lt>
+struct AggDataTypeTraits<lt, ArrayGuard<lt>> {
+    using CppType = RunTimeCppType<lt>;
+    using ColumnType = RunTimeColumnType<lt>;
+    using ValueType = typename ColumnType::MutablePtr;
+
+    struct RefType {
+        const ColumnType* column;
+        const size_t row;
+
+        RefType(const ColumnType* c, size_t r) : column(c), row(r) {}
+    };
+
+    static void assign_value(ValueType& value, const RefType& ref) {
+        value = ArrayColumn::static_pointer_cast(ref.column->clone_empty());
+        value->append_datum(ref.column->get(ref.row).template get<CppType>());
+    }
+
+    static void append_value(ColumnType* column, const ValueType& value) {
+        column->append_datum(value->get(0).template get<CppType>());
+    }
+
+    static RefType get_row_ref(const ColumnType& column, size_t row) { return RefType(&column, row); }
+
+    static bool is_equal(const ValueType& lhs, const ValueType& rhs) {
+        return lhs->get(0).template get<CppType>() == rhs->get(0).template get<CppType>();
+    }
+
+    static bool equals(const ValueType& lhs, const ValueType& rhs) {
+        return lhs->get(0).template get<CppType>() == rhs->get(0).template get<CppType>();
+    }
+};
+
 template <LogicalType lt>
 struct AggDataTypeTraits<lt, StringLTGuard<lt>> {
     using ColumnType = RunTimeColumnType<lt>;
@@ -96,6 +134,7 @@ struct AggDataTypeTraits<lt, StringLTGuard<lt>> {
             memcpy(current.data(), input.data, input.size);
         }
     }
+
     static void update_min(ValueType& current, const RefType& input) {
         if (Slice(current.data(), current.size()).compare(input) > 0) {
             current.resize(input.size);
@@ -110,5 +149,4 @@ template <LogicalType lt>
 using AggDataValueType = typename AggDataTypeTraits<lt>::ValueType;
 template <LogicalType lt>
 using AggDataRefType = typename AggDataTypeTraits<lt>::RefType;
-
-} // namespace starrocks
+} // namespace starrocks

be/src/exprs/agg/factory/aggregate_resolver.hpp

@@ -75,7 +75,7 @@ public:
                                                      TYPE_BIGINT,     TYPE_LARGEINT,  TYPE_FLOAT,     TYPE_DOUBLE,
                                                      TYPE_VARCHAR,    TYPE_CHAR,      TYPE_DATE,      TYPE_DATETIME,
                                                      TYPE_DECIMALV2,  TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128,
-                                                     TYPE_DECIMAL256, TYPE_HLL,       TYPE_OBJECT};
+                                                     TYPE_DECIMAL256, TYPE_HLL,       TYPE_OBJECT,    TYPE_ARRAY};
         return kTypes;
     }
 

be/src/exprs/agg/factory/aggregate_resolver_distinct.cpp

@@ -0,0 +1,48 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "column/type_traits.h"
+#include "exprs/agg/distinct.h"
+#include "exprs/agg/factory/aggregate_factory.hpp"
+#include "exprs/agg/factory/aggregate_resolver.hpp"
+#include "types/logical_type.h"
+
+namespace starrocks {
+
+struct DistinctDispatcher {
+    template <LogicalType lt>
+    void operator()(AggregateFuncResolver* resolver) {
+        if constexpr (lt_is_aggregate<lt>) {
+            using DistinctState = DistinctAggregateState<lt, SumResultLT<lt>>;
+            using DistinctState2 = DistinctAggregateStateV2<lt, SumResultLT<lt>>;
+            resolver->add_aggregate_mapping<lt, TYPE_BIGINT, DistinctState>(
+                    "multi_distinct_count", false, AggregateFactory::MakeCountDistinctAggregateFunction<lt>());
+            resolver->add_aggregate_mapping<lt, TYPE_BIGINT, DistinctState2>(
+                    "multi_distinct_count2", false, AggregateFactory::MakeCountDistinctAggregateFunctionV2<lt>());
+
+            resolver->add_aggregate_mapping<lt, SumResultLT<lt>, DistinctState>(
+                    "multi_distinct_sum", false, AggregateFactory::MakeSumDistinctAggregateFunction<lt>());
+            resolver->add_aggregate_mapping<lt, SumResultLT<lt>, DistinctState2>(
+                    "multi_distinct_sum2", false, AggregateFactory::MakeSumDistinctAggregateFunctionV2<lt>());
+        }
+    }
+};
+
+void AggregateFuncResolver::register_distinct() {
+    for (auto type : aggregate_types()) {
+        type_dispatch_all(type, DistinctDispatcher(), this);
+    }
+}
+
+} // namespace starrocks

be/src/exprs/agg/factory/aggregate_resolver_sumcount.cpp

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 #include "column/type_traits.h"
-#include "exprs/agg/distinct.h"
 #include "exprs/agg/factory/aggregate_factory.hpp"
 #include "exprs/agg/factory/aggregate_resolver.hpp"
 #include "exprs/agg/sum.h"
@@ -52,25 +51,6 @@ struct StorageSumDispatcher {
     }
 };
 
-struct DistinctDispatcher {
-    template <LogicalType lt>
-    void operator()(AggregateFuncResolver* resolver) {
-        if constexpr (lt_is_aggregate<lt>) {
-            using DistinctState = DistinctAggregateState<lt, SumResultLT<lt>>;
-            using DistinctState2 = DistinctAggregateStateV2<lt, SumResultLT<lt>>;
-            resolver->add_aggregate_mapping<lt, TYPE_BIGINT, DistinctState>(
-                    "multi_distinct_count", false, AggregateFactory::MakeCountDistinctAggregateFunction<lt>());
-            resolver->add_aggregate_mapping<lt, TYPE_BIGINT, DistinctState2>(
-                    "multi_distinct_count2", false, AggregateFactory::MakeCountDistinctAggregateFunctionV2<lt>());
-
-            resolver->add_aggregate_mapping<lt, SumResultLT<lt>, DistinctState>(
-                    "multi_distinct_sum", false, AggregateFactory::MakeSumDistinctAggregateFunction<lt>());
-            resolver->add_aggregate_mapping<lt, SumResultLT<lt>, DistinctState2>(
-                    "multi_distinct_sum2", false, AggregateFactory::MakeSumDistinctAggregateFunctionV2<lt>());
-        }
-    }
-};
-
 void AggregateFuncResolver::register_sumcount() {
     for (auto type : aggregate_types()) {
         type_dispatch_all(type, SumDispatcher(), this);
@@ -93,10 +73,4 @@ void AggregateFuncResolver::register_sumcount() {
                            AggregateFactory::MakeCountNullableAggregateFunction<true>());
 }
 
-void AggregateFuncResolver::register_distinct() {
-    for (auto type : aggregate_types()) {
-        type_dispatch_all(type, DistinctDispatcher(), this);
-    }
-}
-
 } // namespace starrocks

be/src/exprs/agg/factory/aggregate_resolver_window.cpp

@@ -24,7 +24,7 @@ namespace starrocks {
 struct WindowDispatcher {
     template <LogicalType lt>
     void operator()(AggregateFuncResolver* resolver) {
-        if constexpr (lt_is_aggregate<lt> || is_object_type(lt)) {
+        if constexpr (lt_is_aggregate<lt> || is_object_type(lt) || lt_is_array<lt>) {
             resolver->add_aggregate_mapping_notnull<lt, lt>(
                     "first_value", true, AggregateFactory::MakeFirstValueWindowFunction<lt, false>());
             // use first_value_in for first_value with ingnore nulls.

be/src/exprs/agg/window.h

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #pragma once
+#include "column/array_column.h"
 #include "column/column_helper.h"
 #include "column/nullable_column.h"
 #include "column/vectorized_fwd.h"
@@ -101,6 +102,10 @@ struct ValueWindowStrategy<LT, JsonGuard<LT>> {
     /// The dst Object column hasn't been resized.
     static constexpr bool use_append = true;
 };
+template <LogicalType LT>
+struct ValueWindowStrategy<LT, ArrayGuard<LT>> {
+    static constexpr bool use_append = true;
+};
 
 template <LogicalType LT, typename State, typename T = RunTimeCppType<LT>>
 class ValueWindowFunction : public WindowFunction<State> {
@@ -124,7 +129,7 @@ public:
 
             Column* data_column = nullable_column->mutable_data_column();
             auto* column = down_cast<InputColumnType*>(data_column);
-            auto value = AggregateFunctionStateHelper<State>::data(state).value;
+            auto& value = AggregateFunctionStateHelper<State>::data(state).value;
             for (size_t i = start; i < end; ++i) {
                 AggDataTypeTraits<LT>::append_value(column, value);
             }
@@ -562,8 +567,14 @@ class LeadLagWindowFunction final : public ValueWindowFunction<LT, LeadLagState<
         if (default_column->is_nullable()) {
             this->data(state).default_is_null = true;
         } else {
-            auto value = ColumnHelper::get_const_value<LT>(arg2);
-            AggDataTypeTraits<LT>::assign_value(this->data(state).default_value, value);
+            if constexpr (lt_is_array<LT>) {
+                const auto* column = down_cast<const ArrayColumn*>(ColumnHelper::get_data_column(arg2));
+                AggDataTypeTraits<LT>::assign_value(this->data(state).default_value,
+                                                    AggDataTypeTraits<LT>::get_row_ref(*column, 0));
+            } else {
+                auto value = ColumnHelper::get_const_value<LT>(arg2);
+                AggDataTypeTraits<LT>::assign_value(this->data(state).default_value, value);
+            }
         }
 
         if constexpr (ignoreNulls) {
@@ -669,7 +680,13 @@ class LeadLagWindowFunction final : public ValueWindowFunction<LT, LeadLagState<
                 if (this->data(state).default_is_null) {
                     this->data(state).is_null = true;
                 } else {
-                    this->data(state).value = this->data(state).default_value;
+                    if constexpr (lt_is_array<LT>) {
+                        AggDataTypeTraits<LT>::assign_value(
+                                this->data(state).value,
+                                AggDataTypeTraits<LT>::get_row_ref(*this->data(state).default_value, 0));
+                    } else {
+                        this->data(state).value = this->data(state).default_value;
+                    }
                 }
             } else {
                 const Column* data_column = ColumnHelper::get_data_column(columns[0]);
@@ -686,7 +703,13 @@ class LeadLagWindowFunction final : public ValueWindowFunction<LT, LeadLagState<
                     this->data(state).is_null = true;
                 } else {
                     this->data(state).is_null = false;
-                    this->data(state).value = this->data(state).default_value;
+                    if constexpr (lt_is_array<LT>) {
+                        AggDataTypeTraits<LT>::assign_value(
+                                this->data(state).value,
+                                AggDataTypeTraits<LT>::get_row_ref(*this->data(state).default_value, 0));
+                    } else {
+                        this->data(state).value = this->data(state).default_value;
+                    }
                 }
                 return;
             }

be/src/exprs/arithmetic_expr.cpp

@@ -44,7 +44,7 @@ namespace starrocks {
                                                       \
     virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new CLASS_NAME(*this)); }
 
-static std::optional<LogicalType> eliminate_trivial_cast_for_decimal_mul(const Expr* e) {
+[[maybe_unused]] static std::optional<LogicalType> eliminate_trivial_cast_for_decimal_mul(const Expr* e) {
     DIAGNOSTIC_PUSH
 #if defined(__GNUC__) && !defined(__clang__)
     DIAGNOSTIC_IGNORE("-Wmaybe-uninitialized")

be/src/exprs/cast_expr_array.cpp

@@ -176,7 +176,8 @@ StatusOr<ColumnPtr> CastStringToArray::evaluate_checked(ExprContext* context, Ch
         // return null if not valid array
         if (!is_valid_array(str, stack)) {
             if (_throw_exception_if_err) {
-                return Status::InternalError(fmt::format("invalid array input: {}", str));
+                return Status::InternalError(
+                        fmt::format("invalid array input: {}", std::string_view(str.get_data(), str.get_size())));
             } else {
                 has_null = true;
                 null_column->append(1);

be/src/exprs/decimal_cast_expr.h

@@ -140,6 +140,15 @@ struct DecimalNonDecimalCast<overflow_mode, DecimalType, NonDecimalType, Decimal
     using NonDecimalColumnType = RunTimeColumnType<NonDecimalType>;
 
     static inline ColumnPtr decimal_from(const ColumnPtr& column, int precision, int scale) {
+        if (scale == 0) {
+            return _decimal_from<true>(column, precision, scale);
+        } else {
+            return _decimal_from<false>(column, precision, scale);
+        }
+    }
+
+    template <bool ZeroScale>
+    static inline ColumnPtr _decimal_from(const ColumnPtr& column, int precision, int scale) {
         const auto num_rows = column->size();
         typename DecimalColumnType::MutablePtr result = DecimalColumnType::create(precision, scale, num_rows);
         const auto data = &ColumnHelper::cast_to_raw<NonDecimalType>(column.get())->immutable_data().front();
@@ -163,9 +172,16 @@ struct DecimalNonDecimalCast<overflow_mode, DecimalType, NonDecimalType, Decimal
                         DecimalV3Cast::from_integer<SignedBooleanType, DecimalCppType, check_overflow<overflow_mode>>(
                                 (SignedBooleanType)(data[i]), scale_factor, &result_data[i]);
             } else if constexpr (lt_is_integer<NonDecimalType>) {
-                overflow =
-                        DecimalV3Cast::from_integer<NonDecimalCppType, DecimalCppType, check_overflow<overflow_mode>>(
-                                data[i], scale_factor, &result_data[i]);
+                if constexpr (ZeroScale) {
+                    // Fast path for integer-to-decimal conversion with scale 0.
+                    overflow =
+                            DecimalV3Cast::to_decimal_trivial<NonDecimalCppType, DecimalCppType,
+                                                              check_overflow<overflow_mode>>(data[i], &result_data[i]);
+                } else {
+                    overflow = DecimalV3Cast::from_integer<NonDecimalCppType, DecimalCppType,
+                                                           check_overflow<overflow_mode>>(data[i], scale_factor,
+                                                                                          &result_data[i]);
+                }
             } else if constexpr (lt_is_float<NonDecimalType>) {
                 overflow = DecimalV3Cast::from_float<NonDecimalCppType, DecimalCppType>(data[i], scale_factor,
                                                                                         &result_data[i]);
@@ -218,6 +234,7 @@ struct DecimalNonDecimalCast<overflow_mode, DecimalType, NonDecimalType, Decimal
                 }
             }
         }
+
         if constexpr (check_overflow<overflow_mode>) {
             ColumnBuilder<DecimalType> builder(std::move(result), std::move(null_column), has_null);
             return builder.build(column->is_constant());

be/src/exprs/function_context.cpp

@@ -20,10 +20,14 @@
 #include "column/map_column.h"
 #include "column/struct_column.h"
 #include "column/type_traits.h"
+#ifndef MACOS_DISABLE_JAVA
 #include "exprs/agg/java_udaf_function.h"
+#endif
 #include "runtime/runtime_state.h"
 #include "types/logical_type_infra.h"
+#ifndef MACOS_DISABLE_JAVA
 #include "udf/java/java_udf.h"
+#endif
 #include "util/bloom_filter.h"
 
 namespace starrocks {
@@ -38,7 +42,7 @@ FunctionContext* FunctionContext::create_context(RuntimeState* state, MemPool* p
     ctx->_mem_pool = pool;
     ctx->_return_type = return_type;
     ctx->_arg_types = arg_types;
-#if !defined(BUILD_FORMAT_LIB)
+#if !defined(MACOS_DISABLE_JAVA) && !defined(BUILD_FORMAT_LIB)
     ctx->_jvm_udaf_ctxs = std::make_unique<JavaUDAFContext>();
 #endif
     return ctx;
@@ -54,7 +58,7 @@ FunctionContext* FunctionContext::create_context(RuntimeState* state, MemPool* p
     ctx->_mem_pool = pool;
     ctx->_return_type = return_type;
     ctx->_arg_types = arg_types;
-#if !defined(BUILD_FORMAT_LIB)
+#if !defined(MACOS_DISABLE_JAVA) && !defined(BUILD_FORMAT_LIB)
     ctx->_jvm_udaf_ctxs = std::make_unique<JavaUDAFContext>();
 #endif
     ctx->_is_distinct = is_distinct;
@@ -137,10 +141,12 @@ void* FunctionContext::get_function_state(FunctionStateScope scope) const {
 }
 
 void FunctionContext::release_mems() {
+#ifndef MACOS_DISABLE_JAVA
     if (_jvm_udaf_ctxs != nullptr && _jvm_udaf_ctxs->states) {
         auto env = JVMFunctionHelper::getInstance().getEnv();
         _jvm_udaf_ctxs->states->clear(this, env);
     }
+#endif
 }
 
 void FunctionContext::set_error(const char* error_msg, const bool is_udf) {

be/src/exprs/function_context.h

@@ -35,6 +35,12 @@ class RuntimeState;
 class Column;
 class Slice;
 struct JavaUDAFContext;
+#if defined(MACOS_DISABLE_JAVA)
+// On macOS build, Java is disabled. Provide an empty definition so that
+// std::unique_ptr<JavaUDAFContext> has a complete type and can be destroyed
+// without pulling in JNI headers.
+struct JavaUDAFContext {};
+#endif
 struct NgramBloomFilterState;
 
 class FunctionContext {

be/src/exprs/string_functions.cpp

@@ -3468,6 +3468,76 @@ StatusOr<ColumnPtr> StringFunctions::regexp_extract(FunctionContext* context, co
     return regexp_extract_general(context, options, columns);
 }
 
+// Helper function to extract whole match (group 0) using RE2::Match
+// This is shared by both overloaded extract_regex_matches functions
+template <typename IndexType>
+static void extract_whole_matches(const re2::StringPiece& str_sp, const re2::RE2& regex, BinaryColumn* str_col,
+                                  IndexType& index, int max_matches) {
+    re2::StringPiece input = str_sp;
+    std::vector<re2::StringPiece> matches(max_matches);
+    size_t pos = 0;
+
+    while (pos <= input.size()) {
+        re2::StringPiece remaining = input.substr(pos);
+        if (regex.Match(remaining, 0, remaining.size(), RE2::UNANCHORED, &matches[0], max_matches)) {
+            // matches[0] contains the whole match (group 0)
+            str_col->append(Slice(matches[0].data(), matches[0].size()));
+            index += 1;
+            // Move past this match
+            pos = matches[0].data() - input.data() + matches[0].size();
+            if (matches[0].size() == 0) {
+                pos++; // Avoid infinite loop on zero-length matches
+            }
+        } else {
+            break;
+        }
+    }
+}
+
+// Helper function to extract regex matches and append to column
+// This reduces code duplication across regexp_extract_all_* functions
+static void extract_regex_matches(const Slice& str_value, const re2::RE2& regex, int group, BinaryColumn* str_col,
+                                  uint32_t& index, int max_matches) {
+    re2::StringPiece str_sp(str_value.get_data(), str_value.get_size());
+
+    if (group == 0) {
+        // Extract the whole match (group 0)
+        extract_whole_matches(str_sp, regex, str_col, index, max_matches);
+    } else {
+        // Extract specific capture group
+        re2::StringPiece find[group];
+        const RE2::Arg* args[group];
+        RE2::Arg argv[group];
+
+        for (size_t i = 0; i < group; i++) {
+            argv[i] = &find[i];
+            args[i] = &argv[i];
+        }
+        while (re2::RE2::FindAndConsumeN(&str_sp, regex, args, group)) {
+            str_col->append(Slice(find[group - 1].data(), find[group - 1].size()));
+            index += 1;
+        }
+    }
+}
+
+// Overloaded version for pre-allocated arrays (used by regexp_extract_all_const)
+static void extract_regex_matches(const Slice& str_value, const re2::RE2& regex, int group, BinaryColumn* str_col,
+                                  uint64_t& index, const std::unique_ptr<re2::StringPiece[]>& find,
+                                  const std::unique_ptr<const RE2::Arg*[]>& args, int max_matches) {
+    re2::StringPiece str_sp(str_value.get_data(), str_value.get_size());
+
+    if (group == 0) {
+        // Extract the whole match (group 0) - reuse common logic
+        extract_whole_matches(str_sp, regex, str_col, index, max_matches);
+    } else {
+        // Extract specific capture group using pre-allocated arrays
+        while (re2::RE2::FindAndConsumeN(&str_sp, regex, args.get(), group)) {
+            str_col->append(Slice(find[group - 1].data(), find[group - 1].size()));
+            index += 1;
+        }
+    }
+}
+
 static ColumnPtr regexp_extract_all_general(FunctionContext* context, re2::RE2::Options* options,
                                             const Columns& columns) {
     auto content_viewer = ColumnViewer<TYPE_VARCHAR>(columns[0]);
@@ -3483,7 +3553,7 @@ static ColumnPtr regexp_extract_all_general(FunctionContext* context, re2::RE2::
     uint32_t index = 0;
 
     for (int row = 0; row < size; ++row) {
-        if (content_viewer.is_null(row) || ptn_viewer.is_null(row)) {
+        if (content_viewer.is_null(row) || ptn_viewer.is_null(row) || group_viewer.is_null(row)) {
             offset_col->append(index);
             nl_col->append(1);
             continue;
@@ -3500,7 +3570,7 @@ static ColumnPtr regexp_extract_all_general(FunctionContext* context, re2::RE2::
 
         nl_col->append(0);
         auto group = group_viewer.value(row);
-        if (group <= 0) {
+        if (group < 0) {
             offset_col->append(index);
             continue;
         }
@@ -3511,21 +3581,7 @@ static ColumnPtr regexp_extract_all_general(FunctionContext* context, re2::RE2::
             continue;
         }
 
-        auto str_value = content_viewer.value(row);
-        re2::StringPiece str_sp(str_value.get_data(), str_value.get_size());
-
-        re2::StringPiece find[group];
-        const RE2::Arg* args[group];
-        RE2::Arg argv[group];
-
-        for (size_t i = 0; i < group; i++) {
-            argv[i] = &find[i];
-            args[i] = &argv[i];
-        }
-        while (re2::RE2::FindAndConsumeN(&str_sp, local_re, args, group)) {
-            str_col->append(Slice(find[group - 1].data(), find[group - 1].size()));
-            index += 1;
-        }
+        extract_regex_matches(content_viewer.value(row), local_re, group, str_col.get(), index, max_matches);
         offset_col->append(index);
     }
 
@@ -3547,7 +3603,7 @@ static ColumnPtr regexp_extract_all_const_pattern(re2::RE2* const_re, const Colu
     uint32_t index = 0;
 
     for (int row = 0; row < size; ++row) {
-        if (content_viewer.is_null(row)) {
+        if (content_viewer.is_null(row) || group_viewer.is_null(row)) {
             offset_col->append(index);
             nl_col->append(1);
             continue;
@@ -3555,7 +3611,7 @@ static ColumnPtr regexp_extract_all_const_pattern(re2::RE2* const_re, const Colu
 
         nl_col->append(0);
         auto group = group_viewer.value(row);
-        if (group <= 0) {
+        if (group < 0) {
             offset_col->append(index);
             continue;
         }
@@ -3566,21 +3622,7 @@ static ColumnPtr regexp_extract_all_const_pattern(re2::RE2* const_re, const Colu
             continue;
         }
 
-        auto str_value = content_viewer.value(row);
-        re2::StringPiece str_sp(str_value.get_data(), str_value.get_size());
-
-        re2::StringPiece find[group];
-        const RE2::Arg* args[group];
-        RE2::Arg argv[group];
-
-        for (size_t i = 0; i < group; i++) {
-            argv[i] = &find[i];
-            args[i] = &argv[i];
-        }
-        while (re2::RE2::FindAndConsumeN(&str_sp, *const_re, args, group)) {
-            str_col->append(Slice(find[group - 1].data(), find[group - 1].size()));
-            index += 1;
-        }
+        extract_regex_matches(content_viewer.value(row), *const_re, group, str_col.get(), index, max_matches);
         offset_col->append(index);
     }
 
@@ -3612,7 +3654,7 @@ static ColumnPtr regexp_extract_all_const(re2::RE2* const_re, const Columns& col
 
     uint64_t index = 0;
     int max_matches = 1 + const_re->NumberOfCapturingGroups();
-    if (group <= 0 || group >= max_matches) {
+    if (group < 0 || group >= max_matches) {
         offset_col->append_value_multiple_times(&index, size);
         auto array = ArrayColumn::create(NullableColumn::create(std::move(str_col), NullColumn::create(0, 0)),
                                          std::move(offset_col));
@@ -3623,26 +3665,27 @@ static ColumnPtr regexp_extract_all_const(re2::RE2* const_re, const Columns& col
         return NullableColumn::create(std::move(array), std::move(nl_col));
     }
 
-    re2::StringPiece find[group];
-    const RE2::Arg* args[group];
-    RE2::Arg argv[group];
+    // Prepare arguments for FindAndConsumeN (only needed when group > 0)
+    std::unique_ptr<re2::StringPiece[]> find;
+    std::unique_ptr<const RE2::Arg*[]> args;
+    std::unique_ptr<RE2::Arg[]> argv;
 
-    for (size_t i = 0; i < group; i++) {
-        argv[i] = &find[i];
-        args[i] = &argv[i];
-    }
-    for (int row = 0; row < size; ++row) {
-        if (content_viewer.is_null(row)) {
-            offset_col->append(index);
-            continue;
+    if (group > 0) {
+        find = std::make_unique<re2::StringPiece[]>(group);
+        args = std::make_unique<const RE2::Arg*[]>(group);
+        argv = std::make_unique<RE2::Arg[]>(group);
+
+        for (size_t i = 0; i < group; i++) {
+            argv[i] = &find[i];
+            args[i] = &argv[i];
         }
+    }
 
-        auto str_value = content_viewer.value(row);
-        re2::StringPiece str_sp(str_value.get_data(), str_value.get_size());
-        while (re2::RE2::FindAndConsumeN(&str_sp, *const_re, args, group)) {
-            str_col->append(Slice(find[group - 1].data(), find[group - 1].size()));
-
-            index += 1;
+    // focuses only on iteration and offset management
+    for (int row = 0; row < size; ++row) {
+        if (!content_viewer.is_null(row)) {
+            extract_regex_matches(content_viewer.value(row), *const_re, group, str_col.get(), index, find, args,
+                                  max_matches);
         }
         offset_col->append(index);
     }

be/src/exprs/time_functions.cpp

@@ -3897,6 +3897,39 @@ StatusOr<ColumnPtr> TimeFunctions::time_format(FunctionContext* context, const s
     return builder.build(ColumnHelper::is_all_const(columns));
 }
 
+constexpr static const int64_t MAX_TIME = 3023999L;
+
+static int64_t from_seconds_with_limit(int64_t time) {
+    if (time > MAX_TIME) {
+        return MAX_TIME;
+    }
+    if (time < -MAX_TIME) {
+        return -MAX_TIME;
+    }
+    return time;
+}
+
+StatusOr<ColumnPtr> TimeFunctions::sec_to_time(FunctionContext* context, const starrocks::Columns& columns) {
+    const auto& bigint_column = columns[0];
+
+    RETURN_IF_COLUMNS_ONLY_NULL(columns);
+
+    auto bigint_viewer = ColumnViewer<TYPE_BIGINT>(bigint_column);
+    const size_t size = bigint_column->size();
+    auto builder = ColumnBuilder<TYPE_TIME>(size);
+
+    for (size_t i = 0; i < size; ++i) {
+        if (bigint_viewer.is_null(i)) {
+            builder.append_null();
+            continue;
+        }
+        auto time = static_cast<double>(from_seconds_with_limit(bigint_viewer.value(i)));
+        builder.append(time);
+    }
+
+    return builder.build(ColumnHelper::is_all_const(columns));
+}
+
 } // namespace starrocks
 
 #include "gen_cpp/opcode/TimeFunctions.inc"

be/src/exprs/time_functions.h

@@ -766,6 +766,14 @@ public:
      */
     DEFINE_VECTORIZED_FN(time_to_sec);
 
+    /**
+     * return time
+     * @param: [int]
+     * @paramType columns: [BinaryColumn]
+     * @return Int64Column
+     */
+    DEFINE_VECTORIZED_FN(sec_to_time);
+
     /**
      * Returns the date of the first specified DOW (day of week) that occurs after the input date.
      * @param: [timestamp, dow]

be/src/formats/parquet/column_converter.cpp

@@ -147,7 +147,7 @@ public:
         auto& src_null_data = src_nullable_column->null_column()->get_data();
         auto& dst_null_data = dst_nullable_column->null_column()->get_data();
 
-        size_t size = src_column->size();
+        size_t size = dst_null_data.size();
         memcpy(dst_null_data.data(), src_null_data.data(), size);
         convert_int_to_int<SourceType, DestType>(src_data.data(), dst_data.data(), size);
         dst_nullable_column->set_has_null(src_nullable_column->has_null());

be/src/formats/parquet/file_reader.h

@@ -20,7 +20,7 @@
 #include <string>
 #include <vector>
 
-#include "cache/block_cache/block_cache.h"
+#include "cache/disk_cache/block_cache.h"
 #include "column/vectorized_fwd.h"
 #include "common/status.h"
 #include "common/statusor.h"

be/src/formats/parquet/metadata.cpp

@@ -470,7 +470,7 @@ StatusOr<FileMetaDataPtr> FileMetaDataParser::get_file_metadata() {
     RETURN_IF_ERROR(_parse_footer(&file_metadata, &file_metadata_size));
     if (file_metadata_size > 0) {
         auto deleter = [](const starrocks::CacheKey& key, void* value) { delete (FileMetaDataPtr*)value; };
-        ObjectCacheWriteOptions options;
+        MemCacheWriteOptions options;
         options.evict_probability = _datacache_options->datacache_evict_probability;
         auto capture = std::make_unique<FileMetaDataPtr>(file_metadata);
         Status st = _cache->insert(metacache_key, (void*)(capture.get()), file_metadata_size, deleter, options,

be/src/formats/parquet/page_reader.cpp

@@ -41,7 +41,7 @@ static constexpr size_t kDefaultPageHeaderSize = 16 * 1024;
 // 16MB is borrowed from Arrow
 static constexpr size_t kMaxPageHeaderSize = 16 * 1024 * 1024;
 
-PageReader::PageReader(io::SeekableInputStream* stream, uint64_t start_offset, uint64_t length, uint64_t num_values,
+PageReader::PageReader(io::SeekableInputStream* stream, size_t start_offset, size_t length, size_t num_values,
                        const ColumnReaderOptions& opts, const tparquet::CompressionCodec::type codec)
         : _stream(stream),
           _finish_offset(start_offset + length),
@@ -92,7 +92,7 @@ Status PageReader::_deal_page_with_cache() {
             return Status::OK();
         }
         RETURN_IF_ERROR(_read_and_decompress_internal(true));
-        ObjectCacheWriteOptions opts{.evict_probability = _opts.datacache_options->datacache_evict_probability};
+        MemCacheWriteOptions opts{.evict_probability = _opts.datacache_options->datacache_evict_probability};
         auto st = _cache->insert(page_cache_key, _cache_buf, opts, &cache_handle);
         if (st.ok()) {
             _page_handle = PageHandle(std::move(cache_handle));

be/src/formats/parquet/parquet_file_writer.cpp

@@ -266,7 +266,8 @@ StatusOr<::parquet::Compression::type> ParquetFileWriter::_convert_compression_t
 
     // Check if arrow supports indicated compression type
     if (!::parquet::IsCodecSupported(converted_type)) {
-        return Status::NotSupported(fmt::format("not supported compression codec {}", converted_type));
+        return Status::NotSupported(
+                fmt::format("not supported compression codec {}", static_cast<int>(converted_type)));
     }
 
     return converted_type;
@@ -455,13 +456,12 @@ Status ParquetFileWriter::init() {
 
 ParquetFileWriter::~ParquetFileWriter() = default;
 
-ParquetFileWriterFactory::ParquetFileWriterFactory(std::shared_ptr<FileSystem> fs,
-                                                   TCompressionType::type compression_type,
-                                                   std::map<std::string, std::string> options,
-                                                   std::vector<std::string> column_names,
-                                                   std::vector<std::unique_ptr<ColumnEvaluator>>&& column_evaluators,
-                                                   std::optional<std::vector<formats::FileColumnId>> field_ids,
-                                                   PriorityThreadPool* executors, RuntimeState* runtime_state)
+ParquetFileWriterFactory::ParquetFileWriterFactory(
+        std::shared_ptr<FileSystem> fs, TCompressionType::type compression_type,
+        std::map<std::string, std::string> options, std::vector<std::string> column_names,
+        std::shared_ptr<std::vector<std::unique_ptr<ColumnEvaluator>>> column_evaluators,
+        std::optional<std::vector<formats::FileColumnId>> field_ids, PriorityThreadPool* executors,
+        RuntimeState* runtime_state)
         : _fs(std::move(fs)),
           _compression_type(compression_type),
           _field_ids(std::move(field_ids)),
@@ -472,7 +472,7 @@ ParquetFileWriterFactory::ParquetFileWriterFactory(std::shared_ptr<FileSystem> f
           _runtime_state(runtime_state) {}
 
 Status ParquetFileWriterFactory::init() {
-    RETURN_IF_ERROR(ColumnEvaluator::init(_column_evaluators));
+    RETURN_IF_ERROR(ColumnEvaluator::init(*_column_evaluators));
     _parsed_options = std::make_shared<ParquetWriterOptions>();
     _parsed_options->column_ids = _field_ids;
     if (_options.contains(ParquetWriterOptions::USE_LEGACY_DECIMAL_ENCODING)) {
@@ -506,8 +506,8 @@ StatusOr<WriterAndStream> ParquetFileWriterFactory::create(const std::string& pa
     auto rollback_action = [fs = _fs, path = path]() {
         WARN_IF_ERROR(ignore_not_found(fs->delete_file(path)), "fail to delete file");
     };
-    auto column_evaluators = ColumnEvaluator::clone(_column_evaluators);
-    auto types = ColumnEvaluator::types(_column_evaluators);
+    auto column_evaluators = ColumnEvaluator::clone(*_column_evaluators);
+    auto types = ColumnEvaluator::types(*_column_evaluators);
     auto async_output_stream =
             std::make_unique<io::AsyncFlushOutputStream>(std::move(file), _executors, _runtime_state);
     auto parquet_output_stream = std::make_shared<parquet::AsyncParquetOutputStream>(async_output_stream.get());

be/src/formats/parquet/parquet_file_writer.h

@@ -162,7 +162,7 @@ class ParquetFileWriterFactory : public FileWriterFactory {
 public:
     ParquetFileWriterFactory(std::shared_ptr<FileSystem> fs, TCompressionType::type compression_type,
                              std::map<std::string, std::string> options, std::vector<std::string> column_names,
-                             std::vector<std::unique_ptr<ColumnEvaluator>>&& column_evaluators,
+                             std::shared_ptr<std::vector<std::unique_ptr<ColumnEvaluator>>> column_evaluators,
                              std::optional<std::vector<formats::FileColumnId>> field_ids, PriorityThreadPool* executors,
                              RuntimeState* runtime_state);
 
@@ -178,7 +178,7 @@ private:
     std::shared_ptr<ParquetWriterOptions> _parsed_options;
 
     std::vector<std::string> _column_names;
-    std::vector<std::unique_ptr<ColumnEvaluator>> _column_evaluators;
+    std::shared_ptr<std::vector<std::unique_ptr<ColumnEvaluator>>> _column_evaluators;
     PriorityThreadPool* _executors = nullptr;
     RuntimeState* _runtime_state = nullptr;
 };

be/src/fs/s3/poco_common.cpp

@@ -92,7 +92,6 @@ HTTPSessionPtr makeHTTPSessionImpl(const std::string& host, Poco::UInt16 port, b
 }
 
 EndpointHTTPSessionPool::Base::ObjectPtr EndpointHTTPSessionPool::allocObject() {
-    SCOPED_THREAD_LOCAL_MEM_TRACKER_SETTER(_mem_tracker);
     auto session = makeHTTPSessionImpl(_host, _port, _is_https, true);
     return session;
 }

be/src/fs/s3/poco_common.h

@@ -68,9 +68,7 @@ class EndpointHTTPSessionPool : public PoolBase<Poco::Net::HTTPClientSession> {
 public:
     using Base = PoolBase<Poco::Net::HTTPClientSession>;
     EndpointHTTPSessionPool(std::string host, uint16_t port, bool is_https)
-            : Base(ENDPOINT_POOL_SIZE), _host(std::move(host)), _port(port), _is_https(is_https) {
-        _mem_tracker = GlobalEnv::GetInstance()->poco_connection_pool_mem_tracker();
-    }
+            : Base(ENDPOINT_POOL_SIZE), _host(std::move(host)), _port(port), _is_https(is_https) {}
 
 private:
     ObjectPtr allocObject() override;
@@ -80,7 +78,6 @@ private:
     const std::string _host;
     const uint16_t _port;
     const bool _is_https;
-    MemTracker* _mem_tracker = nullptr;
 };
 
 class HTTPSessionPools {

be/src/gutil/walltime.h

@@ -28,6 +28,11 @@
 #include <mach/clock.h>
 #include <mach/mach.h>
 #include <mach/mach_time.h>
+// macOS system headers may define a function-like macro named current_task(),
+// which conflicts with StarRocks member function names. Undefine it here.
+#ifdef current_task
+#undef current_task
+#endif
 
 #include "gutil/once.h"
 #endif // #if defined(__APPLE__)

be/src/http/action/datacache_action.cpp

@@ -21,15 +21,16 @@
 
 #include <string>
 
-#include "cache/block_cache/block_cache_hit_rate_counter.hpp"
-#include "cache/local_cache_engine.h"
+#include "cache/disk_cache/block_cache_hit_rate_counter.hpp"
+#include "cache/disk_cache/local_disk_cache_engine.h"
+#include "cache/mem_cache/local_mem_cache_engine.h"
 #include "http/http_channel.h"
 #include "http/http_headers.h"
 #include "http/http_request.h"
 #include "http/http_status.h"
 
 #ifdef WITH_STARCACHE
-#include "cache/starcache_engine.h"
+#include "cache/disk_cache/starcache_engine.h"
 #endif
 
 namespace starrocks {
@@ -56,10 +57,8 @@ void DataCacheAction::handle(HttpRequest* req) {
     if (!_check_request(req)) {
         return;
     }
-    if (!_local_cache || !_local_cache->is_initialized()) {
+    if (!_disk_cache || !_disk_cache->is_initialized()) {
         _handle_error(req, strings::Substitute("Cache system is not ready"));
-    } else if (_local_cache->engine_type() != LocalCacheEngineType::STARCACHE) {
-        _handle_error(req, strings::Substitute("No more metrics for current cache engine type"));
     } else if (req->param(ACTION_KEY) == ACTION_STAT) {
         _handle_stat(req);
     } else {
@@ -81,23 +80,29 @@ void DataCacheAction::_handle(HttpRequest* req, const std::function<void(rapidjs
 void DataCacheAction::_handle_stat(HttpRequest* req) {
     _handle(req, [=](rapidjson::Document& root) {
 #ifdef WITH_STARCACHE
+        DataCacheMemMetrics mem_metrics;
+        if (_mem_cache != nullptr) {
+            mem_metrics = _mem_cache->cache_metrics();
+        }
+
         auto& allocator = root.GetAllocator();
-        auto* starcache = reinterpret_cast<StarCacheEngine*>(_local_cache);
+        auto* starcache = reinterpret_cast<StarCacheEngine*>(_disk_cache);
         auto&& metrics = starcache->starcache_metrics(2);
         std::string status = DataCacheStatusUtils::to_string(static_cast<DataCacheStatus>(metrics.status));
 
         rapidjson::Value status_value;
         status_value.SetString(status.c_str(), status.length(), allocator);
         root.AddMember("status", status_value, allocator);
-        root.AddMember("mem_quota_bytes", rapidjson::Value(metrics.mem_quota_bytes), allocator);
-        root.AddMember("mem_used_bytes", rapidjson::Value(metrics.mem_used_bytes), allocator);
+        root.AddMember("mem_quota_bytes", rapidjson::Value(mem_metrics.mem_quota_bytes), allocator);
+        root.AddMember("mem_used_bytes", rapidjson::Value(mem_metrics.mem_used_bytes), allocator);
         root.AddMember("disk_quota_bytes", rapidjson::Value(metrics.disk_quota_bytes), allocator);
         root.AddMember("disk_used_bytes", rapidjson::Value(metrics.disk_used_bytes), allocator);
 
         auto mem_used_rate = 0.0;
-        if (metrics.mem_quota_bytes > 0) {
+        if (mem_metrics.mem_quota_bytes > 0) {
             mem_used_rate =
-                    std::round(double(metrics.mem_used_bytes) / double(metrics.mem_quota_bytes) * 100.0) / 100.0;
+                    std::round(double(mem_metrics.mem_used_bytes) / double(mem_metrics.mem_quota_bytes) * 100.0) /
+                    100.0;
         }
         auto disk_used_rate = 0.0;
         if (metrics.disk_quota_bytes > 0) {

be/src/http/action/datacache_action.h

@@ -28,11 +28,14 @@
 
 namespace starrocks {
 
-class LocalCacheEngine;
+class LocalDiskCacheEngine;
+class LocalMemCacheEngine;
+
 // TODO: support mem metrics
 class DataCacheAction : public HttpHandler {
 public:
-    explicit DataCacheAction(LocalCacheEngine* local_cache) : _local_cache(local_cache) {}
+    explicit DataCacheAction(LocalDiskCacheEngine* disk_cache, LocalMemCacheEngine* mem_cache)
+            : _disk_cache(disk_cache), _mem_cache(mem_cache) {}
     ~DataCacheAction() override = default;
 
     void handle(HttpRequest* req) override;
@@ -44,7 +47,8 @@ private:
     void _handle_app_stat(HttpRequest* req);
     void _handle_error(HttpRequest* req, const std::string& error_msg);
 
-    LocalCacheEngine* _local_cache;
+    LocalDiskCacheEngine* _disk_cache;
+    LocalMemCacheEngine* _mem_cache;
 };
 
 } // namespace starrocks

be/src/http/action/update_config_action.cpp

@@ -44,7 +44,7 @@
 #include "agent/agent_common.h"
 #include "agent/agent_server.h"
 #include "cache/datacache.h"
-#include "cache/object_cache/page_cache.h"
+#include "cache/mem_cache/page_cache.h"
 #include "common/configbase.h"
 #include "common/status.h"
 #include "exec/workgroup/scan_executor.h"
@@ -115,7 +115,7 @@ Status UpdateConfigAction::update_config(const std::string& name, const std::str
             return Status::OK();
         });
         _config_callback.emplace("datacache_mem_size", [&]() -> Status {
-            LocalCacheEngine* cache = DataCache::GetInstance()->local_mem_cache();
+            LocalMemCacheEngine* cache = DataCache::GetInstance()->local_mem_cache();
             if (cache == nullptr || !cache->is_initialized()) {
                 return Status::InternalError("Local cache is not initialized");
             }
@@ -127,10 +127,10 @@ Status UpdateConfigAction::update_config(const std::string& name, const std::str
                 LOG(WARNING) << "Failed to update datacache mem size";
                 return st;
             }
-            return cache->update_mem_quota(mem_size, true);
+            return cache->update_mem_quota(mem_size);
         });
         _config_callback.emplace("datacache_disk_size", [&]() -> Status {
-            LocalCacheEngine* cache = DataCache::GetInstance()->local_disk_cache();
+            LocalDiskCacheEngine* cache = DataCache::GetInstance()->local_disk_cache();
             if (cache == nullptr || !cache->is_initialized()) {
                 return Status::InternalError("Local cache is not initialized");
             }
@@ -149,7 +149,7 @@ Status UpdateConfigAction::update_config(const std::string& name, const std::str
             return cache->update_disk_spaces(spaces);
         });
         _config_callback.emplace("datacache_inline_item_count_limit", [&]() -> Status {
-            LocalCacheEngine* cache = DataCache::GetInstance()->local_disk_cache();
+            LocalDiskCacheEngine* cache = DataCache::GetInstance()->local_disk_cache();
             if (cache == nullptr || !cache->is_initialized()) {
                 return Status::InternalError("Local cache is not initialized");
             }

be/src/io/cache_input_stream.cpp

@@ -125,7 +125,7 @@ Status CacheInputStream::_read_from_cache(const int64_t offset, const int64_t si
     Status res;
     int64_t read_local_cache_ns = 0;
     BlockBuffer block;
-    ReadCacheOptions options;
+    DiskCacheReadOptions options;
     size_t read_size = 0;
     {
         options.use_adaptor = _enable_cache_io_adaptor;
@@ -153,14 +153,13 @@ Status CacheInputStream::_read_from_cache(const int64_t offset, const int64_t si
         read_size = block_size;
 
         if (res.ok() && _enable_populate_cache) {
-            WriteCacheOptions options;
-            options.async = _enable_async_populate_mode;
-            options.evict_probability = _datacache_evict_probability;
-            options.priority = _priority;
-            options.ttl_seconds = _ttl_seconds;
-            options.frequency = _frequency;
-            options.allow_zero_copy = true;
-            _write_cache(block_offset, block.buffer, &options);
+            DiskCacheWriteOptions write_options;
+            write_options.async = _enable_async_populate_mode;
+            write_options.priority = _priority;
+            write_options.ttl_seconds = _ttl_seconds;
+            write_options.frequency = _frequency;
+            write_options.allow_zero_copy = true;
+            _write_cache(block_offset, block.buffer, &write_options);
         }
     }
 
@@ -203,7 +202,7 @@ Status CacheInputStream::_read_from_cache(const int64_t offset, const int64_t si
     return res;
 }
 
-Status CacheInputStream::_read_peer_cache(off_t offset, size_t size, IOBuffer* iobuf, ReadCacheOptions* options) {
+Status CacheInputStream::_read_peer_cache(off_t offset, size_t size, IOBuffer* iobuf, DiskCacheReadOptions* options) {
     options->remote_host = _peer_host;
     options->remote_port = _peer_port;
     return _cache->read_buffer_from_remote_cache(_cache_key, offset, size, iobuf, options);
@@ -445,9 +444,8 @@ void CacheInputStream::_populate_to_cache(const char* p, int64_t offset, int64_t
     int64_t end = std::min((offset + count + _block_size - 1) / _block_size * _block_size, _size);
     p -= (offset - begin);
     auto f = [sb, this](const char* buf, size_t off, size_t size) {
-        WriteCacheOptions options;
+        DiskCacheWriteOptions options;
         options.async = _enable_async_populate_mode;
-        options.evict_probability = _datacache_evict_probability;
         options.priority = _priority;
         options.ttl_seconds = _ttl_seconds;
         options.frequency = _frequency;
@@ -474,7 +472,7 @@ void CacheInputStream::_populate_to_cache(const char* p, int64_t offset, int64_t
     return;
 }
 
-void CacheInputStream::_write_cache(int64_t offset, const IOBuffer& iobuf, WriteCacheOptions* options) {
+void CacheInputStream::_write_cache(int64_t offset, const IOBuffer& iobuf, DiskCacheWriteOptions* options) {
     DCHECK(offset % _block_size == 0);
     if (_already_populated_blocks.contains(offset / _block_size)) {
         // Already populate in CacheInputStream's lifecycle, ignore this time

be/src/io/cache_input_stream.h

@@ -17,8 +17,8 @@
 #include <memory>
 #include <string>
 
-#include "cache/block_cache/block_cache.h"
-#include "cache/block_cache/io_buffer.h"
+#include "cache/disk_cache/block_cache.h"
+#include "cache/disk_cache/io_buffer.h"
 #include "io/shared_buffered_input_stream.h"
 
 namespace starrocks::io {
@@ -76,8 +76,6 @@ public:
 
     void set_enable_cache_io_adaptor(bool v) { _enable_cache_io_adaptor = v; }
 
-    void set_datacache_evict_probability(int32_t v) { _datacache_evict_probability = v; }
-
     void set_priority(const int8_t priority) { _priority = priority; }
 
     void set_frequency(const int8_t frequency) { _frequency = frequency; }
@@ -108,9 +106,9 @@ protected:
     virtual Status _read_blocks_from_remote(const int64_t offset, const int64_t size, char* out);
     Status _read_from_cache(const int64_t offset, const int64_t size, const int64_t block_offset,
                             const int64_t block_size, char* out);
-    Status _read_peer_cache(off_t offset, size_t size, IOBuffer* iobuf, ReadCacheOptions* options);
+    Status _read_peer_cache(off_t offset, size_t size, IOBuffer* iobuf, DiskCacheReadOptions* options);
     void _populate_to_cache(const char* src, int64_t offset, int64_t count, const SharedBufferPtr& sb);
-    void _write_cache(int64_t offset, const IOBuffer& iobuf, WriteCacheOptions* options);
+    void _write_cache(int64_t offset, const IOBuffer& iobuf, DiskCacheWriteOptions* options);
 
     void _deduplicate_shared_buffer(const SharedBufferPtr& sb);
     bool _can_ignore_populate_error(const Status& status) const;
@@ -128,7 +126,6 @@ protected:
     bool _enable_async_populate_mode = false;
     bool _enable_block_buffer = false;
     bool _enable_cache_io_adaptor = false;
-    int32_t _datacache_evict_probability = 100;
 
     std::string _peer_host;
     int32_t _peer_port = 0;

be/src/runtime/decimalv2_value.h

@@ -149,6 +149,10 @@ public:
     operator int32_t() const { return static_cast<int32_t>(operator int64_t()); }
 
     operator size_t() const { return static_cast<size_t>(operator int64_t()); }
+#if defined(__APPLE__)
+    // Disambiguate C-style casts to uint64_t on libc++
+    operator uint64_t() const { return static_cast<uint64_t>(operator int64_t()); }
+#endif
 
     operator float() const { return (float)operator double(); }
 

be/src/runtime/decimalv3.h

@@ -44,6 +44,13 @@ template <>
 struct unsigned_type<int128_t> {
     using type = uint128_t;
 };
+// On macOS with libc++, std::make_unsigned cannot be specialized for user types.
+// Provide an explicit mapping for int256_t to itself, which is sufficient for
+// decimal operations that only require absolute value and arithmetic.
+template <>
+struct unsigned_type<int256_t> {
+    using type = int256_t;
+};
 
 template <typename T, bool check_overflow>
 class DecimalV3Arithmetics {
@@ -295,7 +302,8 @@ public:
 
         if constexpr (rule == ROUND_HALF_UP || rule == ROUND_HALF_EVEN) {
             //TODO(by satanson): ROUND_HALF_UP is different from ROUND_HALF_EVEN
-            need_round = std::abs(remainder) >= (divisor >> 1);
+            auto abs_remainder = (remainder >= 0) ? remainder : -remainder;
+            need_round = abs_remainder >= (divisor >> 1);
         } else if constexpr (rule == ROUND_FLOOR) {
             need_round = remainder > 0 && quotient > 0;
         } else if constexpr (rule == ROUND_CEILING) {

be/src/runtime/exec_env.cpp

@@ -241,7 +241,6 @@ Status GlobalEnv::_init_mem_tracker() {
     _consistency_mem_tracker =
             regist_tracker(MemTrackerType::CONSISTENCY, consistency_mem_limit, process_mem_tracker());
     _datacache_mem_tracker = regist_tracker(MemTrackerType::DATACACHE, -1, process_mem_tracker());
-    _poco_connection_pool_mem_tracker = regist_tracker(MemTrackerType::POCO_CONNECTION_POOL, -1, process_mem_tracker());
     _replication_mem_tracker = regist_tracker(MemTrackerType::REPLICATION, -1, process_mem_tracker());
 
     MemChunkAllocator::init_metrics();

be/src/runtime/exec_env.h

@@ -160,7 +160,6 @@ public:
     MemTracker* consistency_mem_tracker() { return _consistency_mem_tracker.get(); }
     MemTracker* replication_mem_tracker() { return _replication_mem_tracker.get(); }
     MemTracker* datacache_mem_tracker() { return _datacache_mem_tracker.get(); }
-    MemTracker* poco_connection_pool_mem_tracker() { return _poco_connection_pool_mem_tracker.get(); }
     MemTracker* jemalloc_metadata_traker() { return _jemalloc_metadata_tracker.get(); }
     std::shared_ptr<MemTracker> get_mem_tracker_by_type(MemTrackerType type);
     std::vector<std::shared_ptr<MemTracker>> mem_trackers() const;
@@ -235,9 +234,6 @@ private:
     // The memory used for datacache
     std::shared_ptr<MemTracker> _datacache_mem_tracker;
 
-    // The memory used for poco connection pool
-    std::shared_ptr<MemTracker> _poco_connection_pool_mem_tracker;
-
     std::map<MemTrackerType, std::shared_ptr<MemTracker>> _mem_tracker_map;
 };