tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Enhancement] revise some hive catalog cache code (#63459) 

Signed-off-by: SevenJ <wenjun7j@gmail.com>
This commit is contained in:
SevenJ 2025-09-24 20:29:46 +08:00 committed by GitHub
parent 7d81781448
commit ffa6a5f504
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 3 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/com/starrocks/common/Config.java
View File

@ -2409,16 +2409,18 @@ public class Config extends ConfigBase {
public static long hive_meta_cache_refresh_interval_s = 60;
/**
* Implicit for users
* Hive metastore cache ttl
*/
@ConfField
public static long hive_meta_cache_ttl_s = 3600L * 24L;
/**
* Implicit for users
* Remote file's metadata from hdfs or s3 cache ttl
*/
@ConfField
public static long remote_file_cache_ttl_s = 3600 * 36L;
public static long remote_file_cache_ttl_s = 3600 * 24L;
/**
* The maximum number of partitions to fetch from the metastore in one RPC.

12
fe/fe-core/src/main/java/com/starrocks/connector/CachingRemoteFileConf.java
View File

@ -22,8 +22,6 @@ import java.util.Map;
public class CachingRemoteFileConf {
private final long cacheTtlSec;
private final long cacheRefreshIntervalSec;
private long cacheMaxSize = 1000000L; //deprecated, use memSizeRatio ratio instead
private final int perQueryCacheMaxSize = 10000;
private final int refreshMaxThreadNum;
private double memSizeRatio = 0.1; // 10% of the total memory
@ -32,8 +30,6 @@ public class CachingRemoteFileConf {
String.valueOf(Config.remote_file_cache_ttl_s)));
this.cacheRefreshIntervalSec = Long.parseLong(conf.getOrDefault("remote_file_cache_refresh_interval_sec",
String.valueOf(Config.remote_file_cache_refresh_interval_s)));
//deprecated, use memSizeRatio ratio instead
this.cacheMaxSize = Long.parseLong(conf.getOrDefault("remote_file_cache_max_num", String.valueOf(cacheMaxSize)));
this.refreshMaxThreadNum = Integer.parseInt(conf.getOrDefault("async_refresh_max_thread_num", "32"));
this.memSizeRatio = Double.parseDouble(conf.getOrDefault("remote_file_cache_memory_ratio", String.valueOf(memSizeRatio)));
}
@ -46,14 +42,6 @@ public class CachingRemoteFileConf {
return cacheRefreshIntervalSec;
}
public long getCacheMaxSize() {
return cacheMaxSize;
}
public int getPerQueryCacheMaxSize() {
return perQueryCacheMaxSize;
}
public int getRefreshMaxThreadNum() {
return refreshMaxThreadNum;
}

2
fe/fe-core/src/main/java/com/starrocks/connector/hive/HiveMetadataFactory.java
View File

@ -35,7 +35,6 @@ public class HiveMetadataFactory {
private final IHiveMetastore metastore;
private final RemoteFileIO remoteFileIO;
private final long perQueryMetastoreMaxNum;
private final long perQueryCacheRemotePathMaxNum; //deprecated
private final double perQueryCacheRemotePathMaxMemoryRatio;
private final ExecutorService pullRemoteFileExecutor;
private final Executor updateRemoteFilesExecutor;
@ -65,7 +64,6 @@ public class HiveMetadataFactory {
this.metastore = metastore;
this.remoteFileIO = remoteFileIO;
this.perQueryMetastoreMaxNum = hmsConf.getPerQueryCacheMaxNum();
this.perQueryCacheRemotePathMaxNum = fileConf.getPerQueryCacheMaxSize(); // deprecated
this.perQueryCacheRemotePathMaxMemoryRatio = 0.0; //It is duplicated with Catalog level CachingRemoteFileIO, set as 0.
this.pullRemoteFileExecutor = pullRemoteFileExecutor;
this.updateRemoteFilesExecutor = updateRemoteFilesExecutor;

2
fe/fe-core/src/main/java/com/starrocks/connector/hudi/HudiMetadataFactory.java
View File

@ -40,7 +40,6 @@ public class HudiMetadataFactory {
private final IHiveMetastore metastore;
private final RemoteFileIO remoteFileIO;
private final long perQueryMetastoreMaxNum;
private final long perQueryCacheRemotePathMaxNum; //deprecated
private final double perQueryCacheRemotePathMaxMemoryRatio;
private final ExecutorService pullRemoteFileExecutor;
private final boolean isRecursive;
@ -62,7 +61,6 @@ public class HudiMetadataFactory {
this.metastore = metastore;
this.remoteFileIO = remoteFileIO;
this.perQueryMetastoreMaxNum = hmsConf.getPerQueryCacheMaxNum();
this.perQueryCacheRemotePathMaxNum = fileConf.getPerQueryCacheMaxSize();
this.perQueryCacheRemotePathMaxMemoryRatio = 0.0; //It is duplicated with Catalog level CachingRemoteFileIO, set as 0.
this.pullRemoteFileExecutor = pullRemoteFileExecutor;
this.isRecursive = isRecursive;