tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[BugFix] Fix Child Class Loader of JNI Readers (#60163) 

Signed-off-by: yan zhang <dirtysalt1987@gmail.com>
This commit is contained in:
yan zhang 2025-06-24 18:44:17 +08:00 committed by GitHub
parent 05d01b32f6
commit ba48ea9526
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 27 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
java-extensions/common-runtime/pom.xml
View File

@ -19,7 +19,6 @@
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bundle</artifactId>
<version>${aws-v2-sdk.version}</version>
</dependency>
</dependencies>

5
java-extensions/hive-reader/pom.xml
View File

@ -32,11 +32,6 @@
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>

15
java-extensions/hive-reader/src/main/java/com/starrocks/hive/reader/HiveScannerFactory.java
View File

@ -17,24 +17,11 @@ package com.starrocks.hive.reader;
import com.starrocks.jni.connector.ScannerFactory;
import com.starrocks.jni.connector.ScannerHelper;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
public class HiveScannerFactory implements ScannerFactory {
static ClassLoader classLoader;
static {
String basePath = System.getenv("STARROCKS_HOME");
List<File> preloadFiles = new ArrayList<>();
preloadFiles.add(new File(basePath + "/lib/jni-packages/starrocks-hadoop-ext.jar"));
File dir = new File(basePath + "/lib/hive-reader-lib");
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
dir = new File(basePath + "/lib/common-runtime-lib");
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
classLoader = ScannerHelper.createChildFirstClassLoader(preloadFiles, "hive scanner");
classLoader = ScannerHelper.createModuleClassLoader("hive-reader-lib");
}
@Override

5
java-extensions/hudi-reader/pom.xml
View File

@ -34,11 +34,6 @@
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>

15
java-extensions/hudi-reader/src/main/java/com/starrocks/hudi/reader/HudiSliceScannerFactory.java
View File

@ -17,24 +17,11 @@ package com.starrocks.hudi.reader;
import com.starrocks.jni.connector.ScannerFactory;
import com.starrocks.jni.connector.ScannerHelper;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
public class HudiSliceScannerFactory implements ScannerFactory {
static ClassLoader classLoader;
static {
String basePath = System.getenv("STARROCKS_HOME");
List<File> preloadFiles = new ArrayList<>();
preloadFiles.add(new File(basePath + "/lib/jni-packages/starrocks-hadoop-ext.jar"));
File dir = new File(basePath + "/lib/hudi-reader-lib");
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
dir = new File(basePath + "/lib/common-runtime-lib");
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
classLoader = ScannerHelper.createChildFirstClassLoader(preloadFiles, "hudi scanner");
classLoader = ScannerHelper.createModuleClassLoader("hudi-reader-lib");
}
/**

5
java-extensions/iceberg-metadata-reader/pom.xml
View File

@ -27,11 +27,6 @@
<artifactId>hadoop-ext</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>

13
java-extensions/iceberg-metadata-reader/src/main/java/com/starrocks/connector/iceberg/IcebergMetadataScannerFactory.java
View File

@ -17,22 +17,11 @@ package com.starrocks.connector.iceberg;
import com.starrocks.jni.connector.ScannerFactory;
import com.starrocks.jni.connector.ScannerHelper;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
public class IcebergMetadataScannerFactory implements ScannerFactory {
static ClassLoader classLoader;
static {
String basePath = System.getenv("STARROCKS_HOME");
File dir = new File(basePath + "/lib/iceberg-reader-lib");
List<File> preloadFiles = new ArrayList<>(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
dir = new File(basePath + "/lib/common-runtime-lib");
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
classLoader = ScannerHelper.createChildFirstClassLoader(preloadFiles, "iceberg metadata scanner");
classLoader = ScannerHelper.createModuleClassLoader("iceberg-reader-lib");
}
@Override

3
java-extensions/java-utils/src/main/java/com/starrocks/utils/loader/ChildFirstClassLoader.java
View File

@ -39,8 +39,7 @@ public class ChildFirstClassLoader extends URLClassLoader {
super(urls, null);
this.parentLoader = new ParentClassLoader(parent);
// load native method class from parent
this.parentFirstClass = new ArrayList<>(
Arrays.asList("com.starrocks.utils.NativeMethodHelper", "org.slf4j.ILoggerFactory", "org.slf4j.Logger"));
this.parentFirstClass = new ArrayList<>(Arrays.asList("com.starrocks.utils.NativeMethodHelper"));
}
@Override

16
java-extensions/jni-connector/src/main/java/com/starrocks/jni/connector/ScannerHelper.java
View File

@ -20,7 +20,9 @@ import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.function.Function;
public class ScannerHelper {
@ -39,6 +41,20 @@ public class ScannerHelper {
return classLoader;
}
public static ClassLoader createModuleClassLoader(String moduleName) {
String basePath = System.getenv("STARROCKS_HOME");
List<File> preloadFiles = new ArrayList<>();
preloadFiles.add(new File(basePath + "/lib/jni-packages/starrocks-hadoop-ext.jar"));
File dir = new File(basePath + "/lib/" + moduleName);
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
dir = new File(basePath + "/lib/common-runtime-lib");
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
dir = new File(basePath + "/lib/hadoop/common");
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
ClassLoader classLoader = ScannerHelper.createChildFirstClassLoader(preloadFiles, moduleName);
return classLoader;
}
private static void parseKeyValuePairs(String value, String itemSep, String pairSep, Function<String[], Void> addHandler,
Function<String, Void> errorHandler) {
if (value == null) {

5
java-extensions/kudu-reader/pom.xml
View File

@ -33,11 +33,6 @@
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>

14
java-extensions/kudu-reader/src/main/java/com/starrocks/kudu/reader/KuduSplitScannerFactory.java
View File

@ -17,23 +17,11 @@ package com.starrocks.kudu.reader;
import com.starrocks.jni.connector.ScannerFactory;
import com.starrocks.jni.connector.ScannerHelper;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
public class KuduSplitScannerFactory implements ScannerFactory {
static ClassLoader classLoader;
static {
String basePath = System.getenv("STARROCKS_HOME");
List<File> preloadFiles = new ArrayList<>();
File dir = new File(basePath + "/lib/kudu-reader-lib");
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
dir = new File(basePath + "/lib/common-runtime-lib");
preloadFiles.addAll(Arrays.asList(Objects.requireNonNull(dir.listFiles())));
classLoader = ScannerHelper.createChildFirstClassLoader(preloadFiles, "kudu scanner");
classLoader = ScannerHelper.createModuleClassLoader("kudu-reader-lib");
}
/**

5
java-extensions/paimon-reader/pom.xml
View File

@ -32,11 +32,6 @@
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
<dependency>
<groupId>io.trino.hive</groupId>
<artifactId>hive-apache</artifactId>

6
java-extensions/pom.xml
View File

@ -460,6 +460,12 @@
<artifactId>kryo-shaded</artifactId>
<version>${kryo.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bundle</artifactId>
<version>${aws-v2-sdk.version}</version>
</dependency>
</dependencies>
</dependencyManagement>