[Feature] Broker support COS posix bucket (#46597)
Signed-off-by: wyb <wybb86@gmail.com>
This commit is contained in:
wyb
GitHub
parent
61f31a26df
commit
96e44d4d9e
|
|
@ -25,14 +25,12 @@ export STARROCKS_HOME=${ROOT}/../..
|
|||
|
||||
. ${STARROCKS_HOME}/env.sh
|
||||
|
||||
export BROKER_HOME=$ROOT
|
||||
|
||||
# prepare thrift
|
||||
mkdir -p ${BROKER_HOME}/src/main/resources/thrift
|
||||
mkdir -p ${BROKER_HOME}/src/main/thrift
|
||||
|
||||
cp ${BROKER_HOME}/../../gensrc/thrift/FileBrokerService.thrift ${BROKER_HOME}/src/main/resources/thrift/
|
||||
export BROKER_HOME=$ROOT/
|
||||
export BROKER_CORE_HOME=$ROOT/src/broker-core/
|
||||
export HADOOP_COS_HOME=$ROOT/src/hadoop-cos-shaded/
|
||||
|
||||
echo ${BROKER_HOME}
|
||||
cd ${BROKER_HOME}/src
|
||||
$MVN_CMD clean
|
||||
$MVN_CMD package -DskipTests
|
||||
|
||||
|
|
@ -41,7 +39,7 @@ BROKER_OUTPUT=${BROKER_HOME}/output/apache_hdfs_broker/
|
|||
rm -rf ${BROKER_OUTPUT}
|
||||
|
||||
install -d ${BROKER_OUTPUT}/bin ${BROKER_OUTPUT}/conf \
|
||||
${BROKER_OUTPUT}lib/
|
||||
${BROKER_OUTPUT}/lib/
|
||||
|
||||
cp -r ${STARROCKS_THIRDPARTY}/installed/jindosdk/*.jar ${BROKER_OUTPUT}/lib/
|
||||
|
||||
|
|
@ -49,10 +47,11 @@ cp -r -p ${BROKER_HOME}/bin/*.sh ${BROKER_OUTPUT}/bin/
|
|||
cp -r -p ${BROKER_HOME}/conf/*.conf ${BROKER_OUTPUT}/conf/
|
||||
cp -r -p ${BROKER_HOME}/conf/*.xml ${BROKER_OUTPUT}/conf/
|
||||
cp -r -p ${BROKER_HOME}/conf/log4j.properties ${BROKER_OUTPUT}/conf/
|
||||
cp -r -p ${BROKER_HOME}/target/lib/* ${BROKER_OUTPUT}/lib/
|
||||
cp -r -p ${BROKER_HOME}/target/apache_hdfs_broker.jar ${BROKER_OUTPUT}/lib/
|
||||
cp -r -p ${HADOOP_COS_HOME}/target/hadoop-cos-shaded*.jar ${BROKER_OUTPUT}/lib/
|
||||
cp -r -p ${BROKER_CORE_HOME}/target/lib/* ${BROKER_OUTPUT}/lib/
|
||||
cp -r -p ${BROKER_CORE_HOME}/target/apache_hdfs_broker.jar ${BROKER_OUTPUT}/lib/
|
||||
|
||||
echo ""
|
||||
echo "***************************************"
|
||||
echo "**Successfully build StarRocks Broker**"
|
||||
echo "***************************************"
|
||||
echo "***************************************"
|
||||
|
|
|
|||
|
|
@ -23,178 +23,92 @@ under the License.
|
|||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>com</groupId>
|
||||
<artifactId>starrocks-apache-hdfs-broker</artifactId>
|
||||
<parent>
|
||||
<groupId>com</groupId>
|
||||
<artifactId>starrocks-apache-hdfs-broker</artifactId>
|
||||
<version>3.4.0</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>broker-core</artifactId>
|
||||
<version>3.4.0</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>starrocks-apache-hdfs-broker</name>
|
||||
<url>https://github.com/StarRocks</url>
|
||||
|
||||
<properties>
|
||||
<starrocks.home>${basedir}/../../</starrocks.home>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<maven.compiler.source>1.8</maven.compiler.source>
|
||||
<maven.compiler.target>1.8</maven.compiler.target>
|
||||
<hadoop.version>3.3.6</hadoop.version>
|
||||
<guava.version>32.0.1-jre</guava.version>
|
||||
<zookeeper.version>3.9.1</zookeeper.version>
|
||||
<protobuf.version>3.16.3</protobuf.version>
|
||||
<thrift.version>0.14.1</thrift.version>
|
||||
<tomcat.version>8.5.96</tomcat.version>
|
||||
<log4j.version>2.17.1</log4j.version>
|
||||
<jackson.version>2.13.4.2</jackson.version>
|
||||
<starrocks.home>${basedir}/../../../../</starrocks.home>
|
||||
</properties>
|
||||
|
||||
<profiles>
|
||||
<!-- for custom internal repository -->
|
||||
<profile>
|
||||
<id>custom-env</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env.CUSTOM_MAVEN_REPO</name>
|
||||
</property>
|
||||
</activation>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>custom-nexus</id>
|
||||
<url>${env.CUSTOM_MAVEN_REPO}</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>custom-nexus</id>
|
||||
<url>${env.CUSTOM_MAVEN_REPO}</url>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
</profile>
|
||||
|
||||
<!-- for general repository -->
|
||||
<profile>
|
||||
<id>general-env</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>!env.CUSTOM_MAVEN_REPO</name>
|
||||
</property>
|
||||
</activation>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>central</id>
|
||||
<name>central maven repo https</name>
|
||||
<url>https://repo.maven.apache.org/maven2</url>
|
||||
</repository>
|
||||
<!-- for java-cup -->
|
||||
<repository>
|
||||
<id>cloudera-thirdparty</id>
|
||||
<url>https://repository.cloudera.com/content/repositories/third-party/</url>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>cloudera</id>
|
||||
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
|
||||
</repository>
|
||||
<!-- for bdb je -->
|
||||
<repository>
|
||||
<id>oracleReleases</id>
|
||||
<url>https://download.oracle.com/maven</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
<pluginRepositories>
|
||||
<!-- for cup-maven-plugin -->
|
||||
<pluginRepository>
|
||||
<id>spring-plugins</id>
|
||||
<url>http://repo.spring.io/plugins-release/</url>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<dependencies>
|
||||
<!-- https://mvnrepository.com/artifact/commons-cli/commons-cli -->
|
||||
<dependency>
|
||||
<groupId>commons-cli</groupId>
|
||||
<artifactId>commons-cli</artifactId>
|
||||
<version>1.4</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>1.9</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-collections/commons-collections -->
|
||||
<dependency>
|
||||
<groupId>commons-collections</groupId>
|
||||
<artifactId>commons-collections</artifactId>
|
||||
<version>3.2.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-configuration/commons-configuration -->
|
||||
<dependency>
|
||||
<groupId>commons-configuration</groupId>
|
||||
<artifactId>commons-configuration</artifactId>
|
||||
<version>1.6</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-daemon/commons-daemon -->
|
||||
<dependency>
|
||||
<groupId>commons-daemon</groupId>
|
||||
<artifactId>commons-daemon</artifactId>
|
||||
<version>1.0.13</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>2.7</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-lang/commons-lang -->
|
||||
<dependency>
|
||||
<groupId>commons-lang</groupId>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
<version>2.6</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<version>3.3.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-logging/commons-logging -->
|
||||
<dependency>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
<version>1.1.3</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>${guava.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-annotations -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-annotations</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-auth -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-auth</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
|
|
@ -207,34 +121,29 @@ under the License.
|
|||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17-cloudera6</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper -->
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<version>${zookeeper.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs-client</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>com.fasterxml.jackson.core</artifactId>
|
||||
|
|
@ -247,14 +156,12 @@ under the License.
|
|||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/junit/junit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.13.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
|
|
@ -262,7 +169,6 @@ under the License.
|
|||
<dependency>
|
||||
<groupId>org.apache.thrift</groupId>
|
||||
<artifactId>libthrift</artifactId>
|
||||
<version>${thrift.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.apache.tomcat.embed</groupId>
|
||||
|
|
@ -279,49 +185,42 @@ under the License.
|
|||
<dependency>
|
||||
<groupId>org.apache.tomcat.embed</groupId>
|
||||
<artifactId>tomcat-embed-core</artifactId>
|
||||
<version>${tomcat.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.tomcat/tomcat-annotations-api -->
|
||||
<dependency>
|
||||
<groupId>org.apache.tomcat</groupId>
|
||||
<artifactId>tomcat-annotations-api</artifactId>
|
||||
<version>${tomcat.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<version>${log4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<version>${log4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>${protobuf.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.30</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>1.7.5</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
|
|
@ -329,28 +228,24 @@ under the License.
|
|||
<dependency>
|
||||
<groupId>javax.annotation</groupId>
|
||||
<artifactId>javax.annotation-api</artifactId>
|
||||
<version>1.3.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-aws</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aliyun -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-aliyun</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.ksyun.kmr/hadoop-ks3 -->
|
||||
<dependency>
|
||||
<groupId>com.ksyun.kmr</groupId>
|
||||
<artifactId>hadoop-ks3</artifactId>
|
||||
<version>3.1.1-1.1.2</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
|
@ -374,27 +269,11 @@ under the License.
|
|||
<executable>mkdir</executable>
|
||||
<arguments>
|
||||
<argument>-p</argument>
|
||||
<argument>${basedir}/src/main/resources/thrift</argument>
|
||||
<argument>${basedir}/src/main/thrift</argument>
|
||||
</arguments>
|
||||
<skip>${skip.plugin}</skip>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>copy-thrift</id>
|
||||
<phase>process-resources</phase>
|
||||
<goals>
|
||||
<goal>exec</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<executable>cp</executable>
|
||||
<arguments>
|
||||
<argument>${starrocks.home}/gensrc/thrift/FileBrokerService.thrift</argument>
|
||||
<argument>${basedir}/src/main/resources/thrift/</argument>
|
||||
</arguments>
|
||||
<skip>${skip.plugin}</skip>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
|
|
@ -149,6 +149,10 @@ public class FileSystemManager {
|
|||
private static final String FS_COS_ENDPOINT = "fs.cosn.bucket.endpoint_suffix";
|
||||
private static final String FS_COS_IMPL_DISABLE_CACHE = "fs.cosn.impl.disable.cache";
|
||||
private static final String FS_COS_IMPL = "fs.cosn.impl";
|
||||
private static final String FS_ABSTRACTFILESYSTEM_COSN_IMPL = "fs.AbstractFileSystem.cosn.impl";
|
||||
private static final String FS_COSN_TRSF_FS_ABSTRACTFILESYSTEM_OFS_IMPL = "fs.cosn.trsf.fs.AbstractFileSystem.ofs.impl";
|
||||
private static final String FS_COSN_TRSF_FS_OFS_IMPL = "fs.cosn.trsf.fs.ofs.impl";
|
||||
private static final String FS_COSN_CREDENTIALS_PROVIDER = "fs.cosn.credentials.provider";
|
||||
|
||||
// arguments for obs
|
||||
private static final String FS_OBS_ACCESS_KEY = "fs.obs.access.key";
|
||||
|
|
@ -819,8 +823,43 @@ public class FileSystemManager {
|
|||
conf.set(FS_COS_ENDPOINT, endpoint);
|
||||
conf.set(FS_COS_IMPL, "org.apache.hadoop.fs.CosFileSystem");
|
||||
conf.set(FS_COS_IMPL_DISABLE_CACHE, disableCache);
|
||||
FileSystem cosFileSystem = FileSystem.get(pathUri.getUri(), conf);
|
||||
fileSystem.setFileSystem(cosFileSystem);
|
||||
|
||||
// Too many configuration items, so we directly pass through the properties.
|
||||
for (Map.Entry<String, String> entry : properties.entrySet()) {
|
||||
conf.set(entry.getKey(), entry.getValue());
|
||||
}
|
||||
|
||||
String authentication = properties.getOrDefault(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
|
||||
AUTHENTICATION_SIMPLE);
|
||||
UserGroupInformation ugi = null;
|
||||
if (authentication.equalsIgnoreCase(AUTHENTICATION_KERBEROS)) {
|
||||
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, AUTHENTICATION_KERBEROS);
|
||||
conf.set(CommonConfigurationKeysPublic.HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED, "true");
|
||||
conf.set(FS_ABSTRACTFILESYSTEM_COSN_IMPL, "org.apache.hadoop.fs.CosN");
|
||||
conf.set(FS_COSN_TRSF_FS_ABSTRACTFILESYSTEM_OFS_IMPL, "com.qcloud.chdfs.fs.CHDFSDelegateFSAdapter");
|
||||
conf.set(FS_COSN_TRSF_FS_OFS_IMPL, "com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter");
|
||||
conf.set(FS_COSN_CREDENTIALS_PROVIDER, "org.apache.hadoop.fs.auth.RangerCredentialsProvider");
|
||||
UserGroupInformation.setConfiguration(conf);
|
||||
|
||||
String principal = properties.getOrDefault(KERBEROS_PRINCIPAL,"");
|
||||
String keytab = properties.getOrDefault(KERBEROS_KEYTAB,"");
|
||||
if (!principal.isEmpty() && !keytab.isEmpty()) {
|
||||
ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab);
|
||||
}
|
||||
}
|
||||
|
||||
FileSystem cosFileSystem;
|
||||
if (ugi != null) {
|
||||
cosFileSystem = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
|
||||
@Override
|
||||
public FileSystem run() throws Exception {
|
||||
return FileSystem.get(pathUri.getUri(), conf);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
cosFileSystem = FileSystem.get(pathUri.getUri(), conf);
|
||||
}
|
||||
fileSystem.setFileSystem(cosFileSystem, authentication.equalsIgnoreCase(AUTHENTICATION_KERBEROS));
|
||||
}
|
||||
return fileSystem;
|
||||
} catch (Exception e) {
|
||||
|
|
@ -0,0 +1 @@
|
|||
../../../../../../../../gensrc/thrift/FileBrokerService.thrift
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>com</groupId>
|
||||
<artifactId>starrocks-apache-hdfs-broker</artifactId>
|
||||
<version>3.4.0</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>hadoop-cos-shaded</artifactId>
|
||||
<version>3.3.0-8.3.7</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
<starrocks.home>${basedir}/../../../../</starrocks.home>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<!-- https://mvnrepository.com/artifact/com.qcloud.cos/hadoop-cos -->
|
||||
<dependency>
|
||||
<groupId>com.qcloud.cos</groupId>
|
||||
<artifactId>hadoop-cos</artifactId>
|
||||
<version>3.3.0-8.3.7</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.qcloud/cos_api-bundle -->
|
||||
<dependency>
|
||||
<groupId>com.qcloud</groupId>
|
||||
<artifactId>cos_api-bundle</artifactId>
|
||||
<version>5.6.137.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.qcloud/cosn-ranger-interface -->
|
||||
<dependency>
|
||||
<groupId>com.qcloud</groupId>
|
||||
<artifactId>cosn-ranger-interface</artifactId>
|
||||
<version>1.0.5</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.qcloud/chdfs_hadoop_plugin_network -->
|
||||
<dependency>
|
||||
<groupId>com.qcloud</groupId>
|
||||
<artifactId>chdfs_hadoop_plugin_network</artifactId>
|
||||
<version>3.5</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.qcloud/hadoop-ranger-client-for-hadoop -->
|
||||
<dependency>
|
||||
<groupId>com.qcloud</groupId>
|
||||
<artifactId>hadoop-ranger-client-for-hadoop</artifactId>
|
||||
<version>3.3.0-4.1</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.5.2</version>
|
||||
<configuration>
|
||||
<relocations>
|
||||
<relocation>
|
||||
<!-- org.apache.hadoop.security.proto.SecurityProtos conflicts in hadoop-ranger-client-for-hadoop and hadoop-common-3.3 -->
|
||||
<pattern>org.apache.hadoop.security.proto</pattern>
|
||||
<shadedPattern>shaded.starrocks.org.apache.hadoop.security.proto</shadedPattern>
|
||||
</relocation>
|
||||
</relocations>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<!--set for ecplise lifecycle -->
|
||||
<plugin>
|
||||
<groupId>org.eclipse.m2e</groupId>
|
||||
<artifactId>lifecycle-mapping</artifactId>
|
||||
<version>1.0.0</version>
|
||||
<configuration>
|
||||
<lifecycleMappingMetadata>
|
||||
<pluginExecutions>
|
||||
<pluginExecution>
|
||||
<pluginExecutionFilter>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<versionRange>[3.1,)</versionRange>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
<goal>testCompile</goal>
|
||||
</goals>
|
||||
</pluginExecutionFilter>
|
||||
<action>
|
||||
<ignore></ignore>
|
||||
</action>
|
||||
</pluginExecution>
|
||||
</pluginExecutions>
|
||||
</lifecycleMappingMetadata>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,369 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>com</groupId>
|
||||
<artifactId>starrocks-apache-hdfs-broker</artifactId>
|
||||
<version>3.4.0</version>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
<modules>
|
||||
<module>hadoop-cos-shaded</module>
|
||||
<module>broker-core</module>
|
||||
</modules>
|
||||
|
||||
<name>apache_hdfs_broker</name>
|
||||
|
||||
<properties>
|
||||
<starrocks.home>${basedir}/../../../</starrocks.home>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<maven.compiler.source>1.8</maven.compiler.source>
|
||||
<maven.compiler.target>1.8</maven.compiler.target>
|
||||
<hadoop.version>3.3.6</hadoop.version>
|
||||
<guava.version>32.0.1-jre</guava.version>
|
||||
<zookeeper.version>3.9.1</zookeeper.version>
|
||||
<protobuf.version>3.16.3</protobuf.version>
|
||||
<thrift.version>0.14.1</thrift.version>
|
||||
<tomcat.version>8.5.96</tomcat.version>
|
||||
<log4j.version>2.17.1</log4j.version>
|
||||
<jackson.version>2.13.4.2</jackson.version>
|
||||
</properties>
|
||||
|
||||
<profiles>
|
||||
<!-- for custom internal repository -->
|
||||
<profile>
|
||||
<id>custom-env</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env.CUSTOM_MAVEN_REPO</name>
|
||||
</property>
|
||||
</activation>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>custom-nexus</id>
|
||||
<url>${env.CUSTOM_MAVEN_REPO}</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>custom-nexus</id>
|
||||
<url>${env.CUSTOM_MAVEN_REPO}</url>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
</profile>
|
||||
|
||||
<!-- for general repository -->
|
||||
<profile>
|
||||
<id>general-env</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>!env.CUSTOM_MAVEN_REPO</name>
|
||||
</property>
|
||||
</activation>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>central</id>
|
||||
<name>central maven repo https</name>
|
||||
<url>https://repo.maven.apache.org/maven2</url>
|
||||
</repository>
|
||||
<!-- for java-cup -->
|
||||
<repository>
|
||||
<id>cloudera-thirdparty</id>
|
||||
<url>https://repository.cloudera.com/content/repositories/third-party/</url>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>cloudera</id>
|
||||
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
|
||||
</repository>
|
||||
<!-- for bdb je -->
|
||||
<repository>
|
||||
<id>oracleReleases</id>
|
||||
<url>https://download.oracle.com/maven</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
<pluginRepositories>
|
||||
<!-- for cup-maven-plugin -->
|
||||
<pluginRepository>
|
||||
<id>spring-plugins</id>
|
||||
<url>http://repo.spring.io/plugins-release/</url>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<!-- https://mvnrepository.com/artifact/commons-cli/commons-cli -->
|
||||
<dependency>
|
||||
<groupId>commons-cli</groupId>
|
||||
<artifactId>commons-cli</artifactId>
|
||||
<version>1.4</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<version>1.9</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-collections/commons-collections -->
|
||||
<dependency>
|
||||
<groupId>commons-collections</groupId>
|
||||
<artifactId>commons-collections</artifactId>
|
||||
<version>3.2.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-configuration/commons-configuration -->
|
||||
<dependency>
|
||||
<groupId>commons-configuration</groupId>
|
||||
<artifactId>commons-configuration</artifactId>
|
||||
<version>1.6</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-daemon/commons-daemon -->
|
||||
<dependency>
|
||||
<groupId>commons-daemon</groupId>
|
||||
<artifactId>commons-daemon</artifactId>
|
||||
<version>1.0.13</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>2.7</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-lang/commons-lang -->
|
||||
<dependency>
|
||||
<groupId>commons-lang</groupId>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
<version>2.6</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<version>3.3.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/commons-logging/commons-logging -->
|
||||
<dependency>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
<version>1.1.3</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>${guava.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-annotations -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-annotations</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-auth -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-auth</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/log4j/log4j -->
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>1.2.17-cloudera6</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper -->
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<version>${zookeeper.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs-client</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdfs</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>com.fasterxml.jackson.core</artifactId>
|
||||
<groupId>jackson-databind</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/junit/junit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.13.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.thrift/libthrift -->
|
||||
<dependency>
|
||||
<groupId>org.apache.thrift</groupId>
|
||||
<artifactId>libthrift</artifactId>
|
||||
<version>${thrift.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.apache.tomcat.embed</groupId>
|
||||
<artifactId>tomcat-embed-core</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.tomcat</groupId>
|
||||
<artifactId>tomcat-annotations-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.tomcat.embed/tomcat-embed-core -->
|
||||
<dependency>
|
||||
<groupId>org.apache.tomcat.embed</groupId>
|
||||
<artifactId>tomcat-embed-core</artifactId>
|
||||
<version>${tomcat.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.tomcat/tomcat-annotations-api -->
|
||||
<dependency>
|
||||
<groupId>org.apache.tomcat</groupId>
|
||||
<artifactId>tomcat-annotations-api</artifactId>
|
||||
<version>${tomcat.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<version>${log4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<version>${log4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>${protobuf.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.30</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>1.7.5</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- support jdk9 -->
|
||||
<dependency>
|
||||
<groupId>javax.annotation</groupId>
|
||||
<artifactId>javax.annotation-api</artifactId>
|
||||
<version>1.3.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-aws</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.qcloud.cos/hadoop-cos -->
|
||||
<dependency>
|
||||
<groupId>com</groupId>
|
||||
<artifactId>hadoop-cos</artifactId>
|
||||
<version>1.0.0</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aliyun -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-aliyun</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.ksyun.kmr/hadoop-ks3 -->
|
||||
<dependency>
|
||||
<groupId>com.ksyun.kmr</groupId>
|
||||
<artifactId>hadoop-ks3</artifactId>
|
||||
<version>3.1.1-1.1.2</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
</project>
|
||||
Loading…
Reference in New Issue