tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Refactor] Use thirdparty jar packages as the source of broker dependency (#17222) 

Signed-off-by: miomiocat <284487410@qq.com>
This commit is contained in:
miomiocat 2023-02-06 12:39:39 +08:00 committed by GitHub
parent fc9e869c17
commit db19fc4454
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
fs_brokers/apache_hdfs_broker/build.sh
View File

@ -43,25 +43,8 @@ rm -rf ${BROKER_OUTPUT}
install -d ${BROKER_OUTPUT}/bin ${BROKER_OUTPUT}/conf \
${BROKER_OUTPUT}lib/
# download aliyun hadoop oss and tencent hadoop cos jar, this can not be found in maven repository, so we download it manually,
# here we update oss library from hadoop-aliyun-2.7.2.jar to jindosdk-4.6.2 to consistent with the oss library in FE and BE
wget http://cdn-thirdparty.starrocks.com/broker_thirdparty_jars.tar.gz
tar xzf broker_thirdparty_jars.tar.gz
rm broker_thirdparty_jars/hadoop-aliyun-2.7.2.jar
mv broker_thirdparty_jars/*.jar ${BROKER_OUTPUT}/lib/
rm -r broker_thirdparty_jars broker_thirdparty_jars.tar.gz
# jindosdk-4.6.2.tar.gz is very large, about 931M, so we cache it in local disk
if [ ! -d ${BROKER_HOME}/jindosdk-4.6.2 ]; then
echo "download aliyun jindo..."
wget https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/4.6.2/jindosdk-4.6.2.tar.gz
tar xvf jindosdk-4.6.2.tar.gz
fi
cp -r jindosdk-4.6.2/lib/*.jar ${BROKER_OUTPUT}/lib/
if [ -f ${BROKER_HOME}/jindosdk-4.6.2.tar.gz ]; then
rm -r ${BROKER_HOME}/jindosdk-4.6.2.tar.gz
fi
cp -r ${STARROCKS_THIRDPARTY}/installed/broker_thirdparty_jars/*.jar ${BROKER_OUTPUT}/lib/
cp -r ${STARROCKS_THIRDPARTY}/installed/jindosdk/*.jar ${BROKER_OUTPUT}/lib/
cp -r -p ${BROKER_HOME}/bin/*.sh ${BROKER_OUTPUT}/bin/
cp -r -p ${BROKER_HOME}/conf/*.conf ${BROKER_OUTPUT}/conf/

7
thirdparty/build-thirdparty.sh vendored
View File

@ -860,10 +860,11 @@ build_aliyun_jindosdk() {
cp -r $TP_SOURCE_DIR/$JINDOSDK_SOURCE/lib/*.jar $TP_INSTALL_DIR/jindosdk
}
build_tencent_cos_jars() {
build_broker_thirdparty_jars() {
check_if_source_exist $BROKER_THIRDPARTY_JARS_SOURCE
mkdir -p $TP_INSTALL_DIR/$BROKER_THIRDPARTY_JARS_SOURCE
cp -r $TP_SOURCE_DIR/$BROKER_THIRDPARTY_JARS_SOURCE/*cos* $TP_INSTALL_DIR/$BROKER_THIRDPARTY_JARS_SOURCE
cp -r $TP_SOURCE_DIR/$BROKER_THIRDPARTY_JARS_SOURCE/* $TP_INSTALL_DIR/$BROKER_THIRDPARTY_JARS_SOURCE
rm $TP_INSTALL_DIR/$BROKER_THIRDPARTY_JARS_SOURCE/hadoop-aliyun-2.7.2.jar
}
build_aws_cpp_sdk() {
@ -1027,7 +1028,7 @@ build_ragel
build_hyperscan
build_mariadb
build_aliyun_jindosdk
build_tencent_cos_jars
build_broker_thirdparty_jars
build_aws_cpp_sdk
build_vpack
build_opentelemetry