[Feature] Add confluet schema registry module in thirdparty (#17709)

* [Feature] Add confluet schema registry module in thirdparty Signed-off-by: Zaorang Yang <zaorangy@gmail.com>
2023-02-20 12:36:58 +08:00 · 2023-02-20 12:36:58 +08:00 · ee4d84e26c
parent 96e4c4d23a
commit ee4d84e26c
7 changed files with 229 additions and 1 deletions
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@ -324,6 +324,14 @@ set_target_properties(opentelemetry_trace PROPERTIES IMPORTED_LOCATION ${THIRDPA
 add_library(opentelemetry_resources STATIC IMPORTED GLOBAL)
 set_target_properties(opentelemetry_resources PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libopentelemetry_resources.a)

+add_library(jansson STATIC IMPORTED GLOBAL)
+set_target_properties(jansson PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libjansson.a)
+
+add_library(avro STATIC IMPORTED GLOBAL)
+set_target_properties(avro PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libavro.a)
+
+add_library(serdes STATIC IMPORTED GLOBAL)
+set_target_properties(serdes PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libserdes.a)

 add_library(opentelemetry_exporter_jaeger_trace STATIC IMPORTED GLOBAL)
 set_target_properties(opentelemetry_exporter_jaeger_trace PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libopentelemetry_exporter_jaeger_trace.a)
@ -689,6 +697,9 @@ set(STARROCKS_DEPENDENCIES
    opentelemetry_resources
    opentelemetry_common
    opentelemetry_exporter_jaeger_trace
+    jansson
+    avro
+    serdes
    ${WL_END_GROUP}
 )

--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@ -988,6 +988,52 @@ build_streamvbyte() {
    make install
 }

+# jansson
+build_jansson() {
+    check_if_source_exist $JANSSON_SOURCE
+    cd $TP_SOURCE_DIR/$JANSSON_SOURCE/
+    mkdir -p build
+    cd build
+    $CMAKE_CMD .. -DCMAKE_INSTALL_PREFIX=${TP_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=lib
+    ${BUILD_SYSTEM} -j$PARALLEL
+    ${BUILD_SYSTEM} install
+}
+
+# avro-c
+build_avro_c() {
+    check_if_source_exist $AVRO_SOURCE
+    cd $TP_SOURCE_DIR/$AVRO_SOURCE/lang/c
+    mkdir -p build
+    cd build
+    $CMAKE_CMD .. -DCMAKE_INSTALL_PREFIX=${TP_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=lib64 -DCMAKE_BUILD_TYPE=Release
+    ${BUILD_SYSTEM} -j$PARALLEL    
+    ${BUILD_SYSTEM} install
+    rm ${TP_INSTALL_DIR}/lib64/libavro.so*
+}
+
+# serders
+build_serdes() {
+    OLD_CFLAGS=$CFLAGS
+    unset CFLAGS
+    export CFLAGS="-O3 -fno-omit-frame-pointer -fPIC -g"
+    check_if_source_exist $SERDES_SOURCE
+    cd $TP_SOURCE_DIR/$SERDES_SOURCE
+    export LIBS="-lrt -lpthread -lcurl -ljansson -lrdkafka -lrdkafka++ -lavro -lssl -lcrypto -ldl" 
+    ./configure --prefix=${TP_INSTALL_DIR} \
+                --libdir=${TP_INSTALL_DIR}/lib \
+                --CFLAGS="-I ${TP_INSTALL_DIR}/include"  \
+                --CXXFLAGS="-I ${TP_INSTALL_DIR}/include" \
+                --LDFLAGS="-L ${TP_INSTALL_DIR}/lib -L ${TP_INSTALL_DIR}/lib64" \
+                --enable-static \
+                --disable-shared
+
+    ${BUILD_SYSTEM} -j$PARALLEL
+    ${BUILD_SYSTEM} install
+    rm ${TP_INSTALL_DIR}/lib/libserdes.so*
+    unset LIBS
+    export CFLAGS=$OLD_CFLAGS
+}
+
 export CXXFLAGS="-O3 -fno-omit-frame-pointer -Wno-class-memaccess -fPIC -g"
 export CPPFLAGS="-I ${TP_INCLUDE_DIR}"
 # https://stackoverflow.com/questions/42597685/storage-size-of-timespec-isnt-known
@ -1038,6 +1084,9 @@ build_benchmark
 build_fast_float
 build_cachelib
 build_streamvbyte
+build_jansson
+build_avro_c
+build_serdes

 if [[ "${MACHINE_TYPE}" != "aarch64" ]]; then
    build_breakpad
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@ -432,6 +432,22 @@ fi
 cd -
 echo "Finished patching $VPACK_SOURCE"

+# patch avro-c
+cd $TP_SOURCE_DIR/$AVRO_SOURCE/lang/c
+if [ ! -f $PATCHED_MARK ] && [ $AVRO_SOURCE = "avro-release-1.10.2" ]; then
+    patch -p0 < $TP_PATCH_DIR/avro-1.10.2.c.patch
+    cp $TP_PATCH_DIR/avro-1.10.2.c.findjansson.patch $TP_SOURCE_DIR/$AVRO_SOURCE/lang/c/Findjansson.cmake
+    touch $PATCHED_MARK
+fi
+cd -
+echo "Finished patching $AVRO_SOURCE-c"

+# patch serdes
+cd $TP_SOURCE_DIR/$SERDES_SOURCE
+if [ ! -f $PATCHED_MARK ] && [ $SERDES_SOURCE = "libserdes-7.3.1" ]; then
+    patch -p0 < $TP_PATCH_DIR/libserdes-7.3.1.patch
+    touch $PATCHED_MARK
+fi
+echo "Finished patching $SERDES_SOURCE"
 cd -

--- a/thirdparty/patches/avro-1.10.2.c.findjansson.patch
+++ b/thirdparty/patches/avro-1.10.2.c.findjansson.patch
@ -0,0 +1,17 @@
+find_path(JANSSON_INCLUDE_DIR
+    NAMES jansson.h
+    HINTS ${JANSSON_ROOT_DIR}/include)
+
+find_library(JANSSON_LIBRARIES
+    NAMES jansson
+    HINTS ${JANSSON_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Jansson DEFAULT_MSG
+    JANSSON_LIBRARIES
+    JANSSON_INCLUDE_DIR)
+
+mark_as_advanced(
+    JANSSON_ROOT_DIR
+    JANSSON_LIBRARIES
+    JANSSON_INCLUDE_DIR)
--- a/thirdparty/patches/avro-1.10.2.c.patch
+++ b/thirdparty/patches/avro-1.10.2.c.patch
@ -0,0 +1,84 @@
+--- CMakeLists.txt
+++ CMakeLists.txt
+@@ -18,7 +18,7 @@
+ #
+ cmake_minimum_required(VERSION 3.1)
+ project(AvroC C)
+-enable_testing()
+# enable_testing()
+ 
+ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR})
+ 
+@@ -136,12 +136,14 @@ include_directories(${AvroC_SOURCE_DIR}/
+ 
+ 
+ # Enable codecs
+-
+set ( ZLIB_INCLUDE_DIR $ENV{TP_INSTALL_DIR}/include )
+set ( ZLIB_LIBRARIES $ENV{TP_INSTALL_DIR}/lib/libz.a )
+ find_package(ZLIB)
+ if (ZLIB_FOUND)
+     set(ZLIB_PKG zlib)
+     add_definitions(-DDEFLATE_CODEC)
+-    include_directories(${ZLIB_INCLUDE_DIRS})
+    include_directories(${ZLIB_INCLUDE_DIR})
+    link_directories(${ZLIB_LIBRARIES})
+     message("Enabled deflate codec")
+ else (ZLIB_FOUND)
+     set(ZLIB_PKG "")
+@@ -149,17 +151,17 @@ else (ZLIB_FOUND)
+     message("Disabled deflate codec. zlib not found.")
+ endif (ZLIB_FOUND)
+ 
+-find_package(Snappy)
+-if (SNAPPY_FOUND AND ZLIB_FOUND)  # Snappy borrows crc32 from zlib
+-    set(SNAPPY_PKG libsnappy)
+-    add_definitions(-DSNAPPY_CODEC)
+-    include_directories(${SNAPPY_INCLUDE_DIRS})
+-    message("Enabled snappy codec")
+-else (SNAPPY_FOUND AND ZLIB_FOUND)
+-    set(SNAPPY_PKG "")
+-    set(SNAPPY_LIBRARIES "")
+-    message("Disabled snappy codec. libsnappy not found or zlib not found.")
+-endif (SNAPPY_FOUND AND ZLIB_FOUND)
+# find_package(Snappy)
+# if (SNAPPY_FOUND AND ZLIB_FOUND)  # Snappy borrows crc32 from zlib
+#     set(SNAPPY_PKG libsnappy)
+#     add_definitions(-DSNAPPY_CODEC)
+#     include_directories(${SNAPPY_INCLUDE_DIRS})
+#     message("Enabled snappy codec")
+# else (SNAPPY_FOUND AND ZLIB_FOUND)
+#     set(SNAPPY_PKG "")
+#     set(SNAPPY_LIBRARIES "")
+#     message("Disabled snappy codec. libsnappy not found or zlib not found.")
+# endif (SNAPPY_FOUND AND ZLIB_FOUND)
+ 
+ find_package(PkgConfig)
+ pkg_check_modules(LZMA liblzma)
+@@ -179,11 +181,14 @@ set(CODEC_LIBRARIES ${ZLIB_LIBRARIES} ${
+ set(CODEC_PKG "@ZLIB_PKG@ @LZMA_PKG@ @SNAPPY_PKG@")
+ 
+ # Jansson JSON library
+-pkg_check_modules(JANSSON jansson>=2.3)
+# pkg_check_modules(JANSSON jansson>=2.3)
+set ( JANSSON_INCLUDE_DIR $ENV{TP_INSTALL_DIR}/include )
+set ( JANSSON_LIBRARIES $ENV{TP_INSTALL_DIR}/lib/libjansson.a )
+find_package(jansson)
+ if (JANSSON_FOUND)
+     set(JANSSON_PKG libjansson)
+-    include_directories(${JANSSON_INCLUDE_DIRS})
+-    link_directories(${JANSSON_LIBRARY_DIRS})
+    include_directories(${JANSSON_INCLUDE_DIR})
+    link_directories(${JANSSON_LIBRARIES})
+ else (JANSSON_FOUND)
+     message(FATAL_ERROR "libjansson >=2.3 not found")
+ endif (JANSSON_FOUND)
+@@ -191,7 +196,7 @@ endif (JANSSON_FOUND)
+ 
+ add_subdirectory(src)
+ add_subdirectory(examples)
+-add_subdirectory(tests)
+# add_subdirectory(tests)
+ add_subdirectory(docs)
+ 
+ add_custom_target(pretty
--- a/thirdparty/patches/libserdes-7.3.1.patch
+++ b/thirdparty/patches/libserdes-7.3.1.patch
@ -0,0 +1,33 @@
+--- Makefile
+++ Makefile
+@@ -10,7 +10,7 @@ BUILD_NUMBER ?= 1
+ 
+ .PHONY:
+ 
+-all: mklove-check libs check
+all: mklove-check libs
+ 
+ include mklove/Makefile.base
+ 
+@@ -22,16 +22,9 @@ LIBSUBDIRS=	src $(LIBSUBDIRS_y)
+ libs:
+ 	@(for d in $(LIBSUBDIRS); do $(MAKE) -C $$d || exit $?; done)
+ 
+-file-check: examples
+-check: file-check
+-	@(for d in $(LIBSUBDIRS); do $(MAKE) -C $$d $@ || exit $?; done)
+-
+ install:
+ 	@(for d in $(LIBSUBDIRS); do $(MAKE) -C $$d $@ || exit $?; done)
+ 
+-examples tests: .PHONY libs
+-	$(MAKE) -C $@
+-
+ clean:
+ 	@$(MAKE) -C tests $@
+ 	@$(MAKE) -C examples $@
+@@ -47,3 +40,4 @@ archive:
+ 		-o $(PACKAGE_NAME)-$(VERSION).tar.gz HEAD
+ 	git archive --prefix=$(PACKAGE_NAME)-$(VERSION)/ \
+ 		-o $(PACKAGE_NAME)-$(VERSION).zip HEAD
+
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@ -345,8 +345,26 @@ BROKER_THIRDPARTY_JARS_NAME="broker_thirdparty_jars.tar.gz"
 BROKER_THIRDPARTY_JARS_SOURCE="broker_thirdparty_jars"
 BROKER_THIRDPARTY_JARS_MD5SUM="8e9a8c2ef3b19709dd0cc37e26a60c83"

+# jansson
+JANSSON_DOWNLOAD="https://github.com/akheron/jansson/releases/download/v2.14/jansson-2.14.tar.gz"
+JANSSON_NAME="jansson-2.14.tar.gz"
+JANSSON_SOURCE="jansson-2.14"
+JANSSON_MD5SUM="6cbfc54c2ab3b4d7284e188e185c2b0b"
+
+# avro
+AVRO_DOWNLOAD="https://github.com/apache/avro/archive/refs/tags/release-1.10.2.tar.gz"
+AVRO_NAME="avro-release-1.10.2.tar.gz"
+AVRO_SOURCE="avro-release-1.10.2"
+AVRO_MD5SUM="55b9c200976366fd62f1201231f3a5eb"
+
+# serdes
+SERDES_DOWNLOAD="https://github.com/confluentinc/libserdes/archive/refs/tags/v7.3.1.tar.gz"
+SERDES_NAME="libserdes-7.3.1.tar.gz"
+SERDES_SOURCE="libserdes-7.3.1"
+SERDES_MD5SUM="61012487a8845f37540710ac4ac2f7ab"
+
 # all thirdparties which need to be downloaded is set in array TP_ARCHIVES
 TP_ARCHIVES="LIBEVENT OPENSSL THRIFT PROTOBUF GFLAGS GLOG GTEST RAPIDJSON SIMDJSON SNAPPY GPERFTOOLS ZLIB LZ4 BZIP CURL \
            RE2 BOOST LEVELDB BRPC ROCKSDB LIBRDKAFKA PULSAR FLATBUFFERS ARROW BROTLI ZSTD S2 BITSHUFFLE CROARINGBITMAP \
            JEMALLOC CCTZ FMT RYU BREAK_PAD HADOOP JDK RAGEL HYPERSCAN MARIADB JINDOSDK AWS_SDK_CPP VPACK OPENTELEMETRY \
-            BENCHMARK FAST_FLOAT CACHELIB STREAMVBYTE BROKER_THIRDPARTY_JARS"
+            BENCHMARK FAST_FLOAT CACHELIB STREAMVBYTE BROKER_THIRDPARTY_JARS JANSSON AVRO SERDES"