tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Tool] [BugFix] Fix building format-sdk compile bugs (#59365) 

Signed-off-by: shuming.li <ming.moriarty@gmail.com>
This commit is contained in:
shuming.li 2025-05-27 16:03:16 +08:00 committed by GitHub
parent d2620451d4
commit 522f719bf6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 34 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
format-sdk/build.sh Normal file → Executable file
View File

2
format-sdk/src/main/cpp/CMakeLists.txt
View File

@ -59,7 +59,7 @@ include_directories(
set(JAVA_HOME ${THIRDPARTY_DIR}/open_jdk/) set(JAVA_HOME ${THIRDPARTY_DIR}/open_jdk/)
add_library(jvm SHARED IMPORTED) add_library(jvm SHARED IMPORTED)
FILE(GLOB_RECURSE LIB_JVM ${JAVA_HOME}/jre/lib/*/libjvm.so) FILE(GLOB_RECURSE LIB_JVM ${JAVA_HOME}/lib/*/libjvm.so)
set_target_properties(jvm PROPERTIES IMPORTED_LOCATION ${LIB_JVM}) set_target_properties(jvm PROPERTIES IMPORTED_LOCATION ${LIB_JVM})
include_directories(${JAVA_HOME}/include) include_directories(${JAVA_HOME}/include)
include_directories(${JAVA_HOME}/include/linux) include_directories(${JAVA_HOME}/include/linux)

4
format-sdk/src/main/cpp/convert/binary_converter.h
View File

@ -49,7 +49,7 @@ public:
const arrow::MemoryPool* pool) const arrow::MemoryPool* pool)
: ColumnConverter(arrow_type, sr_field, pool){}; : ColumnConverter(arrow_type, sr_field, pool){};
arrow::Status toSrColumn(const std::shared_ptr<arrow::Array> array, ColumnPtr& column) override { arrow::Status toSrColumn(const std::shared_ptr<arrow::Array> array, MutableColumnPtr& column) override {
if (!column->is_nullable() && array->null_count() > 0) { if (!column->is_nullable() && array->null_count() > 0) {
return arrow::Status::Invalid("Column ", column->get_name(), return arrow::Status::Invalid("Column ", column->get_name(),
" is non-nullable, but there are some null data in array."); " is non-nullable, but there are some null data in array.");
@ -101,7 +101,7 @@ public:
return arrow::Status::OK(); return arrow::Status::OK();
} }
arrow::Result<std::shared_ptr<arrow::Array>> toArrowArray(const std::shared_ptr<Column>& column) override { arrow::Result<std::shared_ptr<arrow::Array>> toArrowArray(const ColumnPtr& column) override {
using ArrowBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType; using ArrowBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
std::unique_ptr<ArrowBuilderType> builder = std::unique_ptr<ArrowBuilderType> builder =

12
format-sdk/src/main/cpp/convert/column_converter.cpp
View File

@ -199,18 +199,18 @@ arrow::Result<std::shared_ptr<arrow::Buffer>> ColumnConverter::convert_null_bitm
return null_bitmap; return null_bitmap;
} }
ColumnPtr ColumnConverter::get_data_column(const ColumnPtr& column) { ColumnPtr ColumnConverter::get_data_column(const Column* column_ptr) {
if (column->is_nullable()) { if (column_ptr->is_nullable()) {
auto* nullable_column = down_cast<const NullableColumn*>(column.get()); auto* nullable_column = down_cast<const NullableColumn*>(column_ptr);
return nullable_column->data_column(); return nullable_column->data_column();
} }
if (column->is_constant()) { if (column_ptr->is_constant()) {
auto* const_column = down_cast<const ConstColumn*>(column.get()); auto* const_column = down_cast<const ConstColumn*>(column_ptr);
return const_column->data_column(); return const_column->data_column();
} }
return column; return column_ptr->get_ptr();
} }
} // namespace starrocks::lake::format } // namespace starrocks::lake::format

6
format-sdk/src/main/cpp/convert/column_converter.h
View File

@ -56,17 +56,17 @@ public:
/** /**
* Convert arrow array to starrocks column. * Convert arrow array to starrocks column.
*/ */
virtual arrow::Status toSrColumn(std::shared_ptr<arrow::Array> array, ColumnPtr& column) = 0; virtual arrow::Status toSrColumn(std::shared_ptr<arrow::Array> array, MutableColumnPtr& column) = 0;
/** /**
* Convert starrocks column to arrow array. * Convert starrocks column to arrow array.
*/ */
virtual arrow::Result<std::shared_ptr<arrow::Array>> toArrowArray(const std::shared_ptr<Column>& column) = 0; virtual arrow::Result<std::shared_ptr<arrow::Array>> toArrowArray(const ColumnPtr& column) = 0;
protected: protected:
arrow::Result<std::shared_ptr<arrow::Buffer>> convert_null_bitmap(const Buffer<uint8_t>& null_bytes); arrow::Result<std::shared_ptr<arrow::Buffer>> convert_null_bitmap(const Buffer<uint8_t>& null_bytes);
static ColumnPtr get_data_column(const ColumnPtr& column); static ColumnPtr get_data_column(const Column* column_ptr);
protected: protected:
const std::shared_ptr<arrow::DataType> _arrow_type; const std::shared_ptr<arrow::DataType> _arrow_type;

25
format-sdk/src/main/cpp/convert/nested_converter.h
View File

@ -51,7 +51,7 @@ public:
const arrow::MemoryPool* pool) const arrow::MemoryPool* pool)
: ColumnConverter(arrow_type, sr_field, pool) {} : ColumnConverter(arrow_type, sr_field, pool) {}
arrow::Status toSrColumn(const std::shared_ptr<arrow::Array> array, ColumnPtr& column) override { arrow::Status toSrColumn(const std::shared_ptr<arrow::Array> array, MutableColumnPtr& column) override {
if (!column->is_nullable() && array->null_count() > 0) { if (!column->is_nullable() && array->null_count() > 0) {
return arrow::Status::Invalid("Column ", column->get_name(), return arrow::Status::Invalid("Column ", column->get_name(),
" is non-nullable, but there are some null data in array."); " is non-nullable, but there are some null data in array.");
@ -61,8 +61,9 @@ public:
const auto& nested_array = arrow::internal::checked_pointer_cast<ArrowArrayType>(array); const auto& nested_array = arrow::internal::checked_pointer_cast<ArrowArrayType>(array);
ARROW_ASSIGN_OR_RAISE(arrow::ArrayVector arrow_children_arrays, get_children_arrays(nested_array)); ARROW_ASSIGN_OR_RAISE(arrow::ArrayVector arrow_children_arrays, get_children_arrays(nested_array));
const auto data_column = arrow::internal::checked_pointer_cast<SrColumnType>(get_data_column(column)); auto data_column = SrColumnType::static_pointer_cast(get_data_column(column.get()));
ARROW_ASSIGN_OR_RAISE(std::vector<starrocks::ColumnPtr> sr_sub_columns, get_children_columns(data_column)); ARROW_ASSIGN_OR_RAISE(std::vector<starrocks::ColumnPtr> sr_sub_columns,
get_children_columns(data_column.get()));
if (arrow_children_arrays.size() != sr_sub_columns.size()) { if (arrow_children_arrays.size() != sr_sub_columns.size()) {
return arrow::Status::Invalid("Can't convert nested array, the array children size(", return arrow::Status::Invalid("Can't convert nested array, the array children size(",
@ -76,10 +77,12 @@ public:
// copy data column // copy data column
for (size_t idx = 0; idx < arrow_children_arrays.size(); ++idx) { for (size_t idx = 0; idx < arrow_children_arrays.size(); ++idx) {
ARROW_RETURN_NOT_OK(_children[idx]->toSrColumn(arrow_children_arrays[idx], sr_sub_columns[idx])); auto mutable_sr_column = sr_sub_columns[idx]->as_mutable_ptr();
ARROW_RETURN_NOT_OK(_children[idx]->toSrColumn(arrow_children_arrays[idx], mutable_sr_column));
} }
// for print sr sub column; // for print sr sub column;
ARROW_ASSIGN_OR_RAISE(std::vector<starrocks::ColumnPtr> sr_sub_columns2, get_children_columns(data_column)); ARROW_ASSIGN_OR_RAISE(std::vector<starrocks::ColumnPtr> sr_sub_columns2,
get_children_columns(data_column.get()));
// copy null bitmap // copy null bitmap
if (column->is_nullable()) { if (column->is_nullable()) {
@ -95,10 +98,11 @@ public:
return arrow::Status::OK(); return arrow::Status::OK();
} }
arrow::Result<std::shared_ptr<arrow::Array>> toArrowArray(const std::shared_ptr<Column>& column) override { arrow::Result<std::shared_ptr<arrow::Array>> toArrowArray(const ColumnPtr& column) override {
// convert data column,include list:offsets, values, map: offsets, keys, values, struct: children columns. // convert data column,include list:offsets, values, map: offsets, keys, values, struct: children columns.
const auto data_column = arrow::internal::checked_pointer_cast<SrColumnType>(get_data_column(column)); const auto data_column = SrColumnType::static_pointer_cast(get_data_column(column.get()));
ARROW_ASSIGN_OR_RAISE(std::vector<starrocks::ColumnPtr> sr_sub_columns, get_children_columns(data_column)); ARROW_ASSIGN_OR_RAISE(std::vector<starrocks::ColumnPtr> sr_sub_columns,
get_children_columns(data_column.get()));
std::vector<std::shared_ptr<arrow::Array>> arrays; std::vector<std::shared_ptr<arrow::Array>> arrays;
arrays.resize(sr_sub_columns.size()); arrays.resize(sr_sub_columns.size());
@ -115,7 +119,7 @@ public:
// convert null bitmap // convert null bitmap
std::shared_ptr<arrow::Buffer> null_bitmap; std::shared_ptr<arrow::Buffer> null_bitmap;
if (column->is_nullable()) { if (column->is_nullable()) {
auto nullable = down_cast<NullableColumn*>(column.get()); auto nullable = down_cast<const NullableColumn*>(column.get());
auto& null_bytes = nullable->immutable_null_column_data(); auto& null_bytes = nullable->immutable_null_column_data();
ARROW_ASSIGN_OR_RAISE(null_bitmap, convert_null_bitmap(null_bytes)); ARROW_ASSIGN_OR_RAISE(null_bitmap, convert_null_bitmap(null_bytes));
} }
@ -144,7 +148,8 @@ private:
template <class SrColumnClass, typename = std::enable_if_t<std::is_same_v<SrColumnClass, ArrayColumn> || template <class SrColumnClass, typename = std::enable_if_t<std::is_same_v<SrColumnClass, ArrayColumn> ||
std::is_same_v<SrColumnClass, MapColumn> || std::is_same_v<SrColumnClass, MapColumn> ||
std::is_same_v<SrColumnClass, StructColumn>>> std::is_same_v<SrColumnClass, StructColumn>>>
arrow::Result<Columns> get_children_columns(const std::shared_ptr<SrColumnClass> data_column) {
arrow::Result<Columns> get_children_columns(const SrColumnClass* data_column) {
if constexpr (std::is_same_v<SrColumnClass, ArrayColumn>) { if constexpr (std::is_same_v<SrColumnClass, ArrayColumn>) {
Columns all_sub_columns = {data_column->offsets_column(), data_column->elements_column()}; Columns all_sub_columns = {data_column->offsets_column(), data_column->elements_column()};
return all_sub_columns; return all_sub_columns;

6
format-sdk/src/main/cpp/convert/primitive_converter.h
View File

@ -68,7 +68,7 @@ public:
} }
}; };
arrow::Status toSrColumn(const std::shared_ptr<arrow::Array> array, ColumnPtr& column) override { arrow::Status toSrColumn(const std::shared_ptr<arrow::Array> array, MutableColumnPtr& column) override {
if (!column->is_nullable() && array->null_count() > 0) { if (!column->is_nullable() && array->null_count() > 0) {
return arrow::Status::Invalid("Column ", column->get_name(), return arrow::Status::Invalid("Column ", column->get_name(),
" is non-nullable, but there are some null data in array."); " is non-nullable, but there are some null data in array.");
@ -79,7 +79,7 @@ public:
// copy data column // copy data column
const auto& real_arrow_type = arrow::internal::checked_pointer_cast<ArrowType>(_arrow_type); const auto& real_arrow_type = arrow::internal::checked_pointer_cast<ArrowType>(_arrow_type);
const auto& real_array = arrow::internal::checked_pointer_cast<const ArrowArrayType>(array); const auto& real_array = arrow::internal::checked_pointer_cast<const ArrowArrayType>(array);
const auto data_column = arrow::internal::checked_pointer_cast<SrColumnType>(get_data_column(column)); auto data_column = SrColumnType::dynamic_pointer_cast(get_data_column(column.get()));
if constexpr (SR_TYPE == TYPE_DATE || SR_TYPE == TYPE_DATETIME) { if constexpr (SR_TYPE == TYPE_DATE || SR_TYPE == TYPE_DATETIME) {
for (size_t i = 0; i < num_rows; ++i) { for (size_t i = 0; i < num_rows; ++i) {
SrCppType value; SrCppType value;
@ -154,7 +154,7 @@ public:
return arrow::Status::OK(); return arrow::Status::OK();
} }
arrow::Result<std::shared_ptr<arrow::Array>> toArrowArray(const std::shared_ptr<Column>& column) override { arrow::Result<std::shared_ptr<arrow::Array>> toArrowArray(const ColumnPtr& column) override {
using ArrowBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType; using ArrowBuilderType = typename arrow::TypeTraits<ArrowType>::BuilderType;
const auto& real_arrow_type = arrow::internal::checked_pointer_cast<ArrowType>(_arrow_type); const auto& real_arrow_type = arrow::internal::checked_pointer_cast<ArrowType>(_arrow_type);

3
format-sdk/src/main/cpp/convert/starrocks_arrow_converter.cpp
View File

@ -69,7 +69,8 @@ public:
auto chunk = ChunkHelper::new_chunk(*_sr_schema, recordBatch->num_rows()); auto chunk = ChunkHelper::new_chunk(*_sr_schema, recordBatch->num_rows());
for (size_t idx = 0; idx < column_size; ++idx) { for (size_t idx = 0; idx < column_size; ++idx) {
ARROW_RETURN_NOT_OK(_converters[idx]->toSrColumn(recordBatch->column(idx), chunk->columns()[idx])); auto mutable_column = chunk->columns()[idx]->as_mutable_ptr();
ARROW_RETURN_NOT_OK(_converters[idx]->toSrColumn(recordBatch->column(idx), mutable_column));
} }
return chunk; return chunk;

2
thirdparty/build-thirdparty.sh vendored
View File

@ -1382,6 +1382,8 @@ build_icu() {
# Use a subshell to prevent LD_LIBRARY_PATH from affecting the external environment # Use a subshell to prevent LD_LIBRARY_PATH from affecting the external environment
( (
export LD_LIBRARY_PATH=${STARROCKS_GCC_HOME}/lib:${STARROCKS_GCC_HOME}/lib64:${LD_LIBRARY_PATH:-} export LD_LIBRARY_PATH=${STARROCKS_GCC_HOME}/lib:${STARROCKS_GCC_HOME}/lib64:${LD_LIBRARY_PATH:-}
export CFLAGS="-O3 -fno-omit-frame-pointer -fPIC"
export CXXFLAGS="-O3 -fno-omit-frame-pointer -fPIC"
./runConfigureICU Linux --prefix=$TP_INSTALL_DIR --enable-static --disable-shared ./runConfigureICU Linux --prefix=$TP_INSTALL_DIR --enable-static --disable-shared
make -j$PARALLEL make -j$PARALLEL
make install make install