tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Enhancement] make JIT as compile option (#50928) 

Signed-off-by: stdpain <drfeng08@gmail.com>
This commit is contained in:
stdpain 2024-09-19 13:54:33 +08:00 committed by GitHub
parent 5bc037dadd
commit d2e2d8ccc7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 147 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
be/CMakeLists.txt
View File

@ -497,7 +497,6 @@ if ("${USE_STAROS}" STREQUAL "ON")
endif()
# LLVM
# TODO(yueyang): Adjust libraries during the implementation of JIT.
set (LLVM_LIBRARIES
LLVMRuntimeDyld
LLVMBitstreamReader
@ -558,6 +557,7 @@ foreach(lib IN ITEMS ${LLVM_LIBRARIES})
set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/llvm/lib/lib${lib}.a)
endforeach()
# Check Clang-Tidy
include(cmake_modules/FindClangTidy.cmake)
@ -677,6 +677,11 @@ if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new")
endif()
if (STARROCKS_JIT_ENABLE)
message(STATUS "Enable JIT")
set(CXX_GCC_FLAGS "${CXX_GCC_FLAGS} -DSTARROCKS_JIT_ENABLE")
endif()
# For any gcc builds:
# -g: Enable symbols for profiler tools
# -Wno-unused-local-typedefs: Do not warn for local typedefs that are unused.
@ -837,14 +842,20 @@ set(STARROCKS_DEPENDENCIES ${STARROCKS_DEPENDENCIES}
)
set_target_properties(aws-cpp-sdk-core PROPERTIES INTERFACE_LINK_LIBRARIES AWS::aws-crt-cpp)
if (STARROCKS_JIT_ENABLE)
set(STARROCKS_DEPENDENCIES
${STARROCKS_DEPENDENCIES}
${WL_START_GROUP}
${LLVM_LIBRARIES}
${WL_END_GROUP}
)
endif()
# Set thirdparty libraries
set(STARROCKS_DEPENDENCIES
${STARROCKS_DEPENDENCIES}
build_version
${WL_START_GROUP}
${LLVM_LIBRARIES}
${WL_END_GROUP}
${WL_START_GROUP}
clucene-core
clucene-shared
clucene-contribs-lib

11
be/src/exprs/CMakeLists.txt
View File

@ -29,9 +29,6 @@ set(EXPR_FILES
agg/factory/aggregate_resolver_utility.cpp
agg/factory/aggregate_resolver_variance.cpp
agg/factory/aggregate_resolver_window.cpp
jit/ir_helper.cpp
jit/jit_engine.cpp
jit/jit_expr.cpp
anyval_util.cpp
base64.cpp
binary_functions.cpp
@ -106,4 +103,12 @@ set(EXPR_FILES
match_expr.cpp
)
if(STARROCKS_JIT_ENABLE)
set(EXPR_FILES ${EXPR_FILES}
jit/ir_helper.cpp
jit/jit_engine.cpp
jit/jit_expr.cpp)
endif()
add_library(Exprs ${EXPR_FILES})

23
be/src/exprs/arithmetic_expr.cpp
View File

@ -14,9 +14,6 @@
#include "exprs/arithmetic_expr.h"
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/Value.h>
#include <optional>
#include "column/type_traits.h"
@ -27,12 +24,18 @@
#include "exprs/binary_function.h"
#include "exprs/decimal_binary_function.h"
#include "exprs/decimal_cast_expr.h"
#include "exprs/jit/ir_helper.h"
#include "exprs/overflow.h"
#include "exprs/unary_function.h"
#include "runtime/runtime_state.h"
#include "types/logical_type.h"
#ifdef STARROCKS_JIT_ENABLE
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/Value.h>
#include "exprs/jit/ir_helper.h"
#endif
namespace starrocks {
#define DEFINE_CLASS_CONSTRUCTOR(CLASS_NAME) \
@ -124,6 +127,7 @@ public:
return VectorizedStrictBinaryFunction<ArithmeticOp>::template evaluate<Type>(l, r);
}
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override {
return state->can_jit_expr(CompilableExprType::ARITHMETIC) && IRHelper::support_jit(Type);
@ -148,6 +152,7 @@ public:
return ArithmeticOp::template generate_ir<CppType>(context, jit_ctx->module, jit_ctx->builder, datums);
}
}
#endif
std::string debug_string() const override {
std::stringstream out;
@ -196,6 +201,8 @@ public:
}
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override {
return state->can_jit_expr(CompilableExprType::DIV) && Type != TYPE_LARGEINT && IRHelper::support_jit(Type);
}
@ -234,6 +241,7 @@ public:
return ArithmeticOp::template generate_ir<CppType>(context, jit_ctx->module, jit_ctx->builder, datums);
}
}
#endif
std::string debug_string() const override {
std::stringstream out;
@ -290,6 +298,7 @@ public:
return VectorizedMod::template evaluate<Type>(l, r);
}
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override {
return state->can_jit_expr(CompilableExprType::MOD) && Type != TYPE_LARGEINT && IRHelper::support_jit(Type);
@ -329,6 +338,7 @@ public:
return ArithmeticOp::template generate_ir<CppType>(context, jit_ctx->module, jit_ctx->builder, datums);
}
}
#endif
std::string debug_string() const override {
std::stringstream out;
@ -350,6 +360,7 @@ public:
using ArithmeticBitNot = ArithmeticUnaryOperator<BitNotOp, Type>;
return VectorizedStrictUnaryFunction<ArithmeticBitNot>::template evaluate<Type>(l);
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override {
return state->can_jit_expr(CompilableExprType::ARITHMETIC) && IRHelper::support_jit(Type);
@ -381,6 +392,7 @@ public:
datum.value = ArithmeticBitNot::generate_ir(jit_ctx->builder, datum.value);
return datum;
}
#endif
std::string debug_string() const override {
std::stringstream out;
@ -404,6 +416,8 @@ public:
return VectorizedStrictBinaryFunction<ArithmeticOp>::template evaluate<Type, TYPE_BIGINT, Type>(l, r);
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override {
return state->can_jit_expr(CompilableExprType::ARITHMETIC) && IRHelper::support_jit(Type);
}
@ -439,6 +453,7 @@ public:
return ArithmeticOp::template generate_ir<CppType, RunTimeCppType<TYPE_BIGINT>, CppType>(
context, jit_ctx->module, jit_ctx->builder, datums);
}
#endif
std::string debug_string() const override {
std::stringstream out;

23
be/src/exprs/arithmetic_operation.h
View File

@ -14,17 +14,20 @@
#pragma once
#include <llvm/IR/Constants.h>
#include <llvm/IR/Value.h>
#include "column/type_traits.h"
#include "common/status.h"
#include "exprs/expr_context.h"
#include "exprs/jit/ir_helper.h"
#include "runtime/decimalv3.h"
#include "types/logical_type.h"
#include "util/guard.h"
#ifdef STARROCKS_JIT_ENABLE
#include <llvm/IR/Constants.h>
#include <llvm/IR/Value.h>
#include "exprs/jit/ir_helper.h"
#endif
namespace starrocks {
struct AddOp {};
struct SubOp {};
@ -194,7 +197,7 @@ struct ArithmeticBinaryOperator {
static_assert(is_binary_op<Op>, "Invalid binary operators");
}
}
#ifdef STARROCKS_JIT_ENABLE
template <typename ResultType>
static StatusOr<LLVMDatum> generate_ir(ExprContext* context, const llvm::Module& module, llvm::IRBuilder<>& b,
const std::vector<LLVMDatum>& datums) {
@ -343,6 +346,7 @@ struct ArithmeticBinaryOperator {
return result;
}
#endif
};
TYPE_GUARD(DivModOpGuard, is_divmod_op, DivOp, ModOp)
@ -365,7 +369,7 @@ struct ArithmeticBinaryOperator<Op, TYPE_DECIMALV2, DivModOpGuard<Op>, guard::Gu
static_assert(is_divmod_op<Op>, "Invalid float operators");
}
}
#ifdef STARROCKS_JIT_ENABLE
template <typename ResultType>
static StatusOr<LLVMDatum> generate_ir(ExprContext* context, const llvm::Module& module, llvm::IRBuilder<>& b,
const std::vector<LLVMDatum>& datums) {
@ -378,6 +382,7 @@ struct ArithmeticBinaryOperator<Op, TYPE_DECIMALV2, DivModOpGuard<Op>, guard::Gu
// JIT compile of DecimalV2 type is not supported.
return Status::NotSupported("JIT compile of DecimalV2 type is not supported.");
}
#endif
};
TYPE_GUARD(DecimalOpGuard, is_decimal_op, AddOp, SubOp, ReverseSubOp, MulOp, DivOp, ModOp, ReverseModOp)
@ -581,12 +586,13 @@ struct ArithmeticBinaryOperator<Op, Type, DecimalOpGuard<Op>, DecimalLTGuard<Typ
return apply<check_overflow, LType, RType, ResultType>(l, r, result);
}
}
#ifdef STARROCKS_JIT_ENABLE
llvm::Value* generate_ir(llvm::IRBuilder<>& b, const std::vector<llvm::Value*>& args) const {
// TODO(Yueyang): Support JIT compile of DecimalV3 type.
LOG(WARNING) << "JIT compile of DecimalV3 type is not supported.";
return nullptr;
}
#endif
};
template <typename Op, LogicalType Type>
@ -613,7 +619,7 @@ struct ArithmeticUnaryOperator {
static_assert(is_bitnot_op<Op>, "Invalid unary operators");
}
}
#ifdef STARROCKS_JIT_ENABLE
static llvm::Value* generate_ir(llvm::IRBuilder<>& b, llvm::Value* l) {
if constexpr (is_bitnot_op<Op>) {
return b.CreateNot(l);
@ -621,6 +627,7 @@ struct ArithmeticUnaryOperator {
static_assert(is_bitnot_op<Op>, "Invalid unary operators");
}
}
#endif
};
template <LogicalType Type, typename = guard::Guard>

17
be/src/exprs/binary_predicate.cpp
View File

@ -14,22 +14,25 @@
#include "exprs/binary_predicate.h"
#include <llvm/IR/Constants.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/Value.h>
#include "column/array_column.h"
#include "column/column_builder.h"
#include "column/column_viewer.h"
#include "column/type_traits.h"
#include "exprs/binary_function.h"
#include "exprs/jit/ir_helper.h"
#include "exprs/unary_function.h"
#include "runtime/runtime_state.h"
#include "storage/column_predicate.h"
#include "types/logical_type.h"
#include "types/logical_type_infra.h"
#ifdef STARROCKS_JIT_ENABLE
#include <llvm/IR/Constants.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/Value.h>
#include "exprs/jit/ir_helper.h"
#endif
namespace starrocks {
template <LogicalType ltype>
@ -131,6 +134,7 @@ public:
ASSIGN_OR_RETURN(auto r, _children[1]->evaluate_checked(context, ptr));
return VectorizedStrictBinaryFunction<OP>::template evaluate<Type, TYPE_BOOLEAN>(l, r);
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override {
return state->can_jit_expr(CompilableExprType::CMP) && IRHelper::support_jit(Type);
@ -215,6 +219,7 @@ public:
_children[1]->jit_func_name(state) + "}" + (is_constant() ? "c:" : "") + (is_nullable() ? "n:" : "") +
type().debug_string();
}
#endif
std::string debug_string() const override {
std::stringstream out;
@ -426,6 +431,7 @@ public:
return builder.build(ColumnHelper::is_all_const(list));
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override {
return state->can_jit_expr(CompilableExprType::CMP) && IRHelper::support_jit(Type);
@ -464,6 +470,7 @@ public:
return "{" + _children[0]->jit_func_name(state) + "<=>" + _children[1]->jit_func_name(state) + "}" +
(is_constant() ? "c:" : "") + (is_nullable() ? "n:" : "") + type().debug_string();
}
#endif
std::string debug_string() const override {
std::stringstream out;

7
be/src/exprs/case_expr.cpp
View File

@ -23,13 +23,16 @@
#include "column/type_traits.h"
#include "column/vectorized_fwd.h"
#include "common/object_pool.h"
#include "exprs/jit/ir_helper.h"
#include "gutil/casts.h"
#include "runtime/runtime_state.h"
#include "simd/mulselector.h"
#include "types/logical_type_infra.h"
#include "util/percentile_value.h"
#ifdef STARROCKS_JIT_ENABLE
#include "exprs/jit/ir_helper.h"
#endif
namespace starrocks {
/**
@ -71,6 +74,7 @@ public:
return _children.size() % 2 == 1 ? Status::OK() : Status::InvalidArgument("case when children is error!");
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override {
if (_has_case_expr) {
return state->can_jit_expr(CompilableExprType::CASE) && IRHelper::support_jit(WhenType) &&
@ -240,6 +244,7 @@ public:
out << "}" << (is_constant() ? "c:" : "") << (is_nullable() ? "n:" : "") << type().debug_string();
return out.str();
}
#endif
std::string debug_string() const override {
std::stringstream out;

10
be/src/exprs/cast_expr.cpp
View File

@ -14,10 +14,13 @@
#include "exprs/cast_expr.h"
#ifdef STARROCKS_JIT_ENABLE
#include <llvm/ADT/APInt.h>
#include <llvm/IR/Constants.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/Value.h>
#endif
#include <ryu/ryu.h>
#include <limits>
@ -38,7 +41,6 @@
#include "exprs/binary_function.h"
#include "exprs/column_ref.h"
#include "exprs/decimal_cast_expr.h"
#include "exprs/jit/ir_helper.h"
#include "exprs/unary_function.h"
#include "gutil/casts.h"
#include "gutil/strings/substitute.h"
@ -54,6 +56,10 @@
#include "util/mysql_global.h"
#include "util/numeric_types.h"
#ifdef STARROCKS_JIT_ENABLE
#include "exprs/jit/ir_helper.h"
#endif
namespace starrocks {
#define THROW_RUNTIME_ERROR_WITH_TYPE(TYPE) \
@ -1118,6 +1124,7 @@ public:
}
return result_column;
};
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override {
return state->can_jit_expr(CompilableExprType::CAST) && !AllowThrowException && FromType != TYPE_LARGEINT &&
@ -1184,6 +1191,7 @@ public:
return datum;
}
}
#endif
std::string debug_string() const override {
std::stringstream out;

12
be/src/exprs/compound_predicate.cpp
View File

@ -16,11 +16,14 @@
#include "common/object_pool.h"
#include "exprs/binary_function.h"
#include "exprs/jit/ir_helper.h"
#include "exprs/predicate.h"
#include "exprs/unary_function.h"
#include "runtime/runtime_state.h"
#ifdef STARROCKS_JIT_ENABLE
#include "exprs/jit/ir_helper.h"
#endif
namespace starrocks {
#define DEFINE_COMPOUND_CONSTRUCT(CLASS) \
@ -60,6 +63,7 @@ public:
return VectorizedLogicPredicateBinaryFunction<AndNullImpl, AndImpl>::template evaluate<TYPE_BOOLEAN>(l, r);
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override { return state->can_jit_expr(CompilableExprType::LOGICAL); }
JitScore compute_jit_score(RuntimeState* state) const override {
@ -94,6 +98,7 @@ public:
return "{" + _children[0]->jit_func_name(state) + " & " + _children[1]->jit_func_name(state) + "}" +
(is_constant() ? "c:" : "") + (is_nullable() ? "n:" : "") + type().debug_string();
}
#endif
std::string debug_string() const override {
std::stringstream out;
@ -137,6 +142,8 @@ public:
return VectorizedLogicPredicateBinaryFunction<OrNullImpl, OrImpl>::template evaluate<TYPE_BOOLEAN>(l, r);
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override { return state->can_jit_expr(CompilableExprType::LOGICAL); }
JitScore compute_jit_score(RuntimeState* state) const override {
@ -171,6 +178,7 @@ public:
return "{" + _children[0]->jit_func_name(state) + " | " + _children[1]->jit_func_name(state) + "}" +
(is_constant() ? "c:" : "") + (is_nullable() ? "n:" : "") + type().debug_string();
}
#endif
std::string debug_string() const override {
std::stringstream out;
@ -195,6 +203,7 @@ public:
return VectorizedStrictUnaryFunction<CompoundPredNot>::template evaluate<TYPE_BOOLEAN>(l);
}
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override { return state->can_jit_expr(CompilableExprType::LOGICAL); }
@ -226,6 +235,7 @@ public:
return "{!" + _children[0]->jit_func_name(state) + "}" + (is_constant() ? "c:" : "") +
(is_nullable() ? "n:" : "") + type().debug_string();
}
#endif
std::string debug_string() const override {
std::stringstream out;

2
be/src/exprs/decimal_binary_function.h
View File

@ -205,6 +205,7 @@ struct DecimalBinaryFunction {
}
}
#ifdef STARROCKS_JIT_ENABLE
template <LogicalType LhsType, LogicalType RhsType, LogicalType ResultType>
static llvm::Value* generate_ir(llvm::IRBuilder<>& b, llvm::Module& module, llvm::Value* l, llvm::Value* r,
int l_scale, int r_scale) {
@ -217,6 +218,7 @@ struct DecimalBinaryFunction {
return nullptr;
}
#endif
template <LogicalType LhsType, LogicalType RhsType, LogicalType ResultType>
static inline ColumnPtr const_const(const ColumnPtr& lhs, const ColumnPtr& rhs) {

18
be/src/exprs/expr.cpp
View File

@ -34,7 +34,6 @@
#include "exprs/expr.h"
#include <llvm/IR/Value.h>
#include <thrift/protocol/TDebugProtocol.h>
#include <sstream>
@ -66,9 +65,6 @@
#include "exprs/info_func.h"
#include "exprs/is_null_predicate.h"
#include "exprs/java_function_call_expr.h"
#include "exprs/jit/ir_helper.h"
#include "exprs/jit/jit_engine.h"
#include "exprs/jit/jit_expr.h"
#include "exprs/lambda_function.h"
#include "exprs/literal.h"
#include "exprs/map_apply_expr.h"
@ -82,6 +78,14 @@
#include "types/logical_type.h"
#include "util/failpoint/fail_point.h"
#ifdef STARROCKS_JIT_ENABLE
#include <llvm/IR/Value.h>
#include "exprs/jit/ir_helper.h"
#include "exprs/jit/jit_engine.h"
#include "exprs/jit/jit_expr.h"
#endif
#pragma clang diagnostic push
#pragma ide diagnostic ignored "EndlessLoop"
using std::vector;
@ -231,6 +235,7 @@ Status Expr::create_expr_tree(ObjectPool* pool, const TExpr& texpr, ExprContext*
return status;
}
#ifdef STARROCKS_JIT_ENABLE
Status Expr::prepare_jit_expr(RuntimeState* state, ExprContext* context) {
if (this->node_type() == TExprNodeType::JIT_EXPR) {
RETURN_IF_ERROR(((JITExpr*)this)->prepare_impl(state, context));
@ -240,6 +245,7 @@ Status Expr::prepare_jit_expr(RuntimeState* state, ExprContext* context) {
}
return Status::OK();
}
#endif
Status Expr::create_expr_trees(ObjectPool* pool, const std::vector<TExpr>& texprs, std::vector<ExprContext*>* ctxs,
RuntimeState* state, bool can_jit) {
@ -260,6 +266,7 @@ Status Expr::create_tree_from_thrift_with_jit(ObjectPool* pool, const std::vecto
return status;
}
#ifdef STARROCKS_JIT_ENABLE
bool replaced = false;
status = (*root_expr)->replace_compilable_exprs(root_expr, pool, state, replaced);
if (!status.ok()) {
@ -272,6 +279,7 @@ Status Expr::create_tree_from_thrift_with_jit(ObjectPool* pool, const std::vecto
// The node was replaced, so we need to update the context.
*ctx = pool->add(new ExprContext(*root_expr));
}
#endif
return status;
}
@ -723,6 +731,7 @@ ColumnRef* Expr::get_column_ref() {
return nullptr;
}
#ifdef STARROCKS_JIT_ENABLE
StatusOr<LLVMDatum> Expr::generate_ir(ExprContext* context, JITContext* jit_ctx) {
if (this->is_compilable(context->_runtime_state)) {
return this->generate_ir_impl(context, jit_ctx);
@ -843,6 +852,7 @@ bool Expr::should_compile(RuntimeState* state) const {
}
return true;
}
#endif
bool Expr::support_ngram_bloom_filter(ExprContext* context) const {
bool support = false;

5
be/src/exprs/expr.h
View File

@ -215,6 +215,7 @@ public:
// Get the first column ref in expr.
ColumnRef* get_column_ref();
#ifdef STARROCKS_JIT_ENABLE
StatusOr<LLVMDatum> generate_ir(ExprContext* context, JITContext* jit_ctx);
virtual StatusOr<LLVMDatum> generate_ir_impl(ExprContext* context, JITContext* jit_ctx);
@ -246,6 +247,7 @@ public:
// The valuable expressions get 1 score per expression, others get 0 score per expression, including
// comparison expr, logical expr, branch expr, div and mod.
virtual JitScore compute_jit_score(RuntimeState* state) const;
#endif
// Return true if this expr or any of its children support ngram bloom filter, otherwise return flase
virtual bool support_ngram_bloom_filter(ExprContext* context) const;
@ -357,8 +359,9 @@ protected:
out << expr_name << "(" << Expr::debug_string() << ")";
return out.str();
}
#ifdef STARROCKS_JIT_ENABLE
Status prepare_jit_expr(RuntimeState* state, ExprContext* context);
#endif
private:
// Create a new vectorized expr

3
be/src/exprs/expr_context.cpp
View File

@ -215,6 +215,7 @@ Status ExprContext::rewrite_jit_expr(ObjectPool* pool) {
if (_runtime_state == nullptr || !_runtime_state->is_jit_enabled()) {
return Status::OK();
}
#ifdef STARROCKS_JIT_ENABLE
bool replaced = false;
auto st = _root->replace_compilable_exprs(&_root, pool, _runtime_state, replaced);
if (!st.ok()) {
@ -225,6 +226,8 @@ Status ExprContext::rewrite_jit_expr(ObjectPool* pool) {
if (replaced) { // only prepare jit_expr
WARN_IF_ERROR(_root->prepare_jit_expr(_runtime_state, this), "prepare rewritten expr failed");
}
#endif
return Status::OK();
}

8
be/src/exprs/literal.cpp
View File

@ -19,11 +19,14 @@
#include "column/const_column.h"
#include "column/vectorized_fwd.h"
#include "common/statusor.h"
#include "exprs/jit/ir_helper.h"
#include "gutil/port.h"
#include "gutil/strings/fastmem.h"
#include "types/constexpr.h"
#ifdef STARROCKS_JIT_ENABLE
#include "exprs/jit/ir_helper.h"
#endif
namespace starrocks {
#define CASE_TYPE_COLUMN(NODE_TYPE, CHECK_TYPE, LITERAL_VALUE) \
@ -165,6 +168,8 @@ StatusOr<ColumnPtr> VectorizedLiteral::evaluate_checked(ExprContext* context, Ch
return column;
}
#ifdef STARROCKS_JIT_ENABLE
bool VectorizedLiteral::is_compilable(RuntimeState* state) const {
return IRHelper::support_jit(_type.type);
}
@ -187,6 +192,7 @@ StatusOr<LLVMDatum> VectorizedLiteral::generate_ir_impl(ExprContext* context, JI
}
return datum;
}
#endif
std::string VectorizedLiteral::debug_string() const {
std::stringstream out;

2
be/src/exprs/literal.h
View File

@ -30,6 +30,7 @@ public:
StatusOr<ColumnPtr> evaluate_checked(ExprContext* context, Chunk* ptr) override;
#ifdef STARROCKS_JIT_ENABLE
bool is_compilable(RuntimeState* state) const override;
JitScore compute_jit_score(RuntimeState* state) const override;
@ -37,6 +38,7 @@ public:
std::string jit_func_name_impl(RuntimeState* state) const override;
StatusOr<LLVMDatum> generate_ir_impl(ExprContext* context, JITContext* jit_ctx) override;
#endif
std::string debug_string() const override;

7
be/src/runtime/exec_env.cpp
View File

@ -53,7 +53,6 @@
#include "exec/workgroup/scan_executor.h"
#include "exec/workgroup/scan_task_queue.h"
#include "exec/workgroup/work_group.h"
#include "exprs/jit/jit_engine.h"
#include "fs/fs_s3.h"
#include "gen_cpp/BackendService.h"
#include "gen_cpp/TFileBrokerService.h"
@ -101,6 +100,10 @@
#include "util/priority_thread_pool.hpp"
#include "util/starrocks_metrics.h"
#ifdef STARROCKS_JIT_ENABLE
#include "exprs/jit/jit_engine.h"
#endif
namespace starrocks {
// Calculate the total memory limit of all load tasks on this BE
@ -550,11 +553,13 @@ Status ExecEnv::init(const std::vector<StorePath>& store_paths, bool as_cn) {
_spill_dir_mgr = std::make_shared<spill::DirManager>();
RETURN_IF_ERROR(_spill_dir_mgr->init(config::spill_local_storage_dir));
#ifdef STARROCKS_JIT_ENABLE
auto jit_engine = JITEngine::get_instance();
status = jit_engine->init();
if (!status.ok()) {
LOG(WARNING) << "Failed to init JIT engine: " << status.message();
}
#endif
RETURN_IF_ERROR(PythonEnvManager::getInstance().init(config::python_envs));
PythonEnvManager::getInstance().start_background_cleanup_thread();

9
be/src/runtime/runtime_state.cpp
View File

@ -46,7 +46,6 @@
#include "common/status.h"
#include "exec/exec_node.h"
#include "exec/pipeline/query_context.h"
#include "exprs/jit/jit_engine.h"
#include "fs/fs_util.h"
#ifdef USE_STAROS
#include "fslib/star_cache_handler.h"
@ -62,6 +61,10 @@
#include "util/timezone_utils.h"
#include "util/uid_util.h"
#ifdef STARROCKS_JIT_ENABLE
#include "exprs/jit/jit_engine.h"
#endif
namespace starrocks {
// for ut only
@ -523,8 +526,12 @@ Status RuntimeState::reset_epoch() {
}
bool RuntimeState::is_jit_enabled() const {
#ifdef STARROCKS_JIT_ENABLE
return JITEngine::get_instance()->support_jit() && _query_options.__isset.jit_level &&
_query_options.jit_level != 0;
#else
return false;
#endif
}
void RuntimeState::update_load_datacache_metrics(TReportExecStatusParams* load_params) const {

5
build.sh
View File

@ -176,6 +176,10 @@ fi
if [[ -z ${JEMALLOC_DEBUG} ]]; then
JEMALLOC_DEBUG=OFF
fi
if [[ -z ${ENABLE_JIT} ]]; then
ENABLE_JIT=ON
fi
if [[ -z ${CCACHE} ]] && [[ -x "$(command -v ccache)" ]]; then
CCACHE=ccache
fi
@ -389,6 +393,7 @@ if [ ${BUILD_BE} -eq 1 ] ; then
-DUSE_STAROS=${USE_STAROS} \
-DENABLE_FAULT_INJECTION=${ENABLE_FAULT_INJECTION} \
-DWITH_TENANN=${WITH_TENANN} \
-DSTARROCKS_JIT_ENABLE=${ENABLE_JIT} \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
time ${BUILD_SYSTEM} -j${PARALLEL}

1
run-be-ut.sh
View File

@ -183,6 +183,7 @@ ${CMAKE_CMD} -G "${CMAKE_GENERATOR}" \
-DWITH_GCOV=${WITH_GCOV} \
-DWITH_STARCACHE=${WITH_STARCACHE} \
-DWITH_BRPC_KEEPALIVE=${WITH_BRPC_KEEPALIVE} \
-DSTARROCKS_JIT_ENABLE=ON \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON ../
${BUILD_SYSTEM} -j${PARALLEL}