[Tool] Docker-Based Development Environment to build local code from any host environment (#60151)
Signed-off-by: Deng Liu <d.liu@celonis.com>
This commit is contained in:
Deng Liu
GitHub
parent
dec41ae9d0
commit
e17fd1517a
|
|
@ -0,0 +1,143 @@
|
|||
# Docker-based Makefile for StarRocks
|
||||
# Usage: make -f Makefile.docker <target>
|
||||
|
||||
.PHONY: help shell build-fe build-be build-all clean-build test-fe test-be test-all clean docker-pull
|
||||
|
||||
# Configuration
|
||||
DOCKER_IMAGE ?= starrocks/dev-env-ubuntu:latest
|
||||
DOCKER_BUILD_SCRIPT = ./build-in-docker.sh
|
||||
DOCKER_DEV_SCRIPT = ./docker-dev.sh
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "Docker-based build targets for StarRocks"
|
||||
@echo ""
|
||||
@echo "Development:"
|
||||
@echo " shell Open interactive development shell"
|
||||
@echo " docker-pull Pull the latest Docker image"
|
||||
@echo ""
|
||||
@echo "Building:"
|
||||
@echo " build-fe Build Frontend only"
|
||||
@echo " build-be Build Backend only"
|
||||
@echo " build-all Build both Frontend and Backend"
|
||||
@echo " clean-build Clean and build everything"
|
||||
@echo ""
|
||||
@echo "Testing:"
|
||||
@echo " test-fe Run Frontend tests"
|
||||
@echo " test-be Run Backend tests"
|
||||
@echo " test-all Run all tests"
|
||||
@echo ""
|
||||
@echo "Maintenance:"
|
||||
@echo " clean Clean build artifacts"
|
||||
@echo " docker-clean Clean Docker containers and volumes"
|
||||
@echo ""
|
||||
@echo "Environment Variables:"
|
||||
@echo " DOCKER_IMAGE Docker image to use (default: $(DOCKER_IMAGE))"
|
||||
@echo ""
|
||||
@echo "Examples:"
|
||||
@echo " make -f Makefile.docker shell"
|
||||
@echo " make -f Makefile.docker build-fe"
|
||||
@echo " make -f Makefile.docker DOCKER_IMAGE=starrocks/dev-env-ubuntu:latest build-all"
|
||||
|
||||
# Development targets
|
||||
shell:
|
||||
@echo "Opening development shell..."
|
||||
$(DOCKER_DEV_SCRIPT) shell
|
||||
|
||||
docker-pull:
|
||||
@echo "Pulling Docker image: $(DOCKER_IMAGE)"
|
||||
docker pull $(DOCKER_IMAGE)
|
||||
|
||||
# Build targets
|
||||
build-fe:
|
||||
@echo "Building Frontend..."
|
||||
$(DOCKER_DEV_SCRIPT) build-fe
|
||||
|
||||
build-be:
|
||||
@echo "Building Backend..."
|
||||
$(DOCKER_DEV_SCRIPT) build-be
|
||||
|
||||
build-all:
|
||||
@echo "Building all components..."
|
||||
$(DOCKER_DEV_SCRIPT) build-all
|
||||
|
||||
clean-build:
|
||||
@echo "Clean building all components..."
|
||||
$(DOCKER_DEV_SCRIPT) clean-build
|
||||
|
||||
# Test targets
|
||||
test-fe:
|
||||
@echo "Running Frontend tests..."
|
||||
$(DOCKER_DEV_SCRIPT) test-fe
|
||||
|
||||
test-be:
|
||||
@echo "Running Backend tests..."
|
||||
$(DOCKER_DEV_SCRIPT) test-be
|
||||
|
||||
test-all:
|
||||
@echo "Running all tests..."
|
||||
$(DOCKER_DEV_SCRIPT) test-all
|
||||
|
||||
# Maintenance targets
|
||||
clean:
|
||||
@echo "Cleaning build artifacts..."
|
||||
rm -rf output/
|
||||
rm -rf fe/fe-core/target/
|
||||
rm -rf java-extensions/*/target/
|
||||
|
||||
docker-clean:
|
||||
@echo "Cleaning Docker containers and volumes..."
|
||||
docker-compose -f docker-compose.dev.yml down -v --remove-orphans || true
|
||||
docker container prune -f
|
||||
docker volume prune -f
|
||||
|
||||
# Advanced build targets with specific options
|
||||
build-fe-debug:
|
||||
@echo "Building Frontend in debug mode..."
|
||||
BUILD_TYPE=Debug $(DOCKER_BUILD_SCRIPT) --fe
|
||||
|
||||
build-be-debug:
|
||||
@echo "Building Backend in debug mode..."
|
||||
BUILD_TYPE=Debug $(DOCKER_BUILD_SCRIPT) --be
|
||||
|
||||
build-be-asan:
|
||||
@echo "Building Backend with AddressSanitizer..."
|
||||
BUILD_TYPE=Asan $(DOCKER_BUILD_SCRIPT) --be
|
||||
|
||||
build-be-gcov:
|
||||
@echo "Building Backend with code coverage..."
|
||||
$(DOCKER_BUILD_SCRIPT) --be --with-gcov
|
||||
|
||||
# Parallel build targets
|
||||
build-fe-fast:
|
||||
@echo "Building Frontend with maximum parallelism..."
|
||||
$(DOCKER_BUILD_SCRIPT) --fe -j $$(nproc)
|
||||
|
||||
build-be-fast:
|
||||
@echo "Building Backend with maximum parallelism..."
|
||||
$(DOCKER_BUILD_SCRIPT) --be -j $$(nproc)
|
||||
|
||||
build-all-fast:
|
||||
@echo "Building all components with maximum parallelism..."
|
||||
$(DOCKER_BUILD_SCRIPT) --fe --be -j $$(nproc)
|
||||
|
||||
# Docker Compose targets
|
||||
compose-shell:
|
||||
@echo "Starting development shell with Docker Compose..."
|
||||
docker-compose -f docker-compose.dev.yml run --rm starrocks-dev
|
||||
|
||||
compose-build-fe:
|
||||
@echo "Building Frontend with Docker Compose..."
|
||||
docker-compose -f docker-compose.dev.yml run --rm build-fe
|
||||
|
||||
compose-build-be:
|
||||
@echo "Building Backend with Docker Compose..."
|
||||
docker-compose -f docker-compose.dev.yml run --rm build-be
|
||||
|
||||
compose-test-fe:
|
||||
@echo "Running Frontend tests with Docker Compose..."
|
||||
docker-compose -f docker-compose.dev.yml run --rm test-fe
|
||||
|
||||
compose-test-be:
|
||||
@echo "Running Backend tests with Docker Compose..."
|
||||
docker-compose -f docker-compose.dev.yml run --rm test-be
|
||||
|
|
@ -0,0 +1,363 @@
|
|||
#!/usr/bin/env bash
|
||||
# Copyright 2021-present StarRocks, Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
##############################################################
|
||||
# Docker-based build script for StarRocks
|
||||
# This script uses the official StarRocks dev-env Docker image
|
||||
# to build the project in a consistent Ubuntu environment.
|
||||
#
|
||||
# Usage:
|
||||
# ./build-in-docker.sh --help
|
||||
# Examples:
|
||||
# ./build-in-docker.sh # build all (FE + BE)
|
||||
# ./build-in-docker.sh --fe # build Frontend only
|
||||
# ./build-in-docker.sh --be # build Backend only
|
||||
# ./build-in-docker.sh --fe --clean # clean and build Frontend
|
||||
# ./build-in-docker.sh --fe --be --clean # clean and build both
|
||||
# ./build-in-docker.sh --shell # open interactive shell
|
||||
# ./build-in-docker.sh --test # run tests
|
||||
##############################################################
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
DOCKER_IMAGE="${STARROCKS_DEV_ENV_IMAGE:-starrocks/dev-env-ubuntu:latest}"
|
||||
CONTAINER_NAME="starrocks-build-$(whoami)-$(id -u)-$(date +%s)"
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BUILD_ARGS=""
|
||||
INTERACTIVE_SHELL=false
|
||||
RUN_TESTS=false
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Logging functions
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Help function
|
||||
usage() {
|
||||
cat << EOF
|
||||
Docker-based build script for StarRocks
|
||||
|
||||
Usage: $0 [DOCKER_OPTIONS] [BUILD_OPTIONS]
|
||||
|
||||
DOCKER-SPECIFIC OPTIONS:
|
||||
--shell Open interactive shell in dev environment
|
||||
--test Run tests after building
|
||||
--image IMAGE Use specific Docker image (default: $DOCKER_IMAGE)
|
||||
--help, -h Show this help message
|
||||
|
||||
BUILD OPTIONS (passed through to build.sh):
|
||||
All options supported by build.sh are automatically passed through, including:
|
||||
--fe Build Frontend only
|
||||
--be Build Backend only
|
||||
--spark-dpp Build Spark DPP application
|
||||
--hive-udf Build Hive UDF
|
||||
--clean Clean before building
|
||||
--enable-shared-data Build Backend with shared-data feature support
|
||||
--with-gcov Build Backend with gcov
|
||||
--without-gcov Build Backend without gcov (default)
|
||||
--with-bench Build Backend with bench
|
||||
--with-clang-tidy Build Backend with clang-tidy
|
||||
--without-java-ext Build Backend without java-extensions
|
||||
--without-starcache Build Backend without starcache library
|
||||
--without-tenann Build without vector index tenann library
|
||||
--without-avx2 Build Backend without avx2 instruction
|
||||
--disable-java-check-style Disable Java checkstyle checks
|
||||
-j N Build with N parallel jobs
|
||||
... and any other build.sh options
|
||||
|
||||
EXAMPLES:
|
||||
$0 # Build all components
|
||||
$0 --fe --clean # Clean and build Frontend
|
||||
$0 --be --with-gcov # Build Backend with gcov
|
||||
$0 --shell # Open interactive shell
|
||||
$0 --test # Build and run tests
|
||||
$0 --image starrocks/dev-env-ubuntu:latest # Use different image
|
||||
$0 --be --new-future-option # Any new build.sh option works automatically
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
STARROCKS_DEV_ENV_IMAGE Docker image to use (default: $DOCKER_IMAGE)
|
||||
DOCKER_BUILD_OPTS Additional Docker run options
|
||||
|
||||
NOTE: This script automatically passes through all unrecognized options to build.sh,
|
||||
so new build.sh options work without updating this script.
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
# Docker-specific options (handled by this script)
|
||||
--shell)
|
||||
INTERACTIVE_SHELL=true
|
||||
shift
|
||||
;;
|
||||
--test)
|
||||
RUN_TESTS=true
|
||||
shift
|
||||
;;
|
||||
--image)
|
||||
DOCKER_IMAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help|-h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
# Options with arguments (pass through to build.sh)
|
||||
-j)
|
||||
BUILD_ARGS="$BUILD_ARGS $1 $2"
|
||||
shift 2
|
||||
;;
|
||||
# All other options (pass through to build.sh)
|
||||
*)
|
||||
BUILD_ARGS="$BUILD_ARGS $1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# Check if Docker is available
|
||||
check_docker() {
|
||||
if ! command -v docker &> /dev/null; then
|
||||
log_error "Docker is not installed or not in PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! docker info &> /dev/null; then
|
||||
log_error "Docker daemon is not running or not accessible"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Pull Docker image if needed
|
||||
pull_image() {
|
||||
log_info "Checking Docker image: $DOCKER_IMAGE"
|
||||
|
||||
if ! docker image inspect "$DOCKER_IMAGE" &> /dev/null; then
|
||||
log_info "Pulling Docker image: $DOCKER_IMAGE"
|
||||
if ! docker pull "$DOCKER_IMAGE"; then
|
||||
log_error "Failed to pull Docker image: $DOCKER_IMAGE"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
log_info "Docker image already available: $DOCKER_IMAGE"
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup Docker run command
|
||||
setup_docker_run() {
|
||||
# Base Docker run options
|
||||
DOCKER_RUN_OPTS=(
|
||||
--rm
|
||||
--name "$CONTAINER_NAME"
|
||||
--volume "$REPO_ROOT:/workspace"
|
||||
--volume "$HOME/.m2:/tmp/.m2"
|
||||
--workdir /workspace
|
||||
--user "$(id -u):$(id -g)"
|
||||
--env "HOME=/tmp"
|
||||
--env "STARROCKS_HOME=/workspace"
|
||||
--env "STARROCKS_THIRDPARTY=/var/local/thirdparty"
|
||||
--env "MAVEN_OPTS=-Dmaven.repo.local=/tmp/.m2/repository"
|
||||
)
|
||||
|
||||
# Add any additional Docker options from environment
|
||||
if [[ -n "${DOCKER_BUILD_OPTS:-}" ]]; then
|
||||
read -ra ADDITIONAL_OPTS <<< "$DOCKER_BUILD_OPTS"
|
||||
DOCKER_RUN_OPTS+=("${ADDITIONAL_OPTS[@]}")
|
||||
fi
|
||||
|
||||
# Interactive mode for shell
|
||||
if [[ "$INTERACTIVE_SHELL" == "true" ]]; then
|
||||
DOCKER_RUN_OPTS+=(--interactive --tty)
|
||||
fi
|
||||
}
|
||||
|
||||
# Run interactive shell
|
||||
run_shell() {
|
||||
log_info "Starting interactive shell in StarRocks dev environment"
|
||||
log_info "Container: $CONTAINER_NAME"
|
||||
log_info "Image: $DOCKER_IMAGE"
|
||||
log_info "Workspace: /workspace (mounted from $REPO_ROOT)"
|
||||
|
||||
docker run "${DOCKER_RUN_OPTS[@]}" "$DOCKER_IMAGE" /bin/bash
|
||||
}
|
||||
|
||||
# Check if protobuf version mismatch exists
|
||||
check_protobuf_version_mismatch() {
|
||||
log_info "Checking for protobuf version compatibility..."
|
||||
|
||||
# Check if generated protobuf files exist
|
||||
if [[ ! -d "gensrc/build/gen_cpp" ]] || [[ -z "$(find gensrc/build/gen_cpp -name "*.pb.cc" 2>/dev/null)" ]]; then
|
||||
log_info "No generated protobuf files found - clean generation needed"
|
||||
return 0 # Need to clean/generate
|
||||
fi
|
||||
|
||||
# Get protobuf version from Docker container
|
||||
local container_protoc_version
|
||||
container_protoc_version=$(docker run --rm "${DOCKER_RUN_OPTS[@]}" "$DOCKER_IMAGE" bash -c "/var/local/thirdparty/installed/bin/protoc --version 2>/dev/null | cut -d' ' -f2" 2>/dev/null || echo "unknown")
|
||||
|
||||
# Try to detect version mismatch by checking for common error patterns in a test compilation
|
||||
local test_result
|
||||
test_result=$(docker run --rm "${DOCKER_RUN_OPTS[@]}" "$DOCKER_IMAGE" bash -c "
|
||||
cd /workspace &&
|
||||
echo '#include \"gensrc/build/gen_cpp/data.pb.h\"' > /tmp/test_protobuf.cpp &&
|
||||
echo 'int main() { return 0; }' >> /tmp/test_protobuf.cpp &&
|
||||
g++ -I/var/local/thirdparty/installed/include -I. -c /tmp/test_protobuf.cpp -o /tmp/test_protobuf.o 2>&1 || echo 'PROTOBUF_MISMATCH'
|
||||
" 2>/dev/null)
|
||||
|
||||
if [[ "$test_result" == *"PROTOBUF_MISMATCH"* ]] || [[ "$test_result" == *"incompatible version"* ]] || [[ "$test_result" == *"generated_message_tctable_decl.h"* ]]; then
|
||||
log_warning "Protobuf version mismatch detected (container: $container_protoc_version)"
|
||||
log_warning "Generated files were created with different protobuf version"
|
||||
return 0 # Need to clean
|
||||
fi
|
||||
|
||||
log_success "Protobuf version compatibility check passed (container: $container_protoc_version)"
|
||||
return 1 # No need to clean
|
||||
}
|
||||
|
||||
# Clean generated files only when necessary
|
||||
clean_generated_files_if_needed() {
|
||||
if check_protobuf_version_mismatch; then
|
||||
log_info "Cleaning generated files due to protobuf version mismatch..."
|
||||
docker run "${DOCKER_RUN_OPTS[@]}" "$DOCKER_IMAGE" bash -c "make -C gensrc clean || true"
|
||||
docker run "${DOCKER_RUN_OPTS[@]}" "$DOCKER_IMAGE" bash -c "rm -rf fe/fe-core/target/generated-sources || true"
|
||||
log_success "Generated files cleaned - will regenerate with correct protobuf version"
|
||||
else
|
||||
log_info "Skipping gensrc clean - no protobuf version mismatch detected"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run build
|
||||
run_build() {
|
||||
local build_cmd="./build.sh"
|
||||
|
||||
if [[ -n "$BUILD_ARGS" ]]; then
|
||||
build_cmd="$build_cmd$BUILD_ARGS"
|
||||
fi
|
||||
|
||||
log_info "Starting build in Docker container"
|
||||
log_info "Container: $CONTAINER_NAME"
|
||||
log_info "Image: $DOCKER_IMAGE"
|
||||
log_info "Build command: $build_cmd"
|
||||
|
||||
# Smart cleaning - only clean when protobuf version mismatch detected
|
||||
clean_generated_files_if_needed
|
||||
|
||||
# Run the build
|
||||
if docker run "${DOCKER_RUN_OPTS[@]}" "$DOCKER_IMAGE" bash -c "$build_cmd"; then
|
||||
log_success "Build completed successfully!"
|
||||
|
||||
# Show output directory contents
|
||||
if [[ -d "$REPO_ROOT/output" ]]; then
|
||||
log_info "Build artifacts in output directory:"
|
||||
ls -la "$REPO_ROOT/output/"
|
||||
fi
|
||||
|
||||
return 0
|
||||
else
|
||||
log_error "Build failed!"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Run tests
|
||||
run_tests() {
|
||||
log_info "Running tests in Docker container"
|
||||
|
||||
# Determine which tests to run based on build arguments
|
||||
local run_fe_tests=false
|
||||
local run_be_tests=false
|
||||
|
||||
if [[ "$BUILD_ARGS" == *"--fe"* ]] && [[ "$BUILD_ARGS" != *"--be"* ]]; then
|
||||
run_fe_tests=true
|
||||
elif [[ "$BUILD_ARGS" == *"--be"* ]] && [[ "$BUILD_ARGS" != *"--fe"* ]]; then
|
||||
run_be_tests=true
|
||||
else
|
||||
# Default: run both if no specific component specified
|
||||
run_fe_tests=true
|
||||
run_be_tests=true
|
||||
fi
|
||||
|
||||
if [[ "$run_fe_tests" == "true" ]]; then
|
||||
log_info "Running Frontend tests..."
|
||||
docker run "${DOCKER_RUN_OPTS[@]}" "$DOCKER_IMAGE" bash -c "./run-fe-ut.sh || echo 'FE tests completed with some failures'"
|
||||
fi
|
||||
|
||||
if [[ "$run_be_tests" == "true" ]]; then
|
||||
log_info "Running Backend tests..."
|
||||
docker run "${DOCKER_RUN_OPTS[@]}" "$DOCKER_IMAGE" bash -c "./run-be-ut.sh || echo 'BE tests completed with some failures'"
|
||||
fi
|
||||
}
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
if docker ps -q -f name="$CONTAINER_NAME" &> /dev/null; then
|
||||
log_info "Cleaning up container: $CONTAINER_NAME"
|
||||
docker stop "$CONTAINER_NAME" &> /dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Main function
|
||||
main() {
|
||||
# Set up cleanup trap
|
||||
trap cleanup EXIT
|
||||
|
||||
# Parse arguments
|
||||
parse_args "$@"
|
||||
|
||||
# Check prerequisites
|
||||
check_docker
|
||||
pull_image
|
||||
setup_docker_run
|
||||
|
||||
# Execute requested action
|
||||
if [[ "$INTERACTIVE_SHELL" == "true" ]]; then
|
||||
run_shell
|
||||
elif [[ "$RUN_TESTS" == "true" ]]; then
|
||||
run_build && run_tests
|
||||
else
|
||||
# Default build args if none specified
|
||||
if [[ -z "$BUILD_ARGS" ]]; then
|
||||
BUILD_ARGS=" --fe --be"
|
||||
fi
|
||||
run_build
|
||||
fi
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
version: '3.8'
|
||||
|
||||
services:
|
||||
starrocks-dev:
|
||||
image: ${STARROCKS_DEV_ENV_IMAGE:-starrocks/dev-env-ubuntu:latest}
|
||||
container_name: starrocks-dev-env
|
||||
volumes:
|
||||
- .:/workspace
|
||||
- ~/.m2:/tmp/.m2
|
||||
- starrocks-maven-cache:/root/.m2
|
||||
- starrocks-build-cache:/workspace/output
|
||||
working_dir: /workspace
|
||||
environment:
|
||||
- STARROCKS_HOME=/workspace
|
||||
- HOME=/tmp
|
||||
user: "${UID:-1000}:${GID:-1000}"
|
||||
stdin_open: true
|
||||
tty: true
|
||||
command: /bin/bash
|
||||
networks:
|
||||
- starrocks-dev
|
||||
|
||||
# Service for building Frontend
|
||||
build-fe:
|
||||
extends: starrocks-dev
|
||||
container_name: starrocks-build-fe
|
||||
command: ./build.sh --fe
|
||||
|
||||
# Service for building Backend
|
||||
build-be:
|
||||
extends: starrocks-dev
|
||||
container_name: starrocks-build-be
|
||||
command: ./build.sh --be
|
||||
|
||||
# Service for running tests
|
||||
test-fe:
|
||||
extends: starrocks-dev
|
||||
container_name: starrocks-test-fe
|
||||
command: ./run-fe-ut.sh
|
||||
|
||||
test-be:
|
||||
extends: starrocks-dev
|
||||
container_name: starrocks-test-be
|
||||
command: ./run-be-ut.sh
|
||||
|
||||
volumes:
|
||||
starrocks-maven-cache:
|
||||
driver: local
|
||||
starrocks-build-cache:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
starrocks-dev:
|
||||
driver: bridge
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
#!/usr/bin/env bash
|
||||
# Simple wrapper for common Docker development tasks
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DOCKER_IMAGE="${STARROCKS_DEV_ENV_IMAGE:-starrocks/dev-env-ubuntu:latest}"
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat << EOF
|
||||
Simple Docker development wrapper for StarRocks
|
||||
|
||||
Usage: $0 <command> [additional_options]
|
||||
|
||||
COMMANDS:
|
||||
shell Open interactive shell in dev environment
|
||||
build-fe Build Frontend only
|
||||
build-be Build Backend only
|
||||
build-all Build both Frontend and Backend
|
||||
clean-build Clean and build all
|
||||
test-fe Run Frontend tests
|
||||
test-be Run Backend tests
|
||||
test-all Run all tests
|
||||
build Pass through any build.sh options directly
|
||||
|
||||
EXAMPLES:
|
||||
$0 shell # Open development shell
|
||||
$0 build-fe # Build Frontend
|
||||
$0 clean-build # Clean and build everything
|
||||
$0 test-fe # Run Frontend tests
|
||||
$0 build --be --with-gcov # Pass through custom build options
|
||||
$0 build --fe --new-option # Any new build.sh option works
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
shell)
|
||||
log_info "Opening development shell..."
|
||||
./build-in-docker.sh --shell
|
||||
;;
|
||||
build-fe)
|
||||
log_info "Building Frontend..."
|
||||
./build-in-docker.sh --fe
|
||||
;;
|
||||
build-be)
|
||||
log_info "Building Backend..."
|
||||
./build-in-docker.sh --be
|
||||
;;
|
||||
build-all)
|
||||
log_info "Building all components..."
|
||||
./build-in-docker.sh --fe --be
|
||||
;;
|
||||
clean-build)
|
||||
log_info "Clean building all components..."
|
||||
./build-in-docker.sh --fe --be --clean
|
||||
;;
|
||||
test-fe)
|
||||
log_info "Running Frontend tests..."
|
||||
docker run --rm -v "$REPO_ROOT:/workspace" -w /workspace "$DOCKER_IMAGE" ./run-fe-ut.sh
|
||||
;;
|
||||
test-be)
|
||||
log_info "Running Backend tests..."
|
||||
docker run --rm -v "$REPO_ROOT:/workspace" -w /workspace "$DOCKER_IMAGE" ./run-be-ut.sh
|
||||
;;
|
||||
test-all)
|
||||
log_info "Running all tests..."
|
||||
./build-in-docker.sh --test
|
||||
;;
|
||||
build)
|
||||
# Pass through all remaining arguments to build-in-docker.sh
|
||||
shift
|
||||
./build-in-docker.sh "$@"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
@ -0,0 +1,275 @@
|
|||
# Setup development environment with Docker
|
||||
|
||||
This directory provides Docker-based build tools for StarRocks that use the official `starrocks/dev-env-ubuntu:latest` development environment image. This ensures consistent builds across different host systems by using a standardized Ubuntu environment with all required toolchains and dependencies pre-installed.
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### Prerequisites
|
||||
- Docker installed and running
|
||||
- At least 8GB RAM available for Docker
|
||||
- At least 20GB free disk space
|
||||
|
||||
### Simple Commands
|
||||
|
||||
```bash
|
||||
# Open development shell
|
||||
./docker-dev.sh shell
|
||||
|
||||
# Build Frontend only
|
||||
./docker-dev.sh build-fe
|
||||
|
||||
# Build Backend only
|
||||
./docker-dev.sh build-be
|
||||
|
||||
# Build everything
|
||||
./docker-dev.sh build-all
|
||||
|
||||
# Clean build everything
|
||||
./docker-dev.sh clean-build
|
||||
|
||||
# Run Frontend tests
|
||||
./docker-dev.sh test-fe
|
||||
```
|
||||
|
||||
## 📋 Available Tools
|
||||
|
||||
### 1. `build-in-docker.sh` - Full-Featured Build Script
|
||||
|
||||
The main build script that automatically passes through all `build.sh` options:
|
||||
|
||||
```bash
|
||||
# Basic usage
|
||||
./build-in-docker.sh # Build all (FE + BE)
|
||||
./build-in-docker.sh --fe # Build Frontend only
|
||||
./build-in-docker.sh --be # Build Backend only
|
||||
./build-in-docker.sh --fe --be --clean # Clean and build both
|
||||
|
||||
# Advanced options
|
||||
./build-in-docker.sh --be --with-gcov # Build BE with code coverage
|
||||
./build-in-docker.sh --fe --disable-java-check-style # Skip checkstyle
|
||||
./build-in-docker.sh --be -j 8 # Build with 8 parallel jobs
|
||||
|
||||
# Development
|
||||
./build-in-docker.sh --shell # Interactive shell
|
||||
./build-in-docker.sh --test # Build and run tests
|
||||
|
||||
# Custom image
|
||||
./build-in-docker.sh --image starrocks/dev-env-ubuntu:latest --fe
|
||||
|
||||
# Future build.sh options work automatically
|
||||
./build-in-docker.sh --be --new-future-option
|
||||
```
|
||||
|
||||
### 2. `docker-dev.sh` - Simple Wrapper
|
||||
|
||||
Quick commands for common tasks:
|
||||
|
||||
```bash
|
||||
./docker-dev.sh shell # Development shell
|
||||
./docker-dev.sh build-fe # Build Frontend
|
||||
./docker-dev.sh build-be # Build Backend
|
||||
./docker-dev.sh build-all # Build everything
|
||||
./docker-dev.sh clean-build # Clean and build all
|
||||
./docker-dev.sh test-fe # Run FE tests
|
||||
./docker-dev.sh test-be # Run BE tests
|
||||
./docker-dev.sh test-all # Run all tests
|
||||
|
||||
# Pass through any build.sh options
|
||||
./docker-dev.sh build --be --with-gcov
|
||||
./docker-dev.sh build --fe --new-option
|
||||
```
|
||||
|
||||
### 3. `docker-compose.dev.yml` - Docker Compose
|
||||
|
||||
For persistent development environments:
|
||||
|
||||
```bash
|
||||
# Start development shell
|
||||
docker-compose -f docker-compose.dev.yml run --rm starrocks-dev
|
||||
|
||||
# Build Frontend
|
||||
docker-compose -f docker-compose.dev.yml run --rm build-fe
|
||||
|
||||
# Build Backend
|
||||
docker-compose -f docker-compose.dev.yml run --rm build-be
|
||||
|
||||
# Run tests
|
||||
docker-compose -f docker-compose.dev.yml run --rm test-fe
|
||||
docker-compose -f docker-compose.dev.yml run --rm test-be
|
||||
|
||||
# Clean up
|
||||
docker-compose -f docker-compose.dev.yml down -v
|
||||
```
|
||||
|
||||
## 🔧 Configuration
|
||||
|
||||
### Volume Mounts
|
||||
|
||||
The Docker scripts automatically mount:
|
||||
- **Source code**: `$(pwd):/workspace` - Your local repository
|
||||
- **Maven cache**: `~/.m2:/tmp/.m2` - Maven dependencies cache for faster builds
|
||||
|
||||
**Note**: The Maven cache is shared between your host system and the Docker container, so dependencies downloaded during builds are persisted and reused across build sessions.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
```bash
|
||||
# Use different Docker image
|
||||
export STARROCKS_DEV_ENV_IMAGE=starrocks/dev-env-ubuntu:latest
|
||||
|
||||
# Additional Docker options
|
||||
export DOCKER_BUILD_OPTS="--memory=16g --cpus=8"
|
||||
|
||||
# Set user ID for file permissions
|
||||
export UID=$(id -u)
|
||||
export GID=$(id -g)
|
||||
```
|
||||
|
||||
### Custom Build Options
|
||||
|
||||
All original `build.sh` options are supported:
|
||||
|
||||
```bash
|
||||
# Backend build types
|
||||
BUILD_TYPE=Debug ./build-in-docker.sh --be # Debug build
|
||||
BUILD_TYPE=Release ./build-in-docker.sh --be # Release build (default)
|
||||
BUILD_TYPE=Asan ./build-in-docker.sh --be # AddressSanitizer build
|
||||
|
||||
# Feature flags
|
||||
./build-in-docker.sh --be --enable-shared-data # Enable shared data
|
||||
./build-in-docker.sh --be --with-gcov # Code coverage
|
||||
./build-in-docker.sh --be --with-bench # Benchmarks
|
||||
./build-in-docker.sh --be --without-avx2 # Disable AVX2
|
||||
```
|
||||
|
||||
## 📁 Output and Artifacts
|
||||
|
||||
Build artifacts are created in the `output/` directory:
|
||||
|
||||
```
|
||||
output/
|
||||
├── fe/ # Frontend artifacts
|
||||
├── be/ # Backend artifacts
|
||||
└── java-extensions/ # Java extensions
|
||||
```
|
||||
|
||||
The Docker container mounts your local repository and Maven cache, so all build outputs are available on your host system and dependencies are cached for faster subsequent builds.
|
||||
|
||||
**Multi-User Support**: Container names include username and user ID to prevent conflicts on shared development machines (e.g., `starrocks-build-username-1001-1234567890`).
|
||||
|
||||
**Automatic Extensibility**: All unrecognized options are automatically passed through to `build.sh`, so new build options work without updating the Docker scripts.
|
||||
|
||||
## 🐛 Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Permission Issues**
|
||||
```bash
|
||||
# Fix file permissions
|
||||
sudo chown -R $(id -u):$(id -g) output/
|
||||
|
||||
# Or run with correct user
|
||||
export UID=$(id -u) GID=$(id -g)
|
||||
./build-in-docker.sh --fe
|
||||
```
|
||||
|
||||
2. **Out of Memory**
|
||||
```bash
|
||||
# Increase Docker memory limit or reduce parallel jobs
|
||||
./build-in-docker.sh --be -j 2
|
||||
```
|
||||
|
||||
3. **Docker Image Not Found**
|
||||
```bash
|
||||
# Pull the image manually
|
||||
docker pull starrocks/dev-env-ubuntu:latest
|
||||
```
|
||||
|
||||
4. **Build Failures**
|
||||
```bash
|
||||
# Clean build
|
||||
./build-in-docker.sh --clean --fe --be
|
||||
|
||||
# Check logs in interactive shell
|
||||
./build-in-docker.sh --shell
|
||||
```
|
||||
|
||||
### Debug Mode
|
||||
|
||||
For debugging build issues:
|
||||
|
||||
```bash
|
||||
# Open shell and run commands manually
|
||||
./build-in-docker.sh --shell
|
||||
|
||||
# Inside container:
|
||||
./build.sh --fe --clean
|
||||
./run-fe-ut.sh
|
||||
```
|
||||
|
||||
## 🔍 Verification
|
||||
|
||||
Test that everything works:
|
||||
|
||||
```bash
|
||||
# 1. Test Docker setup
|
||||
docker --version
|
||||
docker info
|
||||
|
||||
# 2. Test image availability
|
||||
docker pull starrocks/dev-env-ubuntu:latest
|
||||
|
||||
# 3. Test build scripts
|
||||
./docker-dev.sh shell
|
||||
# Inside container: exit
|
||||
|
||||
# 4. Test simple build
|
||||
./docker-dev.sh build-fe
|
||||
```
|
||||
|
||||
## 📊 Performance Tips
|
||||
|
||||
1. **Use parallel builds**: `./build-in-docker.sh --be -j $(nproc)`
|
||||
2. **Maven cache**: Automatically mounted from `~/.m2` for faster dependency resolution
|
||||
3. **Memory allocation**: Increase Docker memory limit for faster builds
|
||||
4. **SSD storage**: Use SSD for Docker storage driver
|
||||
5. **Persistent volumes**: Use Docker Compose for additional cache persistence
|
||||
|
||||
## 🤝 Integration with IDEs
|
||||
|
||||
### VS Code with Dev Containers
|
||||
|
||||
Create `.devcontainer/devcontainer.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "StarRocks Dev Environment",
|
||||
"image": "starrocks/dev-env-ubuntu:latest",
|
||||
"workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind",
|
||||
"workspaceFolder": "/workspace",
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": [
|
||||
"ms-vscode.cpptools",
|
||||
"redhat.java"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### IntelliJ IDEA
|
||||
|
||||
Use the Docker integration to run builds and tests within the container environment.
|
||||
|
||||
## 📚 Additional Resources
|
||||
- [Original build.sh](../../../../build.sh) for reference
|
||||
|
||||
## 🆘 Support
|
||||
|
||||
If you encounter issues:
|
||||
|
||||
1. Check the troubleshooting section above
|
||||
2. Verify Docker setup and image availability
|
||||
3. Try clean builds with `--clean` flag
|
||||
4. Open an issue with build logs and system information
|
||||
Loading…
Reference in New Issue