tool
/
starrocks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Tool] Inspection pipeline supports feature coverage (#55837)

This commit is contained in:
andyziye 2025-02-18 08:02:19 +05:30 committed by GitHub
parent 3a04cacbca
commit 55671c9656
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 476 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
.github/workflows/ci-merged.yml vendored
View File

@ -22,79 +22,27 @@ jobs:
github.base_ref == 'main' &&
github.repository == 'StarRocks/starrocks' &&
!contains(github.event.pull_request.title, 'cherry-pick') &&
!contains(github.event.pull_request.title, 'backport') && (
contains(github.event.pull_request.labels.*.name, '3.4') ||
contains(github.event.pull_request.labels.*.name, '3.3') ||
contains(github.event.pull_request.labels.*.name, '3.2') ||
contains(github.event.pull_request.labels.*.name, '3.1') ||
contains(github.event.pull_request.labels.*.name, '3.0') )
!contains(github.event.pull_request.title, 'backport')
strategy:
fail-fast: false
matrix:
version: [ '3.4', '3.3', '3.2', '3.1', '3.0' ]
env:
PR_NUMBER: ${{ github.event.number }}
steps:
- name: backport branch-3.4
if: contains(github.event.pull_request.labels.*.name, '3.4')
- name: backport branch-${{ matrix.version }}
if: contains(github.event.pull_request.labels.*.name, matrix.version)
uses: thollander/actions-comment-pull-request@v2
with:
message: |
@Mergifyio backport branch-3.4
@Mergifyio backport branch-${{ matrix.version }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-ecosystem/action-remove-labels@v1
if: contains(github.event.pull_request.labels.*.name, '3.4')
if: contains(github.event.pull_request.labels.*.name, matrix.version)
with:
labels: '3.4'
- name: backport branch-3.3
if: contains(github.event.pull_request.labels.*.name, '3.3')
uses: thollander/actions-comment-pull-request@v2
with:
message: |
@Mergifyio backport branch-3.3
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-ecosystem/action-remove-labels@v1
if: contains(github.event.pull_request.labels.*.name, '3.3')
with:
labels: '3.3'
- name: backport branch-3.2
if: contains(github.event.pull_request.labels.*.name, '3.2')
uses: thollander/actions-comment-pull-request@v2
with:
message: |
@Mergifyio backport branch-3.2
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-ecosystem/action-remove-labels@v1
if: contains(github.event.pull_request.labels.*.name, '3.2')
with:
labels: '3.2'
- name: backport branch-3.1
if: contains(github.event.pull_request.labels.*.name, '3.1')
uses: thollander/actions-comment-pull-request@v2
with:
message: |
@Mergifyio backport branch-3.1
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-ecosystem/action-remove-labels@v1
if: contains(github.event.pull_request.labels.*.name, '3.1')
with:
labels: '3.1'
- name: backport branch-3.0
if: contains(github.event.pull_request.labels.*.name, '3.0')
uses: thollander/actions-comment-pull-request@v2
with:
message: |
@Mergifyio backport branch-3.0
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-ecosystem/action-remove-labels@v1
if: contains(github.event.pull_request.labels.*.name, '3.0')
with:
labels: '3.0'
labels: ${{ matrix.version }}
thirdparty-update-image:
runs-on: [self-hosted, normal]

132
.github/workflows/inspection-be-coverage.yml vendored Normal file
View File

@ -0,0 +1,132 @@
name: BE COVERAGE IN INSPECTION PIPELINE
on:
workflow_call:
inputs:
BRANCH:
description: 'BRANCH'
required: true
type: string
COMMIT_ID:
description: 'COMMIT ID'
required: true
type: string
DIFF_FILE_PATH:
description: 'FILE PATH OF DIFF INFO'
required: true
type: string
outputs:
BE_COV_RES_FILE:
description: "BE COV RESULT FILE"
value: ${{ jobs.REPORT.outputs.BE_COV_RES_FILE }}
permissions:
checks: write
actions: write
contents: write
deployments: write
discussions: write
issues: write
packages: write
pages: write
pull-requests: write
repository-projects: write
security-events: write
statuses: write
env:
IS_INSPECTION: true
BASE_REF: ${{ inputs.BRANCH }}
COMMIT_ID: ${{ inputs.COMMIT_ID }}
DIFF_FILE_PATH: ${{ inputs.DIFF_FILE_PATH }}
bucket_prefix: starrocks
jobs:
REPORT:
runs-on: [ self-hosted, quick ]
env:
OSS_CMD: "ossutil64 --config-file /root/.ossutilconfig"
JAVA_HOME: /var/local/env/jdk1.8.0_202
outputs:
BE_COV_RES_FILE: ${{ steps.incremental_cov_report.outputs.BE_COV_RES_FILE }}
steps:
- name: CLEAN & ENV
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
if [[ ! -e "${DIFF_FILE_PATH}" ]]; then
echo "::error::diff file: ${DIFF_FILE_PATH} not exist!"
exit 1
fi
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ env.COMMIT_ID }}
- name: Download BE UT XML
id: download-ut-xml
run: |
oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${COMMIT_ID}
be_ut_res_path=${oss_path}/flag/be_ut_res
size=$(${OSS_CMD} ls ${be_ut_res_path} | grep "Object Number is" | awk '{print $NF}')
if [[ "$size" == "0" ]]; then
echo "::error::BE UT result not exit!"
exit 1
fi
${OSS_CMD} cp ${be_ut_res_path} .
res=`cat be_ut_res`
if [[ "$res" != "0" ]]; then
echo "::error::BE UT failed!"
exit 1
fi
cd be
${OSS_CMD} cp ${oss_path}/be_ut_coverage.xml . -f 1>/dev/null
- name: Merge BE Coverage
id: merge_report
env:
IGNORE_CODECOV: true
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
export ECI_ID=`./bin/create_eci.sh ${BASE_REF} ${GITHUB_REPOSITORY}`
export TOTAL_COV_XML=/var/local/env/be_total_coverage_${COMMIT_ID}.xml
export COV_LOG=/var/local/env/be_total_coverage_${COMMIT_ID}.log
export RES_FILE=/var/local/env/be_total_coverage_res_${COMMIT_ID}.log
echo "ECI_ID=${ECI_ID}" | tee -a $GITHUB_OUTPUT
echo "COV_XML=${TOTAL_COV_XML}" | tee -a $GITHUB_OUTPUT
echo "COV_LOG=${COV_LOG}" | tee -a $GITHUB_OUTPUT
echo "RES_FILE=${RES_FILE}" | tee -a $GITHUB_OUTPUT
./bin/gen_be_cov.sh ${GITHUB_REPOSITORY} ${BASE_REF} ${COMMIT_ID} ${COMMIT_ID}
# Incremental Total Coverage
- name: Incremental Coverage Report - Total
id: incremental_cov_report
env:
be_path: ${{ github.workspace }}/be
total_xml: ${{ steps.merge_report.outputs.COV_XML }}
run: |
rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
export PATH=$JAVA_HOME/bin:$PATH;
java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
--cover ${total_xml} \
--threshold 80 \
-type cobertura \
--module BE \
-d ${DIFF_FILE_PATH} -dt file | tee "${DIFF_FILE_PATH}_be_cov.log"
echo "BE_COV_RES_FILE=${DIFF_FILE_PATH}_be_cov.log" >> $GITHUB_OUTPUT
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*
eci rm ${{ steps.merge_report.outputs.ECI_ID }} || true
rm -rf ${{ steps.merge_report.outputs.COV_XML }} ${{ steps.merge_report.outputs.COV_LOG }} ${{ steps.merge_report.outputs.RES_FILE }}

128
.github/workflows/inspection-fe-coverage.yml vendored Normal file
View File

@ -0,0 +1,128 @@
name: FE COVERAGE IN INSPECTION PIPELINE
on:
workflow_call:
inputs:
BRANCH:
description: 'BRANCH'
required: true
type: string
COMMIT_ID:
description: 'COMMIT ID'
required: true
type: string
DIFF_FILE_PATH:
description: 'FILE PATH OF DIFF INFO'
required: true
type: string
outputs:
FE_COV_RES_FILE:
description: "FE COV RESULT FILE"
value: ${{ jobs.REPORT.outputs.FE_COV_RES_FILE }}
permissions:
checks: write
actions: write
contents: write
deployments: write
discussions: write
issues: write
packages: write
pages: write
pull-requests: write
repository-projects: write
security-events: write
statuses: write
env:
IS_INSPECTION: true
BASE_REF: ${{ inputs.BRANCH }}
COMMIT_ID: ${{ inputs.COMMIT_ID }}
DIFF_FILE_PATH: ${{ inputs.DIFF_FILE_PATH }}
bucket_prefix: starrocks
jobs:
REPORT:
runs-on: [ self-hosted, quick ]
env:
OSS_CMD: "ossutil64 --config-file /root/.ossutilconfig"
JAVA_HOME: /var/local/env/jdk1.8.0_202
outputs:
FE_COV_RES_FILE: ${{ steps.incremental_cov_report.outputs.FE_COV_RES_FILE }}
steps:
- name: CLEAN & ENV
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
if [[ ! -e "${DIFF_FILE_PATH}" ]]; then
echo "::error::diff file: ${DIFF_FILE_PATH} not exist!"
exit 1
fi
- name: Download FE UT XML
id: download-ut-xml
run: |
oss_path=oss://${bucket_prefix}-ci-release/${BASE_REF}/Release/pr/UT-Report/${COMMIT_ID}
size=$(${OSS_CMD} ls ${oss_path}/fe_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}')
if [[ "$size" == "0" ]]; then
echo "::error::FE UT result not exit!"
exit 1
fi
mkdir fe && cd fe
${OSS_CMD} cp ${oss_path}/ . --recursive
tar zxf fe_ut_report.tar.gz
- name: Merge FE Coverage
id: merge_report
env:
fe_path: ${{ github.workspace }}/fe
run: |
export PATH=$JAVA_HOME/bin:$PATH;
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
if [[ ! -e ${fe_path}/test.exec ]]; then
echo "::error::System test result not exit!"
exit 1
fi
java -jar package/jacococli.jar merge ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/test.exec --destfile ${fe_path}/merge.exec
- name: Generate XML Report
id: generate-xml-report
env:
package_path: ${{ github.workspace }}/ci-tool/package
fe_path: ${{ github.workspace }}/fe
fe_core_path: ${{ github.workspace }}/fe/fe-core
run: |
rm -rf result
export PATH=$JAVA_HOME/bin:$PATH;
cd ${fe_core_path}/target/classes; rm -rf org;
cd com/starrocks && rm -rf thrift proto sql/parser builtins common/Version.class;
cd ${{ github.workspace }};
java -jar $package_path/jacococli.jar report ${fe_path}/merge.exec --classfiles ${fe_core_path}/target/classes/ \
--html ./result --xml ${{ github.workspace }}/coverage.xml \
--sourcefiles ${fe_core_path}/src/main/java/ --encoding utf-8 --name fe-coverage
oss_path=oss://${bucket_prefix}-ci-release/${BASE_REF}/Release/pr/UT-Report/${COMMIT_ID}
${OSS_CMD} cp coverage.xml ${oss_path}/ -f
# Incremental Total Coverage
- name: Incremental Coverage Report - Total
id: incremental_cov_report
env:
fe_path: ${{ github.workspace }}/fe
run: |
rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
export PATH=$JAVA_HOME/bin:$PATH;
java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
--cover ${{ github.workspace }}/result/ \
--threshold 80 -type jacoco \
-d ${DIFF_FILE_PATH} -dt file | tee "${DIFF_FILE_PATH}_fe_cov.log"
echo "FE_COV_RES_FILE=${DIFF_FILE_PATH}_fe_cov.log" >> $GITHUB_OUTPUT
rm -rf coverchecker
rm -rf ci-tools
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*

260
.github/workflows/inspection-pipeline.yml vendored
View File

@ -21,34 +21,34 @@ on:
description: 'TAR PATH(Release & Centos)'
required: false
type: string
CENTOS_ASAN_TAR_PATH:
description: 'TAR PATH(ASAN & Centos)'
required: false
type: string
UBUNTU_TAR_PATH:
description: 'TAR PATH(Release & Ubuntu)'
required: false
type: string
UBUNTU_ASAN_TAR_PATH:
description: 'TAR PATH(ASAN & Ubuntu)'
required: false
type: string
INCLUDE_ADMIT:
description: 'RUN ADMIT?'
type: boolean
default: true
ALL_LINUX:
description: "ALL_LINUX (Default: Ubuntu+Release)"
type: boolean
default: false
IS_REBUILD:
description: 'REBUILD'
type: boolean
default: false
INCLUDE_ADMIT:
description: 'RUN ADMIT?'
type: string
default: 'true'
ALL_LINUX:
description: "ALL_LINUX? (⬜: Ubuntu && Release only)"
type: string
default: 'true'
RETAIN_ENV:
description: 'RETAIN THE ENVIRONMENT?'
WITH_FEATURE_COV:
description: "GEN FEATURE COV?"
type: boolean
default: false
IS_ARM:
description: "ARM"
type: boolean
default: false
COV_BASE_COMMIT:
description: ''
type: string
default: ''
permissions:
checks: write
@ -66,6 +66,7 @@ permissions:
env:
IS_INSPECTION: true
SHARE_PATH: /var/local/env
jobs:
info:
@ -78,35 +79,21 @@ jobs:
BRANCH: ${{ steps.param.outputs.BRANCH }}
PR_NUMBER: ${{ steps.param.outputs.PR_NUMBER }}
CENTOS_TAR_PATH: ${{ steps.param.outputs.CENTOS_TAR_PATH }}
CENTOS_ASAN_TAR_PATH: ${{ steps.param.outputs.CENTOS_ASAN_TAR_PATH }}
UBUNTU_TAR_PATH: ${{ steps.param.outputs.UBUNTU_TAR_PATH }}
UBUNTU_ASAN_TAR_PATH: ${{ steps.param.outputs.UBUNTU_ASAN_TAR_PATH }}
BE_UT_LINUX: ${{ steps.param.outputs.BE_UT_LINUX }}
RETAIN_ENV: ${{ steps.param.outputs.RETAIN_ENV }}
ALL_LINUX: ${{ steps.param.outputs.ALL_LINUX }}
WITH_FEATURE_COV: ${{ steps.param.outputs.WITH_FEATURE_COV }}
INCLUDE_ADMIT: ${{ steps.param.outputs.INCLUDE_ADMIT }}
IS_ARM: ${{ steps.param.outputs.IS_ARM }}
steps:
- name: CRON PARAM
id: param
run: |
BE_UT_LINUX=ubuntu
if [[ "${{ github.event_name }}" == "schedule" ]]; then
if [[ "${{ github.event.schedule }}" == "0 11 * * 1,3,5" ]]; then
branch=branch-3.4
if [[ "$GITHUB_REPOSITORY" == 'StarRocks/starrocks' && `date +%u` -eq 1 ]]; then
all_linux=true
fi
elif [[ "${{ github.event.schedule }}" == "0 11 * * 2,4" ]]; then
branch=branch-3.3
all_linux=false
elif [[ "${{ github.event.schedule }}" == "0 0 * * 1-5" ]]; then
branch=main
if [[ `date +%u` -eq 1 || `date +%u` -eq 4 ]]; then
all_linux=true
fi
else
branch=main
fi
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull
read branch all_linux IS_ARM < <(python3 scripts/inspection_params.py "${{ github.event.schedule }}")
[[ $((`date +%e` % 2)) -eq 1 ]] && BE_UT_LINUX=centos7
base_sha=$(gh api /repos/${REPO}/branches/${branch} | jq -r .commit.sha)
echo "::notice::${branch}(${BE_UT_LINUX} ${base_sha})"
@ -115,16 +102,22 @@ jobs:
echo "PR_NUMBER=${base_sha}" >> $GITHUB_OUTPUT
echo "ALL_LINUX=${all_linux}" >> $GITHUB_OUTPUT
echo "INCLUDE_ADMIT=true" >> $GITHUB_OUTPUT
echo "IS_ARM=${IS_ARM}" >> $GITHUB_OUTPUT
else
echo "BRANCH=${{ inputs.BRANCH }}" >> $GITHUB_OUTPUT
echo "PR_NUMBER=${{ inputs.COMMIT_ID }}" >> $GITHUB_OUTPUT
echo "CENTOS_TAR_PATH=${{ inputs.CENTOS_TAR_PATH }}" >> $GITHUB_OUTPUT
echo "CENTOS_ASAN_TAR_PATH=${{ inputs.CENTOS_ASAN_TAR_PATH }}" >> $GITHUB_OUTPUT
echo "UBUNTU_TAR_PATH=${{ inputs.UBUNTU_TAR_PATH }}" >> $GITHUB_OUTPUT
echo "UBUNTU_ASAN_TAR_PATH=${{ inputs.UBUNTU_ASAN_TAR_PATH }}" >> $GITHUB_OUTPUT
echo "RETAIN_ENV=${{ inputs.RETAIN_ENV }}" >> $GITHUB_OUTPUT
echo "ALL_LINUX=${{ inputs.ALL_LINUX }}" >> $GITHUB_OUTPUT
echo "WITH_FEATURE_COV=${{ inputs.WITH_FEATURE_COV }}" >> $GITHUB_OUTPUT
echo "IS_ARM=${{ inputs.IS_ARM }}" >> $GITHUB_OUTPUT
echo "INCLUDE_ADMIT=false" >> $GITHUB_OUTPUT
# check cov params
if [[ "${{ inputs.WITH_FEATURE_COV }}" == "true" && "${{ inputs.COV_BASE_COMMIT }}" == "" ]]; then
echo "::error::[Coverage] Base commit is missing!"
exit 1
fi
fi
echo "BE_UT_LINUX=${BE_UT_LINUX}" >> $GITHUB_OUTPUT
@ -149,11 +142,12 @@ jobs:
id: run_ut
shell: bash
timeout-minutes: 90
if: matrix.build_type == 'Release' || needs.info.outputs.WITH_FEATURE_COV != 'true'
env:
build_type: ${{ matrix.build_type }}
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/elastic-ut.sh --repository ${{ github.repository }} --branch ${BRANCH} --pr ${PR_NUMBER} --module be --build ${build_type} --linuxdistro ${LINUX}
./bin/elastic-ut.sh --repository ${{ github.repository }} --branch ${BRANCH} --pr ${PR_NUMBER} --module be --build ${build_type} --linuxdistro ${LINUX} --with-gcov
- name: clean ECI
if: always()
@ -164,7 +158,7 @@ jobs:
- name: Upload Log
uses: actions/upload-artifact@v4
if: always() && steps.run_ut.outcome == 'failure'
if: always() && (steps.run_ut.outcome == 'success' || steps.run_ut.outcome == 'failure')
with:
name: BE UT LOG
path: ${{ steps.run_ut.outputs.BE_LOG }}
@ -228,7 +222,7 @@ jobs:
eci rm ${{ steps.run_ut.outputs.ECI_ID }}
- name: Upload log
if: always()
if: always() && (steps.run_ut.outcome == 'success' || steps.run_ut.outcome == 'failure')
uses: actions/upload-artifact@v4
with:
name: FE UT LOG
@ -237,7 +231,7 @@ jobs:
overwrite: true
- name: Download FE UT XML
if: always()
if: always() && (steps.run_ut.outcome == 'success' || steps.run_ut.outcome == 'failure')
id: download-ut-xml
env:
oss_path: ${{ steps.run_ut.outputs.oss_path }}
@ -248,7 +242,7 @@ jobs:
- name: Publish UT Report
uses: mikepenz/action-junit-report@v4
if: always()
if: always() && (steps.run_ut.outcome == 'success' || steps.run_ut.outcome == 'failure')
id: publish_report
env:
github_token: ${{ secrets.GITHUB_TOKEN }}
@ -260,7 +254,7 @@ jobs:
report_paths: ./fe/fe-core/target/surefire-reports/*.xml
- name: Clean ENV
if: always()
if: always() && (steps.run_ut.outcome == 'success' || steps.run_ut.outcome == 'failure')
run: |
rm -rf ${{ steps.run_ut.outputs.FE_REPORT_DIR }}
rm -f ${{ steps.run_ut.outputs.RES_FILE }}
@ -273,7 +267,7 @@ jobs:
name: BUILD
needs:
- info
if: github.event_name == 'schedule' || inputs.IS_REBUILD
if: github.event_name == 'schedule' || inputs.IS_REBUILD || inputs.WITH_FEATURE_COV == 'true'
continue-on-error: true
strategy:
fail-fast: false
@ -284,6 +278,7 @@ jobs:
BRANCH: ${{ needs.info.outputs.BRANCH }}
PR_NUMBER: ${{ needs.info.outputs.PR_NUMBER }}
ALL_LINUX: ${{ needs.info.outputs.ALL_LINUX }}
WITH_FEATURE_COV: ${{ needs.info.outputs.WITH_FEATURE_COV }}
steps:
- name: CLEAN
run: |
@ -295,9 +290,12 @@ jobs:
timeout-minutes: 90
if: (matrix.build_type == 'Release' && matrix.linux == 'ubuntu') || env.ALL_LINUX == 'true'
run: |
if [[ "${WITH_FEATURE_COV}" == "true" ]]; then
export SKIP_SYNC_TAR=true
fi
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/elastic-build.sh --repository ${{ github.repository }} --branch ${BRANCH} --pr ${PR_NUMBER} \
--build ${{ matrix.build_type }} --linuxdistro ${{ matrix.linux }} --with-trivy
--build ${{ matrix.build_type }} --linuxdistro ${{ matrix.linux }} --with-trivy --with-gcov
- name: BUILD RESULT
run: |
@ -331,7 +329,7 @@ jobs:
needs:
- info
- build
if: always() && needs.info.result == 'success'
if: always() && needs.info.result == 'success' && needs.build.result != 'cancelled'
env:
ALL_LINUX: needs.info.outputs.ALL_LINUX
outputs:
@ -455,6 +453,8 @@ jobs:
LINUX_DISTRO: ubuntu
IS_SELF_BUILD: ${{ needs.test_filter.outputs.IS_SELF_BUILD }}
INCLUDE_ADMIT: ${{ needs.info.outputs.INCLUDE_ADMIT }}
WITH_FEATURE_COV: ${{ needs.info.outputs.WITH_FEATURE_COV }}
IS_ARM: ${{ needs.info.outputs.IS_ARM }}
system-test-asan-ubuntu:
name: 'ASAN/Ubuntu'
@ -473,6 +473,88 @@ jobs:
IS_SELF_BUILD: ${{ needs.test_filter.outputs.IS_SELF_BUILD }}
INCLUDE_ADMIT: ${{ needs.info.outputs.INCLUDE_ADMIT }}
diff-info:
runs-on: [self-hosted, quick]
if: github.repository == 'StarRocks/starrocks' && needs.info.outputs.WITH_FEATURE_COV == 'true'
name: Gen Diff File
env:
commit_id: ${{ inputs.COMMIT_ID }}
base_commit_id: ${{ inputs.COV_BASE_COMMIT }}
outputs:
DIFF_FILE: ${{ steps.diff.outputs.DIFF_FILE }}
needs:
- info
- be-ut
- fe-ut
- system-test-release-ubuntu
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: Checkout Code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: diff
id: diff
run: |
diff_file=${SHARE_PATH}/${base_commit_id}_${commit_id}.diff
git diff ${base_commit_id} ${commit_id} > ${diff_file}
echo "DIFF_FILE=${diff_file}" >> $GITHUB_OUTPUT
FE-COV-REPORT:
uses: ./.github/workflows/inspection-fe-coverage.yml
name: 'FE Coverage'
if: github.repository == 'StarRocks/starrocks' && needs.info.outputs.WITH_FEATURE_COV == 'true'
needs:
- info
- diff-info
secrets: inherit
with:
BRANCH: ${{ needs.info.outputs.BRANCH }}
COMMIT_ID: ${{ needs.info.outputs.PR_NUMBER }}
DIFF_FILE_PATH: ${{ needs.diff-info.outputs.DIFF_FILE }}
BE-COV-REPORT:
uses: ./.github/workflows/inspection-be-coverage.yml
name: 'BE Coverage'
if: github.repository == 'StarRocks/starrocks' && needs.info.outputs.WITH_FEATURE_COV == 'true'
needs:
- info
- diff-info
secrets: inherit
with:
BRANCH: ${{ needs.info.outputs.BRANCH }}
COMMIT_ID: ${{ needs.info.outputs.PR_NUMBER }}
DIFF_FILE_PATH: ${{ needs.diff-info.outputs.DIFF_FILE }}
FEATURE-COV:
runs-on: [ self-hosted, normal ]
name: COV INFO
needs:
- info
- FE-COV-REPORT
- BE-COV-REPORT
env:
GH_TOKEN: ${{ github.token }}
steps:
- name: Prepare Tools
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull
- name: INFO
run: |
echo "FE_COV_RES_FILE: ${{ needs.FE-COV-REPORT.outputs.FE_COV_RES_FILE }}"
echo "BE_COV_RES_FILE: ${{ needs.BE-COV-REPORT.outputs.BE_COV_RES_FILE }}"
- name: Result
run: |
cd ci-tool/scripts/feature_coverage
./version_cov.sh --repository ${{ github.repository }} --commit ${{ needs.info.outputs.PR_NUMBER }} --head ${{ inputs.COV_BASE_COMMIT }} --redownload --in_runner
Teardown:
runs-on: [self-hosted, normal]
name: Teardown
@ -502,98 +584,18 @@ jobs:
UBUNTU_RELEASE_RESULT: ${{ needs.system-test-release-ubuntu.result }}
UBUNTU_ASAN_RESULT: ${{ needs.system-test-asan-ubuntu.result }}
BE_UT_LINUX: ${{ needs.info.outputs.BE_UT_LINUX }}
RETAIN_ENV: ${{ needs.info.outputs.RETAIN_ENV }}
steps:
- name: init
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull
- name: Backup SR Info (Release & Centos)
if: env.RELEASE_CONF_FILE != ''
id: backup_release_centos
env:
build_type: Release
linuxdistro: centos7
run: |
cd ci-tool && source lib/init.sh
./bin/backup_log_cores.sh --branch "${BRANCH}" --pr "${PR_NUMBER}" --build "${build_type}" --conf "${RELEASE_CONF_FILE}" --repository "${GITHUB_REPOSITORY}" || true
if [[ "${RETAIN_ENV}" != "true" ]]; then
./bin/elastic-cluster.sh --delete
fi
- name: Backup SR Info (ASAN & Centos)
if: env.ASAN_CONF_FILE != ''
id: backup_asan_centos
env:
build_type: ASAN
linuxdistro: centos7
run: |
cd ci-tool && source lib/init.sh
./bin/backup_log_cores.sh --branch "${BRANCH}" --pr "${PR_NUMBER}" --build "${build_type}" --conf "${ASAN_CONF_FILE}" --repository "${GITHUB_REPOSITORY}" || true
if [[ "${RETAIN_ENV}" != "true" ]]; then
./bin/elastic-cluster.sh --delete
fi
- name: Backup SR Info (Release & Ubuntu)
if: env.UBUNTU_RELEASE_CONF_FILE != ''
id: backup_release_ubuntu
env:
build_type: Release
linuxdistro: ubuntu
run: |
cd ci-tool && source lib/init.sh
./bin/backup_log_cores.sh --branch "${BRANCH}" --pr "${PR_NUMBER}" --build "${build_type}" --conf "${UBUNTU_RELEASE_CONF_FILE}" --repository "${GITHUB_REPOSITORY}" || true
if [[ "${RETAIN_ENV}" != "true" ]]; then
./bin/elastic-cluster.sh --delete
fi
- name: Backup SR Info (ASAN & Ubuntu)
if: env.UBUNTU_ASAN_CONF_FILE != ''
id: backup_asan_ubuntu
env:
build_type: ASAN
linuxdistro: ubuntu
run: |
cd ci-tool && source lib/init.sh
./bin/backup_log_cores.sh --branch "${BRANCH}" --pr "${PR_NUMBER}" --build "${build_type}" --conf "${UBUNTU_ASAN_CONF_FILE}" --repository "${GITHUB_REPOSITORY}" || true
if [[ "${RETAIN_ENV}" != "true" ]]; then
./bin/elastic-cluster.sh --delete
fi
- name: save unstable cases
if: always() && (github.event_name == 'schedule' || (github.event_name != 'schedule' && env.BRANCH != 'main'))
run: |
cd ci-tool && source lib/init.sh
./bin/save_unstable_cases.sh
- name: clean ECI
if: always()
run: |
if [[ "${RELEASE_MYSQL_ECI_ID}" != "" ]]; then
eci rm ${RELEASE_MYSQL_ECI_ID}
fi
if [[ "${ASAN_MYSQL_ECI_ID}" != "" ]]; then
eci rm ${ASAN_MYSQL_ECI_ID}
fi
if [[ "${UBUNTU_RELEASE_MYSQL_ECI_ID}" != "" ]]; then
eci rm ${UBUNTU_RELEASE_MYSQL_ECI_ID}
fi
if [[ "${UBUNTU_ASAN_MYSQL_ECI_ID}" != "" ]]; then
eci rm ${UBUNTU_ASAN_MYSQL_ECI_ID}
fi
- name: Clean
if: always()
run: |
rm -f $RELEASE_CONF_FILE
rm -f $ASAN_CONF_FILE
rm -f $UBUNTU_RELEASE_CONF_FILE
rm -f $UBUNTU_ASAN_CONF_FILE
rm -rf ${{ github.workspace }}/*

57
.github/workflows/inspection-reusable-pipeline.yml vendored
View File

@ -37,6 +37,12 @@ on:
description: 'RUN ADMIT?'
type: string
default: 'true'
WITH_FEATURE_COV:
type: string
default: 'false'
IS_ARM:
type: string
default: 'false'
outputs:
CONF_FILE:
@ -70,6 +76,8 @@ env:
CLUSTER_NAME: ${{ inputs.CLUSTER_NAME }}
IS_SELF_BUILD: ${{ inputs.IS_SELF_BUILD }}
INCLUDE_ADMIT: ${{ inputs.INCLUDE_ADMIT }}
WITH_FEATURE_COV: ${{ inputs.WITH_FEATURE_COV }}
IS_ARM: ${{ inputs.IS_ARM }}
jobs:
deploy:
@ -88,7 +96,8 @@ jobs:
id: apply_resource
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/elastic-cluster.sh --template ${CLUSTER_NAME} --linuxdistro ${LINUX_DISTRO}
[[ "${IS_ARM}" == "true" ]] && arm_arg="--arm"
./bin/elastic-cluster.sh --template ${CLUSTER_NAME} --linuxdistro ${LINUX_DISTRO} ${arm_arg}
conf_name=/var/local/env/${PR_NUMBER}-${BUILD_TYPE}-${LINUX_DISTRO}-starrocks_deploy.conf
cp conf/starrocks_deploy.conf ${conf_name}
echo "deploy_conf_file=${conf_name}" >> $GITHUB_OUTPUT
@ -99,7 +108,8 @@ jobs:
run: |
cd ci-tool && source lib/init.sh
echo ${TAR_PATH}
./bin/deploy-cluster.sh -c ${CLUSTER_NAME} --output ${TAR_PATH}
[[ "${IS_ARM}" == "true" ]] && arm_arg="--arm"
./bin/deploy-cluster.sh -c ${CLUSTER_NAME} --output ${TAR_PATH} --with-coverage ${arm_arg}
- name: Clean ENV
if: always()
@ -189,7 +199,7 @@ jobs:
- name: Restart FE (${{needs.deploy.outputs.fe}})
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/restart-fe.sh --repository ${{ github.repository }}
./bin/restart-fe.sh --repository ${{ github.repository }} --pr ${PR_NUMBER} --branch ${BRANCH} --build Release
- name: Clean ENV
if: always()
@ -219,7 +229,8 @@ jobs:
- name: Restart BE (${{needs.deploy.outputs.be_list}})
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/system-cov-be.sh --pr ${PR_NUMBER} --branch ${BRANCH} --repository ${{ github.repository }} --skip-gcov
[[ "${WITH_FEATURE_COV}" != "true" ]] && skip_gcov="--skip-gcov"
./bin/system-cov-be.sh --pr ${PR_NUMBER} --branch ${BRANCH} --repository ${{ github.repository }} ${skip_gcov}
- name: Clean ENV
if: always()
@ -281,5 +292,41 @@ jobs:
- name: Clean ENV
if: always()
run: rm -rf ${{ github.workspace }}/*
clean:
runs-on: [ self-hosted, normal ]
name: CLEAN ENV
needs:
- deploy
- admit
- SQL-Tester
if: always()
env:
conf_file: ${{ needs.deploy.outputs.deploy_conf_file }}
mysql_eci_id: ${{ needs.SQL-Tester.outputs.MYSQL_ECI_ID }}
steps:
- name: init
run: rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull
- name: Backup Cluster Info
if: always() && env.conf_file != ''
env:
linuxdistro: ${LINUX_DISTRO}
run: |
rm -rf ${{ github.workspace }}/*
cd ci-tool && source lib/init.sh
./bin/backup_log_cores.sh \
--branch "${BRANCH}" \
--pr "${PR_NUMBER}" \
--build "${BUILD_TYPE}" \
--conf "${conf_file}" \
--repository "${GITHUB_REPOSITORY}" || true
./bin/elastic-cluster.sh --delete
- name: Delete MySQL Env
if: always() && env.mysql_eci_id != ''
run: eci rm ${mysql_eci_id}
- name: Delete Conf
if: always() && env.conf_file != ''
run: rm -rf ${conf_file}

85
.github/workflows/pr-checker.yml vendored
View File

@ -117,7 +117,7 @@ jobs:
echo "::error::Branch PR does not contain the backport option!"
exit 1
backport-check:
backport-check-subtask:
needs: title-check
runs-on: ubuntu-latest
if: >
@ -127,80 +127,39 @@ jobs:
startsWith(github.event.pull_request.title, '[UT]')) &&
!contains(github.event.pull_request.title, 'cherry-pick') &&
!contains(github.event.pull_request.title, 'backport')
strategy:
fail-fast: false
matrix:
version: [ '3.4', '3.3', '3.2', '3.1', '3.0' ]
steps:
- name: add branch-3.4 label
if: contains(toJson(github.event.pull_request.body), '[x] 3.4')
- name: add ${{ matrix.version }} label
if: contains(toJson(github.event.pull_request.body), format('[x] {0}', matrix.version))
uses: actions-ecosystem/action-add-labels@v1
with:
labels: '3.4'
labels: ${{ matrix.version }}
- name: add branch-3.3 label
if: contains(toJson(github.event.pull_request.body), '[x] 3.3')
uses: actions-ecosystem/action-add-labels@v1
with:
labels: '3.3'
- name: add branch-3.2 label
if: contains(toJson(github.event.pull_request.body), '[x] 3.2')
uses: actions-ecosystem/action-add-labels@v1
with:
labels: '3.2'
- name: add branch-3.1 label
if: contains(toJson(github.event.pull_request.body), '[x] 3.1')
uses: actions-ecosystem/action-add-labels@v1
with:
labels: '3.1'
- name: add branch-3.0 label
if: contains(toJson(github.event.pull_request.body), '[x] 3.0')
uses: actions-ecosystem/action-add-labels@v1
with:
labels: '3.0'
- name: remove branch-3.4 label
if: contains(toJson(github.event.pull_request.body), '[ ] 3.4') && contains(github.event.pull_request.labels.*.name, '3.4')
- name: remove ${{ matrix.version }} label
if: contains(toJson(github.event.pull_request.body), format('[ ] {0}', matrix.version)) && contains(github.event.pull_request.labels.*.name, matrix.version)
uses: actions-ecosystem/action-remove-labels@v1
with:
labels: '3.4'
- name: remove branch-3.3 label
if: contains(toJson(github.event.pull_request.body), '[ ] 3.3') && contains(github.event.pull_request.labels.*.name, '3.3')
uses: actions-ecosystem/action-remove-labels@v1
with:
labels: '3.3'
- name: remove branch-3.2 label
if: contains(toJson(github.event.pull_request.body), '[ ] 3.2') && contains(github.event.pull_request.labels.*.name, '3.2')
uses: actions-ecosystem/action-remove-labels@v1
with:
labels: '3.2'
- name: remove branch-3.1 label
if: contains(toJson(github.event.pull_request.body), '[ ] 3.1') && contains(github.event.pull_request.labels.*.name, '3.1')
uses: actions-ecosystem/action-remove-labels@v1
with:
labels: '3.1'
- name: remove branch-3.0 label
if: contains(toJson(github.event.pull_request.body), '[ ] 3.0') && contains(github.event.pull_request.labels.*.name, '3.0')
uses: actions-ecosystem/action-remove-labels@v1
with:
labels: '3.0'
labels: ${{ matrix.version }}
- name: check-done
if: >
(startsWith(github.event.pull_request.title, '[BugFix]') ||
startsWith(github.event.pull_request.title, '[Enhancement]') ||
startsWith(github.event.pull_request.title, '[Doc]') ||
startsWith(github.event.pull_request.title, '[UT]')) &&
!contains(github.event.pull_request.title, 'cherry-pick') &&
!contains(github.event.pull_request.title, 'backport') &&
contains(toJson(github.event.pull_request.body), '[ ] I have checked the version labels')
if: contains(toJson(github.event.pull_request.body), '[ ] I have checked the version labels')
run: |
echo "::error::You must mark this checkbox - I have checked the version labels which the pr will be auto-backported to the target branch"
exit 1
backport-check:
runs-on: ubuntu-latest
needs: backport-check-subtask
if: always() && needs.backport-check-subtask.result != 'skipped'
steps:
- name: Check subtask status
if: needs.backport-check-subtask.result == 'failure'
run: |
exit 1
behavior-check:
runs-on: ubuntu-latest
if: >