starrocks/.github/workflows/ci-report.yml

name: CI Report

on:
  workflow_run:
    workflows: [ "CI PIPELINE", "CI PIPELINE - BRANCH" ]
    types:
      - completed

permissions:
  checks: write
  actions: write
  contents: write
  deployments: write
  discussions: write
  issues: write
  packages: write
  pages: write
  pull-requests: write
  repository-projects: write
  security-events: write
  statuses: write

jobs:
  INFO:
    runs-on: [ self-hosted, quick ]
    if: >
      github.event.workflow_run.event == 'pull_request' &&
      (github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure')      
    outputs:
      PR_NUMBER: ${{ steps.pr_details.outputs.PR_NUMBER }}
      HEAD_SHA: ${{ steps.pr_details.outputs.HEAD_SHA }}
      BASE_REF: ${{ steps.pr_details.outputs.BASE_REF }}
    steps:
      - run: |
          sleep 30          

      - name: Download workflow artifact - PR
        uses: dawidd6/action-download-artifact@v2
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          run_id: ${{ github.event.workflow_run.id }}
          name: 'pr_num'

      - name: Download workflow artifact - HEAD REF
        uses: dawidd6/action-download-artifact@v2
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          run_id: ${{ github.event.workflow_run.id }}
          name: 'head_sha'

      - name: Read the pr num file
        id: pr_num_reader
        uses: juliangruber/read-file-action@v1
        with:
          path: ./pr_num.txt

      - name: Read the pr head ref
        id: head_sha_reader
        uses: juliangruber/read-file-action@v1
        with:
          path: ./head_sha.txt

      - name: PR DETAILS
        id: pr_details
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ steps.pr_num_reader.outputs.content }}
          HEAD_SHA: ${{ steps.head_sha_reader.outputs.content }}
          REPO: ${{ github.repository }}
        run: |
          PR_NUMBER=`echo ${PR_NUMBER}`
          HEAD_SHA=`echo ${HEAD_SHA}`
          base_ref=`gh pr view ${PR_NUMBER} -R ${REPO} --json baseRefName -q .baseRefName`
          echo "BASE_REF=${base_ref}" >> $GITHUB_OUTPUT
          echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT
          echo "HEAD_SHA=${HEAD_SHA}" >> $GITHUB_OUTPUT          

      - name: Set Workflow Name
        id: set_workflow_name
        env:
          pr: ${{ steps.pr_details.outputs.PR_NUMBER }}
        run: echo "workflow_name=\"CI Report - ${pr}\"" >> $GITHUB_OUTPUT

  FE-REPORT:
    runs-on: [ self-hosted, quick ]
    needs: INFO
    env:
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
      branch: ${{ needs.INFO.outputs.BASE_REF }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}          

      - name: INFO
        id: info
        run: |
          repo="${{ github.repository }}"
          bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
          echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT          

      - name: Download FE UT XML
        id: download-ut-xml
        env:
          bucket_prefix: ${{ steps.info.outputs.bucket_prefix }}
        run: |
          oss_path=oss://${bucket_prefix}-ci-release/$branch/Release/pr/UT-Report/${PR_NUMBER}
          size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/fe_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}')
          echo "size=${size}" >> $GITHUB_OUTPUT
          if [[ "$size" != "0" ]]; then
            mkdir fe && cd fe
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/ . --recursive
            tar zxf fe_ut_report.tar.gz
          else
            rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
            export JAVA_HOME=/var/local/env/jdk1.8.0_202;
            export PATH=$JAVA_HOME/bin:$PATH;
            java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
              --cover /var/local/env/empty_cov_result --github-token ${{ secrets.PAT }} \
              --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco
          fi          

      - name: Publish UT Report
        uses: mikepenz/action-junit-report@v3
        id: publish_report
        if: steps.download-ut-xml.outputs.size != '0'
        env:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          token: ${{ secrets.GITHUB_TOKEN }}
        with:
          check_name: 'FE UT Report'
          detailed_summary: true
          fail_on_failure: true
          commit: ${{ github.event.workflow_run.head_sha }}
          report_paths: ./fe/fe-core/target/surefire-reports/*.xml

      - name: Merge FE Coverage
        id: merge_report
        if: steps.publish_report.outcome == 'success'
        env:
          fe_path: ${{ github.workspace }}/fe
        run: |
          export JAVA_HOME=/var/local/env/jdk1.8.0_202;
          export PATH=$JAVA_HOME/bin:$PATH;
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          if [[ -e ${fe_path}/test.exec ]]; then
            java -jar package/jacococli.jar merge ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/test.exec --destfile ${fe_path}/merge.exec
          else
            cp ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/merge.exec
          fi          

      - name: Generate XML Report
        id: generate-xml-report
        env:
          package_path: ${{ github.workspace }}/ci-tool/package
          fe_path: ${{ github.workspace }}/fe
          fe_core_path: ${{ github.workspace }}/fe/fe-core
          bucket_prefix: ${{ steps.info.outputs.bucket_prefix }}
        if: steps.merge_report.outcome == 'success'
        run: |
          rm -rf result
          export JAVA_HOME=/var/local/env/jdk1.8.0_202;
          export PATH=$JAVA_HOME/bin:$PATH;
          cd ${fe_core_path}/target/classes;
          rm -rf org;
          cd com/starrocks && rm -rf thrift proto sql/parser builtins common/Version.class;
          cd ${{ github.workspace }};
          java -jar $package_path/jacococli.jar report ${fe_path}/merge.exec --classfiles ${fe_core_path}/target/classes/ \
            --html ./result --xml ${{ github.workspace }}/coverage.xml \
            --sourcefiles ${fe_core_path}/src/main/java/ --encoding utf-8 --name fe-coverage
          oss_path=oss://${bucket_prefix}-ci-release/$branch/Release/pr/UT-Report/${PR_NUMBER}/
          ossutil64 --config-file ~/.ossutilconfig cp coverage.xml ${oss_path} -f          

      # Incremental Total Coverage
      - name: Publish Incremental Coverage Report - Total
        if: steps.generate-xml-report.outcome == 'success'
        env:
          fe_path: ${{ github.workspace }}/fe
        run: |
          rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
          export JAVA_HOME=/var/local/env/jdk1.8.0_202;
          export PATH=$JAVA_HOME/bin:$PATH;
          if [[ -e "${fe_path}/diff.txt" ]]; then
            java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
              --cover ${{ github.workspace }}/result/ --github-token ${{ secrets.PAT }} \
              --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco \
              -d ${fe_path}/diff.txt -dt file
          else
            java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
              --cover ${{ github.workspace }}/result/ --github-token ${{ secrets.PAT }} \
              --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco
          fi
          rm -rf coverchecker
          rm -rf ci-tools          

      # total coverage
      - name: Coverage Report
        uses: codecov/codecov-action@v3
        if: steps.generate-xml-report.outcome == 'success'
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ${{ github.workspace }}/coverage.xml
          dry_run: false
          name: fe-total
          flags: fe-total
          fail_ci_if_error: false
          verbose: true
          override_pr: ${{ needs.INFO.outputs.PR_NUMBER }}
          override_branch: ${{ needs.INFO.outputs.BASE_REF }}
          override_commit: ${{ needs.INFO.outputs.HEAD_SHA }}

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ github.workspace }}/*          

  BE-REPORT:
    runs-on: [ self-hosted, quick ]
    needs: INFO
    env:
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
      BRANCH: ${{ needs.INFO.outputs.BASE_REF }}
      GH_TOKEN: ${{ github.token }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}          

      - name: INFO
        id: info
        run: |
          repo="${{ github.repository }}"
          bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
          echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT          

      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Download BE UT XML
        id: download-ut-xml
        env:
          bucket_prefix: ${{ steps.info.outputs.bucket_prefix }}
        run: |
          oss_path=oss://${bucket_prefix}-ci-release/$BRANCH/ASAN/pr/UT-Report/${PR_NUMBER}
          size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/be_ut_coverage.xml | grep "Object Number is" | awk '{print $NF}')
          echo "size=${size}" >> $GITHUB_OUTPUT
          if [[ "$size" != "0" ]]; then
            cd be
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/be_ut_coverage.xml . -f 1>/dev/null
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/base_version.txt . -f 1>/dev/null
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/diff.txt . -f 1>/dev/null
            echo "base_version=`cat base_version.txt`" >> $GITHUB_OUTPUT
          else
            rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
            export JAVA_HOME=/var/local/env/jdk1.8.0_202;
            export PATH=$JAVA_HOME/bin:$PATH;
            java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
              --cover /var/local/env/be_empty_coverage.xml --github-token ${{ secrets.PAT }} --pr ${PR_NUMBER} \
              --repo ${{ github.repository }} --threshold 80 --github-url api.github.com -type cobertura --module BE
          fi          

      - name: Publish UT Report
        uses: mikepenz/action-junit-report@v3
        id: publish_report
        if: steps.download-ut-xml.outcome == 'success' && steps.download-ut-xml.outputs.size != '0'
        env:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          token: ${{ secrets.GITHUB_TOKEN }}
        with:
          check_name: 'BE UT Report'
          detailed_summary: true
          fail_on_failure: true
          commit: ${{ github.event.workflow_run.head_sha }}
          report_paths: ./be/be_ut_coverage.xml

      - name: Checkout PR Code
        if: steps.publish_report.outcome == 'success'
        env:
          CODE_PATH: ${{ github.workspace }}
          BASE_VERSION: ${{ steps.download-ut-xml.outputs.base_version }}
        run: |
          cd /var/lib/ci-tool/ && git pull && ./bin/checkout.sh          

      # Incremental Total Coverage
      - name: Publish Incremental Coverage Report - Total
        if: steps.publish_report.outcome == 'success'
        env:
          be_path: ${{ github.workspace }}/be
        run: |
          rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
          export JAVA_HOME=/var/local/env/jdk1.8.0_202;
          export PATH=$JAVA_HOME/bin:$PATH;
          java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
            --cover ${be_path}/be_ut_coverage.xml --github-token ${{ secrets.PAT }} --repo ${{ github.repository }} \
            --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type cobertura --module BE \
            -d ${be_path}/diff.txt -dt file          

      # total coverage
      - name: Coverage Report
        uses: codecov/codecov-action@v3
        if: steps.publish_report.outcome == 'success'
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ${{ github.workspace }}/be/be_ut_coverage.xml
          dry_run: false
          name: be-total
          flags: be-total
          fail_ci_if_error: false
          verbose: true
          override_pr: ${{ needs.INFO.outputs.PR_NUMBER }}
          override_branch: ${{ needs.INFO.outputs.BASE_REF }}
          override_commit: ${{ needs.INFO.outputs.HEAD_SHA }}

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ github.workspace }}/*          

  SQL-Tester-REPORT:
    runs-on: [ self-hosted, quick ]
    needs: INFO
    env:
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
      branch: ${{ needs.INFO.outputs.BASE_REF }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}          

      - name: INFO
        id: info
        run: |
          repo="${{ github.repository }}"
          bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
          echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT          

      - name: Download SQL-Tester XML
        id: download-SQL-Tester-xml
        env:
          bucket_prefix: ${{ steps.info.outputs.bucket_prefix }}
        run: |
          mkdir sql-tester-result && cd sql-tester-result
          oss_path=oss://${bucket_prefix}-ci-release/$branch/Release/pr/SQL-Tester-XML/${PR_NUMBER}/
          size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}')
          echo "size=${size}" >> $GITHUB_OUTPUT
          if [[ "$size" != "0" ]]; then
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive
          fi          

      - name: Prepare Tools
        id: prepare-tools
        if: steps.download-SQL-Tester-xml.outputs.size != '0'
        run: |
          mkdir -p .actions/nose-report-action
          cd .actions/nose-report-action
          git clone https://github.com/StarRocks/action-junit-report.git .          

      - name: Publish SQL-Tester Report
        uses: ./.actions/nose-report-action
        if: steps.prepare-tools.outcome == 'success'
        with:
          check_name: 'SQL-Tester Report'
          fail_on_failure: true
          detailed_summary: true
          token: ${{ secrets.GITHUB_TOKEN }}
          commit: ${{ github.event.workflow_run.head_sha }}
          report_paths: 'sql-tester-result/*.xml'

  Admit-REPORT:
    runs-on: [ self-hosted, quick ]
    needs: INFO
    env:
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
      branch: ${{ needs.INFO.outputs.BASE_REF }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}          

      - name: INFO
        id: info
        run: |
          repo="${{ github.repository }}"
          bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
          echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT          

      - name: Download Admit XML
        id: download-admit-xml
        env:
          bucket_prefix: ${{ steps.info.outputs.bucket_prefix }}
        run: |
          mkdir admit-result && cd admit-result
          oss_path=oss://${bucket_prefix}-ci-release/$branch/Release/pr/Admit-XML/${PR_NUMBER}/
          size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}')
          echo "size=${size}" >> $GITHUB_OUTPUT
          if [[ "$size" != "0" ]]; then
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive
          fi          

      - name: Prepare Tools
        if: steps.download-admit-xml.outputs.size != '0'
        id: prepare-tools
        run: |
          mkdir -p .actions/nose-report-action
          cd .actions/nose-report-action
          git clone https://github.com/StarRocks/action-junit-report.git .          

      - name: Publish Admit Report
        uses: ./.actions/nose-report-action
        if: steps.prepare-tools.outcome == 'success'
        with:
          check_name: 'Admit Report'
          fail_on_failure: true
          detailed_summary: true
          commit: ${{ github.event.workflow_run.head_sha }}
          token: ${{ secrets.GITHUB_TOKEN }}
          report_paths: 'admit-result/*.xml'

  NOTIFICATION:
    runs-on: [ self-hosted, quick ]
    needs: [ INFO, FE-REPORT, Admit-REPORT, BE-REPORT, SQL-Tester-REPORT ]
    if: always() && needs.INFO.outputs.PR_NUMBER != ''
    env:
      PR_ID: ${{ needs.INFO.outputs.PR_NUMBER }}
      WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
      FEISHU_APP_ID: ${{ secrets.FEISHU_APP_ID }}
      FEISHU_APP_SECRET: ${{ secrets.FEISHU_APP_SECRET }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}          

      - name: NOTIFY
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          ./bin/notify.sh          

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ github.workspace }}/*