ci-report.yml

name: CI Report
run-name: CI Report(#${{ github.event.workflow_run.id }})

on:
  workflow_run:
    workflows: [ "CI PIPELINE", "CI PIPELINE - BRANCH" ]
    types:
      - completed

permissions:
  checks: write
  actions: write
  contents: write
  deployments: write
  discussions: write
  issues: write
  packages: write
  pages: write
  pull-requests: write
  repository-projects: write
  security-events: write
  statuses: write

env:
  HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
  REPO: ${{ github.repository }}

jobs:
  INFO:
    runs-on: [ self-hosted, quick ]
    if: >
      github.event.workflow_run.event == 'pull_request' &&
      (github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure')
    outputs:
      PR_NUMBER: ${{ steps.pr_details.outputs.PR_NUMBER }}
      BASE_REF: ${{ steps.pr_details.outputs.BASE_REF }}
      PR_STATE: ${{ steps.pr_details.outputs.PR_STATE }}
      BUCKET_PREFIX: ${{ steps.bucket_info.outputs.bucket_prefix }}
    steps:
      - run: |
          sleep 10
      
      - name: INFO
        id: bucket_info
        run: |
          bucket_prefix=`echo ${REPO%/*} | tr '[:upper:]' '[:lower:]'`
          echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT
      
      - name: Download workflow artifact - PR
        uses: dawidd6/action-download-artifact@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          run_id: ${{ github.event.workflow_run.id }}
          name: 'pr_num'

      - name: Read the pr num file
        id: pr_num_reader
        uses: juliangruber/read-file-action@v1
        with:
          path: ./pr_num.txt

      - name: PR DETAILS
        id: pr_details
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ steps.pr_num_reader.outputs.content }}
          REPO: ${{ github.repository }}
        run: |
          PR_NUMBER=`echo ${PR_NUMBER}`
          pr_info=`gh pr view ${PR_NUMBER} -R ${REPO} --json baseRefName,state`
          base_ref=$(echo ${pr_info} | jq -r .baseRefName)
          pr_state=$(echo ${pr_info} | jq -r .state)
          echo "BASE_REF=${base_ref}" >> $GITHUB_OUTPUT
          echo "PR_STATE=${pr_state}" >> $GITHUB_OUTPUT
          echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT

      - name: INFO
        id: info
        run: |
          echo ${{ steps.pr_details.outputs.PR_NUMBER }}
          echo ${{ steps.pr_details.outputs.BASE_REF }}

  FE-REPORT:
    runs-on: [ self-hosted, quick ]
    needs: INFO
    if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED'
    env:
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
      BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
      bucket_prefix: ${{ needs.INFO.outputs.BUCKET_PREFIX }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}

      - name: Download FE UT XML
        id: download-ut-xml
        run: |
          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
          size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/fe_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}')
          echo "size=${size}" >> $GITHUB_OUTPUT
          if [[ "$size" != "0" ]]; then
            mkdir fe && cd fe
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/ . --recursive
            tar zxf fe_ut_report.tar.gz
          elif [[ "${BASE_REF}" == "main" ]]; then
            rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
            export JAVA_HOME=/var/local/env/jdk1.8.0_202;
            export PATH=$JAVA_HOME/bin:$PATH;
            java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
              --cover /var/local/env/empty_cov_result --github-token ${{ secrets.GITHUB_TOKEN }} \
              --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco
          fi

      - name: Publish UT Report
        uses: mikepenz/action-junit-report@v4
        id: publish_report
        if: steps.download-ut-xml.outputs.size != '0'
        env:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          token: ${{ secrets.GITHUB_TOKEN }}
        with:
          check_name: 'FE UT Report'
          detailed_summary: true
          fail_on_failure: true
          commit: ${{ github.event.workflow_run.head_sha }}
          report_paths: ./fe/fe-core/target/surefire-reports/*.xml

      - name: Merge FE Coverage
        id: merge_report
        if: steps.publish_report.outcome == 'success' && env.BASE_REF == 'main'
        env:
          fe_path: ${{ github.workspace }}/fe
        run: |
          export JAVA_HOME=/var/local/env/jdk1.8.0_202;
          export PATH=$JAVA_HOME/bin:$PATH;
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          if [[ -e ${fe_path}/test.exec ]]; then
            java -jar package/jacococli.jar merge ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/test.exec --destfile ${fe_path}/merge.exec
          else
            cp ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/merge.exec
          fi

      - name: Generate XML Report
        id: generate-xml-report
        env:
          package_path: ${{ github.workspace }}/ci-tool/package
          fe_path: ${{ github.workspace }}/fe
          fe_core_path: ${{ github.workspace }}/fe/fe-core
        if: steps.merge_report.outcome == 'success'
        run: |
          rm -rf result
          export JAVA_HOME=/var/local/env/jdk1.8.0_202;
          export PATH=$JAVA_HOME/bin:$PATH;
          cd ${fe_core_path}/target/classes;
          rm -rf org;
          cd com/starrocks && rm -rf thrift proto sql/parser builtins common/Version.class;
          cd ${{ github.workspace }};
          java -jar $package_path/jacococli.jar report ${fe_path}/merge.exec --classfiles ${fe_core_path}/target/classes/ \
            --html ./result --xml ${{ github.workspace }}/coverage.xml \
            --sourcefiles ${fe_core_path}/src/main/java/ --encoding utf-8 --name fe-coverage
          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}/
          ossutil64 --config-file ~/.ossutilconfig cp coverage.xml ${oss_path} -f

      # Incremental Total Coverage
      - name: Publish Incremental Coverage Report - Total
        if: steps.generate-xml-report.outcome == 'success'
        env:
          fe_path: ${{ github.workspace }}/fe
        run: |
          rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
          export JAVA_HOME=/var/local/env/jdk1.8.0_202;
          export PATH=$JAVA_HOME/bin:$PATH;
          if [[ -e "${fe_path}/diff.txt" ]]; then
            java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
              --cover ${{ github.workspace }}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \
              --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco \
              -d ${fe_path}/diff.txt -dt file
          else
            java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
              --cover ${{ github.workspace }}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \
              --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco
          fi
          rm -rf coverchecker
          rm -rf ci-tools

      # total coverage
      - name: Coverage Report
        uses: codecov/codecov-action@v3
        if: steps.generate-xml-report.outcome == 'success'
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ${{ github.workspace }}/coverage.xml
          dry_run: false
          name: fe-total
          flags: fe-total
          fail_ci_if_error: false
          verbose: true
          override_pr: ${{ env.PR_NUMBER }}
          override_branch: ${{ env.BASE_REF }}
          override_commit: ${{ env.HEAD_SHA }}

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ github.workspace }}/*

  BE-REPORT:
    runs-on: [ self-hosted, quick ]
    needs: INFO
    if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED'
    env:
      GH_TOKEN: ${{ github.token }}
      REPO: ${{ github.repository }}
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
      BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
      bucket_prefix: ${{ needs.INFO.outputs.BUCKET_PREFIX }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Download BE UT XML
        id: download-ut-xml
        run: |
          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
          be_ut_res_path=${oss_path}/flag/be_ut_res
          size=$(ossutil64 --config-file ~/.ossutilconfig ls ${be_ut_res_path} | grep "Object Number is" | awk '{print $NF}')
          echo "size=${size}" >> $GITHUB_OUTPUT
          if [[ "$size" != "0" ]]; then
            ossutil64 --config-file ~/.ossutilconfig cp ${be_ut_res_path} . >/dev/null
            res=`cat be_ut_res`
            if [[ "$res" != "0" ]]; then
              echo "::error::BE UT failed!"
              exit 1
            fi
            cd be
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/be_ut_coverage.xml . -f 1>/dev/null
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/base_version.txt . -f 1>/dev/null
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/diff.txt . -f 1>/dev/null
            echo "base_version=`cat base_version.txt`" >> $GITHUB_OUTPUT
          else
            rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
            export JAVA_HOME=/var/local/env/jdk1.8.0_202;
            export PATH=$JAVA_HOME/bin:$PATH;
            java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
              --cover /var/local/env/be_empty_coverage.xml --github-token ${{ secrets.GITHUB_TOKEN }} --pr ${PR_NUMBER} \
              --repo ${{ github.repository }} --threshold 80 --github-url api.github.com -type cobertura --module BE
          fi

      - name: Merge BE Total Coverage
        id: merge
        if: github.repository == 'StarRocks/starrocks' && steps.download-ut-xml.outputs.size != '0' && steps.download-ut-xml.outcome == 'success'
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          
          export ECI_ID=`./bin/create_eci.sh ${BASE_REF} ${REPO}`
          export TOTAL_COV_XML=/var/local/env/be_total_coverage_${PR_NUMBER}.xml
          export COV_LOG=/var/local/env/be_total_coverage_${PR_NUMBER}.log
          export RES_FILE=/var/local/env/be_total_coverage_res_${PR_NUMBER}.log
               
          echo ${ECI_ID}
          echo "ECI_ID=${ECI_ID}" >> $GITHUB_OUTPUT
          echo "COV_XML=${TOTAL_COV_XML}" >> $GITHUB_OUTPUT
          echo "COV_LOG=${COV_LOG}" >> $GITHUB_OUTPUT
          echo "RES_FILE=${RES_FILE}" >> $GITHUB_OUTPUT
          
          ./bin/gen_be_cov.sh ${REPO} ${BASE_REF} ${PR_NUMBER} ${HEAD_SHA}

      # Incremental Total Coverage
      - name: Publish Incremental Coverage Report - Total
        if: always() && steps.download-ut-xml.outputs.size != '0' && steps.download-ut-xml.outcome == 'success'
        env:
          be_path: ${{ github.workspace }}/be
          merge_outcome: ${{ steps.merge.outcome }}
          total_xml: ${{ steps.merge.outputs.COV_XML }}
        run: |
          rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
          export JAVA_HOME=/var/local/env/jdk1.8.0_202;
          export PATH=$JAVA_HOME/bin:$PATH;
          if [[ "${{ github.repository }}" != "StarRocks/starrocks" ]] || [[ "${merge_outcome}" == "failure" ]]; then
              total_xml=${be_path}/be_ut_coverage.xml
          fi
          java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
            --cover ${total_xml} --github-token ${{ secrets.GITHUB_TOKEN }} --repo ${{ github.repository }} \
            --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type cobertura --module BE \
            -d ${be_path}/diff.txt -dt file

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ github.workspace }}/*
          eci rm ${{ steps.merge.outputs.ECI_ID }} || true
          rm -rf ${{ steps.merge.outputs.COV_LOG }} ${{ steps.merge.outputs.COV_XML }} ${{ steps.merge.outputs.RES_FILE }}

  JAVA-EXTENSION-REPORT:
    runs-on: [ self-hosted, quick ]
    needs: INFO
    if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED'
    env:
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
      BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
      bucket_prefix: ${{ needs.INFO.outputs.BUCKET_PREFIX }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}

      - name: Download Java Extension UT XML
        id: download-ut-xml
        run: |
          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
          size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/java_extension_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}')
          echo "size=${size}" >> $GITHUB_OUTPUT
          if [[ "$size" != "0" ]]; then
            mkdir java-extensions && cd java-extensions
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/java_extension_ut_report.tar.gz . --recursive
            tar zxf java_extension_ut_report.tar.gz
          elif [[ "${BASE_REF}" == "main" ]]; then
            rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
            export JAVA_HOME=/var/local/env/jdk1.8.0_202;
            export PATH=$JAVA_HOME/bin:$PATH;
            java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
              --cover /var/local/env/java_extension_empty_coverage --github-token ${{ secrets.GITHUB_TOKEN }} \
              --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco \
              --module "Java-Extensions"
          fi

      - name: Publish UT Report
        uses: mikepenz/action-junit-report@v4
        id: publish_report
        if: steps.download-ut-xml.outputs.size != '0'
        env:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          token: ${{ secrets.GITHUB_TOKEN }}
        with:
          check_name: 'Java Extensions UT Report'
          detailed_summary: true
          fail_on_failure: true
          commit: ${{ github.event.workflow_run.head_sha }}
          report_paths: ./java-extensions/xml_dir/*.xml

      # Incremental Total Coverage
      - name: Publish Incremental Coverage Report - Total
        if: steps.publish_report.outcome == 'success' && env.BASE_REF == 'main'
        env:
          code_path: ${{ github.workspace }}/java-extensions
        run: |
          rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
          export JAVA_HOME=/var/local/env/jdk1.8.0_202;
          export PATH=$JAVA_HOME/bin:$PATH;
          java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
            --cover ${code_path}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \
            --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco \
            -d ${code_path}/diff.txt -dt file --module "Java-Extensions"
          rm -rf coverchecker
          rm -rf ci-tools

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ github.workspace }}/*
  
  SQL-Tester-REPORT:
    runs-on: [ self-hosted, quick ]
    needs: INFO
    if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED'
    env:
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
      BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}

      - name: INFO
        id: info
        run: |
          repo="${{ github.repository }}"
          bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
          echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT

      - name: Download SQL-Tester XML
        id: download-SQL-Tester-xml
        env:
          bucket_prefix: ${{ steps.info.outputs.bucket_prefix }}
        run: |
          mkdir sql-tester-result && cd sql-tester-result
          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/SQL-Tester-XML/${PR_NUMBER}/
          size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}')
          echo "size=${size}" >> $GITHUB_OUTPUT
          if [[ "$size" != "0" ]]; then
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive
          else
            cp /var/local/env/sqltester_empty_coverage.xml .
          fi

      - name: Prepare Tools
        id: prepare-tools
        run: |
          mkdir -p .actions/nose-report-action
          cd .actions/nose-report-action
          git clone https://github.com/StarRocks/action-junit-report.git .

      - name: Publish SQL-Tester Report
        uses: ./.actions/nose-report-action
        if: steps.prepare-tools.outcome == 'success'
        with:
          check_name: 'SQL-Tester Report'
          fail_on_failure: true
          detailed_summary: true
          token: ${{ secrets.GITHUB_TOKEN }}
          commit: ${{ env.HEAD_SHA }}
          report_paths: 'sql-tester-result/*.xml'

  Admit-REPORT:
    runs-on: [ self-hosted, quick ]
    needs: INFO
    if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED'
    env:
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
      BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
      bucket_prefix: ${{ needs.INFO.outputs.bucket_prefix }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}

      - name: Download Admit XML
        id: download-admit-xml

        run: |
          mkdir admit-result && cd admit-result
          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/Admit-XML/${PR_NUMBER}/
          size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}')
          echo "size=${size}" >> $GITHUB_OUTPUT
          if [[ "$size" != "0" ]]; then
            ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive
          fi

      - name: Prepare Tools
        if: steps.download-admit-xml.outputs.size != '0'
        id: prepare-tools
        run: |
          mkdir -p .actions/nose-report-action
          cd .actions/nose-report-action
          git clone https://github.com/StarRocks/action-junit-report.git .

      - name: Publish Admit Report
        uses: ./.actions/nose-report-action
        if: steps.prepare-tools.outcome == 'success'
        with:
          check_name: 'Admit Report'
          fail_on_failure: true
          detailed_summary: true
          commit: ${{ env.HEAD_SHA }}
          token: ${{ secrets.GITHUB_TOKEN }}
          report_paths: 'admit-result/*.xml'

  NOTIFICATION:
    runs-on: [ self-hosted, quick ]
    needs: [ INFO, FE-REPORT, Admit-REPORT, BE-REPORT, SQL-Tester-REPORT, JAVA-EXTENSION-REPORT ]
    if: always() && needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED'
    env:
      WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
      FEISHU_APP_ID: ${{ secrets.FEISHU_APP_ID }}
      FEISHU_APP_SECRET: ${{ secrets.FEISHU_APP_SECRET }}
      PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}

      - name: NOTIFY
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          ./bin/notify.sh

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ github.workspace }}/*