diff --git a/.github/workflows/ci-merged.yml b/.github/workflows/ci-merged.yml index 890481627b1ff..e66d7420d861f 100644 --- a/.github/workflows/ci-merged.yml +++ b/.github/workflows/ci-merged.yml @@ -85,6 +85,11 @@ jobs: runs-on: [self-hosted, normal] name: Thirdparty Update Image if: github.event.pull_request.merged == true + continue-on-error: false + strategy: + fail-fast: false + matrix: + linux: [ centos7, ubuntu ] env: PR_NUMBER: ${{ github.event.number }} BRANCH: ${{ github.base_ref }} @@ -103,11 +108,11 @@ jobs: - 'thirdparty/**' - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' - - name: update image + - name: update image (${{ matrix.linux }}) if: steps.changes.outputs.thirdparty == 'true' run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - ./bin/elastic-update-image.sh $BRANCH $PR_NUMBER + ./bin/elastic-update-image.sh $BRANCH $PR_NUMBER ${{ matrix.linux }} - name: Clean ENV if: always() diff --git a/.github/workflows/ci-pipeline-branch.yml b/.github/workflows/ci-pipeline-branch.yml index b7475ebc5ffde..50a85de75fdf5 100644 --- a/.github/workflows/ci-pipeline-branch.yml +++ b/.github/workflows/ci-pipeline-branch.yml @@ -128,28 +128,33 @@ jobs: PR_NUMBER: ${{ github.event.number }} BRANCH: ${{ github.base_ref }} REPO: ${{ github.repository }} - outputs: - ip: ${{ steps.update-image.outputs.ip }} - instance_id: ${{ steps.update-image.outputs.instance_id }} - image_cache_id: ${{ steps.update-image.outputs.image_cache_id }} + continue-on-error: true + strategy: + fail-fast: false + matrix: + build_type: [ Release, ASAN ] steps: - name: clean run: | rm -rf ${{ github.workspace }} mkdir -p ${{ github.workspace }} - - name: Checkout Code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Update Image + - name: Update Image (${{ matrix.build_type }}) id: update-image + if: needs.be-checker.outputs.output2 == 'true' + env: + linux_distro: ${{ matrix.build_type }} run: | - if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then - rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - ./bin/run-pr-update-image.sh - fi + cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + ./bin/run-pr-update-image.sh + + - name: Upload Thirdparty Result + uses: actions/upload-artifact@v4 + with: + name: THIRDPARTY-RESULT-${{ matrix.build_type }} + path: image_cache.info + retention-days: 1 + overwrite: true - name: Clean ENV if: always() @@ -160,9 +165,40 @@ jobs: fi rm -rf ${{ github.workspace }}/* + thirdparty-info: + runs-on: [ self-hosted, normal ] + needs: + - thirdparty-update + name: Thirdparty Info + outputs: + centos7_image_cache_id: ${{ steps.info.outputs.centos7_image_cache_id }} + ubuntu_image_cache_id: ${{ steps.info.outputs.ubuntu_image_cache_id }} + steps: + - name: Check Result + run: | + if [[ "${{ needs.thirdparty-update.result }}" == 'failure' ]]; then + echo "::error:: Thirdparty Update Error!" + exit 1 + fi + + - name: Download Thirdparty Artifact + uses: actions/download-artifact@v4 + with: + pattern: THIRDPARTY-RESULT-* + path: outputs + + - name: Read Info + id: info + if: needs.thirdparty-update.result == 'success' + run: | + image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-centos7/image_cache.info" || echo "") + echo "centos7_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT + image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-ubuntu/image_cache.info" || echo "") + echo "ubuntu_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT + be-ut: runs-on: [self-hosted, normal] - needs: [ be-checker, thirdparty-update ] + needs: [ be-checker, thirdparty-info ] timeout-minutes: 180 name: BE UT env: @@ -182,7 +218,7 @@ jobs: run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then - export image_cache_id=${{ needs.thirdparty-update.outputs.image_cache_id }} + export image_cache_id=${{ needs.thirdparty-info.outputs.centos7_image_cache_id }} export image_tag=$BRANCH-$PR_NUMBER fi ./bin/elastic-ut.sh --pr ${PR_NUMBER} --module be --branch ${{ steps.branch.outputs.branch }} --repository ${{ github.repository }} @@ -198,7 +234,7 @@ jobs: if: always() with: name: BE UT LOG - path: ${{ steps.run_ut.outputs.RES_LOG }} + path: ${{ steps.run_ut.outputs.BE_LOG }} retention-days: 1 overwrite: true @@ -207,6 +243,7 @@ jobs: run: | rm -f ${{ steps.run_ut.outputs.RES_FILE }} rm -f ${{ steps.run_ut.outputs.RES_LOG }} + rm -rf ${{ steps.run_ut.outputs.BE_LOG }} rm -rf ${{ github.workspace }}/* fe-checker: @@ -432,7 +469,6 @@ jobs: env: PR_NUMBER: ${{ github.event.number }} BRANCH: ${{ github.base_ref }} - CONF_FILE: ${{ needs.deploy.outputs.deploy_conf_file }} steps: - name: upload info if: always() diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 480a3308af734..01ddc3c588d00 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -136,24 +136,37 @@ jobs: runs-on: [self-hosted, normal] needs: [ be-checker, clang-format ] name: Thirdparty Update + continue-on-error: true + strategy: + fail-fast: false + matrix: + linux: [ centos7, ubuntu ] env: PR_NUMBER: ${{ github.event.number }} BRANCH: ${{ github.base_ref }} REPO: ${{ github.repository }} - outputs: - image_cache_id: ${{ steps.update-image.outputs.image_cache_id }} steps: - name: clean run: | rm -rf ${{ github.workspace }} mkdir -p ${{ github.workspace }} - - name: Update Image + - name: Update Image (${{ matrix.linux }}) id: update-image if: needs.be-checker.outputs.thirdparty_filter == 'true' + env: + linux_distro: ${{ matrix.linux }} run: | cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh ./bin/run-pr-update-image.sh + + - name: Upload Thirdparty Result + uses: actions/upload-artifact@v4 + with: + name: THIRDPARTY-RESULT-${{ matrix.linux }} + path: image_cache.info + retention-days: 1 + overwrite: true - name: Clean ENV if: always() && needs.be-checker.outputs.thirdparty_filter == 'true' @@ -162,9 +175,40 @@ jobs: ./bin/elastic-cluster.sh --delete rm -rf ${{ github.workspace }}/* + thirdparty-info: + runs-on: [self-hosted, normal] + needs: + - thirdparty-update + name: Thirdparty Info + outputs: + centos7_image_cache_id: ${{ steps.info.outputs.centos7_image_cache_id }} + ubuntu_image_cache_id: ${{ steps.info.outputs.ubuntu_image_cache_id }} + steps: + - name: Check Result + run: | + if [[ "${{ needs.thirdparty-update.result }}" == 'failure' ]]; then + echo "::error:: Thirdparty Update Error!" + exit 1 + fi + + - name: Download Thirdparty Artifact + uses: actions/download-artifact@v4 + with: + pattern: THIRDPARTY-RESULT-* + path: outputs + + - name: Read Info + id: info + if: needs.thirdparty-update.result == 'success' + run: | + image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-centos7/image_cache.info" || echo "") + echo "centos7_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT + image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-ubuntu/image_cache.info" || echo "") + echo "ubuntu_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT + be-ut: runs-on: [self-hosted, normal] - needs: [ be-checker, thirdparty-update ] + needs: thirdparty-info timeout-minutes: 90 name: BE UT env: @@ -177,8 +221,8 @@ jobs: timeout-minutes: 90 run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - if [[ "${{ needs.be-checker.outputs.thirdparty_filter }}" == 'true' ]]; then - export image_cache_id=${{ needs.thirdparty-update.outputs.image_cache_id }} + if [[ "${{ needs.thirdparty-info.outputs.centos7_image_cache_id }}" != '' ]]; then + export image_cache_id=${{ needs.thirdparty-info.outputs.centos7_image_cache_id }} export image_tag=$BRANCH-$PR_NUMBER fi ./bin/elastic-ut.sh --pr ${PR_NUMBER} --module be --repository ${{ github.repository }} --with-gcov @@ -208,7 +252,7 @@ jobs: clang-tidy: runs-on: [self-hosted, normal] - needs: [ be-checker, thirdparty-update ] + needs: [ be-checker, thirdparty-info ] if: success() && needs.be-checker.outputs.src_filter == 'true' timeout-minutes: 90 name: Clang-Tidy @@ -234,7 +278,7 @@ jobs: run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh if [[ "${{ needs.be-checker.outputs.thirdparty_filter }}" == 'true' ]]; then - export image_cache_id=${{ needs.thirdparty-update.outputs.image_cache_id }} + export image_cache_id=${{ needs.thirdparty-info.outputs.centos7_image_cache_id }} export image_tag=$BRANCH-$PR_NUMBER fi ./bin/elastic-build.sh --pr ${PR_NUMBER} --repository ${{ github.repository }} --be --clang-tidy @@ -477,7 +521,7 @@ jobs: build: runs-on: [self-hosted, normal] - needs: [be-checker, fe-checker, test-checker, clang-tidy, fe-ut, thirdparty-update] + needs: [be-checker, fe-checker, test-checker, clang-tidy, fe-ut, thirdparty-info] name: BUILD env: PR_NUMBER: ${{ github.event.number }} @@ -527,7 +571,7 @@ jobs: run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh if [[ "${{ needs.be-checker.outputs.thirdparty_filter }}" == 'true' ]]; then - export image_cache_id=${{ needs.thirdparty-update.outputs.image_cache_id }} + export image_cache_id=${{ needs.thirdparty-info.outputs.centos7_image_cache_id }} export image_tag=$BRANCH-$PR_NUMBER fi diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml new file mode 100644 index 0000000000000..50f2b53cc6070 --- /dev/null +++ b/.github/workflows/ci-report.yml @@ -0,0 +1,433 @@ +name: CI Report +run-name: CI Report(#${{ github.event.workflow_run.id }}) + +on: + workflow_run: + workflows: [ "CI PIPELINE", "CI PIPELINE - BRANCH" ] + types: + - completed + +permissions: + checks: write + actions: write + contents: write + deployments: write + discussions: write + issues: write + packages: write + pages: write + pull-requests: write + repository-projects: write + security-events: write + statuses: write + +env: + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + +jobs: + INFO: + runs-on: [ self-hosted, quick ] + if: > + github.event.workflow_run.event == 'pull_request' && + (github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure') + outputs: + PR_NUMBER: ${{ steps.pr_details.outputs.PR_NUMBER }} + BASE_REF: ${{ steps.pr_details.outputs.BASE_REF }} + PR_STATE: ${{ steps.pr_details.outputs.PR_STATE }} + steps: + - run: | + sleep 10 + + - name: Download workflow artifact - PR + uses: dawidd6/action-download-artifact@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ github.event.workflow_run.id }} + name: 'pr_num' + + - name: Read the pr num file + id: pr_num_reader + uses: juliangruber/read-file-action@v1 + with: + path: ./pr_num.txt + + - name: PR DETAILS + id: pr_details + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ steps.pr_num_reader.outputs.content }} + REPO: ${{ github.repository }} + run: | + PR_NUMBER=`echo ${PR_NUMBER}` + pr_info=`gh pr view ${PR_NUMBER} -R ${REPO} --json baseRefName,state` + base_ref=$(echo ${pr_info} | jq -r .baseRefName) + pr_state=$(echo ${pr_info} | jq -r .state) + echo "BASE_REF=${base_ref}" >> $GITHUB_OUTPUT + echo "PR_STATE=${pr_state}" >> $GITHUB_OUTPUT + echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT + + - name: INFO + id: info + run: | + echo ${{ steps.pr_details.outputs.PR_NUMBER }} + echo ${{ steps.pr_details.outputs.BASE_REF }} + + FE-REPORT: + runs-on: [ self-hosted, quick ] + needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + BASE_REF: ${{ needs.INFO.outputs.BASE_REF }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: INFO + id: info + run: | + repo="${{ github.repository }}" + bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT + + - name: Download FE UT XML + id: download-ut-xml + env: + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + run: | + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER} + size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/fe_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}') + echo "size=${size}" >> $GITHUB_OUTPUT + if [[ "$size" != "0" ]]; then + mkdir fe && cd fe + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/ . --recursive + tar zxf fe_ut_report.tar.gz + elif [[ "${BASE_REF}" == "main" ]]; then + rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover /var/local/env/empty_cov_result --github-token ${{ secrets.GITHUB_TOKEN }} \ + --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco + fi + + - name: Publish UT Report + uses: mikepenz/action-junit-report@v3 + id: publish_report + if: steps.download-ut-xml.outputs.size != '0' + env: + github_token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} + with: + check_name: 'FE UT Report' + detailed_summary: true + fail_on_failure: true + commit: ${{ github.event.workflow_run.head_sha }} + report_paths: ./fe/fe-core/target/surefire-reports/*.xml + + - name: Merge FE Coverage + id: merge_report + if: steps.publish_report.outcome == 'success' && env.BASE_REF == 'main' + env: + fe_path: ${{ github.workspace }}/fe + run: | + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + if [[ -e ${fe_path}/test.exec ]]; then + java -jar package/jacococli.jar merge ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/test.exec --destfile ${fe_path}/merge.exec + else + cp ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/merge.exec + fi + + - name: Generate XML Report + id: generate-xml-report + env: + package_path: ${{ github.workspace }}/ci-tool/package + fe_path: ${{ github.workspace }}/fe + fe_core_path: ${{ github.workspace }}/fe/fe-core + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + if: steps.merge_report.outcome == 'success' + run: | + rm -rf result + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + cd ${fe_core_path}/target/classes; + rm -rf org; + cd com/starrocks && rm -rf thrift proto sql/parser builtins common/Version.class; + cd ${{ github.workspace }}; + java -jar $package_path/jacococli.jar report ${fe_path}/merge.exec --classfiles ${fe_core_path}/target/classes/ \ + --html ./result --xml ${{ github.workspace }}/coverage.xml \ + --sourcefiles ${fe_core_path}/src/main/java/ --encoding utf-8 --name fe-coverage + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}/ + ossutil64 --config-file ~/.ossutilconfig cp coverage.xml ${oss_path} -f + + # Incremental Total Coverage + - name: Publish Incremental Coverage Report - Total + if: steps.generate-xml-report.outcome == 'success' + env: + fe_path: ${{ github.workspace }}/fe + run: | + rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + if [[ -e "${fe_path}/diff.txt" ]]; then + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover ${{ github.workspace }}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \ + --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco \ + -d ${fe_path}/diff.txt -dt file + else + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover ${{ github.workspace }}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \ + --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco + fi + rm -rf coverchecker + rm -rf ci-tools + + # total coverage + - name: Coverage Report + uses: codecov/codecov-action@v3 + if: steps.generate-xml-report.outcome == 'success' + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ github.workspace }}/coverage.xml + dry_run: false + name: fe-total + flags: fe-total + fail_ci_if_error: false + verbose: true + override_pr: ${{ env.PR_NUMBER }} + override_branch: ${{ env.BASE_REF }} + override_commit: ${{ env.HEAD_SHA }} + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ github.workspace }}/* + + BE-REPORT: + runs-on: [ self-hosted, quick ] + needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + BASE_REF: ${{ needs.INFO.outputs.BASE_REF }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: INFO + id: info + run: | + bucket_prefix=`echo ${REPO%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download BE UT XML + id: download-ut-xml + env: + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + run: | + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER} + be_ut_res_path=${oss_path}/flag/be_ut_res + size=$(ossutil64 --config-file ~/.ossutilconfig ls ${be_ut_res_path} | grep "Object Number is" | awk '{print $NF}') + echo "size=${size}" >> $GITHUB_OUTPUT + if [[ "$size" != "0" ]]; then + ossutil64 --config-file ~/.ossutilconfig cp ${be_ut_res_path} . >/dev/null + res=`cat be_ut_res` + if [[ "$res" != "0" ]]; then + echo "::error::BE UT failed!" + exit 1 + fi + cd be + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/be_ut_coverage.xml . -f 1>/dev/null + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/base_version.txt . -f 1>/dev/null + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/diff.txt . -f 1>/dev/null + echo "base_version=`cat base_version.txt`" >> $GITHUB_OUTPUT + else + rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover /var/local/env/be_empty_coverage.xml --github-token ${{ secrets.GITHUB_TOKEN }} --pr ${PR_NUMBER} \ + --repo ${{ github.repository }} --threshold 80 --github-url api.github.com -type cobertura --module BE + fi + + - name: Merge BE Total Coverage + id: merge + if: github.repository == 'StarRocks/starrocks' && steps.download-ut-xml.outputs.size != '0' && steps.download-ut-xml.outcome == 'success' + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + + export ECI_ID=`./bin/create_eci.sh ${BASE_REF} ${REPO}` + export TOTAL_COV_XML=/var/local/env/be_total_coverage_${PR_NUMBER}.xml + export COV_LOG=/var/local/env/be_total_coverage_${PR_NUMBER}.log + export RES_FILE=/var/local/env/be_total_coverage_res_${PR_NUMBER}.log + + echo ${ECI_ID} + echo "ECI_ID=${ECI_ID}" >> $GITHUB_OUTPUT + echo "COV_XML=${TOTAL_COV_XML}" >> $GITHUB_OUTPUT + echo "COV_LOG=${COV_LOG}" >> $GITHUB_OUTPUT + echo "RES_FILE=${RES_FILE}" >> $GITHUB_OUTPUT + + ./bin/gen_be_cov.sh ${REPO} ${BASE_REF} ${PR_NUMBER} ${HEAD_SHA} + + # Incremental Total Coverage + - name: Publish Incremental Coverage Report - Total + if: always() && steps.download-ut-xml.outputs.size != '0' && steps.download-ut-xml.outcome == 'success' + env: + be_path: ${{ github.workspace }}/be + merge_outcome: ${{ steps.merge.outcome }} + total_xml: ${{ steps.merge.outputs.COV_XML }} + run: | + rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + if [[ "${{ github.repository }}" != "StarRocks/starrocks" ]] || [[ "${merge_outcome}" == "failure" ]]; then + total_xml=${be_path}/be_ut_coverage.xml + fi + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover ${total_xml} --github-token ${{ secrets.GITHUB_TOKEN }} --repo ${{ github.repository }} \ + --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type cobertura --module BE \ + -d ${be_path}/diff.txt -dt file + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ github.workspace }}/* + eci rm ${{ steps.merge.outputs.ECI_ID }} || true + rm -rf ${{ steps.merge.outputs.COV_LOG }} ${{ steps.merge.outputs.COV_XML }} ${{ steps.merge.outputs.RES_FILE }} + + SQL-Tester-REPORT: + runs-on: [ self-hosted, quick ] + needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + BASE_REF: ${{ needs.INFO.outputs.BASE_REF }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: INFO + id: info + run: | + repo="${{ github.repository }}" + bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT + + - name: Download SQL-Tester XML + id: download-SQL-Tester-xml + env: + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + run: | + mkdir sql-tester-result && cd sql-tester-result + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/SQL-Tester-XML/${PR_NUMBER}/ + size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}') + echo "size=${size}" >> $GITHUB_OUTPUT + if [[ "$size" != "0" ]]; then + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive + else + cp /var/local/env/sqltester_empty_coverage.xml . + fi + + - name: Prepare Tools + id: prepare-tools + run: | + mkdir -p .actions/nose-report-action + cd .actions/nose-report-action + git clone https://github.com/StarRocks/action-junit-report.git . + + - name: Publish SQL-Tester Report + uses: ./.actions/nose-report-action + if: steps.prepare-tools.outcome == 'success' + with: + check_name: 'SQL-Tester Report' + fail_on_failure: true + detailed_summary: true + token: ${{ secrets.GITHUB_TOKEN }} + commit: ${{ env.HEAD_SHA }} + report_paths: 'sql-tester-result/*.xml' + + Admit-REPORT: + runs-on: [ self-hosted, quick ] + needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + BASE_REF: ${{ needs.INFO.outputs.BASE_REF }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: INFO + id: info + run: | + repo="${{ github.repository }}" + bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT + + - name: Download Admit XML + id: download-admit-xml + env: + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + run: | + mkdir admit-result && cd admit-result + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/Admit-XML/${PR_NUMBER}/ + size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}') + echo "size=${size}" >> $GITHUB_OUTPUT + if [[ "$size" != "0" ]]; then + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive + fi + + - name: Prepare Tools + if: steps.download-admit-xml.outputs.size != '0' + id: prepare-tools + run: | + mkdir -p .actions/nose-report-action + cd .actions/nose-report-action + git clone https://github.com/StarRocks/action-junit-report.git . + + - name: Publish Admit Report + uses: ./.actions/nose-report-action + if: steps.prepare-tools.outcome == 'success' + with: + check_name: 'Admit Report' + fail_on_failure: true + detailed_summary: true + commit: ${{ env.HEAD_SHA }} + token: ${{ secrets.GITHUB_TOKEN }} + report_paths: 'admit-result/*.xml' + + NOTIFICATION: + runs-on: [ self-hosted, quick ] + needs: [ INFO, FE-REPORT, Admit-REPORT, BE-REPORT, SQL-Tester-REPORT ] + if: always() && needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + FEISHU_APP_ID: ${{ secrets.FEISHU_APP_ID }} + FEISHU_APP_SECRET: ${{ secrets.FEISHU_APP_SECRET }} + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: NOTIFY + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + ./bin/notify.sh + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ github.workspace }}/* diff --git a/.github/workflows/inspection-pipeline.yml b/.github/workflows/inspection-pipeline.yml index 957ad1638ea7e..aebed95554ab8 100644 --- a/.github/workflows/inspection-pipeline.yml +++ b/.github/workflows/inspection-pipeline.yml @@ -538,7 +538,7 @@ jobs: fi - name: save unstable cases - if: always() && ((github.event_name == 'schedule' && env.BRANCH == 'main') || (github.event_name != 'schedule' && env.BRANCH != 'main')) + if: always() && (github.event_name == 'schedule' || (github.event_name != 'schedule' && env.BRANCH != 'main')) run: | cd ci-tool && source lib/init.sh ./bin/save_unstable_cases.sh diff --git a/.github/workflows/inspection-reusable-pipeline.yml b/.github/workflows/inspection-reusable-pipeline.yml index e1a3f49ad6644..6d9b0b1c07884 100644 --- a/.github/workflows/inspection-reusable-pipeline.yml +++ b/.github/workflows/inspection-reusable-pipeline.yml @@ -230,7 +230,7 @@ jobs: needs: - restart-fe - restart-be - if: always() && inputs.INCLUDE_ADMIT == 'true' && inputs.BUILD_TYPE == 'Release' && needs.restart-be.result == 'success' && needs.restart-fe.result == 'success' + if: always() && env.INCLUDE_ADMIT == 'true' && env.BUILD_TYPE == 'Release' && needs.restart-be.result == 'success' && needs.restart-fe.result == 'success' timeout-minutes: 40 env: FE_NODE: ${{ needs.restart-be.outputs.FE_NODE }}