Update ion-java-performance-regression-detector.yml #454
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will build a Java project with Maven | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven | |
| name: Ion Java performance regression detector | |
| on: | |
| [pull_request, push] | |
| permissions: | |
| contents: read | |
| jobs: | |
| detect-regression: | |
| name: Detect Regression | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 11 | |
| - name: Checkout ion-java from the new commit. | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| path: ion-java-new | |
| - name: Build ion-java from the new commit | |
| run: cd ion-java-new && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal | |
| - name: Checkout ion-java-benchmark-cli | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 | |
| with: | |
| repository: amazon-ion/ion-java-benchmark-cli | |
| ref: master | |
| path: ion-java-benchmark-cli | |
| - name: Build ion-java-benchmark-cli | |
| run: cd ion-java-benchmark-cli && mvn clean install | |
| - name: Checkout ion-data-generator | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 | |
| with: | |
| repository: amazon-ion/ion-data-generator | |
| ref: main | |
| path: ion-data-generator | |
| - name: Build ion-data-generator | |
| run: cd ion-data-generator && mvn clean install | |
| - name: Check the version of ion-java. | |
| run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version | |
| - name: Generate test Ion Data | |
| run: | | |
| mkdir -p testData | |
| java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedStruct.isl testData/testStructs.10n | |
| java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedList.isl testData/testLists.10n | |
| java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/sexp.isl testData/testSexps.10n | |
| java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema01.isl testData/realWorldData01.10n | |
| java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema02.isl testData/realWorldData02.10n | |
| java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema03.isl testData/realWorldData03.10n | |
| - name: Upload test Ion Data to artifacts | |
| uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 | |
| with: | |
| name: test Ion Data | |
| path: testData | |
| - name: Benchmark ion-java from the new commit | |
| run: | | |
| mkdir -p benchmarkResults | |
| cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults | |
| - name: Upload benchmark results to artifacts | |
| uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 | |
| with: | |
| name: Benchmark result | |
| path: benchmarkResults | |
| - name: Clean maven dependencies repository | |
| run : rm -r /home/runner/.m2 | |
| - name: Checkout the current commit | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 | |
| with: | |
| repository: amazon-ion/ion-java | |
| ref: master | |
| path: ion-java | |
| - name: Build ion-java from the previous commit | |
| run: cd ion-java && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal | |
| - name: Build ion-java-benchmark-cli | |
| run: cd ion-java-benchmark-cli && mvn clean install | |
| - name: Check the version of ion-java | |
| run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version | |
| - name: Create directories for test data and benchmark results | |
| run: | | |
| mkdir -p benchmarkResults | |
| mkdir -p testData | |
| - name: Download test Ion Data from artifacts | |
| uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 | |
| with: | |
| name: test Ion Data | |
| path: testData | |
| - name: Download benchmark results of ion-java from the new commit from artifacts | |
| uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 | |
| with: | |
| name: Benchmark result | |
| path: benchmarkResults | |
| - name: Benchmark ion-java from the previous commit and add the generated benchmark results to the existing directories | |
| run: cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults | |
| - name: Upload new benchmark results directory to artifacts | |
| uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 | |
| with: | |
| name: Benchmark result | |
| path: benchmarkResults | |
| - name: Detect performance regression | |
| id: regression_result | |
| run: | | |
| result=true | |
| cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "no regression detected" ]; then result=false; fi; done | |
| echo "::set-output name=regression-result::$result" | |
| if [ "$result" = "true" ]; then echo "No regression detected!" >> $GITHUB_STEP_SUMMARY; fi | |
| - name: Upload comparison reports to the benchmark results directory | |
| uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 | |
| with: | |
| name: Benchmark result | |
| path: benchmarkResults | |
| - name: Fail the workflow if regression happened | |
| env: | |
| regression_detect: ${{steps.regression_result.outputs.regression-result}} | |
| if: ${{ env.regression_detect == 'false' }} | |
| run: | | |
| cd benchmarkResults && echo "| Benchmark command | GC Allocation Rate | Heap Usage | Speed |" >> $GITHUB_STEP_SUMMARY && echo "| ----------- | ----------- |----------- | ----------- |" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion) && if [ "$regressionDetection" != "no regression detected" ]; then command=$(echo $FILE | sed "s/_/ /g") && read gc heapUsage speed <<< $( echo ${regressionDetection} | awk -F", " '{print $1" "$2" "$3}' ) && echo "|$command|$gc|$heapUsage|$speed|" >> $GITHUB_STEP_SUMMARY; fi; done | |
| exit 1 |