diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index 4de76d1bd5..2db68dfb5a 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -5,146 +5,114 @@ name: Ion Java performance regression detector on: pull_request: - paths: - - 'src/*' -permissions: - contents: read +env: + data_size: '50000' + test_data_id: 'generated-test-data' jobs: - detect-regression: - name: Detect Regression + generate-test-data: + name: Generate Test Data runs-on: ubuntu-latest - steps: - - name: Set up JDK 11 - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 - with: - distribution: 'corretto' - java-version: 11 - - - name: Checkout ion-java from the new commit. - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 - with: - ref: ${{ github.event.pull_request.head.sha }} - path: ion-java-new - - - name: Build ion-java from the new commit - run: cd ion-java-new && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal - - - name: Checkout ion-java-benchmark-cli - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 - with: - repository: amazon-ion/ion-java-benchmark-cli - ref: master - path: ion-java-benchmark-cli - - - name: Build ion-java-benchmark-cli - run: cd ion-java-benchmark-cli && mvn clean install - - name: Checkout ion-data-generator - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 + uses: actions/checkout@v4 with: repository: amazon-ion/ion-data-generator ref: main - path: ion-data-generator - name: Build ion-data-generator - run: cd ion-data-generator && mvn clean install + run: mvn clean install - - name: Check the version of ion-java. - run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version - - - name: Generate test Ion Data + - name: Generate test data + env: + jar_file: target/ion-data-generator-1.0-SNAPSHOT.jar + schema_dir: tst/com/amazon/ion/workflow run: | mkdir -p testData - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedStruct.isl testData/testStructs.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedList.isl testData/testLists.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/sexp.isl testData/testSexps.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema01.isl testData/realWorldData01.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema02.isl testData/realWorldData02.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema03.isl testData/realWorldData03.10n - - - name: Upload test Ion Data to artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: test Ion Data - path: testData + for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03 + do + java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n + done - - name: Benchmark ion-java from the new commit - run: | - mkdir -p benchmarkResults - cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults + - name: Upload test Ion Data to artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{env.test_data_id}} + path: testData - - name: Upload benchmark results to artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: Benchmark result - path: benchmarkResults + detect-regression: + name: Detect Regression + runs-on: ubuntu-latest + needs: generate-test-data + strategy: + matrix: + test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03'] + usage-patterns: [' read --api dom --format ion_binary --results-format ion ', ' read --api streaming --format ion_binary --results-format ion ', ' write --api streaming --format ion_binary --results-format ion ', ' read --api streaming --format json --results-format ion '] + fail-fast: false - - name: Clean maven dependencies repository - run : rm -r /home/runner/.m2 - - name: Checkout the current commit - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 + steps: + - name: Set up JDK 11 + uses: actions/setup-java@v3 with: - repository: amazon-ion/ion-java - ref: master - path: ion-java - - - name: Build ion-java from the previous commit - run: cd ion-java && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal + distribution: 'corretto' + java-version: 11 - - name: Build ion-java-benchmark-cli - run: cd ion-java-benchmark-cli && mvn clean install + - name: Checkout ion-java-benchmark-cli + uses: actions/checkout@v4 + with: + repository: amazon-ion/ion-java-benchmark-cli + ref: master + path: ion-java-benchmark-cli - - name: Check the version of ion-java - run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version + - name: Checkout ion-java from the previous commit + uses: actions/checkout@v4 + with: + ref: ${{ github.base_ref }} + path: baseline - - name: Create directories for test data and benchmark results - run: | - mkdir -p benchmarkResults - mkdir -p testData + - name: Checkout ion-java from the new commit. + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + path: new - name: Download test Ion Data from artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@v3 with: name: test Ion Data path: testData - - name: Download benchmark results of ion-java from the new commit from artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: Benchmark result - path: benchmarkResults + # Get the benchmark results from the base commit + - name: Build ion-java from the base commit + working-directory: baseline + run: | + git submodule init && git submodule update && ./gradlew clean publishToMavenLocal - - name: Benchmark ion-java from the previous commit and add the generated benchmark results to the existing directories - run: cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults + - name: Benchmark ion-java from the base commit + working-directory: ion-java-benchmark-cli + run: | + mvn clean install + java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} ${{matrix.test-data}} + - - name: Upload new benchmark results directory to artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: Benchmark result - path: benchmarkResults + - name: Build ion-java from the new commit + working-directory: new + run: | + git submodule init && git submodule update && ./gradlew clean publishToMavenLocal - - name: Detect performance regression - id: regression_result + - name: Benchmark ion-java from the new commit + working-directory: ion-java-benchmark-cli run: | - result=true - cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "no regression detected" ]; then result=false; fi; done - echo "::set-output name=regression-result::$result" - if [ "$result" = "true" ]; then echo "No regression detected!" >> $GITHUB_STEP_SUMMARY; fi + mvn clean install + java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} ./testData/${{matrix.test-data}} - - name: Upload comparison reports to the benchmark results directory - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + #Detect regression + + - name: Upload benchmark results to artifacts + uses: actions/upload-artifact@v3 with: name: Benchmark result path: benchmarkResults - - name: Fail the workflow if regression happened - env: - regression_detect: ${{steps.regression_result.outputs.regression-result}} - if: ${{ env.regression_detect == 'false' }} - run: | - cd benchmarkResults && echo "| Benchmark command | GC Allocation Rate | Heap Usage | Speed |" >> $GITHUB_STEP_SUMMARY && echo "| ----------- | ----------- |----------- | ----------- |" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion) && if [ "$regressionDetection" != "no regression detected" ]; then command=$(echo $FILE | sed "s/_/ /g") && read gc heapUsage speed <<< $( echo ${regressionDetection} | awk -F", " '{print $1" "$2" "$3}' ) && echo "|$command|$gc|$heapUsage|$speed|" >> $GITHUB_STEP_SUMMARY; fi; done - exit 1 \ No newline at end of file