Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor regression detection workflow. #42

Merged
merged 1 commit into from
Sep 29, 2023
Merged

Refactor regression detection workflow. #42

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 74 additions & 106 deletions .github/workflows/ion-java-performance-regression-detector.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,146 +5,114 @@ name: Ion Java performance regression detector

on:
pull_request:
paths:
- 'src/*'

permissions:
contents: read
env:
data_size: '50000'
test_data_id: 'generated-test-data'

jobs:
detect-regression:
name: Detect Regression
generate-test-data:
name: Generate Test Data
runs-on: ubuntu-latest

steps:
- name: Set up JDK 11
uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0
with:
distribution: 'corretto'
java-version: 11

- name: Checkout ion-java from the new commit.
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0
with:
ref: ${{ github.event.pull_request.head.sha }}
path: ion-java-new

- name: Build ion-java from the new commit
run: cd ion-java-new && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal

- name: Checkout ion-java-benchmark-cli
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0
with:
repository: amazon-ion/ion-java-benchmark-cli
ref: master
path: ion-java-benchmark-cli

- name: Build ion-java-benchmark-cli
run: cd ion-java-benchmark-cli && mvn clean install

- name: Checkout ion-data-generator
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0
uses: actions/checkout@v4
with:
repository: amazon-ion/ion-data-generator
ref: main
path: ion-data-generator

- name: Build ion-data-generator
run: cd ion-data-generator && mvn clean install
run: mvn clean install

- name: Check the version of ion-java.
run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version

- name: Generate test Ion Data
- name: Generate test data
env:
jar_file: target/ion-data-generator-1.0-SNAPSHOT.jar
schema_dir: tst/com/amazon/ion/workflow
run: |
mkdir -p testData
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedStruct.isl testData/testStructs.10n
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedList.isl testData/testLists.10n
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/sexp.isl testData/testSexps.10n
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema01.isl testData/realWorldData01.10n
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema02.isl testData/realWorldData02.10n
java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema03.isl testData/realWorldData03.10n

- name: Upload test Ion Data to artifacts
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: test Ion Data
path: testData
for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03
do
java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n
done

- name: Benchmark ion-java from the new commit
run: |
mkdir -p benchmarkResults
cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults
- name: Upload test Ion Data to artifacts
uses: actions/upload-artifact@v3
with:
name: ${{env.test_data_id}}
path: testData

- name: Upload benchmark results to artifacts
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: Benchmark result
path: benchmarkResults
detect-regression:
name: Detect Regression
runs-on: ubuntu-latest
needs: generate-test-data
strategy:
matrix:
test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03']
usage-patterns: [' read --api dom --format ion_binary --results-format ion ', ' read --api streaming --format ion_binary --results-format ion ', ' write --api streaming --format ion_binary --results-format ion ', ' read --api streaming --format json --results-format ion ']
fail-fast: false

- name: Clean maven dependencies repository
run : rm -r /home/runner/.m2

- name: Checkout the current commit
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0
steps:
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
repository: amazon-ion/ion-java
ref: master
path: ion-java

- name: Build ion-java from the previous commit
run: cd ion-java && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal
distribution: 'corretto'
java-version: 11

- name: Build ion-java-benchmark-cli
run: cd ion-java-benchmark-cli && mvn clean install
- name: Checkout ion-java-benchmark-cli
uses: actions/checkout@v4
with:
repository: amazon-ion/ion-java-benchmark-cli
ref: master
path: ion-java-benchmark-cli

- name: Check the version of ion-java
run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version
- name: Checkout ion-java from the previous commit
uses: actions/checkout@v4
with:
ref: ${{ github.base_ref }}
path: baseline

- name: Create directories for test data and benchmark results
run: |
mkdir -p benchmarkResults
mkdir -p testData
- name: Checkout ion-java from the new commit.
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
path: new

- name: Download test Ion Data from artifacts
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
uses: actions/download-artifact@v3
with:
name: test Ion Data
path: testData

- name: Download benchmark results of ion-java from the new commit from artifacts
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: Benchmark result
path: benchmarkResults
# Get the benchmark results from the base commit
- name: Build ion-java from the base commit
working-directory: baseline
run: |
git submodule init && git submodule update && ./gradlew clean publishToMavenLocal

- name: Benchmark ion-java from the previous commit and add the generated benchmark results to the existing directories
run: cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults
- name: Benchmark ion-java from the base commit
working-directory: ion-java-benchmark-cli
run: |
mvn clean install
java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} ${{matrix.test-data}}


- name: Upload new benchmark results directory to artifacts
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: Benchmark result
path: benchmarkResults
- name: Build ion-java from the new commit
working-directory: new
run: |
git submodule init && git submodule update && ./gradlew clean publishToMavenLocal

- name: Detect performance regression
id: regression_result
- name: Benchmark ion-java from the new commit
working-directory: ion-java-benchmark-cli
run: |
result=true
cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "no regression detected" ]; then result=false; fi; done
echo "::set-output name=regression-result::$result"
if [ "$result" = "true" ]; then echo "No regression detected!" >> $GITHUB_STEP_SUMMARY; fi
mvn clean install
java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} ./testData/${{matrix.test-data}}

- name: Upload comparison reports to the benchmark results directory
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
#Detect regression

- name: Upload benchmark results to artifacts
uses: actions/upload-artifact@v3
with:
name: Benchmark result
path: benchmarkResults

- name: Fail the workflow if regression happened
env:
regression_detect: ${{steps.regression_result.outputs.regression-result}}
if: ${{ env.regression_detect == 'false' }}
run: |
cd benchmarkResults && echo "| Benchmark command | GC Allocation Rate | Heap Usage | Speed |" >> $GITHUB_STEP_SUMMARY && echo "| ----------- | ----------- |----------- | ----------- |" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion) && if [ "$regressionDetection" != "no regression detected" ]; then command=$(echo $FILE | sed "s/_/ /g") && read gc heapUsage speed <<< $( echo ${regressionDetection} | awk -F", " '{print $1" "$2" "$3}' ) && echo "|$command|$gc|$heapUsage|$speed|" >> $GITHUB_STEP_SUMMARY; fi; done
exit 1
Loading