Nightly Publish #1207
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Publish | |
| on: | |
| schedule: | |
| - cron: '0 10 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: 'staging/snapshot, default is snapshot' | |
| required: true | |
| default: 'snapshot' | |
| jobs: | |
| build: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ${{ matrix.operating-system }} | |
| strategy: | |
| matrix: | |
| operating-system: [ macos-12, ubuntu-latest ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 11 | |
| # Enable gradle cache: https://github.com/actions/cache/blob/master/examples.md#java---gradle | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: check disk space | |
| run: df -h | |
| - name: Build with Gradle | |
| run: ./gradlew -Dnightly=true build :jacoco:testCodeCoverageReport | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: reports | |
| path: ${{ github.workspace }}/**/build/reports/**/* | |
| test-pytorch: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ${{ matrix.operating-system }} | |
| strategy: | |
| matrix: | |
| operating-system: [ macos-latest, ubuntu-latest, windows-latest ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 11 | |
| # Enable gradle cache: https://github.com/actions/cache/blob/master/examples.md#java---gradle | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Test with Gradle | |
| run: ./gradlew :integration:test "-Dai.djl.default_engine=PyTorch" | |
| test-tensorflow: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ${{ matrix.operating-system }} | |
| strategy: | |
| matrix: | |
| operating-system: [ macos-latest, ubuntu-latest, windows-latest ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 11 | |
| # Enable gradle cache: https://github.com/actions/cache/blob/master/examples.md#java---gradle | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Test with Gradle | |
| run: ./gradlew :integration:test "-Dai.djl.default_engine=TensorFlow" | |
| test-aarch64: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: [ self-hosted, aarch64 ] | |
| container: amazonlinux:2 | |
| timeout-minutes: 30 | |
| needs: create-runners | |
| steps: | |
| - name: Setup Environment | |
| run: | | |
| yum -y update | |
| yum install -y tar gzip | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: 11 | |
| distribution: corretto | |
| architecture: aarch64 | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Test with Gradle | |
| run: | | |
| ./gradlew :integration:test "-Dai.djl.default_engine=PyTorch" | |
| ./gradlew :integration:clean | |
| ./gradlew :integration:test "-Dai.djl.default_engine=OnnxRuntime" | |
| ./gradlew :integration:clean | |
| test-cuda-118: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: [ self-hosted, gpu ] | |
| container: | |
| image: nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu18.04 | |
| options: --gpus all --runtime=nvidia | |
| timeout-minutes: 30 | |
| needs: create-runners | |
| steps: | |
| - name: Setup Environment | |
| run: | | |
| apt-get update | |
| apt-get install -y software-properties-common wget locales libfontconfig1 | |
| locale-gen en_US.UTF-8 | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: 11 | |
| distribution: corretto | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Test with Gradle | |
| run: | | |
| ./gradlew :integration:test | |
| ./gradlew :engines:tensorflow:tensorflow-model-zoo:test | |
| ./gradlew :integration:clean :integration:test -Dai.djl.default_engine=PyTorch | |
| ./gradlew :integration:clean :integration:test -Dai.djl.default_engine=TensorFlow | |
| ./gradlew :engines:mxnet:mxnet-model-zoo:test | |
| ./gradlew :engines:pytorch:pytorch-model-zoo:test | |
| ./gradlew :engines:tensorflow:tensorflow-model-zoo:test | |
| ./gradlew :engines:tensorrt:test | |
| ./gradlew :engines:onnxruntime:onnxruntime-engine:test | |
| publish: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: ubuntu-latest | |
| needs: [ build, test-pytorch, test-tensorflow, test-aarch64, test-cuda-118 ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 11 | |
| # Enable gradle cache: https://github.com/actions/cache/blob/master/examples.md#java---gradle | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Publish to snapshot repository | |
| if: ${{ github.event.inputs.mode == '' || github.event.inputs.mode == 'snapshot' }} | |
| run: | | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.11.0 -Psnapshot | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.12.1 -Psnapshot | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.13.1 -Psnapshot | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.0.1 -Psnapshot | |
| ./gradlew clean engines:ml:xgboost:publish -Pgpu -Psnapshot | |
| ./gradlew clean publish -Psnapshot | |
| cd bom | |
| ./gradlew publish -Psnapshot | |
| env: | |
| ORG_GRADLE_PROJECT_signingKey: ${{ secrets.ORG_GRADLE_PROJECT_signingKey }} | |
| ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.ORG_GRADLE_PROJECT_signingPassword }} | |
| ORG_GRADLE_PROJECT_ossrhUsername: ${{ secrets.ORG_GRADLE_PROJECT_ossrhUsername }} | |
| ORG_GRADLE_PROJECT_ossrhPassword: ${{ secrets.ORG_GRADLE_PROJECT_ossrhPassword }} | |
| - name: Publish to staging repository | |
| if: ${{ github.event.inputs.mode == 'staging' }} | |
| run: | | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.11.0 -P${{ github.event.inputs.mode }} | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.12.1 -P${{ github.event.inputs.mode }} | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=1.13.1 -P${{ github.event.inputs.mode }} | |
| ./gradlew clean engines:pytorch:pytorch-jni:publish -Ppt_version=2.0.1 -P${{ github.event.inputs.mode }} | |
| ./gradlew clean engines:ml:xgboost:publish -Pgpu -P${{ github.event.inputs.mode }} | |
| ./gradlew clean publish -P${{ github.event.inputs.mode }} | |
| cd bom | |
| ./gradlew publish -P${{ github.event.inputs.mode }} | |
| env: | |
| ORG_GRADLE_PROJECT_signingKey: ${{ secrets.ORG_GRADLE_PROJECT_signingKey }} | |
| ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.ORG_GRADLE_PROJECT_signingPassword }} | |
| ORG_GRADLE_PROJECT_ossrhUsername: ${{ secrets.ORG_GRADLE_PROJECT_ossrhUsername }} | |
| ORG_GRADLE_PROJECT_ossrhPassword: ${{ secrets.ORG_GRADLE_PROJECT_ossrhPassword }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1-node16 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| create-runners: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: [ self-hosted, scheduler ] | |
| steps: | |
| - name: Create new Graviton instance | |
| id: create_aarch64 | |
| run: | | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \ | |
| https://api.github.com/repos/deepjavalibrary/djl/actions/runners/registration-token \ | |
| --fail \ | |
| | jq '.token' | tr -d '"' ) | |
| ./start_instance.sh action_graviton $token djl | |
| - name: Create new GPU instance 1 | |
| id: create_gpu_1 | |
| run: | | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \ | |
| https://api.github.com/repos/deepjavalibrary/djl/actions/runners/registration-token \ | |
| --fail \ | |
| | jq '.token' | tr -d '"' ) | |
| ./start_instance.sh action_gpu $token djl | |
| outputs: | |
| aarch64_instance_id: ${{ steps.create_aarch64.outputs.action_graviton_instance_id }} | |
| gpu_1_instance_id: ${{ steps.create_gpu_1.outputs.action_gpu_instance_id }} | |
| stop-runners: | |
| if: ${{ github.repository == 'deepjavalibrary/djl' && always() }} | |
| runs-on: [ self-hosted, scheduler ] | |
| needs: [ create-runners, test-aarch64, test-cuda-118 ] | |
| steps: | |
| - name: Stop all instances | |
| run: | | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| instance_id=${{ needs.create-runners.outputs.aarch64_instance_id }} | |
| ./stop_instance.sh $instance_id | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| instance_id=${{ needs.create-runners.outputs.gpu_1_instance_id }} | |
| ./stop_instance.sh $instance_id |