main.yml

name: Python Continuous Integration
on:
  push:
    branches: [ "main" ]
  pull_request:
    branches: [ "main" ]


env:
  PNPM_VERSION: 8.5.1
  PYTHON_VERSION: 3.10.13
  NODE_VERSION: 18
  AZURE_RESOURCE_GROUP_NAME: "azure-ml-yolo"
  AZURE_LOCATION: "northeurope"
  AZURE_ML_WORKSPACE_NAME: "cats-dogs-yola"
  AZURE_WEBAPP_NAME: "cats-dogs-yola"
  DELETE_WEBAPP: "false"
  DOCKER_IMAGE_NAME: "guillaumechervetlunique/mlopspython"

permissions:
  id-token: write
  contents: write
jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - uses: dorny/paths-filter@v2
      id: filter_python
      with:
        filters: |
          packages:
            - 'packages/**'
          production:
            - 'production/**'
          train:
            - 'train/**'
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Install Poetry dependencies
      run: |
        python -m pip install --upgrade pip
        pip install --user poetry
    - name: Format with Black
      run: |
        poetry install
        poetry run black train
        poetry run black train --check
    - name: Lint with flake8
      run: |
        poetry install
        poetry run flake8 .
  packages_unit_tests:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install --user poetry
    - name: Run unit tests Packages Extraction
      working-directory: packages/extraction
      run: |
        poetry install
        poetry run coverage run -m unittest tests.extraction_tests
        poetry run coverage report
    - name: Run unit tests Packages Inference
      working-directory: packages/inference
      run: |
        poetry install
        #poetry run coverage run -m unittest tests.inference_tests
        #poetry run coverage report
  train_unit_tests:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install --user poetry
    - name: Build packages
      run: |
        chmod +x ./Makefile
        ./Makefile
    - name: Run unit tests Extraction
      working-directory: train/extraction
      run: |
        poetry lock
        poetry install
        #poetry run coverage run -m unittest tests.extraction_test
        #poetry run coverage report
    - name: Run unit tests Label Split Data
      working-directory: train/label_split_data
      run: |
        poetry lock
        poetry install 
        poetry run coverage run -m unittest tests.label_split_data_test
        poetry run coverage report
    #- name: Run unit tests Train
    #  working-directory: train/train
    #  run: |
    #    poetry install --dev
    #    poetry run python -m unittest tests.train_test
    #- name: Run unit tests Evaluate
    #  working-directory: train/test
    #  run: |
    #    poetry install --dev
    #    poetry run python -m unittest tests.evaluate_test
        
  tags:
    runs-on: ubuntu-latest
    needs: [train_unit_tests, packages_unit_tests]
    outputs:
      new_version: ${{ steps.tag.outputs.new_version }}
    steps:
      - uses: actions/checkout@v3
      - name: Bump version and push tag
        id: tag_version
        if: github.ref == 'refs/heads/main'
        uses: mathieudutour/github-tag-action@v6.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN  }}
      - name: Add tag to output step for main branch
        id: tag
        run: |
          if [ '${{ github.ref }}' = 'refs/heads/main' ]; then
            echo "new_version=${{ steps.tag_version.outputs.new_version }}" >> $GITHUB_OUTPUT
          else
            echo "new_version=pr-${{ github.event.number }}-${{ github.run_number }}" >> $GITHUB_OUTPUT
          fi
  train:
    runs-on: ubuntu-latest
    environment: MLOpsPython
    needs: tags
    outputs:
      MODEL_VERSION: ${{ steps.train.outputs.MODEL_VERSION }}
      INTEGRATION_DATASET_VERSION: ${{ steps.train.outputs.INTEGRATION_DATASET_VERSION }}
      EXPERIMENT_ID: ${{ steps.train.outputs.EXPERIMENT_ID }}
      AZURE_RESOURCE_GROUP_NAME: ${{ env.AZURE_RESOURCE_GROUP_NAME }}
      AZURE_ML_WORKSPACE_NAME: ${{ env.AZURE_ML_WORKSPACE_NAME }}
      DOCKER_IMAGE_NAME: ${{ env.DOCKER_IMAGE_NAME }}
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install --user poetry
    - name: Build packages
      run: |
        chmod +x ./Makefile
        ./Makefile
    - name: azure login
      uses: azure/login@v1
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    - name: Run Train Pipeline
      run: |
          poetry install
          az extension add -n ml
          az group create --name ${{ env.AZURE_RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }}
          az ml workspace create -n ${{ env.AZURE_ML_WORKSPACE_NAME }} -g ${{ env.AZURE_RESOURCE_GROUP_NAME }} 
          cwd=$(pwd)
          cd ./extraction
          chmod +x ./init_dataset.sh 
          ./init_dataset.sh ${{ env.AZURE_RESOURCE_GROUP_NAME }} ${{ env.AZURE_ML_WORKSPACE_NAME }} 
          cd $cwd
          cd ./label_split_data
          chmod +x ./init_dataset.sh
          ./init_dataset.sh ${{ env.AZURE_RESOURCE_GROUP_NAME }} ${{ env.AZURE_ML_WORKSPACE_NAME }}
          cd $cwd
          #poetry run python azureml_run_test.py > train_output.txt
          poetry run python azureml_run_pipeline.py \
            --subscription_id ${{ secrets.AZURE_SUBSCRIPTION_ID }}  \
            --resource_group_name ${{ env.AZURE_RESOURCE_GROUP_NAME }} \
            --workspace_name ${{ env.AZURE_ML_WORKSPACE_NAME }} \
            --location ${{ env.AZURE_LOCATION }} \
            --tags "{\"triggering_actor\":\"${{github.triggering_actor}}\"}" \
            > train_output.txt
          cat train_output.txt
      working-directory: train
    - name: download model
      id: train
      run: |
          az extension add -n ml
          MODEL_VERSION=$(python bin/retrieve_output.py ./train/train_output.txt model_version)
          echo "MODEL_VERSION=$MODEL_VERSION" >> $GITHUB_OUTPUT
          INTEGRATION_DATASET_VERSION=$(python bin/retrieve_output.py ./train/train_output.txt integration_dataset_version)
          echo "INTEGRATION_DATASET_VERSION=$INTEGRATION_DATASET_VERSION" >> $GITHUB_OUTPUT
          EXPERIMENT_ID=$(python bin/retrieve_output.py ./train/train_output.txt experiment_id)
          echo "EXPERIMENT_ID=$EXPERIMENT_ID" >> $GITHUB_OUTPUT
        
          mkdir model
          rm -r model
          mkdir model
          cd model
          az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME }} --workspace-name ${{ env.AZURE_ML_WORKSPACE_NAME }}
          # find files recursively and copy them to the current directory root
          find ./ -name '*.keras' -exec cp "{}" ./  \;
          find ./ -name '*.json' -exec cp "{}" ./  \;
          rm -r ./cats-dogs-others
    - name: Upload Model Build Artifact
      uses: actions/upload-artifact@v2
      with:
        name: Publish model
        path: ./model
    - name: release
      uses: actions/create-release@v1
      id: create_release
      if: github.ref == 'refs/heads/main'
      with:
        draft: false
        prerelease: false
        body: |
          ${{ needs.tags.outputs.changelog }}
        release_name: "v${{ needs.tags.outputs.new_version }}"
        tag_name: "v${{ needs.tags.outputs.new_version }}"
      env:
        GITHUB_TOKEN: ${{ github.token }}
    - name: Zip Release
      uses: TheDoctor0/zip-release@0.6.0
      with:
        filename: mlopspython_model.zip
        path: .
        directory: ./model
    - name: upload artifact
      if: github.ref == 'refs/heads/main'
      uses: actions/upload-release-asset@v1
      env:
        GITHUB_TOKEN: ${{ github.token }}
      with:
        upload_url: ${{ steps.create_release.outputs.upload_url }}
        asset_path: ./model/mlopspython_model.zip
        asset_name: mlopspython_model.zip
        asset_content_type: application/zip
  build_docker:
    needs: [train, tags]
    uses: ./.github/workflows/docker.yml
    with:
      image_name: ${{ needs.train.outputs.DOCKER_IMAGE_NAME }}
      image_version: ${{ needs.tags.outputs.new_version }}
      image_build_args: ""
      image_context: ./production/api
      image_file: "./production/api/Dockerfile"
      model_version: ${{ needs.train.outputs.MODEL_VERSION }}
      resource_group: ${{ needs.train.outputs.AZURE_RESOURCE_GROUP_NAME }}
      workspace_name: ${{ needs.train.outputs.AZURE_ML_WORKSPACE_NAME }}
    secrets:
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
  deploy:
    environment: MLOpsPython
    runs-on: ubuntu-latest
    needs: [tags, build_docker]
    steps:
    - name: azure login
      uses: azure/login@v1
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    - name: Deploy container
      run: |
        # https://learn.microsoft.com/en-us/cli/azure/container?view=azure-cli-latest#az-container-create()
        az container create --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --dns-name-label ${{ env.AZURE_WEBAPP_NAME }} --image docker.io/${{ env.DOCKER_IMAGE_NAME }}:${{ needs.tags.outputs.new_version }} --ports 5000
  integration_tests:
    environment: MLOpsPython
    runs-on: ubuntu-latest
    needs: [deploy, train]
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ env.PYTHON_VERSION }}
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install --user poetry
    - name: azure login
      uses: azure/login@v1
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    - name: Run Integration Test
      working-directory: production/integration
      run: |
        poetry install
        az extension add -n ml
        poetry run python azureml_run_pipeline.py \
            --subscription_id ${{ env.AZURE_SUBSCRIPTION_ID }}  \
            --resource_group_name ${{ env.AZURE_RESOURCE_GROUP_NAME }} \
            --workspace_name ${{ env.AZURE_ML_WORKSPACE_NAME }} \
            --experiment_id ${{ needs.train.outputs.EXPERIMENT_ID }} \
            --integration_dataset_name "cats-dogs-others-integration" \
            --integration_dataset_version ${{ needs.train.outputs.INTEGRATION_DATASET_VERSION }} \
            --url "http://${{ env.AZURE_WEBAPP_NAME }}.${{ env.AZURE_LOCATION }}.azurecontainer.io:5000/upload" > integration_output.txt 
        cat integration_output.txt
    - name: Delete Environment
      run: |
          if [ "${{ env.DELETE_WEBAPP }}" = "true" ]; then
            az container delete --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --yes
          fi
  webapp:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - uses: dorny/paths-filter@v2
      id: filter_webapp
      with:
        filters: |
          webapp:
            - 'webapp/**'
    - uses: actions/setup-node@v2
      with:
        node-version: '18'
    - name: Build webapp
      working-directory: production/webapp
      run: |
        npm ci
        npm run build
    - name: Upload WebApp Build Artifact
      uses: actions/upload-artifact@v2
      with:
        name: Publish webapp
        path: ./production/webapp/build