run_data_pipeline.yml

name: Run data pipeline
on:
  workflow_run:
    workflows: ["Increase Data Version"]
    types:
      - completed 
  workflow_dispatch: 

jobs:
  run-dvc-pipeline:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3  

      - name: Setup DVC
        uses: iterative/setup-dvc@v1 

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install python-box ensure pandas pathlib scikit-learn dagshub mlflow numpy

      - name: Run DVC pipeline | retrain model 
        env: 
          DAGSHUB_TOKEN: ${{ secrets.DAGSHUB_TOKEN }}
        run: |
          dvc repro data_update
          dvc repro data_validation
          dvc repro data_transformation
          dvc repro model_train
          dvc repro model_evaluation

      - name: Push updated model to DVC
        env:
            DVC_REMOTE_URL: ${{ secrets.DVC_REMOTE_URL }}
            DAGSHUB_TOKEN: ${{ secrets.DAGSHUB_TOKEN }}
            DAGSHUB_USERNAME: ${{ secrets.DAGSHUB_USERNAME }}
            DVC_ACCESS_KEY: ${{ secrets.DVC_ACCESS_KEY }}
        run: |
          dvc remote modify origin --local access_key_id $DVC_ACCESS_KEY
          dvc remote modify origin --local secret_access_key $DVC_ACCESS_KEY
          dvc push