Merge branch 'AL-python-serverless' into integration #759
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Backend Pipeline | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| - integration | |
| paths: | |
| - backend/** | |
| - .github/workflows/backend.yml | |
| pull_request: | |
| branches: | |
| - develop | |
| - integration | |
| paths: | |
| - backend/** | |
| - .github/workflows/backend.yml | |
| defaults: | |
| run: | |
| working-directory: ./backend | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: pip- | |
| - run: pip install -r requirements.txt | |
| - name: Lint with pylint | |
| run: pylint src/xfd_django | |
| test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '18' | |
| - name: Restore npm cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} | |
| restore-keys: ${{ runner.os }}-node- | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Run site locally | |
| run: | | |
| cp dev.env.example .env | |
| docker compose up -d db backend es | |
| python -m pip install wait-for-it | |
| wait-for-it --service localhost:3000 --timeout=10 | |
| wait-for-it --service localhost:5432 --timeout=10 | |
| wait-for-it --service localhost:9200 --timeout=10 | |
| wait-for-it --service localhost:9300 --timeout=10 | |
| working-directory: ./ | |
| - name: Sync database | |
| run: make syncdb | |
| - name: Package with Serverless | |
| run: npx sls package | |
| env: | |
| SLS_DEBUG: '*' | |
| - name: Run pytest | |
| run: make pytest | |
| test_worker: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '18' | |
| - name: Restore npm cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} | |
| restore-keys: ${{ runner.os }}-node- | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build | |
| run: npx webpack --config webpack.worker.config.js | |
| - name: Run db locally | |
| run: | | |
| cp dev.env.example .env | |
| docker compose up -d db | |
| npm install -g wait-port | |
| wait-port -t 3000 5432 | |
| working-directory: ./ | |
| - name: Test | |
| run: node dist/worker.bundle.js | |
| env: | |
| CROSSFEED_COMMAND_OPTIONS: '{"scanName": "test"}' | |
| DB_USERNAME: crossfeed | |
| DB_PASSWORD: password | |
| build_worker: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '18' | |
| - name: Restore npm cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} | |
| restore-keys: ${{ runner.os }}-node- | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build worker container | |
| run: npm run build-worker | |
| working-directory: ./backend | |
| deploy_staging: | |
| needs: [build_worker, lint, test, test_worker] | |
| runs-on: ubuntu-latest | |
| environment: staging | |
| concurrency: '1' | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/develop' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '18' | |
| - name: Restore npm cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} | |
| restore-keys: ${{ runner.os }}-node- | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Ensure domain exists | |
| run: npx sls create_domain --stage=staging | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| SLS_DEBUG: '*' | |
| - name: Deploy backend | |
| run: npx sls deploy --stage=staging | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| SLS_DEBUG: '*' | |
| - name: Deploy worker | |
| run: npm run deploy-worker-staging | |
| working-directory: backend | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - name: Run syncdb | |
| run: | | |
| aws lambda invoke --function-name crossfeed-staging-syncdb \ | |
| --region us-east-1 /dev/stdout | |
| working-directory: backend | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - name: Run syncmdl | |
| run: | | |
| aws lambda invoke --function-name crossfeed-staging-syncmdl \ | |
| --region us-east-1 /dev/stdout | |
| working-directory: backend | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| deploy_integration: | |
| needs: [] | |
| runs-on: ubuntu-latest | |
| environment: integration | |
| concurrency: '1' | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/integration' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '18' | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Restore npm cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} | |
| restore-keys: ${{ runner.os }}-node- | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Ensure domain exists | |
| run: npx sls create_domain --stage=integration | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| SLS_DEBUG: '*' | |
| - name: Deploy backend | |
| run: npx sls deploy --stage=integration | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| SLS_DEBUG: '*' | |
| - name: Deploy worker | |
| run: npm run deploy-worker-integration | |
| working-directory: backend | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - name: Run syncdb | |
| run: | | |
| aws lambda invoke --function-name crossfeed-integration-syncdb \ | |
| --region us-east-1 /dev/stdout | |
| working-directory: backend | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - name: Run syncmdl | |
| run: | | |
| aws lambda invoke --function-name crossfeed-integration-syncmdl \ | |
| --region us-east-1 /dev/stdout | |
| working-directory: backend | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} |