Skip to content

Auto Crawling Job

Auto Crawling Job #9

Workflow file for this run

name: Auto Crawling Job
on:
schedule:
- cron: 0 0 * * *
workflow_dispatch:
inputs:
logLevel:
description: 'Log level'
required: true
default: 'warning'
tags:
description: 'Test tags'
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: checkout repo content
uses: actions/checkout@v2 # checkout the repository content
- name: setup python
uses: actions/setup-python@v4
with:
python-version: '3.9' # install the python version needed
- name: install python packages
run: |
cd crawler
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: execute script # run main.py
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
QUEUE_URL: ${{ secrets.QUEUE_URL }}
QUEUE_NAME: ${{ secrets.QUEUE_NAME }}
DATABASE_URL: ${{ secrets.DATABASE_URL }}
run: |
cd crawler
python crawling.py