scrape.yml

name: Scrape latest data

on:
  push:
  workflow_dispatch:
  schedule:
    - cron:  '6,26,46 * * * *'

jobs:
  scheduled:
    runs-on: ubuntu-latest
    steps:
    - name: Check out this repo
      uses: actions/checkout@v2
      with:
          fetch-depth: 2
    - name: Test with pytest
      env: # Or as an environment variable
        QRZ_PSWD: ${{ secrets.QRZ_PSWD }}
        MAPS_API_KEY: ${{ secrets.MAPS_API_KEY }}        
      run: |-
        pip install pytest
        pytest 
    - name: Fetch latest data
      env: # Or as an environment variable
        QRZ_PSWD: ${{ secrets.QRZ_PSWD }}
        MAPS_API_KEY: ${{ secrets.MAPS_API_KEY }}        
      run: |-
        echo "{" > incidents.json
        curl 'https://www.reversebeacon.net/spots.php?h=4f6ae8&cdx=KD0F*&s=0&r=100' | jq '.spots[]|= .[0:6]|.spots ' | sed '/\}/d' | sed '/{/d' >> incidents.json
        echo "," >> incidents.json
        curl 'https://www.reversebeacon.net/spots.php?h=4f6ae8&cdx=KO6B*&s=0&r=100' | jq '.spots[]|= .[0:6]|.spots ' | sed '/\}/d' | sed '/{/d' >> incidents.json
        echo "}" >> incidents.json
        echo "{" > stations_geo.json
        curl 'https://www.reversebeacon.net/spots.php?h=4f6ae8&cdx=KD0F*&s=0&r=100' | jq '.call_info ' | sed '/\}/d' | sed '/{/d' >> stations_geo.json
        echo "," >> stations_geo.json
        curl 'https://www.reversebeacon.net/spots.php?h=4f6ae8&cdx=KO6B*&s=0&r=100' | jq '.call_info ' | sed '/\}/d' | sed '/{/d' >> stations_geo.json
        echo "}" >> stations_geo.json
        python kepler_geo.py > rm_rnb_history.csv
        git config pager.diff false
        cat rm_rnb_history.csv | python nkc.py >> rm_rnb_history_pres.csv
        python map_qso.py -hh >>  rm_rnb_history_pres.csv
        echo "" > qso_locs.txt
        cat qso_update_template.csv > qso_update.csv
    - name: Commit and push if it changed
      run: |-
        git config user.name "Automated"
        git config user.email "actions@users.noreply.github.com"
        git add -A
        timestamp=$(date -u)
        git commit -m "Latest data: ${timestamp}" || exit 0
        git push