expiredDomains.yml

name: "Check Expired Domains From Filterlists"

on:
  workflow_dispatch:
    inputs:
      number_of_KAD_jobs:
        description: "Quantity of KAD jobs"
        required: true
        default: 4
      number_of_KADh_jobs:
        description: "Quantity of KADhosts jobs"
        required: true
        default: 2
  # schedule:
  #   - cron: "55 23 * * 6"

jobs:
  KAD_S:
    runs-on: ubuntu-latest
    container: ghcr.io/filtersheroes/expired_domains_image:latest
    outputs:
      KAD-strategy-matrix: ${{ steps.generate.outputs.KAD-strategy-matrix }}
      KADh-strategy-matrix: ${{ steps.generate.outputs.KADh-strategy-matrix }}
    env:
      NUMBER_OF_KAD_JOBS: ${{ github.event.inputs.number_of_kad_jobs }}
      NUMBER_OF_KADHOSTS_JOBS: ${{ github.event.inputs.number_of_KADh_jobs }}
    steps:
      - uses: actions/checkout@v4
      - run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          echo "$NUMBER_OF_KAD_JOBS"
          numberParts="$NUMBER_OF_KAD_JOBS" ./scripts/CI/CI_LETS_DO_THIS.sh KAD
          numberParts="$NUMBER_OF_KADHOSTS_JOBS" ./scripts/CI/CI_LETS_DO_THIS.sh KADhosts
      - uses: actions/upload-artifact@v3
        with:
          name: E_KAD_S
          path: |
            split/KAD_*
      - uses: actions/upload-artifact@v3
        with:
          name: E_KADH_S
          path: |
            split/KADhosts_*
      - id: generate
        run: |
          KAD_STRATEGY_MATRIX=$(node -e "let r=[]; for(let i = 1; i <= process.env.NUMBER_OF_KAD_JOBS; i++) { r.push(i) }; console.log(JSON.stringify(r));")
          echo "KAD-strategy-matrix=$KAD_STRATEGY_MATRIX" >> $GITHUB_OUTPUT
          KADH_STRATEGY_MATRIX=$(node -e "let r=[]; for(let i = 1; i <= process.env.NUMBER_OF_KADHOSTS_JOBS; i++) { r.push(i) }; console.log(JSON.stringify(r));")
          echo "KADh-strategy-matrix=$KADH_STRATEGY_MATRIX" >> $GITHUB_OUTPUT
  KAD:
    needs: KAD_S
    runs-on: ubuntu-latest
    container: ghcr.io/filtersheroes/expired_domains_image:latest
    strategy:
      matrix:
        KAD: ${{ fromJson(needs.KAD_S.outputs.KAD-strategy-matrix) }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with:
          name: E_KAD_S
          path: split
      - run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          KADfile=KAD_"$(./scripts/CI/getFileNumber.sh ${{ matrix.KAD }} )"
          ./scripts/CI/CI_LETS_DO_THIS.sh "$KADfile"
          echo 'E_KADS<<EOF' >> $GITHUB_ENV
          echo "$(./scripts/CI/getFileListForED.sh $KADfile)" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
          echo "currentKADName=E_$KADfile" >> $GITHUB_ENV
      - uses: actions/upload-artifact@v3
        with:
          name: ${{ env.currentKADName }}
          path: |
            ${{ env.E_KADS }}
  KADhosts:
    needs: KAD_S
    runs-on: ubuntu-latest
    container: ghcr.io/filtersheroes/expired_domains_image:latest
    strategy:
      matrix:
        KADH: ${{ fromJson(needs.KAD_S.outputs.KADh-strategy-matrix) }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with:
          name: E_KADH_S
          path: split
      - run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          KADHfile=KADhosts_"$(./scripts/CI/getFileNumber.sh ${{ matrix.KADH }} )"
          ./scripts/CI/CI_LETS_DO_THIS.sh "$KADHfile"
          echo 'E_KADHS<<EOF' >> $GITHUB_ENV
          echo "$(./scripts/CI/getFileListForED.sh $KADHfile)" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
          echo "currentKADHName=E_$KADHfile" >> $GITHUB_ENV
      - uses: actions/upload-artifact@v3
        with:
          name: ${{ env.currentKADHName }}
          path: |
            ${{ env.E_KADHS }}
  PAF:
    runs-on: ubuntu-latest
    container: ghcr.io/filtersheroes/expired_domains_image:latest
    steps:
      - uses: actions/checkout@v4
      - run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          ./scripts/CI/CI_LETS_DO_THIS.sh PAF_C
          echo 'E_PAF<<EOF' >> $GITHUB_ENV
          echo "$(./scripts/CI/getFileListForED.sh \
          PAF_arrows \
          PAF_backgrounds_self-advertising \
          PAF_contact_feedback_widgets \
          PAF_e_newspaper \
          PAF_newsletters \
          PAF_other_widgets \
          PAF_pop-ups \
          PAF_push \
          PAF_scrolling_videos \
          PAF_backgrounds_self-adv_supp \
          PAF_comeback_titles \
          PAF_contact_feedback_widgets_supp \
          PAF_newsletters_supp \
          PAF_other_elements_supp \
          PAF_pop-ups_supp \
          PAF_push_supp \
          PAF_scrolling_videos_supp \
          PAF_tagged_internal_links)" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@v3
        with:
          name: E_PAF
          path: |
            ${{ env.E_PAF }}
  SocialPASS:
    runs-on: ubuntu-latest
    container: ghcr.io/filtersheroes/expired_domains_image:latest
    steps:
      - uses: actions/checkout@v4
      - run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          ./scripts/CI/CI_LETS_DO_THIS.sh Social_C
          ./scripts/CI/CI_LETS_DO_THIS.sh PASS
          echo 'E_SOCIAL<<EOF' >> $GITHUB_ENV
          echo "$(./scripts/CI/getFileListForED.sh polish_rss_filters polish_rss_filters_supp adblock_social_list social_filters_uB_AG)" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@v3
        with:
          name: E_SOCIAL
          path: |
            ${{ env.E_SOCIAL }}
  Cookies:
    runs-on: ubuntu-latest
    container: ghcr.io/filtersheroes/expired_domains_image:latest
    steps:
      - uses: actions/checkout@v4
      - run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          ./scripts/CI/CI_LETS_DO_THIS.sh Cookies_C
          echo 'E_COOKIES<<EOF' >> $GITHUB_ENV
          echo "$(./scripts/CI/getFileListForED.sh adblock_cookies cookies_uB_AG)" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@v3
        with:
          name: E_COOKIES
          path: |
            ${{ env.E_COOKIES }}
  KAD_limit:
    needs: [KAD]
    runs-on: ubuntu-latest
    container: ghcr.io/filtersheroes/expired_domains_image:latest
    env:
      NUMBER_OF_KAD_JOBS: ${{ github.event.inputs.number_of_kad_jobs }}
    steps:
      - uses: actions/checkout@v4
      - run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          ./scripts/CI/getArtifactNamesForED.py KAD
          npm install @actions/artifact
      - uses: actions/github-script@v6
        continue-on-error: true
        with:
          script: |
            const artifact = require("@actions/artifact")
            const artifactClient = artifact.create()
            const artifactNames = process.env.E_KAD_NAMES.split(" ")
            for (artifactName of artifactNames) {
              await artifactClient.downloadArtifact(artifactName, "./expired-domains", { createArtifactFolder: false })
            }
      - run: |
          ./scripts/CI/CI_LETS_DO_THIS_KAD_limit.sh
          echo 'E_KAD<<EOF' >> $GITHUB_ENV
          echo "$(./scripts/CI/getFileListForED.sh KAD)" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@v3
        with:
          name: E_KAD
          path: |
            ${{ env.E_KAD }}
  deploy:
    needs: [KAD_limit, KADhosts, PAF, SocialPASS, Cookies]
    runs-on: ubuntu-latest
    container: ghcr.io/filtersheroes/expired_domains_image:latest
    env:
      NUMBER_OF_KADHOSTS_JOBS: ${{ github.event.inputs.number_of_KADh_jobs }}
    steps:
      - uses: actions/checkout@v4
      ## See https://github.com/actions/download-artifact/issues/68
      ## Unfortunately currently it's not possible to skip creating extra directory
      ## when we want to download all artifacts at once without specyfing name.
      ## Bad joke...
      - continue-on-error: true
        uses: actions/download-artifact@v3
        with:
          name: E_KAD
          path: expired-domains
      - uses: actions/checkout@v4
      - run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          ./scripts/CI/getArtifactNamesForED.py KADH
          npm install @actions/artifact
      - uses: actions/github-script@v6
        continue-on-error: true
        with:
          script: |
            const artifact = require("@actions/artifact")
            const artifactClient = artifact.create()
            const artifactNames = process.env.E_KADH_NAMES.split(" ")
            for (artifactName of artifactNames) {
              await artifactClient.downloadArtifact(artifactName, "./expired-domains", { createArtifactFolder: false })
            }
      - continue-on-error: true
        uses: actions/download-artifact@v3
        with:
          name: E_PAF
          path: expired-domains
      - continue-on-error: true
        uses: actions/download-artifact@v3
        with:
          name: E_SOCIAL
          path: expired-domains
      - continue-on-error: true
        uses: actions/download-artifact@v3
        with:
          name: E_COOKIES
          path: expired-domains
      - run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          ./scripts/CI/cleanup_push.sh