From 43c5c7e1d5f9257319b2df4124ada9edb2c428f4 Mon Sep 17 00:00:00 2001 From: Jarell <91372088+jarelllama@users.noreply.github.com> Date: Sat, 30 Mar 2024 03:39:01 +0800 Subject: [PATCH] Update --- .github/workflows/build_deploy.yml | 9 ++++++-- .github/workflows/update_readme.yml | 2 +- functions/build_lists.sh | 12 +++++----- functions/check_parked.sh | 36 ++++++++++++++--------------- functions/retrieve_domains.sh | 9 ++++---- functions/test_functions.sh | 14 ++++------- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.github/workflows/build_deploy.yml b/.github/workflows/build_deploy.yml index 58e19a7b5..75d530b71 100644 --- a/.github/workflows/build_deploy.yml +++ b/.github/workflows/build_deploy.yml @@ -51,10 +51,10 @@ jobs: deploy: needs: validate - if: ${{ always() && needs.validate.result == 'success' }} # Deploy only if validate passes + if: ${{ success() || failure() }} uses: ./.github/workflows/build_lists.yml - prune_logs: + prune-logs: needs: deploy if: ${{ success() || failure() }} runs-on: ubuntu-latest @@ -77,3 +77,8 @@ jobs: git add . git diff-index --quiet HEAD || git commit -m "Prune logs" git push -q + + update-readme: + needs: prune-logs + uses: ./.github/workflows/update_readme.yml + if: ${{ always() && needs.deploy.result == 'success' }} # Deploy only if deploy passes \ No newline at end of file diff --git a/.github/workflows/update_readme.yml b/.github/workflows/update_readme.yml index e669269b4..cf431807d 100644 --- a/.github/workflows/update_readme.yml +++ b/.github/workflows/update_readme.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: workflow_call: workflow_run: - workflows: [Build and deploy, Build lists] + workflows: Build lists types: - completed push: diff --git a/functions/build_lists.sh b/functions/build_lists.sh index 5e271b280..120474d65 100644 --- a/functions/build_lists.sh +++ b/functions/build_lists.sh @@ -10,7 +10,7 @@ function main { build_wildcard_domains } -function build_list { +function build_lists { [[ -z "$comment" ]] && comment='#' # Set default comment to '#' mkdir -p "lists/${directory}" # Create directory if not present @@ -43,27 +43,27 @@ function format_list { function build_adblock { syntax='Adblock Plus' && directory='adblock' && comment='!' && before='||' && after='^' - build_list + build_lists } function build_dnsmasq { syntax='Dnsmasq' && directory='dnsmasq' && comment='' && before='local=/' && after='/' - build_list + build_lists } function build_unbound { syntax='Unbound' && directory='unbound' && comment='' && before='local-zone: "' && after='." always_nxdomain' - build_list + build_lists } function build_wildcard_asterisk { syntax='Wildcard Asterisk' && directory='wildcard_asterisk' && comment='' && before='*.' && after='' - build_list + build_lists } function build_wildcard_domains { syntax='Wildcard Domains' && directory='wildcard_domains' && comment='' && before='' && after='' - build_list + build_lists } main diff --git a/functions/check_parked.sh b/functions/check_parked.sh index 153f18cea..01815ab50 100644 --- a/functions/check_parked.sh +++ b/functions/check_parked.sh @@ -23,13 +23,13 @@ function remove_parked_domains { # Split into 12 equal files split -d -l $(($(wc -l < "$raw_file")/12)) "$raw_file" - check_for_parked "x00" & check_for_parked "x01" & - check_for_parked "x02" & check_for_parked "x03" & - check_for_parked "x04" & check_for_parked "x05" & - check_for_parked "x06" & check_for_parked "x07" & - check_for_parked "x08" & check_for_parked "x09" & - check_for_parked "x10" & check_for_parked "x11" & - check_for_parked "x12" & check_for_parked "x13" + check_parked "x00" & check_parked "x01" & + check_parked "x02" & check_parked "x03" & + check_parked "x04" & check_parked "x05" & + check_parked "x06" & check_parked "x07" & + check_parked "x08" & check_parked "x09" & + check_parked "x10" & check_parked "x11" & + check_parked "x12" & check_parked "x13" wait [[ ! -s parked_domains.tmp ]] && return @@ -47,13 +47,13 @@ function add_unparked_domains { # Split into 12 equal files split -d -l $(($(wc -l < "$parked_domains_file")/12)) "$parked_domains_file" - check_for_unparked "x00" & check_for_unparked "x01" & - check_for_unparked "x02" & check_for_unparked "x03" & - check_for_unparked "x04" & check_for_unparked "x05" & - check_for_unparked "x06" & check_for_unparked "x07" & - check_for_unparked "x08" & check_for_unparked "x09" & - check_for_unparked "x10" & check_for_unparked "x11" & - check_for_unparked "x12" & check_for_unparked "x13" + check_unparked "x00" & check_unparked "x01" & + check_unparked "x02" & check_unparked "x03" & + check_unparked "x04" & check_unparked "x05" & + check_unparked "x06" & check_unparked "x07" & + check_unparked "x08" & check_unparked "x09" & + check_unparked "x10" & check_unparked "x11" & + check_unparked "x12" & check_unparked "x13" wait [[ ! -s unparked_domains.tmp ]] && return @@ -67,13 +67,13 @@ function add_unparked_domains { find . -maxdepth 1 -type f -name "x??" -delete # Reset split files before next run } -function check_for_parked { +function check_parked { [[ ! -f "$1" ]] && return # Return if split file not found [[ "$1" == 'x00' ]] && { track=true; count=0; } || track=false # Track progress for first split file while read -r domain; do ((count++)) # Check for parked message in site's HTML - if grep -qiFf "$parked_terms_file" <<< "$(curl -sL --max-time 2 "http://${domain}/" | tr -d '\0')"; then + if grep -qiaFf "$parked_terms_file" <<< "$(curl -sL --max-time 2 "http://${domain}/")"; then printf "Parked: %s\n" "$domain" printf "%s\n" "$domain" >> "parked_domains_${1}.tmp" fi @@ -85,13 +85,13 @@ function check_for_parked { [[ -f "parked_domains_${1}.tmp" ]] && cat "parked_domains_${1}.tmp" >> parked_domains.tmp } -function check_for_unparked { +function check_unparked { [[ ! -f "$1" ]] && return # Return if split file not found [[ "$1" == 'x00' ]] && { track=true; count=0; } || track=false # Track progress for first split file while read -r domain; do ((count++)) # Check for parked message in site's HTML - if ! grep -qiFf "$parked_terms_file" <<< "$(curl -sL --max-time 5 "http://${domain}/" | tr -d '\0')"; then + if ! grep -qiaFf "$parked_terms_file" <<< "$(curl -sL --max-time 5 "http://${domain}/")"; then printf "Unparked: %s\n" "$domain" printf "%s\n" "$domain" >> "unparked_domains_${1}.tmp" fi diff --git a/functions/retrieve_domains.sh b/functions/retrieve_domains.sh index 916a1562d..1063d4538 100644 --- a/functions/retrieve_domains.sh +++ b/functions/retrieve_domains.sh @@ -28,9 +28,8 @@ function main { } function source { - printf "\n" # Check for existing pending domains file - [[ -d data/pending ]] && { use_pending=true; printf "Using existing lists of retrieved domains.\n\n"; } + [[ -d data/pending ]] && { use_pending=true; printf "\nUsing existing lists of retrieved domains.\n"; } [[ -f data/pending/domains_manual.tmp ]] && source_manual # Retrieve manually added domains mkdir -p data/pending source_aa419 @@ -196,6 +195,7 @@ function process_source { [[ -z "$rate_limited" ]] && rate_limited=false [[ -z "$ignore_from_light" ]] && ignore_from_light=false + printf "\n" [[ ! -f "$domains_file" ]] && return # Return if domains file does not exist ! grep -q '[[:alnum:]]' "$domains_file" && { log_source; return; } # Skip to next source if no results retrieved @@ -285,7 +285,7 @@ function process_source { fi total_whitelisted_count=$((whitelisted_count + whitelisted_tld_count)) # Calculate sum of whitelisted domains - filtered_count=$(tr -s '\n' <<< "$pending_domains" | wc -w) # Count number of domains after filtering + filtered_count=$(printf "%s" "$pending_domains" | sed '/^$/d' | wc -w) # Count number of domains after filtering printf "%s\n" "$pending_domains" >> retrieved_domains.tmp # Collate filtered domains [[ "$ignore_from_light" != true ]] && printf "%s\n" "$pending_domains" >> retrieved_light_domains.tmp # Collate filtered domains from light sources log_source @@ -325,8 +325,7 @@ function build { format_list "$raw_file" log_event "$(