️️⚡️ SYNC ⚡️ 2024/09/21 07:34

.cron/Justfile

@@ -0,0 +1,7 @@
+root := `git rev-parse --show-toplevel`
+
+default:
+    pstats memo --ttl=60s -- FORCE_COLOR=1 {{root}}/.cron/jobs/abuseipdb/cron
+    hr
+    pstats memo --ttl=60s -- {{root}}/.cron/jobs/abuseipdb/aggregate
+

.cron/jobs/abuseipdb/aggregate

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Setup
+cd "$(dirname $0)"
+GIT_ROOT=$(git rev-parse --show-toplevel)
+
+. $GIT_ROOT/.cron/scripts/ciutil
+
+DB_PATH=$GIT_ROOT/db
+README_PATH=$GIT_ROOT/README.md
+
+DATE=$(date +%F)
+DATE_DIR=$DB_PATH/$DATE
+LATEST="$DATE_DIR/$DATE.ipv4"
+
+aggregate() {
+  local DAYS=$(expr $1 - 1)
+  local OUTPUT=$2
+  cd $DB_PATH
+  ___
+  echo "ℹ $OUTPUT"; echo
+
+  args=()
+
+  for i in $(seq 0 $DAYS); do
+    day=$(date +%Y-%m-%d -d "$(date) - $i days")
+    file=$day/$day.ipv4
+
+    if [[ -f $file ]]; then
+      args+=("$file")
+      # echo "- $(basename $file)"
+        echo "-  $file (`wc -l < $file` ip)"
+    else
+      echo "❌ $file does not exist"
+    fi
+
+  done
+
+  iprange "${args[@]}" --print-single-ips >| $GIT_ROOT/$OUTPUT.tmp
+
+  TS=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
+  echo "#" >| $GIT_ROOT/$OUTPUT
+  echo "# Aggregated Blocklist for AbuseIPDB: A list of the most reported IP addresses." >> $GIT_ROOT/$OUTPUT
+  echo "#" >> $GIT_ROOT/$OUTPUT
+  echo "# Last updated:        $TS" >> $GIT_ROOT/$OUTPUT
+  echo "# Confidence level:    ~100%" >> $GIT_ROOT/$OUTPUT
+  echo "# Filename:            $OUTPUT" >> $GIT_ROOT/$OUTPUT
+  echo "# Number of ips:       $(wc -l < $GIT_ROOT/$OUTPUT.tmp)" >> $GIT_ROOT/$OUTPUT
+  echo "#" >> $GIT_ROOT/$OUTPUT
+  echo "# Source:              https://github.com/borestad/blocklist-abuseipdb" >> $GIT_ROOT/$OUTPUT
+  echo "# Credits:             https://www.abuseipdb.com - please support them!" >> $GIT_ROOT/$OUTPUT
+  echo "#" >> $GIT_ROOT/$OUTPUT
+
+  cat $GIT_ROOT/$OUTPUT.tmp >> $GIT_ROOT/$OUTPUT
+  echo
+  echo "Total: (`wc -l < $GIT_ROOT/$OUTPUT.tmp` ip)"
+
+  rm -f $GIT_ROOT/$OUTPUT.tmp
+}
+
+update-stats() {
+  echo "✨ Update footer"
+
+  # Delete everything below placeholder
+  sed -i '/ABUSEIPDB-STATS-PLACEHOLDER/q' $README_PATH
+
+  update=$(date -u '+%Y-%m-%d - %H:%M:%S')
+  echo "Last check: \`$update\` (UTC)" >> $README_PATH
+
+  echo '```' >> $README_PATH
+
+  cd $GIT_ROOT && find . -mindepth 1 -maxdepth 1 -iname '*.ipv4' -print0 | sort -zV | xargs -I {} -0 sh -c 'name=$(basename {}); echo "$name ($(wc -l < $name) ip)"' >> $README_PATH
+  echo '```' >> $README_PATH
+}
+
+
+iprange $DB_PATH/**/*.ipv4 --print-single-ips | sponge $GIT_ROOT/abuseipdb-s100-all.ipv4 &
+
+# c = confidence
+aggregate 1   "abuseipdb-s100-1d.ipv4"
+aggregate 3   "abuseipdb-s100-3d.ipv4"
+aggregate 7   "abuseipdb-s100-7d.ipv4"
+aggregate 14  "abuseipdb-s100-14d.ipv4"
+aggregate 30  "abuseipdb-s100-30d.ipv4"
+aggregate 60  "abuseipdb-s100-60d.ipv4"
+aggregate 90  "abuseipdb-s100-90d.ipv4"
+aggregate 120  "abuseipdb-s100-120d.ipv4"
+
+wait
+
+
+
+
+update-stats

.cron/jobs/abuseipdb/cron

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Setup
+cd "$(dirname $0)"
+
+GIT_ROOT=$(git rev-parse --show-toplevel)
+DB_PATH=$GIT_ROOT/db
+mkdir -p $DB_PATH
+TEMPFILE=$(mktemp)
+TEMPDIR=$(mktemp -d)
+
+cd $TEMPDIR
+
+# Debug
+echo "Public IP:"
+echo $(timeout 2s curl --no-progress-meter ipv4.icanhazip.com)
+echo
+
+echo '✔ Debug...'
+date '+%Y/%m/%d %H:%M:%S'
+bkt --ttl=6h -- date '+%Y/%m/%d %H:%M:%S'
+echo
+
+
+echo '✔ Download abuseipdb...'
+# Use a TTL of 6 hours (~4 of 5 requests / day)
+bkt --ttl=6h -- curl https://api.abuseipdb.com/api/v2/blacklist \
+  --get \
+  --max-time 10 \
+  --user-agent "" \
+  --no-progress-meter \
+  -d confidenceMinimum=100 \
+  -d limit=9999999 \
+  -H "Key: $ABUSEIPDB_TOKEN" \
+  -H "Accept: text/plain" \
+  --fail \
+  -w "\n" \
+  -o TEMPFILE.1 || true
+
+echo '✔ Download & decorate with extra sources ...'
+curl https://abuseipdb.tmiland.com/abuseipdb.txt \
+  --compressed --max-time 10 -G -sL -w "\n\n" --fail -o TEMPFILE.2 || true
+
+curl https://raw.githubusercontent.com/LittleJake/ip-blacklist/main/abuseipdb_blacklist_ip_score_100.txt \
+  --compressed --max-time 10 -G -sL -w "\n\n" --fail -o TEMPFILE.3 || true
+
+# Redundancy:
+# - Separate private cache (1 of 5 requests / day) to avoid breaking the 5 free run limit / day
+# - If above urls fail due to github actions being flaky, still have somewhat fresh data.
+echo '✔ Download from cache'
+curl "$CRONSRC_URL" -H "$CRONSRC_HEADER" --compressed --max-time 10 -G -sL -w "\n\n" --fail -o TEMPFILE.4 || true
+
+# echo '✔ Stats'
+# for FILE in TEMPFILE.*; do printf "$FILE "; wc -l < $FILE; done
+
+echo '✔ Squash all sources (by design: fail if no sources worked)'
+grep -h "" TEMPFILE.* >> $TEMPFILE
+
+echo '✔ Validate: Clean comments'
+cat $TEMPFILE | shfmt -mn | sponge $TEMPFILE
+
+echo '✔ Validate: Extract ipv6 data'
+  grep ':' $TEMPFILE | sort | tac | cidr-merger | sponge $TEMPFILE.ipv6
+
+echo '✔ Validate: Extract ipv4 data'
+  grep -v ":" $TEMPFILE | \
+  iprange --print-single-ips \
+  > $TEMPFILE.ipv4
+
+# 3. Validate data
+LINES=`wc -l < $TEMPFILE.ipv4`
+if [[ "$LINES" -gt "1000" ]]; then
+  echo "✔ Validate: File contains: $LINES lines"
+  mv $TEMPFILE.ipv4 $DB_PATH/abuseipdb-s100-latest.ipv4
+  mv $TEMPFILE.ipv6 $DB_PATH/abuseipdb-s100-latest.ipv6
+else
+  echo "❌ Validation failed"
+  echo
+  echo "-----------------------------------------------------"
+  cat $TEMPFILE
+  echo "-----------------------------------------------------"
+  cat $TEMPFILE.ipv4
+  echo "-----------------------------------------------------"
+  exit 1
+fi
+
+echo
+echo '✔ Aggregate: Create folders'
+DATE=$(date +%F)
+DATE_DIR=$DB_PATH/$DATE
+mkdir -pv $DATE_DIR && cd $DATE_DIR
+
+echo '✔ Aggregate: Copy latest to correct date folder'
+cp $DB_PATH/abuseipdb-s100-latest.ipv4 "$DATE_DIR/tmp-$(date +%H-%m-%S).ipv4"
+cp $DB_PATH/abuseipdb-s100-latest.ipv6 "$DATE_DIR/tmp-$(date +%H-%m-%S).ipv6"
+
+echo '✔ Aggregate: Squash ipv4 data'
+iprange --print-single-ips *.ipv4 | sponge $(date +%Y-%m-%d).ipv4
+
+echo '✔ Aggregate: Squash ipv6 data'
+cat *.ipv6 | grep ':' | sort | uniq | sort | sponge $(date +%Y-%m-%d).ipv6
+
+echo
+echo '✔ Cleanup: Remove temp files'
+rm -f tmp*.ipv4
+rm -f tmp*.ipv6

.cron/scripts/ciutil

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Preinstall
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+function exists() { command -v $1 &> /dev/null; }
+
+download() {
+  local url="$1"
+  local software=$(basename "$url")
+  local location=~/.local/bin/$software
+
+  echo "✔ Downloading $software into $location"
+  wget -O $location -q -nv --content-disposition "$url"
+  chmod +x $location 2> /dev/null || true
+}
+
+preinstall-binaries() {
+  # Optimize apt-get install time (no need for mandb on a ci)
+  sudo cp -p /bin/true /usr/bin/mandb 2> /dev/null || true
+  sudo mv -f /var/cache/man /tmp/ 2> /dev/null || true
+
+  mkdir -p ~/.local/bin
+  cp .cron/scripts/ciutil ~/.local/bin/
+
+  for url in $(jq .preinstallBinaries package.json | jq -r .[]); do
+    download "$url" &
+  done
+  wait
+}
+
+install-nodemodules() {
+  if jq -e .dependencies `git rev-parse --show-toplevel`/package.json &> /dev/null; then
+    cd $GITROOT
+
+    # Applying dirty fix for *not* including @types/* in the production ci
+    if [ -f pnpm-lock.yaml ]; then
+      cp pnpm-lock.yaml $RUNNER_TEMPDIR/
+      grep -v "'@types/" $RUNNER_TEMPDIR/pnpm-lock.yaml >| pnpm-lock.yaml
+    fi
+
+    pnpm install --frozen-lockfile --production --prefer-offline --no-verify-store-integrity
+
+    mv $RUNNER_TEMPDIR/pnpm-lock.yaml .
+  else
+    echo "Nothing to install"
+  fi
+}
+
+to_env() {
+  [[ "$2" == *"/"* ]] && (echo "$1 => $2" && mkdir -p $2)
+  export $1=$2
+  echo "$1=$2" >> $GITHUB_ENV
+}
+
+add_path() {
+  echo $1 >> $GITHUB_PATH
+}
+
+set-env-variables() {
+  mkdir -pv ~/.local/bin
+
+  # Action Cache
+  WS=$RUNNER_WORKSPACE
+
+  to_env ACTIONS_CACHE_DIR $WS/.actions-cache
+  to_env ACTIONS_CACHE_BIN_DIR $WS/.actions-cache/bin
+  to_env PNPM_STORE_DIR $WS/.actions-cache/pnpm/pnpm-store
+  to_env EGET_BIN $WS/.actions-cache/bin
+  to_env DENO_DIR $WS/.actions-cache/.deno
+  to_env PNPM_HOME $WS/.actions-cache/pnpm
+
+  # TTL Cache (Cache that will automatically be invalidated)
+  to_env TTL_CACHE_DIR $WS/.cache-ttl
+  to_env BKT_CACHE_DIR $WS/.cache-ttl/bkt
+
+  # Other dirs
+  to_env DEBUG_DIR $WS/.debug
+  to_env RUNNER_TEMPDIR $RUNNER_TEMP/$(date +%s%N)
+
+  # Other env
+  to_env CACHE_PREFIX "$(date +'%U')"
+  to_env NODE_ENV production
+  to_env TERM xterm-256color
+  to_env FORCE_COLOR 1
+
+  # PATH
+  add_path './node_modules/.bin'
+  add_path $HOME/.local/bin
+  add_path $WS/.actions-cache/pnpm
+  add_path $WS/.actions-cache/bin
+  add_path $GITHUB_WORKSPACE/.cron/scripts
+}
+
+
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Debug / Statistics / Information
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+show-cache-structure-default() { dust -b -P -s $ACTIONS_CACHE_DIR && echo; }
+show-cache-structure-ttl() { dust -b -P -s $TTL_CACHE_DIR && echo; }
+
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Formatters / Printers
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+✨() { ___; echo "✨ $1"; echo; }
+
+___() { echo; printf '━%.0s' {1..80}; echo; }
+
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Git Utils
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+has-file-changed() {
+  changed_files="$(git diff-tree -r --name-only --no-commit-id ORIG_HEAD HEAD 2> /dev/null)"
+  if $(echo "$changed_files" | grep --quiet "$1"); then
+    return 0
+  fi
+  return 1
+}
+
+git-squash-to-repository() {
+  DATE=$(date '+%Y/%m/%d %H:%M')
+  LABEL="SYNC"
+
+  git config user.name github-actions
+  git config user.email [email protected]
+
+  if [[ `git status --porcelain` ]]; then
+    MSG="\`️️⚡️ $LABEL ⚡️\` \`$DATE\`"
+    git add .
+    git commit -m "update"
+    git reset $(git commit-tree HEAD^{tree} -m "$MSG")
+    git push -f
+  fi
+}
+
+git-commit-to-repository() {
+  DATE=$(date '+%Y/%m/%d %H:%M')
+  LABEL="CRON"
+
+  echo
+  git diff --stat HEAD
+  echo
+
+  git config user.name github-actions
+  git config user.email [email protected]
+  git pull --rebase --autostash origin $(git rev-parse --abbrev-ref HEAD)
+
+  for file in $(git status -s | cut -c4-); do
+    echo "file: $file";
+    shortname=$(basename $file | sed -E 's/\.(ipset|netset)//g')
+    git add $file
+    stats=$(git diff --cached --shortstat | sed -E 's/ (insertions?|deletions?|changed?)//g' | sed 's/,//g' | xargs)
+    stats=$(echo $stats | sed 's/1 file//g' | xargs)
+    git commit -m "\`️️⚡️ $LABEL ⚡️\` \`$DATE\` \`$stats $shortname\` [ci skip]" || true
+    echo $stats
+  done
+
+  git push
+}
+
+
+# Allow calling for methods from outside
+for i in "$@"; do
+  "$i"
+done