Merge branch 'feat/spamhaus-dqs-asn' into staging

.gitignore

@@ -36,6 +36,7 @@ data/conf/postfix/extra.cf
 data/conf/postfix/sni.map
 data/conf/postfix/sni.map.db
 data/conf/postfix/sql
+data/conf/postfix/dns_blocklists.cf
 data/conf/rspamd/custom/*
 data/conf/rspamd/local.d/*
 data/conf/rspamd/override.d/*

data/Dockerfiles/postfix/Dockerfile

@@ -33,6 +33,7 @@ RUN groupadd -g 102 postfix \
 	syslog-ng-core \
 	syslog-ng-mod-redis \
   tzdata \
+    whois \
 	&& rm -rf /var/lib/apt/lists/* \
 	&& touch /etc/default/locale \
   && printf '#!/bin/bash\n/usr/sbin/postconf -c /opt/postfix/conf "$@"' > /usr/local/sbin/postconf \

data/Dockerfiles/postfix/postfix.sh

@@ -393,6 +393,111 @@ query = SELECT goto FROM spamalias
     AND validity >= UNIX_TIMESTAMP()
 EOF
 
+if [ -n "$SPAMHAUS_DQS_KEY" ]; then
+  echo -e "\e[32mDetected SPAMHAUS_DQS_KEY variable from mailcow.conf...\e[0m"
+  echo -e "\e[33mUsing DQS Blocklists from Spamhaus!\e[0m"
+  cat <<EOF > /opt/postfix/conf/dns_blocklists.cf
+    # Autogenerated by mailcow
+    postscreen_dnsbl_sites = wl.mailspike.net=127.0.0.[18;19;20]*-2
+      hostkarma.junkemailfilter.com=127.0.0.1*-2
+      list.dnswl.org=127.0.[0..255].0*-2
+      list.dnswl.org=127.0.[0..255].1*-4
+      list.dnswl.org=127.0.[0..255].2*-6
+      list.dnswl.org=127.0.[0..255].3*-8
+      ix.dnsbl.manitu.net*2
+      bl.spamcop.net*2
+      bl.suomispam.net*2
+      hostkarma.junkemailfilter.com=127.0.0.2*3
+      hostkarma.junkemailfilter.com=127.0.0.4*2
+      hostkarma.junkemailfilter.com=127.0.1.2*1
+      backscatter.spameatingmonkey.net*2
+      bl.ipv6.spameatingmonkey.net*2
+      bl.spameatingmonkey.net*2
+      b.barracudacentral.org=127.0.0.2*7
+      bl.mailspike.net=127.0.0.2*5
+      bl.mailspike.net=127.0.0.[10;11;12]*4
+      dnsbl.sorbs.net=127.0.0.10*8
+      dnsbl.sorbs.net=127.0.0.5*6
+      dnsbl.sorbs.net=127.0.0.7*3
+      dnsbl.sorbs.net=127.0.0.8*2
+      dnsbl.sorbs.net=127.0.0.6*2
+      dnsbl.sorbs.net=127.0.0.9*2
+      ${SPAMHAUS_DQS_KEY}.zen.dq.spamhaus.net=127.0.0.[4..7]*6
+      ${SPAMHAUS_DQS_KEY}.zen.dq.spamhaus.org=127.0.0.[10;11]*8
+      ${SPAMHAUS_DQS_KEY}.zen.dq.spamhaus.org=127.0.0.3*4
+      ${SPAMHAUS_DQS_KEY}.zen.dq.spamhaus.org=127.0.0.2*3
+      ${SPAMHAUS_DQS_KEY}.dbl.dq.spamhaus.net=127.0.0.3*4
+      ${SPAMHAUS_DQS_KEY}.zrd.dq.spamhaus.net=127.0.0.2*3
+EOF
+
+else
+  if curl -s http://fuzzy.mailcow.email/asn_list.txt | grep $(whois -h whois.radb.net $(curl -s http://ipv4.mailcow.email) | grep -i origin | tr -s " " | cut -d " " -f2 | head -1) > /dev/null; then
+  echo -e "\e[31mThe AS of your IP is listed as a banned AS from Spamhaus!\e[0m"
+  echo -e "\e[33mNo SPAMHAUS_DQS_KEY found... Skipping Spamhaus blocklists entirely!\e[0m"
+  cat <<EOF > /opt/postfix/conf/dns_blocklists.cf
+    # Autogenerated by mailcow
+    postscreen_dnsbl_sites = wl.mailspike.net=127.0.0.[18;19;20]*-2
+      hostkarma.junkemailfilter.com=127.0.0.1*-2
+      list.dnswl.org=127.0.[0..255].0*-2
+      list.dnswl.org=127.0.[0..255].1*-4
+      list.dnswl.org=127.0.[0..255].2*-6
+      list.dnswl.org=127.0.[0..255].3*-8
+      ix.dnsbl.manitu.net*2
+      bl.spamcop.net*2
+      bl.suomispam.net*2
+      hostkarma.junkemailfilter.com=127.0.0.2*3
+      hostkarma.junkemailfilter.com=127.0.0.4*2
+      hostkarma.junkemailfilter.com=127.0.1.2*1
+      backscatter.spameatingmonkey.net*2
+      bl.ipv6.spameatingmonkey.net*2
+      bl.spameatingmonkey.net*2
+      b.barracudacentral.org=127.0.0.2*7
+      bl.mailspike.net=127.0.0.2*5
+      bl.mailspike.net=127.0.0.[10;11;12]*4
+      dnsbl.sorbs.net=127.0.0.10*8
+      dnsbl.sorbs.net=127.0.0.5*6
+      dnsbl.sorbs.net=127.0.0.7*3
+      dnsbl.sorbs.net=127.0.0.8*2
+      dnsbl.sorbs.net=127.0.0.6*2
+      dnsbl.sorbs.net=127.0.0.9*2
+EOF
+  else
+  echo -e "\e[32mThe AS of your IP is NOT listed as a banned AS from Spamhaus!\e[0m"
+  echo -e "\e[33mUsing the open Spamhaus blocklists.\e[0m"
+  cat <<EOF > /opt/postfix/conf/dns_blocklists.cf
+    # Autogenerated by mailcow
+    postscreen_dnsbl_sites = wl.mailspike.net=127.0.0.[18;19;20]*-2
+      hostkarma.junkemailfilter.com=127.0.0.1*-2
+      list.dnswl.org=127.0.[0..255].0*-2
+      list.dnswl.org=127.0.[0..255].1*-4
+      list.dnswl.org=127.0.[0..255].2*-6
+      list.dnswl.org=127.0.[0..255].3*-8
+      ix.dnsbl.manitu.net*2
+      bl.spamcop.net*2
+      bl.suomispam.net*2
+      hostkarma.junkemailfilter.com=127.0.0.2*3
+      hostkarma.junkemailfilter.com=127.0.0.4*2
+      hostkarma.junkemailfilter.com=127.0.1.2*1
+      backscatter.spameatingmonkey.net*2
+      bl.ipv6.spameatingmonkey.net*2
+      bl.spameatingmonkey.net*2
+      b.barracudacentral.org=127.0.0.2*7
+      bl.mailspike.net=127.0.0.2*5
+      bl.mailspike.net=127.0.0.[10;11;12]*4
+      dnsbl.sorbs.net=127.0.0.10*8
+      dnsbl.sorbs.net=127.0.0.5*6
+      dnsbl.sorbs.net=127.0.0.7*3
+      dnsbl.sorbs.net=127.0.0.8*2
+      dnsbl.sorbs.net=127.0.0.6*2
+      dnsbl.sorbs.net=127.0.0.9*2
+      zen.spamhaus.org=127.0.0.[10;11]*8
+      zen.spamhaus.org=127.0.0.[4..7]*6
+      zen.spamhaus.org=127.0.0.3*4
+      zen.spamhaus.org=127.0.0.2*3
+EOF
+  fi
+fi
+
 sed -i '/User overrides/q' /opt/postfix/conf/main.cf
 echo >> /opt/postfix/conf/main.cf
 touch /opt/postfix/conf/extra.cf

data/conf/postfix/main.cf

@@ -40,34 +40,6 @@ postscreen_blacklist_action = drop
 postscreen_cache_cleanup_interval = 24h
 postscreen_cache_map = proxy:btree:$data_directory/postscreen_cache
 postscreen_dnsbl_action = enforce
-postscreen_dnsbl_sites = wl.mailspike.net=127.0.0.[18;19;20]*-2
-  hostkarma.junkemailfilter.com=127.0.0.1*-2
-  list.dnswl.org=127.0.[0..255].0*-2
-  list.dnswl.org=127.0.[0..255].1*-4
-  list.dnswl.org=127.0.[0..255].2*-6
-  list.dnswl.org=127.0.[0..255].3*-8
-  ix.dnsbl.manitu.net*2
-  bl.spamcop.net*2
-  bl.suomispam.net*2
-  hostkarma.junkemailfilter.com=127.0.0.2*3
-  hostkarma.junkemailfilter.com=127.0.0.4*2
-  hostkarma.junkemailfilter.com=127.0.1.2*1
-  backscatter.spameatingmonkey.net*2
-  bl.ipv6.spameatingmonkey.net*2
-  bl.spameatingmonkey.net*2
-  b.barracudacentral.org=127.0.0.2*7
-  bl.mailspike.net=127.0.0.2*5
-  bl.mailspike.net=127.0.0.[10;11;12]*4
-  dnsbl.sorbs.net=127.0.0.10*8
-  dnsbl.sorbs.net=127.0.0.5*6
-  dnsbl.sorbs.net=127.0.0.7*3
-  dnsbl.sorbs.net=127.0.0.8*2
-  dnsbl.sorbs.net=127.0.0.6*2
-  dnsbl.sorbs.net=127.0.0.9*2
-  zen.spamhaus.org=127.0.0.[10;11]*8
-  zen.spamhaus.org=127.0.0.[4..7]*6
-  zen.spamhaus.org=127.0.0.3*4
-  zen.spamhaus.org=127.0.0.2*3
 postscreen_dnsbl_threshold = 6
 postscreen_dnsbl_ttl = 5m
 postscreen_greet_action = enforce

docker-compose.yml

@@ -297,7 +297,7 @@ services:
             - dovecot
 
     postfix-mailcow:
-      image: mailcow/postfix:1.68
+      image: mailcow/postfix:1.69
       depends_on:
         - mysql-mailcow
       volumes:
@@ -317,6 +317,7 @@ services:
         - REDIS_SLAVEOF_IP=${REDIS_SLAVEOF_IP:-}
         - REDIS_SLAVEOF_PORT=${REDIS_SLAVEOF_PORT:-}
         - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
+        - SPAMHAUS_DQS_KEY=${SPAMHAUS_DQS_KEY:-}
       cap_add:
         - NET_BIND_SERVICE
       ports:

generate_config.sh

@@ -21,7 +21,7 @@ if grep --help 2>&1 | head -n 1 | grep -q -i "busybox"; then echo "BusyBox grep
 if cp --help 2>&1 | head -n 1 | grep -q -i "busybox"; then echo "BusyBox cp detected, please install coreutils, \"apk add --no-cache --upgrade coreutils\""; exit 1; fi
 if sed --help 2>&1 | head -n 1 | grep -q -i "busybox"; then echo "BusyBox sed detected, please install gnu sed, \"apk add --no-cache --upgrade sed\""; exit 1; fi
 
-for bin in openssl curl docker git awk sha1sum; do
+for bin in openssl curl docker git awk sha1sum grep cut whois; do
   if [[ -z $(which ${bin}) ]]; then echo "Cannot find ${bin}, exiting..."; exit 1; fi
 done
 
@@ -58,6 +58,23 @@ else
   exit 1
 fi
 
+detect_bad_asn() {
+  if [[ curl -s http://fuzzy.mailcow.email/asn_list.txt |  grep $(whois -h whois.radb.net $(curl -s http://ipv4.mailcow.email) | grep -i origin | tr -s " " | cut -d " " -f2 | head -1) ]]; then
+    if ! $SPAMHAUS_DQS_KEY; then
+      echo -e "\e[31mYour server's public IP uses an AS that is blocked by Spamhaus to use their DNS blocklists for Postfix."
+      echo -e "\e[31mmailcow did not detected a value for the variable SPAMHAUS_DQS_KEY inside mailcow.conf!"
+      echo ""
+      echo -e "\e[31mTo use the Spamhaus DNS Blocklists again, you will need to create a FREE account for their Data Query Service (DQS) at: https://www.spamhaus.com/free-trial/sign-up-for-a-free-data-query-service-account"
+      echo -e "\e[31mOnce done, enter your DQS API key in mailcow.conf and mailcow will do the rest for you!"
+      sleep 2
+
+    else
+      echo -e "\e[31mYour server's public IP uses an AS that is blocked by Spamhaus to use their DNS blocklists for Postfix."
+      echo -e "\e[33mmailcow detected a Value for the variable SPAMHAUS_DQS_KEY inside mailcow.conf. Postfix will use DQS with the given API key..."
+    fi
+  fi
+}
+
 ### If generate_config.sh is started with --dev or -d it will not check out nightly or master branch and will keep on the current branch
 if [[ ${1} == "--dev" || ${1} == "-d" ]]; then
   SKIP_BRANCH=y
@@ -431,6 +448,13 @@ ACME_CONTACT=
 # root certificates can be placed for validation under mailcow-dockerized/data/web/inc/lib/WebAuthn/rootCertificates
 WEBAUTHN_ONLY_TRUSTED_VENDORS=n
 
+# Spamhaus Data Query Service Key
+# Optional: Leave empty for none
+# Enter your key here if you are using a blocked ASN (OVH, AWS, Cloudflare e.g) for the unregistered Spamhaus Blocklist. 
+# If empty, it will completely disable Spamhaus blocklists if it detects that you are running on a server using a blocked AS.
+# Otherwise it will work normally.
+SPAMHAUS_DQS_KEY=
+
 EOF
 
 mkdir -p data/assets/ssl
@@ -503,3 +527,5 @@ else
   echo '?>' >> data/web/inc/app_info.inc.php
   echo -e "\e[33mCannot determine current git repository version...\e[0m"
 fi
+
+detect_bad_asn

update.sh

@@ -255,6 +255,25 @@ elif [ "${DOCKER_COMPOSE_VERSION}" == "standalone" ]; then
 fi
 }
 
+detect_bad_asn() {
+  if curl -s http://fuzzy.mailcow.email/asn_list.txt | grep $(whois -h whois.radb.net $(curl -s http://ipv4.mailcow.email) | grep -i origin | tr -s " " | cut -d " " -f2 | head -1) > /dev/null ; then
+    if [ -z "$SPAMHAUS_DQS_KEY" ]; then
+      echo -e "\e[33mYour server's public IP uses an AS that is blocked by Spamhaus to use their DNS public blocklists for Postfix.\e[0m"
+      echo -e "\e[33mmailcow did not detected a value for the variable SPAMHAUS_DQS_KEY inside mailcow.conf!\e[0m"
+      sleep 2
+      echo ""
+      echo -e "\e[33mTo use the Spamhaus DNS Blocklists again, you will need to create a FREE account for their Data Query Service (DQS) at: https://www.spamhaus.com/free-trial/sign-up-for-a-free-data-query-service-account\e[0m"
+      echo -e "\e[33mOnce done, enter your DQS API key in mailcow.conf and mailcow will do the rest for you!\e[0m"
+      echo ""
+      sleep 2
+
+    else
+      echo -e "\e[33mYour server's public IP uses an AS that is blocked by Spamhaus to use their DNS public blocklists for Postfix.\e[0m"
+      echo -e "\e[32mmailcow detected a Value for the variable SPAMHAUS_DQS_KEY inside mailcow.conf. Postfix will use DQS with the given API key...\e[0m"
+    fi
+  fi
+}
+
 ############## End Function Section ##############
 
 # Check permissions
@@ -301,7 +320,7 @@ umask 0022
 unset COMPOSE_COMMAND
 unset DOCKER_COMPOSE_VERSION
 
-for bin in curl docker git awk sha1sum; do
+for bin in curl docker git awk sha1sum grep cut whois; do
   if [[ -z $(command -v ${bin}) ]]; then 
   echo "Cannot find ${bin}, exiting..." 
   exit 1;
@@ -442,8 +461,11 @@ CONFIG_ARRAY=(
   "ACME_CONTACT"
   "WATCHDOG_VERBOSE"
   "WEBAUTHN_ONLY_TRUSTED_VENDORS"
+  "SPAMHAUS_DQS_KEY"
 )
 
+detect_bad_asn
+
 sed -i --follow-symlinks '$a\' mailcow.conf
 for option in ${CONFIG_ARRAY[@]}; do
   if [[ ${option} == "ADDITIONAL_SAN" ]]; then
@@ -642,6 +664,7 @@ for option in ${CONFIG_ARRAY[@]}; do
     fi
   elif [[ ${option} == "ADDITIONAL_SERVER_NAMES" ]]; then
     if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
       echo '# Additional server names for mailcow UI' >> mailcow.conf
       echo '#' >> mailcow.conf
       echo '# Specify alternative addresses for the mailcow UI to respond to' >> mailcow.conf
@@ -653,25 +676,38 @@ for option in ${CONFIG_ARRAY[@]}; do
     fi
   elif [[ ${option} == "ACME_CONTACT" ]]; then
     if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
       echo '# Lets Encrypt registration contact information' >> mailcow.conf
       echo '# Optional: Leave empty for none' >> mailcow.conf
       echo '# This value is only used on first order!' >> mailcow.conf
       echo '# Setting it at a later point will require the following steps:' >> mailcow.conf
       echo '# https://docs.mailcow.email/troubleshooting/debug-reset_tls/' >> mailcow.conf
       echo 'ACME_CONTACT=' >> mailcow.conf
-  fi
+    fi
   elif [[ ${option} == "WEBAUTHN_ONLY_TRUSTED_VENDORS" ]]; then
     if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
       echo "# WebAuthn device manufacturer verification" >> mailcow.conf
       echo '# After setting WEBAUTHN_ONLY_TRUSTED_VENDORS=y only devices from trusted manufacturers are allowed' >> mailcow.conf
       echo '# root certificates can be placed for validation under mailcow-dockerized/data/web/inc/lib/WebAuthn/rootCertificates' >> mailcow.conf
       echo 'WEBAUTHN_ONLY_TRUSTED_VENDORS=n' >> mailcow.conf
     fi
-elif [[ ${option} == "WATCHDOG_VERBOSE" ]]; then
+  elif [[ ${option} == "SPAMHAUS_DQS_KEY" ]]; then
+    if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
+      echo "# Spamhaus Data Query Service Key" >> mailcow.conf
+      echo '# Optional: Leave empty for none' >> mailcow.conf
+      echo '# Enter your key here if you are using a blocked ASN (OVH, AWS, Cloudflare e.g) for the unregistered Spamhaus Blocklist.' >> mailcow.conf
+      echo '# If empty, it will completely disable Spamhaus blocklists if it detects that you are running on a server using a blocked AS.' >> mailcow.conf
+      echo '# Otherwise it will work as usual.' >> mailcow.conf
+      echo 'SPAMHAUS_DQS_KEY=' >> mailcow.conf
+    fi
+  elif [[ ${option} == "WATCHDOG_VERBOSE" ]]; then
     if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
       echo '# Enable watchdog verbose logging' >> mailcow.conf
       echo 'WATCHDOG_VERBOSE=n' >> mailcow.conf
-  fi
+    fi
   elif ! grep -q ${option} mailcow.conf; then
     echo "Adding new option \"${option}\" to mailcow.conf"
     echo "${option}=n" >> mailcow.conf