Merge pull request #5741 from mailcow/staging

2024-02
mailcow · Feb 15, 2024 · 8d4ef14 · 8d4ef14
2 parents 8ae762a + 8ed6217
commit 8d4ef14
Show file tree

Hide file tree

Showing 30 changed files with 454 additions and 127 deletions.
diff --git a/data/Dockerfiles/netfilter/main.py b/data/Dockerfiles/netfilter/main.py
@@ -114,8 +114,6 @@ def ban(address):
   global lock
 
   refreshF2boptions()
-  BAN_TIME = int(f2boptions['ban_time'])
-  BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
   MAX_ATTEMPTS = int(f2boptions['max_attempts'])
   RETRY_WINDOW = int(f2boptions['retry_window'])
   NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
@@ -150,7 +148,7 @@ def ban(address):
 
   if bans[net]['attempts'] >= MAX_ATTEMPTS:
     cur_time = int(round(time.time()))
-    NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
+    NET_BAN_TIME = calcNetBanTime(bans[net]['ban_counter'])
     logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
     if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1:
       with lock:
@@ -277,22 +275,21 @@ def snat6(snat_target):
       tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
 
 def autopurge():
+  global f2boptions
+
   while not quit_now:
     time.sleep(10)
     refreshF2boptions()
-    BAN_TIME = int(f2boptions['ban_time'])
-    MAX_BAN_TIME = int(f2boptions['max_ban_time'])
-    BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
     MAX_ATTEMPTS = int(f2boptions['max_attempts'])
     QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
     if QUEUE_UNBAN:
       for net in QUEUE_UNBAN:
         unban(str(net))
     for net in bans.copy():
       if bans[net]['attempts'] >= MAX_ATTEMPTS:
-        NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
+        NET_BAN_TIME = calcNetBanTime(bans[net]['ban_counter'])
         TIME_SINCE_LAST_ATTEMPT = time.time() - bans[net]['last_attempt']
-        if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME or TIME_SINCE_LAST_ATTEMPT > MAX_BAN_TIME:
+        if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME:
           unban(net)
 
 def mailcowChainOrder():
@@ -306,6 +303,16 @@ def mailcowChainOrder():
       if quit_now: return
       quit_now, exit_code = tables.checkIPv6ChainOrder()
 
+def calcNetBanTime(ban_counter):
+  global f2boptions
+
+  BAN_TIME = int(f2boptions['ban_time'])
+  MAX_BAN_TIME = int(f2boptions['max_ban_time'])
+  BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
+  NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** ban_counter
+  NET_BAN_TIME = max([BAN_TIME, min([NET_BAN_TIME, MAX_BAN_TIME])])
+  return NET_BAN_TIME
+
 def isIpNetwork(address):
   try:
     ipaddress.ip_network(address, False)

diff --git a/data/Dockerfiles/netfilter/modules/Logger.py b/data/Dockerfiles/netfilter/modules/Logger.py
@@ -13,9 +13,12 @@ def log(self, priority, message):
     tolog['time'] = int(round(time.time()))
     tolog['priority'] = priority
     tolog['message'] = message
-    if self.r is not None:
-      self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
     print(message)
+    if self.r is not None:
+      try:
+        self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
+      except Exception as ex:
+        print('Failed logging to redis: %s'  % (ex))
 
   def logWarn(self, message):
     self.log('warn', message)

diff --git a/data/Dockerfiles/netfilter/modules/NFTables.py b/data/Dockerfiles/netfilter/modules/NFTables.py
@@ -452,6 +452,8 @@ def get_unban_ip_dict(self, ipaddr:str, _family: str):
           continue
 
         rule = _object["rule"]["expr"][0]["match"]
+        if not "payload" in rule["left"]:
+          continue
         left_opt = rule["left"]["payload"]
         if not left_opt["protocol"] == _family:
           continue

diff --git a/data/conf/rspamd/dynmaps/footer.php b/data/conf/rspamd/dynmaps/footer.php
@@ -56,21 +56,42 @@ function getallheaders() {
 error_log("FOOTER: checking for domain " . $domain . ", user " . $username . " and address " . $from . PHP_EOL);
 
 try {
-  $stmt = $pdo->prepare("SELECT `plain`, `html`, `mbox_exclude`, `skip_replies` FROM `domain_wide_footer` 
+  // try get $target_domain if $domain is an alias_domain
+  $stmt = $pdo->prepare("SELECT `target_domain` FROM `alias_domain` 
+    WHERE `alias_domain` = :alias_domain");
+  $stmt->execute(array(
+    ':alias_domain' => $domain
+  ));
+  $alias_domain = $stmt->fetch(PDO::FETCH_ASSOC);
+  if (!$alias_domain) {
+    $target_domain = $domain;
+  } else {
+    $target_domain = $alias_domain['target_domain'];
+  }
+
+  // get footer associated with the domain
+  $stmt = $pdo->prepare("SELECT `plain`, `html`, `mbox_exclude`, `alias_domain_exclude`, `skip_replies` FROM `domain_wide_footer` 
     WHERE `domain` = :domain");
   $stmt->execute(array(
-    ':domain' => $domain
+    ':domain' => $target_domain
   ));
   $footer = $stmt->fetch(PDO::FETCH_ASSOC);
+
+  // check if the sender is excluded
   if (in_array($from, json_decode($footer['mbox_exclude']))){
     $footer = false;
   }
+  if (in_array($domain, json_decode($footer['alias_domain_exclude']))){
+    $footer = false;
+  }
   if (empty($footer)){
     echo $empty_footer;
     exit;
   }
   error_log("FOOTER: " . json_encode($footer) . PHP_EOL);
 
+  // footer will be applied
+  // get custom mailbox attributes to insert into the footer
   $stmt = $pdo->prepare("SELECT `custom_attributes` FROM `mailbox` WHERE `username` = :username");
   $stmt->execute(array(
     ':username' => $username

diff --git a/data/web/debug.php b/data/web/debug.php
@@ -39,9 +39,13 @@
       $StartedAt['month'],
       $StartedAt['day'],
       $StartedAt['year']));
-    $user_tz = new DateTimeZone(getenv('TZ'));
-    $date->setTimezone($user_tz);
-    $started = $date->format('r');
+    try {
+      $user_tz = new DateTimeZone(getenv('TZ'));
+      $date->setTimezone($user_tz);
+      $started = $date->format('r');
+    } catch(Exception $e) {
+      $started = '?';
+    }
   }
   else {
     $started = '?';

diff --git a/data/web/edit.php b/data/web/edit.php
@@ -59,7 +59,8 @@
             'domain_details' => $result,
             'domain_footer' => $domain_footer,
             'mailboxes' => mailbox('get', 'mailboxes', $_GET["domain"]),
-            'aliases' => mailbox('get', 'aliases', $_GET["domain"], 'address')
+            'aliases' => mailbox('get', 'aliases', $_GET["domain"], 'address'),
+            'alias_domains' => mailbox('get', 'alias_domains', $_GET["domain"])
           ];
       }
     }

diff --git a/data/web/inc/functions.mailbox.inc.php b/data/web/inc/functions.mailbox.inc.php
@@ -3438,30 +3438,54 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $footers['plain'] = isset($_data['plain']) ? $_data['plain'] : '';
           $footers['skip_replies'] = isset($_data['skip_replies']) ? (int)$_data['skip_replies'] : 0;
           $footers['mbox_exclude'] = array();
-          if (isset($_data["mbox_exclude"])){
-            if (!is_array($_data["mbox_exclude"])) {
-              $_data["mbox_exclude"] = array($_data["mbox_exclude"]);
-            }
-            foreach ($_data["mbox_exclude"] as $mailbox) {
-              if (!filter_var($mailbox, FILTER_VALIDATE_EMAIL)) {
-                $_SESSION['return'][] = array(
-                  'type' => 'danger',
-                  'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
-                  'msg' => array('username_invalid', $mailbox)
-                );
-                continue;
+          $footers['alias_domain_exclude'] = array();
+          if (isset($_data["exclude"])){
+            if (!is_array($_data["exclude"])) {
+              $_data["exclude"] = array($_data["exclude"]);
+            }
+            foreach ($_data["exclude"] as $exclude) {
+              if (filter_var($exclude, FILTER_VALIDATE_EMAIL)) {
+                $stmt = $pdo->prepare("SELECT `address` FROM `alias` WHERE `address` = :address
+                  UNION
+                  SELECT `username` FROM `mailbox` WHERE `username` = :username");
+                $stmt->execute(array(
+                  ':address' => $exclude,
+                  ':username' => $exclude,
+                ));
+                $row = $stmt->fetch(PDO::FETCH_ASSOC);
+                if(!$row){
+                  $_SESSION['return'][] = array(
+                    'type' => 'danger',
+                    'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                    'msg' => array('username_invalid', $exclude)
+                  );
+                  continue;
+                }
+                array_push($footers['mbox_exclude'], $exclude);
               }
-              $is_now = mailbox('get', 'mailbox_details', $mailbox);            
-              if(empty($is_now)){
+              elseif (is_valid_domain_name($exclude)) {
+                $stmt = $pdo->prepare("SELECT `alias_domain` FROM `alias_domain` WHERE `alias_domain` = :alias_domain");
+                $stmt->execute(array(
+                  ':alias_domain' => $exclude,
+                ));
+                $row = $stmt->fetch(PDO::FETCH_ASSOC);
+                if(!$row){
+                  $_SESSION['return'][] = array(
+                    'type' => 'danger',
+                    'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                    'msg' => array('username_invalid', $exclude)
+                  );
+                  continue;
+                }
+                array_push($footers['alias_domain_exclude'], $exclude);
+              }
+              else {
                 $_SESSION['return'][] = array(
                   'type' => 'danger',
                   'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
-                  'msg' => array('username_invalid', $mailbox)
+                  'msg' => array('username_invalid', $exclude)
                 );
-                continue;
               }
-
-              array_push($footers['mbox_exclude'], $mailbox);
             }
           }
           foreach ($domains as $domain) {
@@ -3486,12 +3510,13 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             try {
               $stmt = $pdo->prepare("DELETE FROM `domain_wide_footer` WHERE `domain`= :domain");
               $stmt->execute(array(':domain' => $domain));
-              $stmt = $pdo->prepare("INSERT INTO `domain_wide_footer` (`domain`, `html`, `plain`, `mbox_exclude`, `skip_replies`) VALUES (:domain, :html, :plain, :mbox_exclude, :skip_replies)");
+              $stmt = $pdo->prepare("INSERT INTO `domain_wide_footer` (`domain`, `html`, `plain`, `mbox_exclude`, `alias_domain_exclude`, `skip_replies`) VALUES (:domain, :html, :plain, :mbox_exclude, :alias_domain_exclude, :skip_replies)");
               $stmt->execute(array(
                 ':domain' => $domain,
                 ':html' => $footers['html'],
                 ':plain' => $footers['plain'],
                 ':mbox_exclude' => json_encode($footers['mbox_exclude']),
+                ':alias_domain_exclude' => json_encode($footers['alias_domain_exclude']),
                 ':skip_replies' => $footers['skip_replies'],
               ));
             }
@@ -4316,6 +4341,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $domaindata['mboxes_in_domain'] = $MailboxDataDomain['count'];
           $domaindata['mboxes_left'] = $row['mailboxes']  - $MailboxDataDomain['count'];
           $domaindata['domain_name'] = $row['domain'];
+          $domaindata['domain_h_name'] = idn_to_utf8($row['domain']);
           $domaindata['description'] = $row['description'];
           $domaindata['max_num_aliases_for_domain'] = $row['aliases'];
           $domaindata['max_num_mboxes_for_domain'] = $row['mailboxes'];
@@ -4648,7 +4674,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           }
 
           try {
-            $stmt = $pdo->prepare("SELECT `html`, `plain`, `mbox_exclude`, `skip_replies` FROM `domain_wide_footer`
+            $stmt = $pdo->prepare("SELECT `html`, `plain`, `mbox_exclude`, `alias_domain_exclude`, `skip_replies` FROM `domain_wide_footer`
               WHERE `domain` = :domain");
             $stmt->execute(array(
               ':domain' => $domain

diff --git a/data/web/inc/init_db.inc.php b/data/web/inc/init_db.inc.php
@@ -3,7 +3,7 @@ function init_db_schema() {
   try {
     global $pdo;
 
-    $db_version = "08012024_1442";
+    $db_version = "09022024_1433";
 
     $stmt = $pdo->query("SHOW TABLES LIKE 'versions'");
     $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
@@ -273,6 +273,7 @@ function init_db_schema() {
           "html" => "LONGTEXT",
           "plain" => "LONGTEXT",
           "mbox_exclude" => "JSON NOT NULL DEFAULT ('[]')",
+          "alias_domain_exclude" => "JSON NOT NULL DEFAULT ('[]')",
           "skip_replies" => "TINYINT(1) NOT NULL DEFAULT '0'"
         ),
         "keys" => array(

diff --git a/data/web/inc/vars.inc.php b/data/web/inc/vars.inc.php
@@ -95,6 +95,7 @@
   'it-it' => 'Italiano (Italian)',
   'ko-kr' => '한국어 (Korean)',
   'lv-lv' => 'latviešu (Latvian)',
+  'nb-no' => 'Norsk (Norwegian)',
   'nl-nl' => 'Nederlands (Dutch)',
   'pl-pl' => 'Język Polski (Polish)',
   'pt-br' => 'Português brasileiro (Brazilian Portuguese)',

diff --git a/data/web/js/site/mailbox.js b/data/web/js/site/mailbox.js
@@ -451,6 +451,10 @@ jQuery(function($){
         dataSrc: function(json){
           $.each(json.data, function(i, item) {
             item.domain_name = escapeHtml(item.domain_name);
+            item.domain_h_name = escapeHtml(item.domain_h_name);
+            if (item.domain_name != item.domain_h_name){
+              item.domain_h_name = item.domain_h_name + '<small class="d-block">' + item.domain_name + '</small>';
+            }
 
             item.aliases = item.aliases_in_domain + " / " + item.max_num_aliases_for_domain;
             item.mailboxes = item.mboxes_in_domain + " / " + item.max_num_mboxes_for_domain;
@@ -489,11 +493,11 @@ jQuery(function($){
 
             if (item.backupmx == 1) {
               if (item.relay_unknown_only == 1) {
-                item.domain_name = '<div class="badge fs-6 bg-info">Relay Non-Local</div> ' + item.domain_name;
+                item.domain_h_name = '<div class="badge fs-7 bg-info">Relay Non-Local</div> ' + item.domain_h_name;
               } else if (item.relay_all_recipients == 1) {
-                item.domain_name = '<div class="badge fs-6 bg-info">Relay All</div> ' + item.domain_name;
+                item.domain_h_name = '<div class="badge fs-7 bg-info">Relay All</div> ' + item.domain_h_name;
               } else {
-                item.domain_name = '<div class="badge fs-6 bg-info">Relay</div> ' + item.domain_name;
+                item.domain_h_name = '<div class="badge fs-7 bg-info">Relay</div> ' + item.domain_h_name;
               }
             }
           });
@@ -521,7 +525,7 @@ jQuery(function($){
         },
         {
           title: lang.domain,
-          data: 'domain_name',
+          data: 'domain_h_name',
           responsivePriority: 3,
           defaultContent: ''
         },

diff --git a/data/web/lang/lang.cs-cz.json b/data/web/lang/lang.cs-cz.json
@@ -547,7 +547,7 @@
         "dns_records": "DNS záznamy",
         "dns_records_24hours": "Upozornění: Změnám v systému DNS může trvat až 24 hodin, než se zde správně zobrazí jejich aktuální stav. Můžete zde snadno zjistit, jak nastavit DNS záznamy a zda jsou všechny záznamy správně uloženy.",
         "dns_records_data": "Správný záznam",
-        "dns_records_docs": "Přečtěte si prosím <a target=\"_blank\" href=\"https://docs.mailcow.email/prerequisite/prerequisite-dns/\">dokumentaci</a>.",
+        "dns_records_docs": "Přečtěte si prosím <a target=\"_blank\" href=\"https://docs.mailcow.email/getstarted/prerequisite-dns\">dokumentaci</a>.",
         "dns_records_name": "Název",
         "dns_records_status": "Současný stav",
         "dns_records_type": "Typ",

diff --git a/data/web/lang/lang.da-dk.json b/data/web/lang/lang.da-dk.json
@@ -459,7 +459,7 @@
         "cname_from_a": "Værdi afledt af A / AAAA-post. Dette understøttes, så længe posten peger på den korrekte ressource.",
         "dns_records": "DNS-poster",
         "dns_records_24hours": "Bemærk, at ændringer, der foretages i DNS, kan tage op til 24 timer for at få deres aktuelle status korrekt reflekteret på denne side. Det er beregnet som en måde for dig let at se, hvordan du konfigurerer dine DNS-poster og kontrollere, om alle dine poster er korrekt gemt i DNS.",
-        "dns_records_docs": "Se også <a target=\"_blank\" href=\"https://docs.mailcow.email/prerequisite/prerequisite-dns/\">dokumentationen</a>.",
+        "dns_records_docs": "Se også <a target=\"_blank\" href=\"https://docs.mailcow.email/getstarted/prerequisite-dns\">dokumentationen</a>.",
         "dns_records_data": "Korrekte data",
         "dns_records_name": "Navn",
         "dns_records_status": "Nuværende tilstand",

diff --git a/data/web/lang/lang.de-de.json b/data/web/lang/lang.de-de.json
@@ -556,7 +556,7 @@
         "dns_records": "DNS-Einträge",
         "dns_records_24hours": "Bitte beachten Sie, dass es bis zu 24 Stunden dauern kann, bis Änderungen an Ihren DNS-Einträgen als aktueller Status auf dieser Seite dargestellt werden. Diese Seite ist nur als Hilfsmittel gedacht, um die korrekten Werte für DNS-Einträge anzuzeigen und zu überprüfen, ob die Daten im DNS hinterlegt sind.",
         "dns_records_data": "Korrekte Daten",
-        "dns_records_docs": "Die <a target=\"_blank\" href=\"https://docs.mailcow.email/prerequisite/prerequisite-dns/\">Online-Dokumentation</a> enthält weitere Informationen zur DNS-Konfiguration.",
+        "dns_records_docs": "Die <a target=\"_blank\" href=\"https://docs.mailcow.email/de/getstarted/prerequisite-dns\">Online-Dokumentation</a> enthält weitere Informationen zur DNS-Konfiguration.",
         "dns_records_name": "Name",
         "dns_records_status": "Aktueller Status",
         "dns_records_type": "Typ",
@@ -613,6 +613,7 @@
         "extended_sender_acl_info": "Der DKIM-Domainkey der externen Absenderdomain sollte in diesen Server importiert werden, falls vorhanden.<br>\r\n  Wird SPF verwendet, muss diesem Server der Versand gestattet werden.<br>\r\n  Wird eine Domain oder Alias-Domain zu diesem Server hinzugefügt, die sich mit der externen Absenderadresse überschneidet, wird der externe Absender hier entfernt.<br>\r\n  Ein Eintrag @domain.tld erlaubt den Versand als *@domain.tld",
         "force_pw_update": "Erzwinge Passwortänderung bei nächstem Login",
         "force_pw_update_info": "Dem Benutzer wird lediglich der Zugang zur %s ermöglicht, App Passwörter funktionieren weiterhin.",
+        "footer_exclude": "von Fußzeile ausschließen",
         "full_name": "Voller Name",
         "gal": "Globales Adressbuch",
         "gal_info": "Das globale Adressbuch enthält alle Objekte einer Domain und kann durch keinen Benutzer geändert werden. Die Verfügbarkeitsinformation in SOGo ist nur bei eingeschaltetem globalen Adressbuch ersichtlich <b>Zum Anwenden einer Änderung muss SOGo neugestartet werden.</b>",
@@ -631,7 +632,6 @@
         "max_quota": "Max. Größe per Mailbox (MiB)",
         "maxage": "Maximales Alter in Tagen einer Nachricht, die kopiert werden soll<br><small>(0 = alle Nachrichten kopieren)</small>",
         "maxbytespersecond": "Max. Übertragungsrate in Bytes/s (0 für unlimitiert)",
-        "mbox_exclude": "Mailboxen ausschließen",
         "mbox_rl_info": "Dieses Limit wird auf den SASL Loginnamen angewendet und betrifft daher alle Absenderadressen, die der eingeloggte Benutzer verwendet. Bei Mailbox Ratelimit überwiegt ein Domain-weites Ratelimit.",
         "mins_interval": "Intervall (min)",
         "multiple_bookings": "Mehrfaches Buchen",