dovecot: replace solr fts with flatcurve (xapian)

data/Dockerfiles/dovecot/Dockerfile

@@ -24,6 +24,7 @@ RUN addgroup -g 5000 vmail \
   envsubst \
   ca-certificates \
   curl \
+  coreutils \
   jq \
   lua \
   lua-cjson \
@@ -103,7 +104,7 @@ RUN addgroup -g 5000 vmail \
   dovecot-submissiond \
   dovecot-pigeonhole-plugin \
   dovecot-pop3d \
-  dovecot-fts-solr \
+  dovecot-fts-flatcurve \
   && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
   && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \
   && chmod +x /usr/local/bin/gosu \
@@ -127,6 +128,7 @@ COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
 COPY quarantine_notify.py /usr/local/bin/quarantine_notify.py
 COPY quota_notify.py /usr/local/bin/quota_notify.py
 COPY repl_health.sh /usr/local/bin/repl_health.sh
+COPY optimize-fts.sh /usr/local/bin/optimize-fts.sh
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -109,14 +109,14 @@ EOF
 
 echo -n ${ACL_ANYONE} > /etc/dovecot/acl_anyone
 
-if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+if [[ "${SKIP_FLATCURVE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
 echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication' > /etc/dovecot/mail_plugins
 echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify listescape replication mail_log' > /etc/dovecot/mail_plugins_imap
 echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 else
-echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_solr listescape replication' > /etc/dovecot/mail_plugins
-echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_solr listescape replication' > /etc/dovecot/mail_plugins_imap
-echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_solr notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_flatcurve notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 fi
 chmod 644 /etc/dovecot/mail_plugins /etc/dovecot/mail_plugins_imap /etc/dovecot/mail_plugins_lmtp /templates/quarantine.tpl
 
@@ -343,6 +343,10 @@ mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT}
 EOF
 fi
 
+if [[ "${SKIP_FLATCURVE}" =~ ^([nN][oO]|[nN])+$ ]]; then
+  sed -i "s/vsz_limit\s*=\s*[0-9]*\s*MB*/vsz_limit=${FTS_HEAP} MB/" /etc/dovecot/conf.d/fts-flatcurve.conf
+  sed -i "s/process_limit\s*=\s*[0-9]*/process_limit=${FTS_PROCS}/" /etc/dovecot/conf.d/fts-flatcurve.conf
+fi
 
 # 401 is user dovecot
 if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
@@ -387,7 +391,8 @@ chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/local/bin/maildir_gc.sh \
   /usr/local/sbin/stop-supervisor.sh \
   /usr/local/bin/quota_notify.py \
-  /usr/local/bin/repl_health.sh
+  /usr/local/bin/repl_health.sh \
+  /usr/local/bin/optimize-fts.sh
 
 # Prepare environment file for cronjobs
 printenv | sed 's/^\(.*\)$/export \1/g' > /source_env.sh

data/Dockerfiles/dovecot/optimize-fts.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ "${SKIP_FLATCURVE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+    exit 0
+else
+    doveadm fts optimize -A
+fi

data/conf/dovecot/conf.d/fts-flatcurve.conf

@@ -0,0 +1,24 @@
+plugin {
+    fts_autoindex = yes
+    fts_autoindex_exclude = \Junk
+    fts_autoindex_exclude2 = \Trash
+    fts = flatcurve
+
+    # These are not flatcurve settings, but required for Dovecot FTS. See
+    # Dovecot FTS Configuration link above for further information.
+    fts_languages = en es de
+    fts_tokenizer_generic = algorithm=simple
+    fts_tokenizers = generic email-address
+
+    # OPTIONAL: Recommended default FTS core configuration
+    fts_filters = normalizer-icu snowball stopwords
+    fts_filters_en = lowercase snowball english-possessive stopwords
+
+}
+
+service indexer-worker {
+  executable = /bin/nice -n 10 /usr/libexec/dovecot/indexer-worker
+  process_limit=5
+
+  vsz_limit=256 MB
+}

data/conf/dovecot/dovecot.conf

@@ -10,6 +10,7 @@
 auth_mechanisms = plain login
 #mail_debug = yes
 #auth_debug = yes
+log_debug = category=fts-flatcurve
 log_path = syslog
 disable_plaintext_auth = yes
 # Uncomment on NFS share
@@ -194,9 +195,6 @@ plugin {
   acl_shared_dict = file:/var/vmail/shared-mailboxes.db
   acl = vfile
   acl_user = %u
-  fts = solr
-  fts_autoindex = yes
-  fts_solr = url=http://solr:8983/solr/dovecot-fts/
   quota = dict:Userquota::proxy::sqlquota
   quota_rule2 = Trash:storage=+100%%
   sieve = /var/vmail/sieve/%u.sieve
@@ -305,6 +303,7 @@ replication_dsync_parameters = -d -l 30 -U -n INBOX
 !include_try /etc/dovecot/extra.conf
 !include_try /etc/dovecot/sogo-sso.conf
 !include_try /etc/dovecot/shared_namespace.conf
+!include_try /etc/dovecot/conf.d/fts-flatcurve.conf
 # </Includes>
 default_client_limit = 10400
 default_vsz_limit = 1024 M

docker-compose.yml

@@ -158,7 +158,7 @@ services:
         - API_KEY_READ_ONLY=${API_KEY_READ_ONLY:-invalid}
         - API_ALLOW_FROM=${API_ALLOW_FROM:-invalid}
         - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized}
-        - SKIP_SOLR=${SKIP_SOLR:-y}
+        - SKIP_FLATCURVE=${SKIP_FLATCURVE:-y}
         - SKIP_CLAMD=${SKIP_CLAMD:-n}
         - SKIP_SOGO=${SKIP_SOGO:-n}
         - ALLOW_ADMIN_EMAIL_LOGIN=${ALLOW_ADMIN_EMAIL_LOGIN:-n}
@@ -221,7 +221,7 @@ services:
             - sogo
 
     dovecot-mailcow:
-      image: mailcow/dovecot:1.29
+      image: mailcow/dovecot:dev
       depends_on:
         - mysql-mailcow
         - netfilter-mailcow
@@ -258,7 +258,9 @@ services:
         - ALLOW_ADMIN_EMAIL_LOGIN=${ALLOW_ADMIN_EMAIL_LOGIN:-n}
         - MAILDIR_GC_TIME=${MAILDIR_GC_TIME:-7200}
         - ACL_ANYONE=${ACL_ANYONE:-disallow}
-        - SKIP_SOLR=${SKIP_SOLR:-y}
+        - SKIP_FLATCURVE=${SKIP_FLATCURVE:-y}
+        - FTS_HEAP=${FTS_HEAP:-512}
+        - FTS_PROCS=${FTS_PROCS:-3}
         - MAILDIR_SUB=${MAILDIR_SUB:-}
         - MASTER=${MASTER:-y}
         - REDIS_SLAVEOF_IP=${REDIS_SLAVEOF_IP:-}
@@ -550,27 +552,6 @@ services:
           aliases:
             - dockerapi
 
-
-    ##### Will be removed soon #####
-    solr-mailcow:
-      image: mailcow/solr:1.8.2
-      restart: always
-      depends_on:
-        - netfilter-mailcow
-      volumes:
-        - solr-vol-1:/opt/solr/server/solr/dovecot-fts/data
-      ports:
-        - "${SOLR_PORT:-127.0.0.1:18983}:8983"
-      environment:
-        - TZ=${TZ}
-        - SOLR_HEAP=${SOLR_HEAP:-1024}
-        - SKIP_SOLR=${SKIP_SOLR:-y}
-      networks:
-        mailcow-network:
-          aliases:
-            - solr
-    ################################
-
     olefy-mailcow:
       image: mailcow/olefy:1.13
       restart: always
@@ -626,7 +607,6 @@ services:
         - netfilter-mailcow
         - watchdog-mailcow
         - dockerapi-mailcow
-        - solr-mailcow
       environment:
         - TZ=${TZ}
       image: robbertkl/ipv6nat
@@ -658,7 +638,6 @@ volumes:
   mysql-socket-vol-1:
   redis-vol-1:
   rspamd-vol-1:
-  solr-vol-1:
   postfix-vol-1:
   crypt-vol-1:
   sogo-web-vol-1:

generate_config.sh

@@ -163,26 +163,6 @@ else
   SKIP_CLAMD=n
 fi
 
-if [ ${MEM_TOTAL} -le "2097152" ]; then
-  echo "Disabling Solr on low-memory system."
-  SKIP_SOLR=y
-elif [ ${MEM_TOTAL} -le "3670016" ]; then
-  echo "Installed memory is <= 3.5 GiB. It is recommended to disable Solr to prevent out-of-memory situations."
-  echo "Solr is a prone to run OOM and should be monitored. The default Solr heap size is 1024 MiB and should be set in mailcow.conf according to your expected load."
-  echo "Solr can be re-enabled by setting SKIP_SOLR=n in mailcow.conf but will refuse to start with less than 2 GB total memory."
-  read -r -p  "Do you want to disable Solr now? [Y/n] " response
-  case $response in
-    [nN][oO]|[nN])
-      SKIP_SOLR=n
-      ;;
-    *)
-      SKIP_SOLR=y
-    ;;
-  esac
-else
-  SKIP_SOLR=n
-fi
-
 if [[ ${SKIP_BRANCH} != y ]]; then
   echo "Which branch of mailcow do you want to use?"
   echo ""
@@ -382,14 +362,21 @@ SKIP_CLAMD=${SKIP_CLAMD}
 
 SKIP_SOGO=n
 
-# Skip Solr on low-memory systems or if you do not want to store a readable index of your mails in solr-vol-1.
+# Skip Flatcurve (FTS) on low-memory systems or if you simply want to disable it.
+
+SKIP_FLATCURVE=n
+
+# Dovecot Indexing (FTS) Process maximum heap size in MB, there is no recommendation, please see Dovecot docs.
+# Flatcurve (Xapian backend) is used as the FTS Indexer. It is supposed to be efficient in CPU and RAM consumption.
+# However: Please always monitor your Resource consumption!
 
-SKIP_SOLR=${SKIP_SOLR}
+FTS_HEAP=512
 
-# Solr heap size in MB, there is no recommendation, please see Solr docs.
-# Solr is a prone to run OOM and should be monitored. Unmonitored Solr setups are not recommended.
+# Controls how many processes the Dovecot indexing process can spawn at max.
+# Too many indexing processes can use a lot of CPU and Disk I/O.
+# Please visit: https://doc.dovecot.org/configuration_manual/service_configuration/#indexer-worker for more informations
 
-SOLR_HEAP=1024
+FTS_PROCS=5
 
 # Allow admins to log into SOGo as email user (without any password)