From cf6594220cdc74426f71f6eeb11a9a7cd089036b Mon Sep 17 00:00:00 2001 From: Niklas Meyer Date: Wed, 26 Jun 2024 11:28:18 +0200 Subject: [PATCH] dovecot: add Flatcurve FTS Engine as EXPERIMENTAL (#5920) * dovecot: experimental added flatcurve backend + switch * dovecot: bump docker image --- data/Dockerfiles/dovecot/Dockerfile | 3 + data/Dockerfiles/dovecot/docker-entrypoint.sh | 55 ++++++++++++++++++- data/Dockerfiles/dovecot/optimize-fts.sh | 7 +++ data/Dockerfiles/solr/solr.sh | 16 +++++- data/conf/dovecot/dovecot.conf | 5 +- docker-compose.yml | 8 ++- 6 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 data/Dockerfiles/dovecot/optimize-fts.sh diff --git a/data/Dockerfiles/dovecot/Dockerfile b/data/Dockerfiles/dovecot/Dockerfile index c33772db4b..65028b1734 100644 --- a/data/Dockerfiles/dovecot/Dockerfile +++ b/data/Dockerfiles/dovecot/Dockerfile @@ -24,6 +24,7 @@ RUN addgroup -g 5000 vmail \ envsubst \ ca-certificates \ curl \ + coreutils \ jq \ lua \ lua-cjson \ @@ -104,6 +105,7 @@ RUN addgroup -g 5000 vmail \ dovecot-pigeonhole-plugin \ dovecot-pop3d \ dovecot-fts-solr \ + dovecot-fts-flatcurve \ && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \ && chmod +x /usr/local/bin/gosu \ @@ -127,6 +129,7 @@ COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh COPY quarantine_notify.py /usr/local/bin/quarantine_notify.py COPY quota_notify.py /usr/local/bin/quota_notify.py COPY repl_health.sh /usr/local/bin/repl_health.sh +COPY optimize-fts.sh /usr/local/bin/optimize-fts.sh ENTRYPOINT ["/docker-entrypoint.sh"] CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf diff --git a/data/Dockerfiles/dovecot/docker-entrypoint.sh b/data/Dockerfiles/dovecot/docker-entrypoint.sh index a9545f338b..b53c06bed4 100755 --- a/data/Dockerfiles/dovecot/docker-entrypoint.sh +++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh @@ -29,6 +29,7 @@ ${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null # Create missing directories [[ ! -d /etc/dovecot/sql/ ]] && mkdir -p /etc/dovecot/sql/ [[ ! -d /etc/dovecot/lua/ ]] && mkdir -p /etc/dovecot/lua/ +[[ ! -d /etc/dovecot/conf.d/ ]] && mkdir -p /etc/dovecot/conf.d/ [[ ! -d /var/vmail/_garbage ]] && mkdir -p /var/vmail/_garbage [[ ! -d /var/vmail/sieve ]] && mkdir -p /var/vmail/sieve [[ ! -d /etc/sogo ]] && mkdir -p /etc/sogo @@ -109,7 +110,14 @@ EOF echo -n ${ACL_ANYONE} > /etc/dovecot/acl_anyone -if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then +if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY]) ]]; then +echo -e "\e[33mActivating Flatcurve as FTS Backend...\e[0m" +echo -e "\e[33mDepending on your previous setup a full reindex might be needed... \e[0m" +echo -e "\e[34mVisit https://docs.mailcow.email/manual-guides/Dovecot/u_e-dovecot-fts/#fts-related-dovecot-commands to learn how to reindex\e[0m" +echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins +echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins_imap +echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_flatcurve notify listescape replication' > /etc/dovecot/mail_plugins_lmtp +elif [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication' > /etc/dovecot/mail_plugins echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify listescape replication mail_log' > /etc/dovecot/mail_plugins_imap echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl notify listescape replication' > /etc/dovecot/mail_plugins_lmtp @@ -239,6 +247,47 @@ function script_deinit() end EOF +# Temporarily set FTS depending on user choice inside mailcow.conf. Will be removed as soon as Solr is dropped +if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])$ ]]; then +cat < /etc/dovecot/conf.d/fts.conf +# Autogenerated by mailcow +plugin { + fts_autoindex = yes + fts_autoindex_exclude = \Junk + fts_autoindex_exclude2 = \Trash + fts = flatcurve + + # These are not flatcurve settings, but required for Dovecot FTS. See + # Dovecot FTS Configuration link above for further information. + fts_languages = en es de + fts_tokenizer_generic = algorithm=simple + fts_tokenizers = generic email-address + + # OPTIONAL: Recommended default FTS core configuration + fts_filters = normalizer-icu snowball stopwords + fts_filters_en = lowercase snowball english-possessive stopwords +} +EOF +elif [[ ! "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])$ ]]; then +cat < /etc/dovecot/conf.d/fts.conf +# Autogenerated by mailcow +plugin { + fts = solr + fts_autoindex = yes + fts_autoindex_exclude = \Junk + fts_autoindex_exclude2 = \Trash + fts_solr = url=http://solr:8983/solr/dovecot-fts/ + + fts_tokenizers = generic email-address + fts_tokenizer_generic = algorithm=simple + + fts_filters = normalizer-icu snowball stopwords + fts_filters_en = lowercase snowball english-possessive stopwords +} +EOF +fi + + # Replace patterns in app-passdb.lua sed -i "s/__DBUSER__/${DBUSER}/g" /etc/dovecot/lua/passwd-verify.lua sed -i "s/__DBPASS__/${DBPASS}/g" /etc/dovecot/lua/passwd-verify.lua @@ -343,7 +392,6 @@ mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT} EOF fi - # 401 is user dovecot if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem @@ -387,7 +435,8 @@ chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \ /usr/local/bin/maildir_gc.sh \ /usr/local/sbin/stop-supervisor.sh \ /usr/local/bin/quota_notify.py \ - /usr/local/bin/repl_health.sh + /usr/local/bin/repl_health.sh \ + /usr/local/bin/optimize-fts.sh # Prepare environment file for cronjobs printenv | sed 's/^\(.*\)$/export \1/g' > /source_env.sh diff --git a/data/Dockerfiles/dovecot/optimize-fts.sh b/data/Dockerfiles/dovecot/optimize-fts.sh new file mode 100644 index 0000000000..a6e8f91da1 --- /dev/null +++ b/data/Dockerfiles/dovecot/optimize-fts.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ && ! "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then + exit 0 +else + doveadm fts optimize -A +fi \ No newline at end of file diff --git a/data/Dockerfiles/solr/solr.sh b/data/Dockerfiles/solr/solr.sh index 1c5c6f51f0..03ab791260 100755 --- a/data/Dockerfiles/solr/solr.sh +++ b/data/Dockerfiles/solr/solr.sh @@ -1,7 +1,15 @@ #!/bin/bash -if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then +if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then + echo "FLATCURVE_EXPERIMENTAL=y, skipping Solr but enabling Flatcurve as FTS for Dovecot!" + echo "Solr will be removed in the future!" + sleep 365d + exit 0 +elif [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then echo "SKIP_SOLR=y, skipping Solr..." + echo "HINT: You could try the newer FTS Backend Flatcurve, which is currently in experimental state..." + echo "Simply set FLATCURVE_EXPERIMENTAL=y inside your mailcow.conf and restart the stack afterwards!" + echo "Solr will be removed in the future!" sleep 365d exit 0 fi @@ -57,5 +65,11 @@ if [[ "${1}" == "--bootstrap" ]]; then exit 0 fi +echo "Starting up Solr..." +echo -e "\e[31mSolr is deprecated! You can try the new FTS System now by enabling FLATCURVE_EXPERIMENTAL=y inside mailcow.conf and restarting the stack\e[0m" +echo -e "\e[31mSolr will be removed completely soon!\e[0m" + +sleep 15 + exec gosu solr solr-foreground diff --git a/data/conf/dovecot/dovecot.conf b/data/conf/dovecot/dovecot.conf index 729686fb18..c230c34950 100644 --- a/data/conf/dovecot/dovecot.conf +++ b/data/conf/dovecot/dovecot.conf @@ -10,6 +10,7 @@ auth_mechanisms = plain login #mail_debug = yes #auth_debug = yes +#log_debug = category=fts-flatcurve # Activate Logging for Flatcurve FTS Searchings log_path = syslog disable_plaintext_auth = yes # Uncomment on NFS share @@ -194,9 +195,6 @@ plugin { acl_shared_dict = file:/var/vmail/shared-mailboxes.db acl = vfile acl_user = %u - fts = solr - fts_autoindex = yes - fts_solr = url=http://solr:8983/solr/dovecot-fts/ quota = dict:Userquota::proxy::sqlquota quota_rule2 = Trash:storage=+100%% sieve = /var/vmail/sieve/%u.sieve @@ -305,6 +303,7 @@ replication_dsync_parameters = -d -l 30 -U -n INBOX !include_try /etc/dovecot/extra.conf !include_try /etc/dovecot/sogo-sso.conf !include_try /etc/dovecot/shared_namespace.conf +!include_try /etc/dovecot/conf.d/fts.conf # default_client_limit = 10400 default_vsz_limit = 1024 M diff --git a/docker-compose.yml b/docker-compose.yml index 49b454ec14..b0f50dd6d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -221,7 +221,7 @@ services: - sogo dovecot-mailcow: - image: mailcow/dovecot:1.29 + image: mailcow/dovecot:1.30 depends_on: - mysql-mailcow - netfilter-mailcow @@ -264,6 +264,7 @@ services: - REDIS_SLAVEOF_IP=${REDIS_SLAVEOF_IP:-} - REDIS_SLAVEOF_PORT=${REDIS_SLAVEOF_PORT:-} - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized} + - FLATCURVE_EXPERIMENTAL=${FLATCURVE_EXPERIMENTAL:-n} ports: - "${DOVEADM_PORT:-127.0.0.1:19991}:12345" - "${IMAP_PORT:-143}:143" @@ -289,7 +290,7 @@ services: ofelia.job-exec.dovecot_sarules.schedule: "@every 24h" ofelia.job-exec.dovecot_sarules.command: "/bin/bash -c \"/usr/local/bin/sa-rules.sh\"" ofelia.job-exec.dovecot_fts.schedule: "@every 24h" - ofelia.job-exec.dovecot_fts.command: "/usr/bin/curl http://solr:8983/solr/dovecot-fts/update?optimize=true" + ofelia.job-exec.dovecot_fts.command: "/bin/bash -c \"/usr/local/bin/gosu vmail /usr/local/bin/optimize-fts.sh\"" ofelia.job-exec.dovecot_repl_health.schedule: "@every 5m" ofelia.job-exec.dovecot_repl_health.command: "/bin/bash -c \"/usr/local/bin/gosu vmail /usr/local/bin/repl_health.sh\"" ulimits: @@ -553,7 +554,7 @@ services: ##### Will be removed soon ##### solr-mailcow: - image: mailcow/solr:1.8.2 + image: mailcow/solr:1.8.3 restart: always depends_on: - netfilter-mailcow @@ -565,6 +566,7 @@ services: - TZ=${TZ} - SOLR_HEAP=${SOLR_HEAP:-1024} - SKIP_SOLR=${SKIP_SOLR:-y} + - FLATCURVE_EXPERIMENTAL=${FLATCURVE_EXPERIMENTAL:-n} networks: mailcow-network: aliases: