[Dovecot] Allow setting ACL_ANYONE in mailcow.conf

.gitignore

@@ -22,6 +22,7 @@ data/conf/rspamd/override.d/*
 data/conf/nginx/*.conf
 data/conf/nginx/*.custom
 data/conf/nginx/*.bak
+data/conf/dovecot/acl_anyone
 data/conf/dovecot/extra.conf
 data/conf/rspamd/custom/*
 data/conf/portainer/

data/Dockerfiles/clamd/Dockerfile

@@ -53,9 +53,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   libjson-c-dev \
   && addgroup --system --gid 700 clamav \
   && adduser --system --no-create-home --home /var/lib/clamav --uid 700 --gid 700 --disabled-login clamav \
-  && mkdir -p /run/clamav /var/lib/clamav \
-  && chown clamav:clamav /run/clamav /var/lib/clamav \
-  && chmod 750 /run/clamav \
   && rm -rf /tmp/* /var/tmp/*
 
 COPY bootstrap.sh ./

data/Dockerfiles/clamd/bootstrap.sh

@@ -14,6 +14,10 @@ if [[ ! -f /var/lib/clamav/whitelist.ign2 ]]; then
   echo "Example-Signature.Ignore-1" > /var/lib/clamav/whitelist.ign2
 fi
 chown clamav:clamav /var/lib/clamav/whitelist.ign2
+mkdir -p /run/clamav /var/lib/clamav
+chown clamav:clamav /run/clamav /var/lib/clamav
+chmod 750 /run/clamav
+chmod 755 /var/lib/clamav
 
 dos2unix /var/lib/clamav/whitelist.ign2
 sed -i '/^\s*$/d' /var/lib/clamav/whitelist.ign2

data/Dockerfiles/dovecot/Dockerfile

@@ -97,6 +97,7 @@ RUN echo '30 3 * * *  vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.
 RUN echo '* * * * *   vmail /usr/local/bin/trim_logs.sh >> /dev/console 2>&1' > /etc/cron.d/trim_logs
 RUN echo '25 * * * *  vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc
 RUN echo '30 1 * * *  root  /usr/local/bin/sa-rules.sh  >> /dev/console 2>&1' > /etc/cron.d/sa-rules
+RUN echo '0 2 * * *   root  /usr/bin/curl http://solr:8983/solr/dovecot/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize 
 COPY trim_logs.sh /usr/local/bin/trim_logs.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY imapsync /usr/local/bin/imapsync

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -85,6 +85,7 @@ map {
 }
 EOF
 
+echo -n ${ACL_ANYONE} > /usr/local/etc/dovecot/acl_anyone
 
 # Create userdb dict for Dovecot
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-userdb.conf

data/Dockerfiles/sogo/bootstrap-sogo.sh

@@ -167,9 +167,17 @@ echo '    </dict>
 chown sogo:sogo -R /var/lib/sogo/
 chmod 600 /var/lib/sogo/GNUstep/Defaults/sogod.plist
 
-# Patch ACLs (comment this out to enable any or authenticated targets for ACL)
-if patch -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
+# Patch ACLs
+if [[ ${ACL_ANYONE} == 'allow' ]]; then
+  #enable any or authenticated targets for ACL
+  if patch -R -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
+    patch -R /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
+  fi
+else
+  #disable any or authenticated targets for ACL
+  if patch -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
     patch /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
+  fi
 fi
 
 # Copy logo, if any

data/Dockerfiles/solr/Dockerfile

@@ -0,0 +1,9 @@
+FROM solr:7-alpine
+USER root
+COPY docker-entrypoint.sh /
+
+RUN apk --no-cache add su-exec curl \
+  && chmod +x /docker-entrypoint.sh \
+  && /docker-entrypoint.sh --bootstrap
+
+ENTRYPOINT ["/docker-entrypoint.sh"]

data/Dockerfiles/solr/docker-entrypoint.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  echo "SKIP_SOLR=y, skipping Solr..."
+  sleep 365d
+  exit 0
+fi
+
+set -e
+
+# allow easier debugging with `docker run -e VERBOSE=yes`
+if [[ "$VERBOSE" = "yes" ]]; then
+  set -x
+fi
+
+# run the optional initdb
+. /opt/docker-solr/scripts/run-initdb
+
+function solr_config() {
+  curl -XPOST http://localhost:8983/solr/dovecot/schema -H 'Content-type:application/json' -d '{
+    "add-field-type":{
+      "name":"long",
+      "class":"solr.TrieLongField"
+    },
+    "add-field-type":{
+      "name":"text",
+      "class":"solr.TextField",
+      "positionIncrementGap":100,
+      "indexAnalyser":{
+        "tokenizer":{
+          "class":"solr.StandardTokenizerFactory"
+        },
+        "filter":{
+          "class":"solr.WordDelimiterFilterFactory",
+          "generateWordParts":1,
+          "generateNumberParts":1,
+          "catenateWorks":1,
+          "catenateNumbers":1,
+          "catenateAll":0
+        },
+        "filter":{
+          "class":"solr.LowerCaseFilterFactory"
+        },
+        "filter":{
+          "class":"solr.KeywordMarkerFilterFactory",
+          "protected":"protwords.txt"
+        }
+      },
+      "queryAnalyzer":{
+        "tokenizer":{
+          "class":"solr.StandardTokenizerFactory"
+        },
+        "filter":{
+          "synonyms":"synonyms.txt",
+          "ignoreCase":true,
+          "expand":true
+        },
+        "filter":{
+          "class":"solr.LowerCaseFilterFactory"
+        },
+        "filter":{
+          "class":"solr.WordDelimiterFilterFactory",
+          "generateWordParts":1,
+          "generateNumberParts":1,
+          "catenateWords":0,
+          "catenateNumbers":0,
+          "catenateAll":0,
+          "splitOnCaseChange":1
+        }
+      }
+    },
+    "add-field":{
+      "name":"uid",
+      "type":"long",
+      "indexed":true,
+      "stored":true,
+      "required":true
+    },
+    "add-field":{
+      "name":"box",
+      "type":"string",
+      "indexed":true,
+      "stored":true,
+      "required":true
+    },
+    "add-field":{
+      "name":"user",
+      "type":"string",
+      "indexed":true,
+      "stored":true,
+      "required":true
+    },
+    "add-field":{
+      "name":"hdr",
+      "type":"text",
+      "indexed":true,
+      "stored":false
+
+    },
+    "add-field":{
+      "name":"body",
+      "type":"text",
+      "indexed":true,
+      "stored":false
+    },
+    "add-field":{
+      "name":"from",
+      "type":"text",
+      "indexed":true,
+      "stored":false
+    },
+    "add-field":{
+      "name":"to",
+      "type":"text",
+      "indexed":true,
+      "stored":false
+    },
+    "add-field":{
+      "name":"cc",
+      "type":"text",
+      "indexed":true,
+      "stored":false
+    },
+    "add-field":{
+      "name":"bcc",
+      "type":"text",
+      "indexed":true,
+      "stored":false
+    },
+    "add-field":{
+      "name":"subject",
+      "type":"text",
+      "indexed":true,
+      "stored":false
+    }
+  }'
+
+  curl -XPOST http://localhost:8983/solr/dovecot/config -H 'Content-type:application/json' -d '{
+    "update-requesthandler":{
+      "name":"/select",
+      "class":"solr.SearchHandler",
+      "defaults":{
+        "wt":"xml"
+      }
+    }
+  }'
+
+  curl -XPOST http://localhost:8983/solr/dovecot/config/updateHandler -d '{
+    "set-property": {
+      "updateHandler.autoSoftCommit.maxDocs":500,
+      "updateHandler.autoSoftCommit.maxTime":120000,
+      "updateHandler.autoCommit.maxDocs":200,
+      "updateHandler.autoCommit.maxTime":1800000,
+      "updateHandler.autoCommit.openSearcher":false
+    }
+  }'
+}
+
+# fixing volume permission
+[[ -d /opt/solr/server/solr/dovecot/data ]] && chown -R solr:solr /opt/solr/server/solr/dovecot/data
+sed -i 's/#SOLR_HEAP="512m"/SOLR_HEAP="'${SOLR_HEAP:-1024}'m"/g' /opt/solr/bin/solr.in.sh
+
+# start a Solr so we can use the Schema API, but only on localhost,
+# so that clients don't see Solr until we have configured it.
+echo "Starting local Solr instance to setup configuration"
+su-exec solr start-local-solr
+
+# keep a sentinel file so we don't try to create the core a second time
+# for example when we restart a container.
+SENTINEL=/opt/docker-solr/core_created
+if [[ -f ${SENTINEL} ]]; then
+  echo "skipping core creation"
+else
+  echo "Creating core \"dovecot\""
+  su-exec solr /opt/solr/bin/solr create -c "dovecot"
+
+  # See https://github.com/docker-solr/docker-solr/issues/27
+  echo "Checking core"
+  while ! wget -O - 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q instanceDir; do
+    echo "Could not find any cores, waiting..."
+    sleep 5
+  done
+  echo "Created core \"dovecot\""
+  touch ${SENTINEL}
+fi
+
+echo "Starting configuration"
+solr_config
+echo "Stopping local Solr"
+su-exec solr stop-local-solr
+if [[ "${1}" == "--bootstrap" ]]; then
+  exit 0
+else
+  exec su-exec solr solr-foreground
+fi

data/conf/dovecot/dovecot.conf

@@ -20,7 +20,7 @@ disable_plaintext_auth = yes
 login_log_format_elements = "user=<%u> method=%m rip=%r lip=%l mpid=%e %c %k"
 mail_home = /var/vmail/%d/%n
 mail_location = maildir:~/
-mail_plugins = quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify
+mail_plugins = quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify fts fts_solr
 mail_attachment_fs = crypt:set_prefix=mail_crypt_global:posix:
 mail_attachment_dir = /var/attachments
 mail_attachment_min_size = 128k
@@ -279,7 +279,7 @@ userdb {
 }
 protocol imap {
   imap_metadata = yes
-  mail_plugins = quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify mail_log
+  mail_plugins = quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify mail_log fts fts_solr
 }
 mail_attribute_dict = file:%h/dovecot-attributes
 protocol lmtp {
@@ -291,9 +291,12 @@ protocol sieve {
 }
 plugin {
   # Allow "any" or "authenticated" to be used in ACLs
-  #acl_anyone = allow
+  acl_anyone = </usr/local/etc/dovecot/acl_anyone
   acl_shared_dict = file:/var/vmail/shared-mailboxes.db
   acl = vfile
+  fts = solr
+  fts_autoindex = yes
+  fts_solr = url=http://solr:8983/solr/dovecot/
   quota = dict:Userquota::proxy::sqlquota
   quota_rule2 = Trash:storage=+100%%
   sieve = /var/vmail/sieve/%u.sieve

data/conf/rspamd/meta_exporter/pipe.php

@@ -51,6 +51,7 @@ $raw_data = mb_convert_encoding($raw_data_content, 'HTML-ENTITIES', "UTF-8");
 $headers = getallheaders();
 
 $qid      = $headers['X-Rspamd-Qid'];
+$subject      = $headers['X-Rspamd-Subject'];
 $score    = $headers['X-Rspamd-Score'];
 $rcpts    = $headers['X-Rspamd-Rcpt'];
 $user     = $headers['X-Rspamd-User'];
@@ -188,10 +189,11 @@ foreach (json_decode($rcpts, true) as $rcpt) {
 foreach ($rcpt_final_mailboxes as $rcpt) {
   error_log("QUARANTINE: quarantine pipe: processing quarantine message for rcpt " . $rcpt);
   try {
-    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`)
+    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `subject`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`)
       VALUES (:qid, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action)");
     $stmt->execute(array(
       ':qid' => $qid,
+      ':subject' => $subject,
       ':score' => $score,
       ':sender' => $sender,
       ':rcpt' => $rcpt,

data/conf/sogo/sogo.conf

@@ -15,6 +15,7 @@
     SOGoFoldersSendEMailNotifications = YES;
     SOGoForwardEnabled = YES;
     SOGoUIAdditionalJSFiles = (js/custom-sogo.js);
+    SOGoEnablePublicAccess = YES;
 
     // Multi-domain setup
     // Domains are isolated, you can define visibility options here.

data/web/inc/functions.inc.php

@@ -1465,4 +1465,39 @@ function getGUID() {
         .substr($charid,16, 4).$hyphen
         .substr($charid,20,12);
 }
+function solr_status() {
+  $curl = curl_init();
+  $endpoint = 'http://solr:8983/solr/admin/cores';
+  $params = array(
+    'action' => 'STATUS',
+    'core' => 'dovecot',
+    'indexInfo' => 'true'
+  );
+  $url = $endpoint . '?' . http_build_query($params);
+  curl_setopt($curl, CURLOPT_URL, $url);
+  curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
+  curl_setopt($curl, CURLOPT_POST, 0);
+  curl_setopt($curl, CURLOPT_TIMEOUT, 20);
+  $response = curl_exec($curl);
+  if ($response === false) {
+    $err = curl_error($curl);
+    curl_close($curl);
+    // logger(array('return' => array(
+      // 'type' => 'danger',
+      // 'log' => array(__FUNCTION__, $action, $service_name, $attr1, $attr2, $extra_headers),
+      // 'msg' => $err,
+    // )));
+    return $err;
+  }
+  else {
+    curl_close($curl);
+    // logger(array('return' => array(
+      // 'type' => 'success',
+      // 'log' => array(__FUNCTION__, $action, $service_name, $attr1, $attr2, $extra_headers),
+    // )));
+    $status = json_decode($response, true);
+    return (!empty($status['status']['dovecot'])) ? $status['status']['dovecot'] : false;
+  }
+  return false;
+}
 ?>

data/web/inc/init_db.inc.php

@@ -3,7 +3,7 @@ function init_db_schema() {
   try {
     global $pdo;
 
-    $db_version = "15122018_0717";
+    $db_version = "14012019_0717";
 
     $stmt = $pdo->query("SHOW TABLES LIKE 'versions'");
     $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
@@ -226,6 +226,7 @@ function init_db_schema() {
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
           "qid" => "VARCHAR(30) NOT NULL",
+          "subject" => "VARCHAR(500)",
           "score" => "FLOAT(8,2)",
           "ip" => "VARBINARY(16)",
           "action" => "CHAR(20) NOT NULL DEFAULT 'unknown'",

docker-compose.yml

@@ -55,7 +55,7 @@ services:
             - redis
 
     clamd-mailcow:
-      image: mailcow/clamd:1.18
+      image: mailcow/clamd:1.20
       build: ./data/Dockerfiles/clamd
       restart: always
       environment:
@@ -128,6 +128,7 @@ services:
         - API_KEY=${API_KEY:-invalid}
         - API_ALLOW_FROM=${API_ALLOW_FROM:-invalid}
         - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized}
+        - SKIP_SOLR=${SKIP_SOLR:-y}
       restart: always
       dns:
         - ${IPV4_NETWORK:-172.22.1}.254
@@ -137,7 +138,7 @@ services:
             - phpfpm
 
     sogo-mailcow:
-      image: mailcow/sogo:1.49
+      image: mailcow/sogo:1.50
       build: ./data/Dockerfiles/sogo
       environment:
         - DBNAME=${DBNAME}
@@ -146,6 +147,7 @@ services:
         - TZ=${TZ}
         - LOG_LINES=${LOG_LINES:-9999}
         - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
+        - ACL_ANYONE=${ACL_ANYONE:-disallow}
       volumes:
         - ./data/conf/sogo/:/etc/sogo/
         - ./data/web/inc/init_db.inc.php:/init_db.inc.php
@@ -161,7 +163,7 @@ services:
             - sogo
 
     dovecot-mailcow:
-      image: mailcow/dovecot:1.52
+      image: mailcow/dovecot:1.54
       build: ./data/Dockerfiles/dovecot
       cap_add:
         - NET_BIND_SERVICE
@@ -182,6 +184,7 @@ services:
         - DBPASS=${DBPASS}
         - TZ=${TZ}
         - MAILDIR_GC_TIME=${MAILDIR_GC_TIME:-1440}
+        - ACL_ANYONE=${ACL_ANYONE:-disallow}
       ports:
         - "${DOVEADM_PORT:-127.0.0.1:19991}:12345"
         - "${IMAP_PORT:-143}:143"
@@ -373,7 +376,7 @@ services:
             - watchdog
 
     dockerapi-mailcow:
-      image: mailcow/dockerapi:1.24
+      image: mailcow/dockerapi:1.25
       restart: always
       build: ./data/Dockerfiles/dockerapi
       oom_kill_disable: true
@@ -389,6 +392,23 @@ services:
           aliases:
             - dockerapi
 
+    solr-mailcow:
+      image: mailcow/solr:1.0
+      build: ./data/Dockerfiles/solr
+      restart: always
+      volumes:
+        - solr-vol-1:/opt/solr/server/solr/dovecot/data
+      dns:
+        - 172.22.1.254
+      dns_search: mailcow-network
+      environment:
+        - SOLR_HEAP=${SOLR_HEAP:-1024}
+        - SKIP_SOLR=${SKIP_SOLR:-y}
+      networks:
+        mailcow-network:
+          aliases:
+            - solr
+
     ipv6nat:
       depends_on:
         - unbound-mailcow
@@ -406,6 +426,7 @@ services:
         - netfilter-mailcow
         - watchdog-mailcow
         - dockerapi-mailcow
+        - solr-mailcow
       image: robbertkl/ipv6nat
       restart: always
       privileged: true
@@ -433,5 +454,6 @@ volumes:
   mysql-socket-vol-1:
   redis-vol-1:
   rspamd-vol-1:
+  solr-vol-1:
   postfix-vol-1:
   crypt-vol-1:

generate_config.sh

@@ -48,6 +48,39 @@ while [ -z "${MAILCOW_TZ}" ]; do
   fi
 done
 
+MEM_TOTAL=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
+
+if [ ${MEM_TOTAL} -le "1572864" ]; then
+  echo "Installed memory is less than 1.5 GiB. It is recommended to disable ClamAV to prevent out-of-memory situations."
+  read -r -p  "Do you want to disable ClamAV now? ClamAV can be re-enabled by setting SKIP_CLAMD=n in mailcow.conf. [Y/n] " response
+  case $response in
+    [nN][oO]|[nN])
+      SKIP_CLAMD=n
+      ;;
+    *)
+      SKIP_CLAMD=y
+    ;;
+  esac
+else
+ SKIP_CLAMD=n
+fi
+
+if [ ${MEM_TOTAL} -le "6815744" ]; then
+  echo "Installed memory is less than 6.5 GiB. It is highly recommended to disable Solr to prevent out-of-memory situations."
+  echo "Solr is a prone to run OOM and should be monitored. The default Solr heap size is 1024 MiB and should be set according to your expected load in mailcow.conf."
+  read -r -p  "Do you want to disable Solr now (recommended)? Solr can be re-enabled by setting SKIP_SOLR=n in mailcow.conf. [Y/n] " response
+  case $response in
+    [nN][oO]|[nN])
+      SKIP_SOLR=n
+      ;;
+    *)
+      SKIP_SOLR=y
+    ;;
+  esac
+else
+  SKIP_SOLR=n
+fi
+
 [ ! -f ./data/conf/rspamd/override.d/worker-controller-password.inc ] && echo '# Placeholder' > ./data/conf/rspamd/override.d/worker-controller-password.inc
 
 cat << EOF > mailcow.conf
@@ -145,6 +178,13 @@ SKIP_IP_CHECK=n
 
 SKIP_CLAMD=n
 
+# Skip Solr on low-memory systems
+SKIP_SOLR=${SKIP_SOLR}
+
+# Solr heap size in MB, there is no recommendation, please see Solr docs.
+# Solr is a prone to run OOM and should be monitored. Unmonitored Solr setups are not recommended.
+SOLR_HEAP=1024
+
 # Enable watchdog (watchdog-mailcow) to restart unhealthy containers (experimental)
 
 USE_WATCHDOG=n

update.sh

@@ -121,6 +121,9 @@ CONFIG_ARRAY=(
   "API_KEY"
   "API_ALLOW_FROM"
   "MAILDIR_GC_TIME"
+  "ACL_ANYONE"
+  "SOLR_HEAP"
+  "SKIP_SOLR"
 )
 
 sed -i '$a\' mailcow.conf
@@ -202,6 +205,26 @@ for option in ${CONFIG_ARRAY[@]}; do
       echo '# Check interval is hourly' >> mailcow.conf
       echo 'MAILDIR_GC_TIME=1440' >> mailcow.conf
     fi
+  elif [[ ${option} == "ACL_ANYONE" ]]; then
+    if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
+      echo '# Set this to "allow" to enable the anyone pseudo user. Disabled by default.
+' >> mailcow.conf
+      echo '# When enabled, ACL can be created, that apply to "All authenticated users"
+' >> mailcow.conf
+      echo '# This should probably only be activated on mail hosts, that are used exclusivly by one organisation.
+' >> mailcow.conf
+      echo '# Otherwise a user might share data with too many other users.
+' >> mailcow.conf
+      echo 'ACL_ANYONE=disallow' >> mailcow.conf
+    fi
+  elif [[ ${option} == "SOLR_HEAP" ]]; then
+    if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
+      echo '# Solr heap size, there is no recommendation, please see Solr docs.' >> mailcow.conf
+      echo '# Solr is a prone to run OOM and should be monitored. Unmonitored Solr setups are not recommended.' >> mailcow.conf
+      echo "SOLR_HEAP=1024" >> mailcow.conf
+  fi
   elif ! grep -q ${option} mailcow.conf; then
     echo "Adding new option \"${option}\" to mailcow.conf"
     echo "${option}=n" >> mailcow.conf