Merge branch 'master' into fix-sender-checks

core/admin/Dockerfile

@@ -7,7 +7,7 @@ COPY requirements-prod.txt requirements.txt
 RUN apk add --no-cache openssl \
  && apk add --no-cache --virtual build-dep openssl-dev libffi-dev python-dev build-base \
  && pip install -r requirements.txt \
- && apk del build-dep
+ && apk del --no-cache build-dep
 
 COPY mailu ./mailu
 COPY migrations ./migrations

core/admin/mailu/__init__.py

@@ -12,7 +12,7 @@ import docker
 import socket
 import uuid
 
-from werkzeug.contrib import fixers
+from werkzeug.contrib import fixers, profiler
 
 # Create application
 app = flask.Flask(__name__)
@@ -57,12 +57,15 @@ default_config = {
     'RECAPTCHA_PUBLIC_KEY': '',
     'RECAPTCHA_PRIVATE_KEY': '',
     # Advanced settings
-    'PASSWORD_SCHEME': 'SHA512-CRYPT',
+    'PASSWORD_SCHEME': 'BLF-CRYPT',
     # Host settings
     'HOST_IMAP': 'imap',
     'HOST_POP3': 'imap',
     'HOST_SMTP': 'smtp',
+    'HOST_WEBMAIL': 'webmail',
+    'HOST_FRONT': 'front',
     'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'),
+    'POD_ADDRESS_RANGE': None
 }
 
 # Load configuration from the environment if available
@@ -80,6 +83,10 @@ if app.config.get("DEBUG"):
     import flask_debugtoolbar
     toolbar = flask_debugtoolbar.DebugToolbarExtension(app)
 
+# Profiler
+if app.config.get("DEBUG"):
+    app.wsgi_app = profiler.ProfilerMiddleware(app.wsgi_app, restrictions=[30])
+
 # Manager commnad
 manager = flask_script.Manager(app)
 manager.add_command('db', flask_migrate.MigrateCommand)
@@ -129,4 +136,5 @@ class PrefixMiddleware(object):
             environ['SCRIPT_NAME'] = prefix
         return self.app(environ, start_response)
 
+
 app.wsgi_app = PrefixMiddleware(fixers.ProxyFix(app.wsgi_app))

core/admin/mailu/internal/views/dovecot.py

@@ -1,14 +1,24 @@
-from mailu import db, models
+from mailu import db, models, app
 from mailu.internal import internal
 
 import flask
+import socket
 
 
 @internal.route("/dovecot/passdb/<user_email>")
 def dovecot_passdb_dict(user_email):
     user = models.User.query.get(user_email) or flask.abort(404)
+    allow_nets = []
+    allow_nets.append(
+        app.config.get("POD_ADDRESS_RANGE") or
+        socket.gethostbyname(app.config["HOST_FRONT"])
+    )
+    allow_nets.append(socket.gethostbyname(app.config["HOST_WEBMAIL"]))
+    print(allow_nets)
     return flask.jsonify({
-        "password": user.password,
+        "password": None,
+        "nopassword": "Y",
+        "allow_nets": ",".join(allow_nets)
     })
 
 

core/admin/mailu/models.py

@@ -288,7 +288,9 @@ class User(Base, Email):
     def get_id(self):
         return self.email
       
-    scheme_dict = {'SHA512-CRYPT': "sha512_crypt",
+    scheme_dict = {'PBKDF2': "pbkdf2_sha512",
+                   'BLF-CRYPT': "bcrypt",
+                   'SHA512-CRYPT': "sha512_crypt",
                    'SHA256-CRYPT': "sha256_crypt",
                    'MD5-CRYPT': "md5_crypt",
                    'CRYPT': "des_crypt"}
@@ -298,8 +300,14 @@ class User(Base, Email):
     )
 
     def check_password(self, password):
+        context = User.pw_context
         reference = re.match('({[^}]+})?(.*)', self.password).group(2)
-        return User.pw_context.verify(password, reference)
+        result = context.verify(password, reference)
+        if result and context.identify(reference) != context.default_scheme():
+            self.set_password(password)
+            db.session.add(self)
+            db.session.commit()
+        return result
 
     def set_password(self, password, hash_scheme=app.config['PASSWORD_SCHEME'], raw=False):
         """Set password for user with specified encryption scheme

core/admin/requirements-prod.txt

@@ -1,6 +1,7 @@
 alembic==0.9.9
 asn1crypto==0.24.0
 Babel==2.5.3
+bcrypt==3.1.4
 blinker==1.4
 certifi==2018.4.16
 cffi==1.11.5

core/admin/requirements.txt

@@ -17,3 +17,4 @@ tabulate
 PyYAML
 PyOpenSSL
 dnspython
+bcrypt

core/dovecot/Dockerfile

@@ -2,7 +2,7 @@ FROM alpine:3.8
 
 RUN apk add --no-cache \
      dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \
-     python3 py3-pip \
+     bash python3 py3-pip \
  && pip3 install --upgrade pip \
  && pip3 install jinja2 podop tenacity
 

core/dovecot/conf/bin/ham

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+tee >(rspamc -h antispam:11334 -P mailu learn_ham /dev/stdin) \
+    | rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /dev/stdin

core/dovecot/conf/bin/mailtrain

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-rspamc -h antispam:11334 -P mailu "learn_$1" /dev/stdin <&0

core/dovecot/conf/bin/spam

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+tee >(rspamc -h antispam:11334 -P mailu learn_spam /dev/stdin) \
+    >(rspamc -h antispam:11334 -P mailu -f 11 fuzzy_add /dev/stdin)

core/dovecot/conf/dovecot.conf

@@ -136,7 +136,8 @@ service managesieve {
 }
 
 plugin {
-  sieve = dict:proxy:/tmp/podop.socket:sieve
+  sieve = file:~/sieve;active=~/.dovecot.sieve
+  sieve_before = dict:proxy:/tmp/podop.socket:sieve
   sieve_plugins = sieve_imapsieve sieve_extprograms
   sieve_extensions = +spamtest +spamtestplus +editheader
   sieve_global_extensions = +vnd.dovecot.execute

core/dovecot/conf/report-ham.sieve

@@ -8,4 +8,4 @@ if string "${mailbox}" "Trash" {
   stop;
 }
 
-execute :pipe "mailtrain" "ham";
+execute :pipe "ham";

core/dovecot/conf/report-spam.sieve

@@ -1,3 +1,3 @@
 require "vnd.dovecot.execute";
 
-execute :pipe "mailtrain" "spam";
+execute :pipe "spam";

core/dovecot/start.py

@@ -36,5 +36,5 @@ for dovecot_file in glob.glob("/conf/*.conf"):
 
 # Run Podop, then postfix
 multiprocessing.Process(target=start_podop).start()
-os.system("chown -R mail:mail /mail /var/lib/dovecot")
+os.system("chown -R mail:mail /mail /var/lib/dovecot /conf")
 os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])

core/nginx/conf/nginx.conf

@@ -91,8 +91,10 @@ http {
       {% endif %}
 
       location {{ WEB_WEBMAIL }} {
+        {% if WEB_WEBMAIL != '/' %}
         rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent;
         rewrite ^{{ WEB_WEBMAIL }}/(.*) /$1 break;
+        {% endif %}
         include /etc/nginx/proxy.conf;
         client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }};
         proxy_pass http://$webmail;

core/postfix/conf/main.cf

@@ -32,7 +32,7 @@ relayhost = {{ RELAYHOST }}
 recipient_delimiter = {{ RECIPIENT_DELIMITER }}
 
 # Only the front server is allowed to perform xclient
-smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }}
+smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
 
 ###############
 # TLS

docs/compose/.env

@@ -120,12 +120,18 @@ WEBSITE=https://mailu.io
 # Advanced settings
 ###################################
 
+# Log driver for front service. Possible values:
+# json-file (default)
+# journald (On systemd platforms, useful for Fail2Ban integration)
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+LOG_DRIVER=json-file
+
 # Docker-compose project name, this will prepended to containers names.
 COMPOSE_PROJECT_NAME=mailu
 
 # Default password scheme used for newly created accounts and changed passwords
-# (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT)
-PASSWORD_SCHEME=SHA512-CRYPT
+PASSWORD_SCHEME=PBKDF2
 
 # Header to take the real ip from
 REAL_IP_HEADER=

docs/compose/docker-compose.yml

@@ -6,6 +6,8 @@ services:
     image: mailu/nginx:$VERSION
     restart: always
     env_file: .env
+    logging:
+      driver: $LOG_DRIVER
     ports:
     - "$BIND_ADDRESS4:80:80"
     - "$BIND_ADDRESS4:443:443"

docs/compose/setup.rst

@@ -26,36 +26,61 @@ for the ``VERSION_TAG`` branch, use:
   wget https://mailu.io/VERSION_TAG/_downloads/docker-compose.yml
   wget https://mailu.io/VERSION_TAG/_downloads/.env
 
-Then open the ``.env`` file to setup the mail server. Modify the ``ROOT`` setting
+Important configuration variables
-to match your setup directory if different from ``/mailu``.
+---------------------------------
 
-Modify the ``VERSION`` configuration in the ``.env`` file to reflect the version you picked.
+Open the ``.env`` file and review the following variable settings:
 
-Set the common configuration values
+- Change ``ROOT`` if you have your setup directory in a different location then ``/mailu``.
------------------------------------
+- Check ``VERSION`` to reflect the version you picked. (``master`` or ``1.5``).
 
-Open the ``.env`` file and set configuration settings after reading the configuration
+Make sure to read the comments in the file and instructions from the :ref:`common_cfg` section.
-documentation. Some settings are specific to the Docker Compose setup.
 
-Modify ``BIND_ADDRESS4`` to match the public IP address assigned to your server.
+TLS certificates
-This address should be configured on one of the network interfaces of the server.
+````````````````
-If the address is not configured directly (NAT) on any of the network interfaces or if
-you would simply like the server to listen on all interfaces, use ``0.0.0.0``.
-
-Modify ``BIND_ADDRESS6`` to match the public IPv6 address assigned to your server.
-The behavior is identical to ``BIND_ADDRESS4``.
 
 Set the ``TLS_FLAVOR`` to one of the following
 values:
 
 - ``cert`` is the default and requires certificates to be setup manually;
-- ``letsencrypt`` will use the Letsencrypt! CA to generate automatic ceriticates;
+- ``letsencrypt`` will use the *Letsencrypt!* CA to generate automatic ceriticates;
 - ``mail`` is similar to ``cert`` except that TLS will only be served for
   emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
 - ``mail-letsencrypt`` is similar to ``letsencrypt`` except that TLS will only be served for
   emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
 - ``notls`` will disable TLS, this is not recommended except for testing.
 
+.. note::
+
+  When using *Letsencrypt!* you have to make sure that the DNS ``A`` and ``AAAA`` records for the
+  all hostnames mentioned in the ``HOSTNAMES`` variable match with the ip adresses of you server.
+  Or else certificate generation will fail! See also: :ref:`dns_setup`.
+
+Bind address
+````````````
+
+Modify ``BIND_ADDRESS4`` and ``BIND_ADDRESS6`` to match the public IP addresses assigned to your server. For IPv6 you will need the ``<global>`` scope address. 
+
+You can find those addresses by running the following:
+
+.. code-block:: bash
+
+  [root@mailu ~]$ ifconfig eth0
+  eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
+          inet 125.189.138.127  netmask 255.255.255.0  broadcast 5.189.138.255
+          inet6 fd21:aab2:717c:cc5a::1  prefixlen 64  scopeid 0x0<global>
+          inet6 fe2f:2a73:43a8:7a1b::1  prefixlen 64  scopeid 0x20<link>
+          ether 00:50:56:3c:b2:23  txqueuelen 1000  (Ethernet)
+          RX packets 174866612  bytes 127773819607 (118.9 GiB)
+          RX errors 0  dropped 0  overruns 0  frame 0
+          TX packets 19905110  bytes 2191519656 (2.0 GiB)
+          TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
+
+If the address is not configured directly (NAT) on any of the network interfaces or if
+you would simply like the server to listen on all interfaces, use ``0.0.0.0`` and ``::``. Note that running is this mode is not supported and can lead to `issues`_.
+
+.. _issues: https://github.com/Mailu/Mailu/issues/641
+
 Enable optional features
 ------------------------
 

docs/configuration.rst

@@ -1,12 +1,20 @@
 Mailu configuration settings
 ============================
 
+.. _common_cfg:
+
 Common configuration
 --------------------
 
 The ``SECRET_KEY`` **must** be changed for every setup and set to a 16 bytes
 randomly generated value. It is intended to secure authentication cookies
-among other critical uses.
+among other critical uses. This can be generated with a utility such as *pwgen*,
+which can be installed on most Linux systems:
+
+.. code-block:: bash
+
+  apt-get install pwgen
+  pwgen 16 1
 
 The ``DOMAIN`` holds the main e-mail domain for the server. This email domain
 is used for bounce emails, for generating the postmaster email and other

docs/contributors/environment.rst

@@ -5,39 +5,51 @@ Docker containers
 -----------------
 
 The development environment is quite similar to the production one. You should always use
-the ``master`` version when developing. Simply add a build directive to the images
+the ``master`` version when developing.
-you are working on in the ``docker-compose.yml``:
 
-.. code-block:: yaml
+Building images
+```````````````
 
-  webdav:
+We supply a separate ``test/build.yml`` file for
-    build: ./optional/radicale
+convenience. To build all Mailu containers:
-    image: mailu/$WEBDAV:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-      - "$ROOT/dav:/data"
-
-  admin:
-    build: ./core/admin
-    image: mailu/admin:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-      - "$ROOT/data:/data"
-      - "$ROOT/dkim:/dkim"
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    depends_on:
-      - redis
-
-
-The build these containers.
 
 .. code-block:: bash
 
-  docker-compose build admin webdav
+  docker-compose -f tests/build.yml build
 
-Then you can simply start the stack as normal, newly-built images will be used.
+The ``build.yml`` file has two variables:
+
+#. ``$DOCKER_ORG``: First part of the image tag. Defaults to *mailu* and needs to be changed
+   only  when pushing to your own Docker hub account.
+#. ``$VERSION``: Last part of the image tag. Defaults to *local* to differentiate from pulled
+   images.
+
+To re-build only specific containers at a later time.
+
+.. code-block:: bash
+
+  docker-compose -f tests/build.yml build admin webdav
+
+If you have to push the images to Docker Hub for testing in Docker Swarm or a remote
+host, you have to define ``DOCKER_ORG`` (usually your Docker user-name) and login to
+the hub.
+
+.. code-block:: bash
+
+  docker login
+  Username: Foo
+  Password: Bar
+  export DOCKER_ORG="Foo"
+  export VERSION="feat-extra-app"
+  docker-compose -f tests/build.yml build
+  docker-compose -f tests/build.yml push
+
+Running containers
+``````````````````
+
+To run the newly created images: ``cd`` to your project directory. Edit ``.env`` to set
+``VERSION`` to the same value as used during the build, which defaults to ``local``.
+After that you can run:
 
 .. code-block:: bash
 

docs/dns.rst

@@ -1,3 +1,5 @@
+.. _dns_setup:
+
 Setting up your DNS
 ===================
 

docs/setup.rst

@@ -32,7 +32,7 @@ user. Make sure you complete the requirements for the flavor you chose.
 
 You should also have at least a DNS hostname and a DNS name for receiving
 emails. Some instructions are provided on the matter in the article
-[Setup your DNS](dns).
+:ref:`dns_setup`.
 
 .. _`MFAshby's fork`: https://github.com/MFAshby/Mailu
 
@@ -68,10 +68,9 @@ Make sure that you test properly before going live!
 - Try to receive an email from an external service
 - Check the logs (``docker-compose logs -f servicenamehere``) to look for
   warnings or errors
-- Use an open relay checker like `mailradar`_
+- Use an open relay checker like `mxtoolbox`_
   to ensure you're not contributing to the spam problem on the internet.
-  All tests there should result in "Relay denied".
 - If using DMARC, be sure to check the reports you get to verify that legitimate
   email is getting through and forgeries are being properly blocked.
 
-  .. _mailradar: http://www.mailradar.com/openrelay/
+  .. _mxtoolbox: https://mxtoolbox.com/diagnostic.aspx

docs/swarm/master/README.md

@@ -0,0 +1,252 @@
+# Install Mailu on a docker swarm
+
+## Prequisites
+
+### Swarm
+
+In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
+
+The main command will be:
+```bash
+docker swarm init --advertise-addr <IP_ADDR>
+```
+See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
+
+If you want to add other managers or workers, please use:
+```bash
+docker swarm join --token xxxxx 
+```
+See https://docs.docker.com/engine/swarm/join-nodes/
+
+You have now a working swarm, and you can check its status with:
+```bash
+core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
+ID                            HOSTNAME            STATUS              AVAILABILITY        MANAGER STATUS      ENGINE VERSION
+xhgeekkrlttpmtgmapt5hyxrb     black-pearl         Ready               Active                                  18.06.0-ce
+sczlqjgfhehsfdjhfhhph1nvb *   coreos-01           Ready               Active              Leader              18.03.1-ce
+mzrm9nbdggsfz4sgq6dhs5i6n     flying-dutchman     Ready               Active                                  18.06.0-ce
+```
+
+### Volume definition
+For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
+
+Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT/certs:/certs" (GlusterFS and nfs shares have been successfully used).
+
+On this example, we are using:
+- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. 
+- the default ingress mode.
+
+### Allow authentification with the mesh routing
+In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
+
+Let's create the mailu_default network:
+```bash
+core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
+core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
+                    "Subnet": "10.0.1.0/24",
+```
+In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 
+In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of  "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
+
+### Limitation with the ingress mode
+With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
+
+This issue is known and discussed here:
+
+https://github.com/moby/moby/issues/25526
+
+A workaround (using network host mode and global deployment) is discussed here:
+
+https://github.com/moby/moby/issues/25526#issuecomment-336363408 
+
+### Don't create an open relay !
+As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
+
+
+## Scalability
+- smtp and imap are scalable
+- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) 
+- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
+
+## Variable substitution and docker-compose.yml
+The docker stack deploy command doesn't support variable substitution in the .yml file itself. 
+As a consequence, we cannot simply use ``` docker stack deploy -c docker.compose.yml mailu ```
+Instead, we will use the following work-around:
+``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ```
+
+We need also to:
+- add a deploy section for every service
+- modify the way the ports are defined for the front service
+- add the POD_ADDRESS_RANGE definition for imap, smtp and antispam services
+
+## Docker compose 
+An example of docker-compose-stack.yml file is available here:
+
+```yaml
+
+version: '3.2'
+
+services:
+
+  front:
+    image: mailu/nginx:$VERSION
+    restart: always
+    env_file: .env
+    ports:
+      - target: 80
+        published: 80
+      - target: 443
+        published: 443
+      - target: 110
+        published: 110
+      - target: 143
+        published: 143
+      - target: 993
+        published: 993
+      - target: 995
+        published: 995
+      - target: 25
+        published: 25
+      - target: 465
+        published: 465
+      - target: 587
+        published: 587
+    volumes:
+      - "$ROOT/certs:/certs"
+    deploy:
+      replicas: 2
+
+  redis:
+    image: redis:alpine
+    restart: always
+    volumes:
+      - "$ROOT/redis:/data"
+    deploy:
+      replicas: 1
+
+  imap:
+    image: mailu/dovecot:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+      - "$ROOT/mail:/mail"
+      - "$ROOT/overrides:/overrides"
+    depends_on:
+      - front
+    deploy:
+      replicas: 2
+
+  smtp:
+    image: mailu/postfix:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+      - "$ROOT/overrides:/overrides"
+    depends_on:
+      - front
+    deploy:
+      replicas: 2
+
+  antispam:
+    image: mailu/rspamd:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+      - "$ROOT/filter:/var/lib/rspamd"
+      - "$ROOT/dkim:/dkim"
+      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+    depends_on:
+      - front
+    deploy:
+      replicas: 1
+
+  antivirus:
+    image: mailu/none:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+      - "$ROOT/filter:/data"
+    deploy:
+      replicas: 1
+
+  webdav:
+    image: mailu/none:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+      - "$ROOT/dav:/data"
+    deploy:
+      replicas: 1
+
+  admin:
+    image: mailu/admin:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+      - "$ROOT/data:/data"
+      - "$ROOT/dkim:/dkim"
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    depends_on:
+      - redis
+    deploy:
+      replicas: 1
+
+  webmail:
+    image: mailu/roundcube:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+      - "$ROOT/webmail:/data"
+    depends_on:
+      - imap
+    deploy:
+      replicas: 2
+
+  fetchmail:
+    image: mailu/fetchmail:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+    deploy:
+      replicas: 1
+
+networks:
+  default:
+    external:
+      name: mailu_default
+```
+
+## Deploy Mailu on the docker swarm
+Run the following command:
+```bash
+echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu
+```
+See how the services are being deployed:
+```bash
+core@coreos-01 ~ $ docker service ls
+ID                  NAME                                 MODE                REPLICAS            IMAGE                                     PORTS
+ywnsetmtkb1l        mailu_antivirus                      replicated          1/1                 mailu/none:master
+pqokiaz0q128        mailu_fetchmail                      replicated          1/1                 mailu/fetchmail:master
+```
+check a specific service:
+```bash
+core@coreos-01 ~ $ docker service ps mailu_fetchmail
+ID                  NAME                IMAGE                 NODE                DESIRED STATE       CURRENT STATE         ERROR               PORTS
+tbu8ppgsdffj        mailu_fetchmail.1   mailu/fetchmail:master   coreos-01           Running             Running 11 days ago
+```
+You might also have a look on the logs:
+```bash
+core@coreos-01 ~ $ docker service logs -f mailu_fetchmail
+```
+
+## Remove the stack
+Run the follwoing command:
+```bash
+core@coreos-01 ~ $ docker stack rm mailu
+```

docs/swarm/master/README_nfs_example.md

@@ -0,0 +1,357 @@
+# Install Mailu on a docker swarm
+
+## Prequisites
+
+### Swarm
+
+In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
+
+The main command will be:
+```bash
+docker swarm init --advertise-addr <IP_ADDR>
+```
+See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
+
+If you want to add other managers or workers, please use:
+```bash
+docker swarm join --token xxxxx 
+```
+See https://docs.docker.com/engine/swarm/join-nodes/
+
+You have now a working swarm, and you can check its status with:
+```bash
+core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
+ID                            HOSTNAME            STATUS              AVAILABILITY        MANAGER STATUS      ENGINE VERSION
+xhgeekkrlttpmtgmapt5hyxrb     black-pearl         Ready               Active                                  18.06.0-ce
+sczlqjgfhehsfdjhfhhph1nvb *   coreos-01           Ready               Active              Leader              18.03.1-ce
+mzrm9nbdggsfz4sgq6dhs5i6n     flying-dutchman     Ready               Active                                  18.06.0-ce
+```
+
+### Volume definition
+For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
+Hereafter we will use a NFS share:
+```bash
+core@coreos-01 ~ $ showmount -e 192.168.0.30
+Export list for 192.168.0.30:
+/mnt/Pool1/pv            192.168.0.0
+```
+
+on the nfs server, I am using the following /etc/exports
+```bash
+$more /etc/exports
+/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 
+```
+on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up)
+```bash
+$mkdir /mnt/Pool1/pv/mailu
+```
+
+On your manager node, mount the nfs share to check that the share is available:
+```bash
+core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/
+```
+If this is ok, you can umount it:
+```bash
+core@coreos-01 ~ $ sudo umount /mnt/local/
+```
+
+
+## Networking mode
+On this example, we are using:
+- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. 
+- the default ingress mode.
+
+### Allow authentification with the mesh routing
+In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
+
+Let's create the mailu_default network:
+```bash
+core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
+core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
+                    "Subnet": "10.0.1.0/24",
+```
+In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 
+In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of  "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
+
+### Limitation with the ingress mode
+With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
+
+This issue is known and discussed here:
+
+https://github.com/moby/moby/issues/25526
+
+A workaround (using network host mode and global deployment) is discussed here:
+
+https://github.com/moby/moby/issues/25526#issuecomment-336363408 
+
+### Don't create an open relay !
+As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
+
+
+## Scalability
+- smtp and imap are scalable
+- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) 
+- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
+
+## Variable substitution and docker-compose.yml
+The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around:
+``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ```
+
+We need also to:
+- change the way we define the volumes (nfs share in our case)
+- add a deploy section for every service
+- the way the ports are defined for the front service
+
+## Docker compose 
+An example of docker-compose-stack.yml file is available here:
+
+```yaml
+
+version: '3.2'
+
+services:
+
+  front:
+    image: mailu/nginx:$VERSION
+    restart: always
+    env_file: .env
+    ports:
+      - target: 80
+        published: 80
+      - target: 443
+        published: 443
+      - target: 110
+        published: 110
+      - target: 143
+        published: 143
+      - target: 993
+        published: 993
+      - target: 995
+        published: 995
+      - target: 25
+        published: 25
+      - target: 465
+        published: 465
+      - target: 587
+        published: 587
+    volumes:
+#      - "$ROOT/certs:/certs"
+      - type: volume
+        source: mailu_certs
+        target: /certs
+    deploy:
+      replicas: 2
+
+  redis:
+    image: redis:alpine
+    restart: always
+    volumes:
+#      - "$ROOT/redis:/data"
+      - type: volume
+        source: mailu_redis
+        target: /data
+    deploy:
+      replicas: 1
+
+  imap:
+    image: mailu/dovecot:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+#      - "$ROOT/mail:/mail"
+      - type: volume
+        source: mailu_mail
+        target: /mail
+#      - "$ROOT/overrides:/overrides"
+      - type: volume
+        source: mailu_overrides
+        target: /overrides
+    depends_on:
+      - front
+    deploy:
+      replicas: 2
+
+  smtp:
+    image: mailu/postfix:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+#      - "$ROOT/overrides:/overrides"
+      - type: volume
+        source: mailu_overrides
+        target: /overrides
+    depends_on:
+      - front
+    deploy:
+      replicas: 2
+
+  antispam:
+    image: mailu/rspamd:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    depends_on:
+      - front
+    volumes:
+#      - "$ROOT/filter:/var/lib/rspamd"
+      - type: volume
+        source: mailu_filter
+        target: /var/lib/rspamd
+#      - "$ROOT/dkim:/dkim"
+      - type: volume
+        source: mailu_dkim
+        target: /dkim
+#      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+      - type: volume
+        source: mailu_overrides_rspamd
+        target: /etc/rspamd/override.d
+    deploy:
+      replicas: 1
+
+  antivirus:
+    image: mailu/none:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+#      - "$ROOT/filter:/data"
+      - type: volume
+        source: mailu_filter
+        target: /data
+    deploy:
+      replicas: 1
+
+  webdav:
+    image: mailu/none:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+#      - "$ROOT/dav:/data"
+      - type: volume
+        source: mailu_dav
+        target: /data
+    deploy:
+      replicas: 1
+
+  admin:
+    image: mailu/admin:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+#      - "$ROOT/data:/data"
+      - type: volume
+        source: mailu_data
+        target: /data
+#      - "$ROOT/dkim:/dkim"
+      - type: volume
+        source: mailu_dkim
+        target: /dkim
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    depends_on:
+      - redis
+    deploy:
+      replicas: 1
+
+  webmail:
+    image: mailu/roundcube:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+#      - "$ROOT/webmail:/data"
+      - type: volume
+        source: mailu_data
+        target: /data
+    depends_on:
+      - imap
+    deploy:
+      replicas: 2
+
+  fetchmail:
+    image: mailu/fetchmail:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+    deploy:
+      replicas: 1
+
+networks:
+  default:
+    external:
+      name: mailu_default
+
+volumes:
+  mailu_filter:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/filter"
+  mailu_dkim:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/dkim"
+  mailu_overrides_rspamd:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/overrides/rspamd"
+  mailu_data:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/data"
+  mailu_mail:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/mail"
+  mailu_overrides:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/overrides"
+  mailu_dav:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/dav"
+  mailu_certs:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/certs"
+  mailu_redis:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/redis"
+```
+
+## Deploy Mailu on the docker swarm
+Run the following command:
+```bash
+echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu
+```
+See how the services are being deployed:
+```bash
+core@coreos-01 ~ $ docker service ls
+ID                  NAME                                 MODE                REPLICAS            IMAGE                                     PORTS
+ywnsetmtkb1l        mailu_antivirus                      replicated          1/1                 mailu/none:master
+pqokiaz0q128        mailu_fetchmail                      replicated          1/1                 mailu/fetchmail:master
+```
+check a specific service:
+```bash
+core@coreos-01 ~ $ docker service ps mailu_fetchmail
+ID                  NAME                IMAGE                 NODE                DESIRED STATE       CURRENT STATE         ERROR               PORTS
+tbu8ppgsdffj        mailu_fetchmail.1   mailu/fetchmail:master   coreos-01           Running             Running 11 days ago
+```
+
+## Remove the stack
+Run the follwoing command:
+```bash
+core@coreos-01 ~ $ docker stack rm mailu
+```

services/rspamd/Dockerfile

@@ -1,6 +1,6 @@
 FROM alpine:3.8
 
-RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip \
+RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates py-pip \
  && pip install --upgrade pip \
  && pip install tenacity
 
@@ -9,10 +9,7 @@ RUN mkdir /run/rspamd
 COPY conf/ /conf
 COPY start.py /start.py
 
-# Temporary fix to remove references to rspamd-fuzzy for now
+EXPOSE 11332/tcp 11334/tcp 11335/tcp
-RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf
-
-EXPOSE 11332/tcp 11334/tcp
 
 VOLUME ["/var/lib/rspamd"]
 

services/rspamd/conf/fuzzy_check.conf

@@ -0,0 +1,34 @@
+rule "local" {
+    # Fuzzy storage server list
+    servers = "localhost:11335";
+    # Default symbol for unknown flags
+    symbol = "LOCAL_FUZZY_UNKNOWN";
+    # Additional mime types to store/check
+    mime_types = ["application/*"];
+    # Hash weight threshold for all maps
+    max_score = 20.0;
+    # Whether we can learn this storage
+    read_only = no;
+    # Ignore unknown flags
+    skip_unknown = yes;
+    # Hash generation algorithm
+    algorithm = "mumhash";
+
+    # Map flags to symbols
+    fuzzy_map = {
+        LOCAL_FUZZY_DENIED {
+            # Local threshold
+            max_score = 20.0;
+            # Flag to match
+            flag = 11;
+        }
+        LOCAL_FUZZY_PROB {
+            max_score = 10.0;
+            flag = 12;
+        }
+        LOCAL_FUZZY_WHITE {
+            max_score = 2.0;
+            flag = 13;
+        }
+    }
+}

services/rspamd/conf/metrics.conf

@@ -0,0 +1,19 @@
+group "fuzzy" {
+    max_score = 12.0;
+    symbol "LOCAL_FUZZY_UNKNOWN" {
+        weight = 5.0;
+        description = "Generic fuzzy hash match";
+    }
+    symbol "LOCAL_FUZZY_DENIED" {
+        weight = 12.0;
+        description = "Denied fuzzy hash";
+    }
+    symbol "LOCAL_FUZZY_PROB" {
+        weight = 5.0;
+        description = "Probable fuzzy hash";
+    }
+    symbol "LOCAL_FUZZY_WHITE" {
+        weight = -2.1;
+        description = "Whitelisted fuzzy hash";
+    }
+}

services/rspamd/conf/worker-controller.inc

@@ -1,3 +1,4 @@
+type = "controller";
 bind_socket = "*:11334";
 password = "mailu";
-secure_ip = "{{ FRONT_ADDRESS }}";
+secure_ip = "{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% endif %}";

services/rspamd/conf/worker-fuzzy.inc

@@ -0,0 +1,6 @@
+type = "fuzzy";
+bind_socket = "*:11335";
+count = 1;
+backend = "redis";
+expire = 90d;
+allow_update = ["127.0.0.1"];

services/rspamd/conf/worker-normal.inc

@@ -1 +1,2 @@
+type = "normal";
 enabled = false;

tests/build.yml

@@ -3,54 +3,54 @@ version: '3'
 services:
 
   front:
-    image: $DOCKER_ORG/nginx:$VERSION
+    image: ${DOCKER_ORG:-mailu}/nginx:${VERSION:-local}
     build: ../core/nginx
 
   imap:
-    image: $DOCKER_ORG/dovecot:$VERSION
+    image: ${DOCKER_ORG:-mailu}/dovecot:${VERSION:-local}
     build: ../core/dovecot
 
   smtp:
-    image: $DOCKER_ORG/postfix:$VERSION
+    image: ${DOCKER_ORG:-mailu}/postfix:${VERSION:-local}
     build: ../core/postfix
 
   antispam:
-    image: $DOCKER_ORG/rspamd:$VERSION
+    image: ${DOCKER_ORG:-mailu}/rspamd:${VERSION:-local}
     build: ../services/rspamd
 
   antivirus:
-    image: $DOCKER_ORG/clamav:$VERSION
+    image: ${DOCKER_ORG:-mailu}/clamav:${VERSION:-local}
     build: ../optional/clamav
 
   webdav:
-    image: $DOCKER_ORG/radicale:$VERSION
+    image: ${DOCKER_ORG:-mailu}/radicale:${VERSION:-local}
     build: ../optional/radicale
 
   admin:
-    image: $DOCKER_ORG/admin:$VERSION
+    image: ${DOCKER_ORG:-mailu}/admin:${VERSION:-local}
     build: ../core/admin
 
   roundcube:
-    image: $DOCKER_ORG/roundcube:$VERSION
+    image: ${DOCKER_ORG:-mailu}/roundcube:${VERSION:-local}
     build: ../webmails/roundcube
 
   rainloop:
-    image: $DOCKER_ORG/rainloop:$VERSION
+    image: ${DOCKER_ORG:-mailu}/rainloop:${VERSION:-local}
     build: ../webmails/rainloop
 
   fetchmail:
-    image: $DOCKER_ORG/fetchmail:$VERSION
+    image: ${DOCKER_ORG:-mailu}/fetchmail:${VERSION:-local}
     build: ../services/fetchmail
 
   none:
-    image: $DOCKER_ORG/none:$VERSION
+    image: ${DOCKER_ORG:-mailu}/none:${VERSION:-local}
     build: ../core/none
 
   docs:
-    image: $DOCKER_ORG/docs:$VERSION
+    image: ${DOCKER_ORG:-mailu}/docs:${VERSION:-local}
     build: ../docs
 
   setup:
-    image: $DOCKER_ORG/setup:$VERSION
+    image: ${DOCKER_ORG:-mailu}/setup:${VERSION:-local}
     build: ../setup
 

tests/compose/core.env

@@ -120,6 +120,12 @@ WEBSITE=https://mailu.io
 # Advanced settings
 ###################################
 
+# Log driver for front service. Possible values:
+# json-file (default)
+# journald (On systemd platforms, useful for Fail2Ban integration)
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+LOG_DRIVER=json-file
+
 # Docker-compose project name, this will prepended to containers names.
 #COMPOSE_PROJECT_NAME=mailu
 

tests/compose/run.yml

@@ -6,6 +6,8 @@ services:
     image: $DOCKER_ORG/nginx:$VERSION
     restart: 'no'
     env_file: $PWD/.env
+    logging:
+      driver: $LOG_DRIVER
     ports:
     - "$BIND_ADDRESS4:80:80"
     - "$BIND_ADDRESS4:443:443"

webmails/rainloop/Dockerfile

@@ -1,20 +1,21 @@
 FROM php:7.2-apache
 
-RUN apt-get update && apt-get install -y \
-      unzip python3 python3-jinja2
-
 ENV RAINLOOP_URL https://github.com/RainLoop/rainloop-webmail/releases/download/v1.12.1/rainloop-community-1.12.1.zip
 
-RUN rm -rf /var/www/html/ \
+RUN apt-get update && apt-get install -y \
+      unzip python3 python3-jinja2 \
+ && rm -rf /var/www/html/ \
  && mkdir /var/www/html \
  && cd /var/www/html \
  && curl -L -O ${RAINLOOP_URL} \
- && unzip *.zip \
+ && unzip -q *.zip \
  && rm -f *.zip \
  && rm -rf data/ \
  && find . -type d -exec chmod 755 {} \; \
  && find . -type f -exec chmod 644 {} \; \
- && chown -R www-data: *
+ && chown -R www-data: * \
+ && apt-get purge -y unzip \
+ && rm -rf /var/lib/apt/lists
 
 COPY include.php /var/www/html/include.php
 COPY php.ini /usr/local/etc/php/conf.d/rainloop.ini

webmails/roundcube/Dockerfile

@@ -1,14 +1,12 @@
 FROM php:7.2-apache
 
-RUN apt-get update && apt-get install -y \
-      zlib1g-dev \
- && docker-php-ext-install zip
-
 ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.7/roundcubemail-1.3.7-complete.tar.gz
 
-RUN echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini
+RUN apt-get update && apt-get install -y \
-
+      zlib1g-dev \
-RUN rm -rf /var/www/html/ \
+ && docker-php-ext-install zip \
+ && echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini \
+ && rm -rf /var/www/html/ \
  && cd /var/www \
  && curl -L -O ${ROUNDCUBE_URL} \
  && tar -xf *.tar.gz \
@@ -17,7 +15,8 @@ RUN rm -rf /var/www/html/ \
  && cd html \
  && rm -rf CHANGELOG INSTALL LICENSE README.md UPGRADING composer.json-dist installer \
  && sed -i 's,mod_php5.c,mod_php7.c,g' .htaccess \
- && chown -R www-data: logs temp
+ && chown -R www-data: logs temp \
+ && rm -rf /var/lib/apt/lists
 
 COPY php.ini /usr/local/etc/php/conf.d/roundcube.ini