Merge branch 'master' into delete-disable

.github/workflows/build_test_deploy.yml

@@ -394,7 +394,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        target: ["setup", "docs", "fetchmail", "webmail", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx"]
+        target: ["setup", "docs", "fetchmail", "webmail", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "oletools", "postfix", "dovecot", "unbound", "nginx"]
     steps:
       - uses: actions/checkout@v3
       - name: Retrieve global variables
@@ -439,7 +439,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        target: ["setup", "docs", "fetchmail", "webmail", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx"]
+        target: ["setup", "docs", "fetchmail", "webmail", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "oletools", "postfix", "dovecot", "unbound", "nginx"]
     steps:
       - uses: actions/checkout@v3
       - name: Retrieve global variables

.gitignore

@@ -10,6 +10,7 @@ pip-selfcheck.json
 /docs/include
 /docs/_build
 /.env
+/.venv
 /docker-compose.yml
 /.idea
 /.vscode

CHANGELOG.md

@@ -10,7 +10,7 @@ These settings tell Mailu that the HTTP header with the remote client IP address
 For more information see the [configuration reference](https://mailu.io/1.9/configuration.html#advanced-settings).
 
 One major change for the docker compose file is that the antispam container needs a fixed hostname [#1837](https://github.com/Mailu/Mailu/issues/1837).
-This is handled when you regenerate the docker-compose file. A fixed hostname is required to retain rspamd history. 
+This is handled when you regenerate the docker compose file. A fixed hostname is required to retain rspamd history.
 
 After changing mailu.env, it is required to recreate all containers for the changes to be propagated.
 
@@ -314,8 +314,8 @@ v1.6.0 - 2019-01-18
 - Enhancement: Reverse proxy - Real ip header and mail-letsencrypt ([#358](https://github.com/Mailu/Mailu/issues/358))
 - Enhancement: Parametrize hosts ([#373](https://github.com/Mailu/Mailu/issues/373))
 - Enhancement: Expose ports in dockerfiles ([#392](https://github.com/Mailu/Mailu/issues/392))
-- Enhancement: Added webmail-imap dependency in docker-compose ([#403](https://github.com/Mailu/Mailu/issues/403))
-- Enhancement: Add environment variables to allow running outside of docker-compose ([#429](https://github.com/Mailu/Mailu/issues/429))
+- Enhancement: Added webmail-imap dependency in docker compose ([#403](https://github.com/Mailu/Mailu/issues/403))
+- Enhancement: Add environment variables to allow running outside of docker compose ([#429](https://github.com/Mailu/Mailu/issues/429))
 - Enhancement: Add original Delivered-To header to received messages ([#433](https://github.com/Mailu/Mailu/issues/433))
 - Enhancement: Use HOST_ADMIN in "Forwarding authentication server" ([#436](https://github.com/Mailu/Mailu/issues/436), [#437](https://github.com/Mailu/Mailu/issues/437))
 - Enhancement: Use POD_ADDRESS_RANGE for Dovecot ([#448](https://github.com/Mailu/Mailu/issues/448))

ISSUE_TEMPLATE.md

@@ -19,7 +19,7 @@ Please put your text outside of the comment blocks to be visible. You can use th
 
 ### Environment
 
-- [ ] docker-compose
+- [ ] docker compose
 - [ ] kubernetes
 - [ ] docker swarm
 
@@ -61,7 +61,7 @@ Just saying "it doesn’t work as expected" is not useful. It's also helpful to
 Often it is very useful to include log fragments of the involved component.
 You can get the logs via `docker logs <container name> --tail 1000`.
 For example for the admin container: `docker logs mailu_admin_1 --tail 1000`
-or using docker-compose `docker-compose -f /mailu/docker-compose.yml logs --tail 1000 admin`
+or using docker compose `docker compose -f /mailu/docker-compose.yml logs --tail 1000 admin`
 
 If you can find the relevant section, please share only the parts that seem relevant. If you have any logs, please enclose them in code tags, like so:
 

README.md

@@ -22,7 +22,7 @@ Main features include:
 - **Web access**, multiple Webmails and administration interface
 - **User features**, aliases, auto-reply, auto-forward, fetched accounts
 - **Admin features**, global admins, announcements, per-domain delegation, quotas
-- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, [Snuffleupagus](https://github.com/jvoisin/snuffleupagus/)
+- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, [Snuffleupagus](https://github.com/jvoisin/snuffleupagus/), block malicious attachments
 - **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing
 - **Freedom**, all FOSS components, no tracker included
 

RELEASE_TEMPLATE.md

@@ -1,5 +1,5 @@
 This is a new automatic release of Mailu. The new version can be seen in the tag name.
-The main version X.Y (e.g. 1.9) will always reflect the latest version of the branch. To update your Mailu installation simply pull the latest images `docker-compose pull && docker-compose up -d`.
+The main version X.Y (e.g. 1.9) will always reflect the latest version of the branch. To update your Mailu installation simply pull the latest images `docker compose pull && docker compose up -d`.
 The pinned version X.Y.Z (e.g. 1.9.1) is not updated. It is pinned to the commit that was used for creating this release. You can use a pinned version to make sure your Mailu installation is not suddenly updated when recreating containers. The pinned version allows the user to manually update. It also allows to go back to a previous pinned version.
 
 To check what was changed:

core/admin/Dockerfile

@@ -22,8 +22,8 @@ RUN set -euxo pipefail \
 
 RUN echo $VERSION >/version
 
-EXPOSE 80/tcp
-HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index
+#EXPOSE 80/tcp
+HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/ping
 
 VOLUME ["/data","/dkim"]
 

core/admin/assets/assets/app.css

@@ -57,3 +57,9 @@ fieldset:disabled .form-control:disabled {
 .input-group-text {
   margin-right: 1em;
 }
+
+/* version string */
+.mailu-version {
+  font-size: 60%;
+  line-height: 0;
+}

core/admin/assets/assets/app.js

@@ -86,9 +86,12 @@ $('document').ready(function() {
         if (value_element.length) {
             value_element = $(value_element[0]);
             var infinity = $(this).data('infinity');
-            var step = $(this).attr('step');
+            var unit = $(this).data('unit');
+            if (typeof unit === 'undefined' || unit === false) {
+                unit=1;
+            }
             $(this).on('input', function() {
-                var num = (infinity && this.value == 0) ? '∞' : (this.value/step).toFixed(2);
+                var num = (infinity && this.value == 0) ? '∞' : (this.value/unit).toFixed(2);
                 if (num.endsWith('.00')) num = num.substr(0, num.length - 3);
                 value_element.text(num);
             }).trigger('input');

core/admin/mailu/__init__.py

@@ -5,9 +5,24 @@ import flask
 import flask_bootstrap
 
 from mailu import utils, debug, models, manage, configuration
+from gunicorn import glogging
+import logging
 
 import hmac
 
+class NoPingFilter(logging.Filter):
+    def filter(self, record):
+        if not (record.args['{host}i'] == 'localhost' and record.args['r'] == 'GET /ping HTTP/1.1'):
+            return True
+
+class Logger(glogging.Logger):
+    def setup(self, cfg):
+        super().setup(cfg)
+
+        # Add filters to Gunicorn logger
+        logger = logging.getLogger("gunicorn.access")
+        logger.addFilter(NoPingFilter())
+
 def create_app_from_config(config):
     """ Create a new application based on the given configuration
     """
@@ -69,11 +84,16 @@ def create_app_from_config(config):
     def format_datetime(value):
         return utils.flask_babel.format_datetime(value) if value else ''
 
+    def ping():
+        return ''
+    app.route('/ping')(ping)
+
     # Import views
-    from mailu import ui, internal, sso
+    from mailu import ui, internal, sso, api
     app.register_blueprint(ui.ui, url_prefix=app.config['WEB_ADMIN'])
     app.register_blueprint(internal.internal, url_prefix='/internal')
     app.register_blueprint(sso.sso, url_prefix='/sso')
+    api.register(app, web_api_root=app.config.get('WEB_API'))
     return app
 
 

core/admin/mailu/api/__init__.py

@@ -0,0 +1,32 @@
+from flask import redirect, url_for, Blueprint
+from flask_restx import apidoc
+from . import v1 as APIv1
+
+def register(app, web_api_root):
+
+    APIv1.app = app
+    # register api bluprint(s)
+    apidoc.apidoc.url_prefix = f'{web_api_root}/v{int(APIv1.VERSION)}'
+    APIv1.api_token = app.config['API_TOKEN']
+    if app.config['API_TOKEN'] != '':
+        app.register_blueprint(APIv1.blueprint, url_prefix=f'{web_api_root}/v{int(APIv1.VERSION)}')
+
+        # add redirect to current api version
+        redirect_api = Blueprint('redirect_api', __name__)
+        @redirect_api.route('/')
+        def redir():
+            return redirect(url_for(f'{APIv1.blueprint.name}.root'))
+        app.register_blueprint(redirect_api, url_prefix=f'{web_api_root}')
+
+        # swagger ui config
+        app.config.SWAGGER_UI_DOC_EXPANSION = 'list'
+        app.config.SWAGGER_UI_OPERATION_ID = True
+        app.config.SWAGGER_UI_REQUEST_DURATION = True
+        app.config.RESTX_MASK_SWAGGER = False
+    else:
+        api = Blueprint('api', __name__)
+        @api.route('/', defaults={'path': ''})
+        @api.route('/<path:path>')
+        def api_token_missing(path):
+            return "<p>Error: API_TOKEN is not configured</p>", 500
+        app.register_blueprint(api, url_prefix=f'{web_api_root}')

core/admin/mailu/api/common.py

@@ -0,0 +1,42 @@
+from .. import models, utils
+from . import v1
+from flask import request
+import flask
+import hmac
+from functools import wraps
+from flask_restx import abort
+from sqlalchemy.sql.expression import label
+
+def fqdn_in_use(name):
+    d = models.db.session.query(label('name', models.Domain.name))
+    a = models.db.session.query(label('name', models.Alternative.name))
+    r = models.db.session.query(label('name', models.Relay.name))
+    u = d.union_all(a).union_all(r).filter_by(name=name)
+    if models.db.session.query(u.exists()).scalar():
+        return True
+    return False
+
+""" Decorator for validating api token for authentication """
+def api_token_authorization(func):
+    @wraps(func)
+    def decorated_function(*args, **kwds):
+        client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr)
+        if utils.limiter.should_rate_limit_ip(client_ip):
+            abort(429, 'Too many attempts from your IP (rate-limit)' )
+        if not request.headers.get('Authorization'):
+            abort(401, 'A valid Bearer token is expected which is provided as request header')
+        #Client provides 'Authentication: Bearer <token>'
+        if (' ' in request.headers.get('Authorization')
+                and not hmac.compare_digest(request.headers.get('Authorization'), 'Bearer ' + v1.api_token)):
+            utils.limiter.rate_limit_ip(client_ip)
+            flask.current_app.logger.warn(f'Invalid API token provided by {client_ip}.')
+            abort(403, 'A valid Bearer token is expected which is provided as request header')
+        #Client provides 'Authentication: <token>'
+        elif (' ' not in request.headers.get('Authorization')
+                and not hmac.compare_digest(request.headers.get('Authorization'), v1.api_token)):
+            utils.limiter.rate_limit_ip(client_ip)
+            flask.current_app.logger.warn(f'Invalid API token provided by {client_ip}.')
+            abort(403, 'A valid Bearer token is expected which is provided as request header')
+        flask.current_app.logger.info(f'Valid API token provided by {client_ip}.')
+        return func(*args, **kwds)
+    return decorated_function

core/admin/mailu/api/v1/__init__.py

@@ -0,0 +1,43 @@
+from flask import Blueprint
+from flask_restx import Api, fields
+
+
+VERSION = 1.0
+api_token = None
+
+blueprint = Blueprint(f'api_v{int(VERSION)}', __name__)
+
+authorization = {
+    'Bearer': {
+        'type': 'apiKey',
+        'in': 'header',
+        'name': 'Authorization'
+    }
+}
+
+api = Api(
+    blueprint, version=f'{VERSION:.1f}',
+    title='Mailu API', default_label='Mailu',
+    validate=True,
+    authorizations=authorization,
+    security='Bearer',
+    doc='/'
+)
+
+response_fields = api.model('Response', {
+    'code': fields.Integer,
+    'message': fields.String,
+})
+
+error_fields = api.model('Error', {
+    'errors': fields.Nested(api.model('Error_Key', {
+        'key': fields.String,
+        'message':fields.String
+    })),
+    'message': fields.String,
+})
+
+from . import domains
+from . import alias
+from . import relay
+from . import user

core/admin/mailu/api/v1/alias.py

@@ -0,0 +1,126 @@
+from flask_restx import Resource, fields, marshal
+from . import api, response_fields
+from .. import common
+from ... import models
+
+db = models.db
+
+alias = api.namespace('alias', description='Alias operations')
+
+alias_fields_update = alias.model('AliasUpdate', {
+    'comment': fields.String(description='a comment'),
+    'destination': fields.List(fields.String(description='alias email address', example='user@example.com')),
+    'wildcard': fields.Boolean(description='enable SQL Like wildcard syntax')
+})
+
+alias_fields = alias.inherit('Alias',alias_fields_update, {
+  'email': fields.String(description='the alias email address', example='user@example.com', required=True),
+  'destination': fields.List(fields.String(description='alias email address', example='user@example.com', required=True)),
+
+})
+
+
+@alias.route('')
+class Aliases(Resource):
+    @alias.doc('list_alias')
+    @alias.marshal_with(alias_fields, as_list=True, skip_none=True, mask=None)
+    @alias.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self):
+        """ List aliases """
+        return models.Alias.query.all()
+
+    @alias.doc('create_alias')
+    @alias.expect(alias_fields)
+    @alias.response(200, 'Success', response_fields)
+    @alias.response(400, 'Input validation exception', response_fields)
+    @alias.response(409, 'Duplicate alias', response_fields)
+    @alias.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self):
+        """ Create a new alias """
+        data = api.payload
+
+        alias_found = models.Alias.query.filter_by(email = data['email']).first()
+        if alias_found:
+          return { 'code': 409, 'message': f'Duplicate alias {data["email"]}'}, 409
+
+        alias_model = models.Alias(email=data["email"],destination=data['destination'])
+        if 'comment' in data:
+          alias_model.comment = data['comment']
+        if 'wildcard' in data:
+          alias_model.wildcard = data['wildcard']
+        db.session.add(alias_model)
+        db.session.commit()
+
+        return {'code': 200, 'message': f'Alias {data["email"]} to destination {data["destination"]} has been created'}, 200
+
+@alias.route('/<string:alias>')
+class Alias(Resource):
+    @alias.doc('find_alias')
+    @alias.response(200, 'Success', alias_fields)
+    @alias.response(404, 'Alias not found', response_fields)
+    @alias.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, alias):
+        """ Find alias """
+        alias_found = models.Alias.query.filter_by(email = alias).first()
+        if alias_found is None:
+          return { 'code': 404, 'message': f'Alias {alias} cannot be found'}, 404
+        else:
+          return marshal(alias_found,alias_fields), 200
+
+    @alias.doc('update_alias')
+    @alias.expect(alias_fields_update)
+    @alias.response(200, 'Success', response_fields)
+    @alias.response(404, 'Alias not found', response_fields)
+    @alias.response(400, 'Input validation exception', response_fields)
+    @alias.doc(security='Bearer')
+    @common.api_token_authorization
+    def patch(self, alias):
+      """ Update alias """
+      data = api.payload
+      alias_found = models.Alias.query.filter_by(email = alias).first()
+      if alias_found is None:
+        return { 'code': 404, 'message': f'Alias {alias} cannot be found'}, 404
+      if 'comment' in data:
+        alias_found.comment = data['comment']
+      if 'destination' in data:
+        alias_found.destination = data['destination']
+      if 'wildcard' in data:
+        alias_found.wildcard = data['wildcard']
+      db.session.add(alias_found)
+      db.session.commit()
+      return {'code': 200, 'message': f'Alias {alias} has been updated'}
+
+    @alias.doc('delete_alias')
+    @alias.response(200, 'Success', response_fields)
+    @alias.response(404, 'Alias not found', response_fields)
+    @alias.doc(security='Bearer')
+    @common.api_token_authorization
+    def delete(self, alias):
+      """ Delete alias """
+      alias_found = models.Alias.query.filter_by(email = alias).first()
+      if alias_found is None:
+        return { 'code': 404, 'message': f'Alias {alias} cannot be found'}, 404
+      db.session.delete(alias_found)
+      db.session.commit()
+      return {'code': 200, 'message': f'Alias {alias} has been deleted'}, 200
+
+@alias.route('/destination/<string:domain>')
+class AliasWithDest(Resource):
+    @alias.doc('find_alias_filter_domain')
+    @alias.response(200, 'Success', alias_fields)
+    @alias.response(404, 'Alias or domain not found', response_fields)
+    @alias.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, domain):
+        """ Find aliases of domain """
+        domain_found = models.Domain.query.filter_by(name=domain).first()
+        if domain_found is None:
+          return { 'code': 404, 'message': f'Domain {domain} cannot be found'}, 404
+        aliases_found = domain_found.aliases
+        if aliases_found.count == 0:
+          return { 'code': 404, 'message': f'No alias can be found for domain {domain}'}, 404
+        else:
+          return marshal(aliases_found, alias_fields), 200

core/admin/mailu/api/v1/domains.py

@@ -0,0 +1,410 @@
+import validators
+from flask_restx import Resource, fields, marshal
+from . import api, response_fields, user
+from .. import common
+from ... import models
+
+db = models.db
+
+dom = api.namespace('domain', description='Domain operations')
+alt = api.namespace('alternative', description='Alternative operations')
+
+domain_fields = api.model('Domain', {
+    'name': fields.String(description='FQDN (e.g. example.com)', example='example.com', required=True),
+    'comment': fields.String(description='a comment'),
+    'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1),
+    'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1),
+    'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0),
+    'signup_enabled': fields.Boolean(description='allow signup'),
+    'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example2.com')),
+})
+
+domain_fields_update = api.model('DomainUpdate', {
+    'comment': fields.String(description='a comment'),
+    'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1),
+    'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1),
+    'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0),
+    'signup_enabled': fields.Boolean(description='allow signup'),
+    'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example2.com')),
+})
+
+domain_fields_get = api.model('DomainGet', {
+    'name': fields.String(description='FQDN (e.g. example.com)', example='example.com', required=True),
+    'comment': fields.String(description='a comment'),
+    'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1),
+    'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1),
+    'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0),
+    'signup_enabled': fields.Boolean(description='allow signup'),
+    'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example2.com')),
+    'dns_autoconfig': fields.List(fields.String(description='DNS client auto-configuration entry')),
+    'dns_mx': fields.String(Description='MX record for domain'),
+    'dns_spf': fields.String(Description='SPF record for domain'),
+    'dns_dkim': fields.String(Description='DKIM record for domain'),
+    'dns_dmarc': fields.String(Description='DMARC record for domain'),
+    'dns_dmarc_report': fields.String(Description='DMARC report record for domain'),
+    'dns_tlsa': fields.String(Description='TLSA record for domain'),
+})
+
+domain_fields_dns = api.model('DomainDNS', {
+    'dns_autoconfig': fields.List(fields.String(description='DNS client auto-configuration entry')),
+    'dns_mx': fields.String(Description='MX record for domain'),
+    'dns_spf': fields.String(Description='SPF record for domain'),
+    'dns_dkim': fields.String(Description='DKIM record for domain'),
+    'dns_dmarc': fields.String(Description='DMARC record for domain'),
+    'dns_dmarc_report': fields.String(Description='DMARC report record for domain'),
+    'dns_tlsa': fields.String(Description='TLSA record for domain'),
+})
+
+manager_fields = api.model('Manager', {
+    'domain_name': fields.String(description='domain managed by manager'),
+    'user_email': fields.String(description='email address of manager'),
+})
+
+manager_fields_create = api.model('ManagerCreate', {
+    'user_email': fields.String(description='email address of manager', required=True),
+})
+
+alternative_fields_update = api.model('AlternativeDomainUpdate', {
+    'domain': fields.String(description='domain FQDN', example='example.com', required=False),
+})
+
+alternative_fields = api.model('AlternativeDomain', {
+    'name': fields.String(description='alternative FQDN', example='example2.com', required=True),
+    'domain': fields.String(description='domain FQDN', example='example.com', required=True),
+})
+
+
+@dom.route('')
+class Domains(Resource):
+    @dom.doc('list_domain')
+    @dom.marshal_with(domain_fields_get, as_list=True, skip_none=True, mask=None)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self):
+        """ List domains """
+        return models.Domain.query.all()
+
+    @dom.doc('create_domain')
+    @dom.expect(domain_fields)
+    @dom.response(200, 'Success', response_fields)
+    @dom.response(400, 'Input validation exception', response_fields)
+    @dom.response(409, 'Duplicate domain/alternative name', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self):
+        """ Create a new domain """
+        data = api.payload
+        if not validators.domain(data['name']):
+            return { 'code': 400, 'message': f'Domain {data["name"]} is not a valid domain'}, 400
+
+        if common.fqdn_in_use(data['name']):
+            return { 'code': 409, 'message': f'Duplicate domain name {data["name"]}'}, 409
+        if 'alternatives' in data:
+            #check if duplicate alternatives are supplied
+            if [x for x in data['alternatives'] if data['alternatives'].count(x) >= 2]:
+                return { 'code': 409, 'message': f'Duplicate alternative domain names in request' }, 409
+            for item in data['alternatives']:
+                if common.fqdn_in_use(item):
+                    return { 'code': 409, 'message': f'Duplicate alternative domain name {item}' }, 409
+                if not validators.domain(item):
+                    return { 'code': 400, 'message': f'Alternative domain {item} is not a valid domain'}, 400
+            for item in data['alternatives']:
+                alternative = models.Alternative(name=item, domain_name=data['name'])
+                models.db.session.add(alternative)
+        domain_new = models.Domain(name=data['name'])
+        if 'comment' in data:
+            domain_new.comment = data['comment']
+        if 'max_users' in data:
+            domain_new.comment = data['max_users']
+        if 'max_aliases' in data:
+            domain_new.comment = data['max_aliases']
+        if 'max_quota_bytes' in data:
+            domain_new.comment = data['max_quota_bytes']
+        if 'signup_enabled' in data:
+            domain_new.comment = data['signup_enabled']
+        models.db.session.add(domain_new)
+        #apply the changes
+        db.session.commit()
+        return  {'code': 200, 'message': f'Domain {data["name"]} has been created'}, 200
+
+@dom.route('/<domain>')
+class Domain(Resource):
+
+    @dom.doc('find_domain')
+    @dom.response(200, 'Success', domain_fields)
+    @dom.response(404, 'Domain not found', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, domain):
+        """ Find domain by name """
+        if not validators.domain(domain):
+            return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400
+        domain_found = models.Domain.query.get(domain)
+        if not domain_found:
+            return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404
+        return marshal(domain_found, domain_fields_get), 200
+
+    @dom.doc('update_domain')
+    @dom.expect(domain_fields_update)
+    @dom.response(200, 'Success', response_fields)
+    @dom.response(400, 'Input validation exception', response_fields)
+    @dom.response(404, 'Domain not found', response_fields)
+    @dom.response(409, 'Duplicate domain/alternative name', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def patch(self, domain):
+        """ Update an existing domain """
+        if not validators.domain(domain):
+            return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400
+        domain_found = models.Domain.query.get(domain)
+        if not domain:
+            return { 'code': 404, 'message': f'Domain {data["name"]} does not exist'}, 404
+        data = api.payload
+
+        if 'alternatives' in data:
+            #check if duplicate alternatives are supplied
+            if [x for x in data['alternatives'] if data['alternatives'].count(x) >= 2]:
+                return { 'code': 409, 'message': f'Duplicate alternative domain names in request' }, 409
+            for item in data['alternatives']:
+                if common.fqdn_in_use(item):
+                    return { 'code': 409, 'message': f'Duplicate alternative domain name {item}' }, 409
+                if not validators.domain(item):
+                    return { 'code': 400, 'message': f'Alternative domain {item} is not a valid domain'}, 400
+            for item in data['alternatives']:
+                alternative = models.Alternative(name=item, domain_name=data['name'])
+                models.db.session.add(alternative)
+
+        if 'comment' in data:
+            domain_found.comment = data['comment']
+        if 'max_users' in data:
+            domain_found.comment = data['max_users']
+        if 'max_aliases' in data:
+            domain_found.comment = data['max_aliases']
+        if 'max_quota_bytes' in data:
+            domain_found.comment = data['max_quota_bytes']
+        if 'signup_enabled' in data:
+            domain_found.comment = data['signup_enabled']
+        models.db.session.add(domain_found)
+
+        #apply the changes
+        db.session.commit()
+        return  {'code': 200, 'message': f'Domain {domain} has been updated'}, 200
+
+    @dom.doc('delete_domain')
+    @dom.response(200, 'Success', response_fields)
+    @dom.response(400, 'Input validation exception', response_fields)
+    @dom.response(404, 'Domain not found', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def delete(self, domain):
+        """ Delete domain """
+        if not validators.domain(domain):
+            return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400
+        domain_found = models.Domain.query.get(domain)
+        if not domain:
+            return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404
+        db.session.delete(domain_found)
+        db.session.commit()
+        return {'code': 200, 'message': f'Domain {domain} has been deleted'}, 200
+
+@dom.route('/<domain>/dkim')
+class Domain(Resource):
+    @dom.doc('generate_dkim')
+    @dom.response(200, 'Success', response_fields)
+    @dom.response(400, 'Input validation exception', response_fields)
+    @dom.response(404, 'Domain not found', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self, domain):
+        """ Generate new DKIM/DMARC keys for domain """
+        if not validators.domain(domain):
+            return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400
+        domain_found = models.Domain.query.get(domain)
+        if not domain_found:
+            return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404
+        domain_found.generate_dkim_key()
+        domain_found.save_dkim_key()
+        return {'code': 200, 'message': f'DKIM/DMARC keys have been generated for domain {domain}'}, 200
+
+@dom.route('/<domain>/manager')
+class Manager(Resource):
+    @dom.doc('list_managers')
+    @dom.marshal_with(manager_fields, as_list=True, skip_none=True, mask=None)
+    @dom.response(400, 'Input validation exception', response_fields)
+    @dom.response(404, 'domain not found', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, domain):
+        """ List managers of domain """
+        if not validators.domain(domain):
+            return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400
+        if not domain:
+            return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404
+        domain = models.Domain.query.filter_by(name=domain)
+        return domain.managers
+
+    @dom.doc('create_manager')
+    @dom.expect(manager_fields_create)
+    @dom.response(200, 'Success', response_fields)
+    @dom.response(400, 'Input validation exception', response_fields)
+    @dom.response(404, 'User or domain not found', response_fields)
+    @dom.response(409, 'Duplicate domain manager', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self, domain):
+        """ Create a new domain manager """
+        data = api.payload
+        if not validators.email(data['user_email']):
+            return {'code': 400, 'message': f'Invalid email address {data["user_email"]}'}, 400
+        if not validators.domain(domain):
+            return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400
+        domain = models.Domain.query.get(domain)
+        if not domain:
+            return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404
+        user = models.User.query.get(data['user_email'])
+        if not user:
+            return { 'code': 404, 'message': f'User {data["user_email"]} does not exist'}, 404
+        if user in domain.managers:
+            return {'code': 409, 'message': f'User {data["user_email"]} is already a manager of the domain {domain} '}, 409
+        domain.managers.append(user)
+        models.db.session.commit()
+        return {'code': 200, 'message': f'User {data["user_email"]} has been added as manager of the domain {domain} '},200
+
+@dom.route('/<domain>/manager/<email>')
+class Domain(Resource):
+    @dom.doc('find_manager')
+    @dom.response(200, 'Success', manager_fields)
+    @dom.response(404, 'Manager not found', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, domain, email):
+        """ Find manager by email address """
+        if not validators.email(email):
+            return {'code': 400, 'message': f'Invalid email address {email}'}, 400
+        if not validators.domain(domain):
+            return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400
+        domain = models.Domain.query.get(domain)
+        if not domain:
+            return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404
+        user = models.User.query.get(email)
+        if not user:
+            return { 'code': 404, 'message': f'User {email} does not exist'}, 404
+        if user in domain.managers:
+            for manager in domain.managers:
+                if manager.email == email:
+                    return marshal(manager, manager_fields),200
+        else:
+            return { 'code': 404, 'message': f'User {email} is not a manager of the domain {domain}'}, 404
+
+
+    @dom.doc('delete_manager')
+    @dom.response(200, 'Success', response_fields)
+    @dom.response(400, 'Input validation exception', response_fields)
+    @dom.response(404, 'Manager not found', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def delete(self, domain, email):
+        if not validators.email(email):
+            return {'code': 400, 'message': f'Invalid email address {email}'}, 400
+        if not validators.domain(domain):
+            return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400
+        domain = models.Domain.query.get(domain)
+        if not domain:
+            return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404
+        user = models.User.query.get(email)
+        if not user:
+            return { 'code': 404, 'message': f'User {email} does not exist'}, 404
+        if user in domain.managers:
+            domain.managers.remove(user)
+            models.db.session.commit()
+            return {'code': 200, 'message': f'User {email} has been removed as a manager of the domain {domain} '},200
+        else:
+            return { 'code': 404, 'message': f'User {email} is not a manager of the domain {domain}'}, 404
+
+@dom.route('/<domain>/users')
+class User(Resource):
+    @dom.doc('list_user_domain')
+    @dom.marshal_with(user.user_fields_get, as_list=True, skip_none=True, mask=None)
+    @dom.response(400, 'Input validation exception', response_fields)
+    @dom.response(404, 'Domain not found', response_fields)
+    @dom.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, domain):
+        """ List users from domain """
+        if not validators.domain(domain):
+            return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400
+        domain_found = models.Domain.query.get(domain)
+        if not domain_found:
+            return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404
+        return  models.User.query.filter_by(domain=domain_found).all()
+
+@alt.route('')
+class Alternatives(Resource):
+
+    @alt.doc('list_alternative')
+    @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=None)
+    @alt.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self):
+      """ List alternatives """
+      return models.Alternative.query.all()
+
+
+    @alt.doc('create_alternative')
+    @alt.expect(alternative_fields)
+    @alt.response(200, 'Success', response_fields)
+    @alt.response(400, 'Input validation exception', response_fields)
+    @alt.response(404, 'Domain not found or missing', response_fields)
+    @alt.response(409, 'Duplicate alternative domain name', response_fields)
+    @alt.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self):
+        """ Create new alternative (for domain) """
+        data = api.payload
+        if not validators.domain(data['name']):
+            return { 'code': 400, 'message': f'Alternative domain {data["name"]} is not a valid domain'}, 400
+        if not validators.domain(data['domain']):
+            return { 'code': 400, 'message': f'Domain {data["domain"]} is not a valid domain'}, 400
+        domain = models.Domain.query.get(data['domain'])
+        if not domain:
+            return { 'code': 404, 'message': f'Domain {data["domain"]} does not exist'}, 404
+        if common.fqdn_in_use(data['name']):
+            return { 'code': 409, 'message': f'Duplicate alternative domain name {data["name"]}'}, 409
+
+        alternative = models.Alternative(name=data['name'], domain_name=data['domain'])
+        models.db.session.add(alternative)
+        db.session.commit()
+        return {'code': 200, 'message': f'Alternative {data["name"]} for domain {data["domain"]} has been created'}, 200
+
+@alt.route('/<string:alt>')
+class Alternative(Resource):
+    @alt.doc('find_alternative')
+    @alt.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, alt):
+        """ Find alternative (of domain) """
+        if not validators.domain(alt):
+            return { 'code': 400, 'message': f'Alternative domain {alt} is not a valid domain'}, 400
+        alternative = models.Alternative.query.filter_by(name=alt).first()
+        if not alternative:
+            return{ 'code': 404, 'message': f'Alternative domain {alt} does not exist'}, 404
+        return marshal(alternative, alternative_fields), 200
+
+    @alt.doc('delete_alternative')
+    @alt.response(200, 'Success', response_fields)
+    @alt.response(400, 'Input validation exception', response_fields)
+    @alt.response(404, 'Alternative/Domain not found or missing', response_fields)
+    @alt.response(409, 'Duplicate domain name', response_fields)
+    @alt.doc(security='Bearer')
+    @common.api_token_authorization
+    def delete(self, alt):
+        """ Delete alternative (for domain) """
+        if not validators.domain(alt):
+            return { 'code': 400, 'message': f'Alternative domain {alt} is not a valid domain'}, 400
+        alternative = models.Alternative.query.filter_by(name=alt).scalar()
+        if not alternative:
+            return { 'code': 404, 'message': f'Alternative domain {alt} does not exist'}, 404
+        domain = alternative.domain_name
+        db.session.delete(alternative)
+        db.session.commit()
+        return {'code': 200, 'message': f'Alternative {alt} for domain {domain} has been deleted'}, 200

core/admin/mailu/api/v1/relay.py

@@ -0,0 +1,118 @@
+from flask_restx import Resource, fields, marshal
+import validators
+
+from . import api, response_fields
+from .. import common
+from ... import models
+
+db = models.db
+
+relay = api.namespace('relay', description='Relay operations')
+
+relay_fields = api.model('Relay', {
+    'name': fields.String(description='relayed domain name', example='example.com', required=True),
+    'smtp': fields.String(description='remote host', example='example.com', required=False),
+    'comment': fields.String(description='a comment', required=False)
+})
+
+relay_fields_update = api.model('RelayUpdate', {
+    'smtp': fields.String(description='remote host', example='example.com', required=False),
+    'comment': fields.String(description='a comment', required=False)
+})
+
+@relay.route('')
+class Relays(Resource):
+    @relay.doc('list_relays')
+    @relay.marshal_with(relay_fields, as_list=True, skip_none=True, mask=None)
+    @relay.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self):
+        "List relays"
+        return models.Relay.query.all()
+
+    @relay.doc('create_relay')
+    @relay.expect(relay_fields)
+    @relay.response(200, 'Success', response_fields)
+    @relay.response(400, 'Input validation exception')
+    @relay.response(409, 'Duplicate relay', response_fields)
+    @relay.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self):
+        """ Create relay """
+        data = api.payload
+
+        if not validators.domain(name):
+            return { 'code': 400, 'message': f'Relayed domain {name} is not a valid domain'}, 400
+
+        if common.fqdn_in_use(data['name']):
+            return { 'code': 409, 'message': f'Duplicate domain {data["name"]}'}, 409
+        relay_model = models.Relay(name=data['name'])
+        if 'smtp' in data:
+            relay_model.smtp = data['smtp']
+        if 'comment' in data:
+            relay_model.comment = data['comment']
+        db.session.add(relay_model)
+        db.session.commit()
+        return {'code': 200, 'message': f'Relayed domain {data["name"]} has been created'}, 200
+
+@relay.route('/<string:name>')
+class Relay(Resource):
+    @relay.doc('find_relay')
+    @relay.response(400, 'Input validation exception', response_fields)
+    @relay.response(404, 'Relay not found', response_fields)
+    @relay.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, name):
+        """ Find relay """
+        if not validators.domain(name):
+            return { 'code': 400, 'message': f'Relayed domain {name} is not a valid domain'}, 400
+
+        relay_found = models.Relay.query.filter_by(name=name).first()
+        if relay_found is None:
+            return { 'code': 404, 'message': f'Relayed domain {name} cannot be found'}, 404
+        return  marshal(relay_found, relay_fields), 200
+
+    @relay.doc('update_relay')
+    @relay.expect(relay_fields_update)
+    @relay.response(200, 'Success', response_fields)
+    @relay.response(400, 'Input validation exception', response_fields)
+    @relay.response(404, 'Relay not found', response_fields)
+    @relay.response(409, 'Duplicate relay', response_fields)
+    @relay.doc(security='Bearer')
+    @common.api_token_authorization
+    def patch(self, name):
+        """ Update relay """
+        data = api.payload
+
+        if not validators.domain(name):
+            return { 'code': 400, 'message': f'Relayed domain {name} is not a valid domain'}, 400
+
+        relay_found = models.Relay.query.filter_by(name=name).first()
+        if relay_found is None:
+            return { 'code': 404, 'message': f'Relayed domain {name} cannot be found'}, 404
+
+        if 'smtp' in data:
+            relay_found.smtp = data['smtp']
+        if 'comment' in data:
+            relay_found.comment = data['comment']
+        db.session.add(relay_found)
+        db.session.commit()
+        return { 'code': 200, 'message': f'Relayed domain {name} has been updated'}, 200
+
+
+    @relay.doc('delete_relay')
+    @relay.response(200, 'Success', response_fields)
+    @relay.response(400, 'Input validation exception', response_fields)
+    @relay.response(404, 'Relay not found', response_fields)
+    @relay.doc(security='Bearer')
+    @common.api_token_authorization
+    def delete(self, name):
+        """ Delete relay """
+        if not validators.domain(name):
+            return { 'code': 400, 'message': f'Relayed domain {name} is not a valid domain'}, 400
+        relay_found = models.Relay.query.filter_by(name=name).first()
+        if relay_found is None:
+            return { 'code': 404, 'message': f'Relayed domain {name} cannot be found'}, 404
+        db.session.delete(relay_found)
+        db.session.commit()
+        return { 'code': 200, 'message': f'Relayed domain {name} has been deleted'}, 200

core/admin/mailu/api/v1/user.py

@@ -0,0 +1,262 @@
+from flask_restx import Resource, fields, marshal
+import validators, datetime
+
+from . import api, response_fields
+from .. import common
+from ... import models
+
+db = models.db
+
+user = api.namespace('user', description='User operations')
+
+user_fields_get = api.model('UserGet', {
+    'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email'),
+    'password': fields.String(description="Hash of the user's password; Example='$bcrypt-sha256$v=2,t=2b,r=12$fmsAdJbYAD1gGQIE5nfJq.$zLkQUEs2XZfTpAEpcix/1k5UTNPm0jO'"),
+    'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'),
+    'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'),
+    'global_admin': fields.Boolean(description='Make the user a global administrator'),
+    'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'),
+    'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'),
+    'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'),
+    'allow_spoofing': fields.Boolean(description='Allow the user to spoof the sender (send email as anyone)'),
+    'forward_enabled': fields.Boolean(description='Enable auto forwarding'),
+    'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'),
+    'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'),
+    'reply_enabled': fields.Boolean(description='Enable automatic replies. This is also known as out of office (ooo) or out of facility (oof) replies'),
+    'reply_subject': fields.String(description='Optional subject for the automatic reply', example='Out of office'),
+    'reply_body': fields.String(description='The body of the automatic reply email', example='Hello, I am out of office. I will respond when I am back.'),
+    'reply_startdate': fields.Date(description='Start date for automatic replies in YYYY-MM-DD format.', example='2022-02-10'),
+    'reply_enddate': fields.Date(description='End date for automatic replies in YYYY-MM-DD format.', example='2022-02-22'),
+    'displayed_name': fields.String(description='The display name of the user within the Admin GUI', example='John Doe'),
+    'spam_enabled': fields.Boolean(description='Enable the spam filter'),
+    'spam_mark_as_read': fields.Boolean(description='Enable marking spam mails as read'),
+    'spam_threshold': fields.Integer(description='The user defined spam filter tolerance', example='80'),
+})
+
+user_fields_post = api.model('UserCreate', {
+    'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email', required=True),
+    'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using BCRYPT-SHA256', example='secret', required=True),
+    'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'),
+    'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'),
+    'global_admin': fields.Boolean(description='Make the user a global administrator'),
+    'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'),
+    'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'),
+    'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'),
+    'allow_spoofing': fields.Boolean(description='Allow the user to spoof the sender (send email as anyone)'),
+    'forward_enabled': fields.Boolean(description='Enable auto forwarding'),
+    'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'),
+    'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'),
+    'reply_enabled': fields.Boolean(description='Enable automatic replies. This is also known as out of office (ooo) or out of facility (oof) replies'),
+    'reply_subject': fields.String(description='Optional subject for the automatic reply', example='Out of office'),
+    'reply_body': fields.String(description='The body of the automatic reply email', example='Hello, I am out of office. I will respond when I am back.'),
+    'reply_startdate': fields.Date(description='Start date for automatic replies in YYYY-MM-DD format.', example='2022-02-10'),
+    'reply_enddate': fields.Date(description='End date for automatic replies in YYYY-MM-DD format.', example='2022-02-22'),
+    'displayed_name': fields.String(description='The display name of the user within the Admin GUI', example='John Doe'),
+    'spam_enabled': fields.Boolean(description='Enable the spam filter'),
+    'spam_mark_as_read': fields.Boolean(description='Enable marking spam mails as read'),
+    'spam_threshold': fields.Integer(description='The user defined spam filter tolerance', example='80'),
+})
+
+user_fields_put = api.model('UserUpdate', {
+    'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using BCRYPT-SHA256', example='secret'),
+    'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'),
+    'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'),
+    'global_admin': fields.Boolean(description='Make the user a global administrator'),
+    'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'),
+    'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'),
+    'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'),
+    'allow_spoofing': fields.Boolean(description='Allow the user to spoof the sender (send email as anyone)'),
+    'forward_enabled': fields.Boolean(description='Enable auto forwarding'),
+    'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'),
+    'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'),
+    'reply_enabled': fields.Boolean(description='Enable automatic replies. This is also known as out of office (ooo) or out of facility (oof) replies'),
+    'reply_subject': fields.String(description='Optional subject for the automatic reply', example='Out of office'),
+    'reply_body': fields.String(description='The body of the automatic reply email', example='Hello, I am out of office. I will respond when I am back.'),
+    'reply_startdate': fields.Date(description='Start date for automatic replies in YYYY-MM-DD format.', example='2022-02-10'),
+    'reply_enddate': fields.Date(description='End date for automatic replies in YYYY-MM-DD format.', example='2022-02-22'),
+    'displayed_name': fields.String(description='The display name of the user within the Admin GUI', example='John Doe'),
+    'spam_enabled': fields.Boolean(description='Enable the spam filter'),
+    'spam_mark_as_read': fields.Boolean(description='Enable marking spam mails as read'),
+    'spam_threshold': fields.Integer(description='The user defined spam filter tolerance', example='80'),
+})
+
+
+@user.route('')
+class Users(Resource):
+    @user.doc('list_users')
+    @user.marshal_with(user_fields_get, as_list=True, skip_none=True, mask=None)
+    @user.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self):
+        "List users"
+        return models.User.query.all()
+
+    @user.doc('create_user')
+    @user.expect(user_fields_post)
+    @user.response(200, 'Success', response_fields)
+    @user.response(400, 'Input validation exception')
+    @user.response(409, 'Duplicate user', response_fields)
+    @user.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self):
+        """ Create user """
+        data = api.payload
+        if not validators.email(data['email']):
+            return { 'code': 400, 'message': f'Provided email address {data["email"]} is not a valid email address'}, 400
+        localpart, domain_name = data['email'].lower().rsplit('@', 1)
+        domain_found = models.Domain.query.get(domain_name)
+        if not domain_found:
+            return { 'code': 404, 'message': f'Domain {domain_name} does not exist'}, 404
+
+        user_new = models.User(email=data['email'])
+        if 'raw_password' in data:
+            user_new.set_password(data['raw_password'])
+        if 'comment' in data:
+            user_new.comment = data['comment']
+        if 'quota_bytes' in data:
+            user_new.quota_bytes = data['quota_bytes']
+        if 'global_admin' in data:
+            user_new.global_admin = data['global_admin']
+        if 'enabled' in data:
+            user_new.enabled = data['enabled']
+        if 'enable_imap' in data:
+            user_new.enable_imap = data['enable_imap']
+        if 'enable_pop' in data:
+            user_new.enable_pop = data['enable_pop']
+        if 'allow_spoofing' in data:
+            user_new.allow_spoofing = data['allow_spoofing']
+        if 'forward_enabled' in data:
+            user_new.forward_enabled = data['forward_enabled']
+        if 'forward_destination' in data:
+            user_new.forward_destination = data['forward_destination']
+        if 'forward_keep' in data:
+            user_new.forward_keep = data['forward_keep']
+        if 'reply_enabled' in data:
+            user_new.reply_enabled = data['reply_enabled']
+        if 'reply_subject' in data:
+            user_new.reply_subject = data['reply_subject']
+        if 'reply_body' in data:
+            user_new.reply_body = data['reply_body']
+        if 'reply_startdate' in data:
+            year, month, day = data['reply_startdate'].split('-')
+            date = datetime.datetime(int(year), int(month), int(day))
+            user_new.reply_startdate = date
+        if 'reply_enddate' in data:
+            year, month, day = data['reply_enddate'].split('-')
+            date = datetime.datetime(int(year), int(month), int(day))
+            user_new.reply_enddate = date
+        if 'displayed_name' in data:
+            user_new.displayed_name = data['displayed_name']
+        if 'spam_enabled' in data:
+            user_new.spam_enabled = data['spam_enabled']
+        if 'spam_mark_as_read' in data:
+            user_new.spam_mark_as_read = data['spam_mark_as_read']
+        if 'spam_threshold' in data:
+            user_new.spam_threshold = data['spam_threshold']
+        db.session.add(user_new)
+        db.session.commit()
+
+        return {'code': 200,'message': f'User {data["email"]} has been created'}, 200
+
+
+@user.route('/<string:email>')
+class User(Resource):
+    @user.doc('find_user')
+    @user.response(400, 'Input validation exception', response_fields)
+    @user.response(404, 'User not found', response_fields)
+    @user.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, email):
+        """ Find user """
+        if not validators.email(email):
+            return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400
+
+        email_found = models.User.query.filter_by(email=email).first()
+        if email_found is None:
+            return { 'code': 404, 'message': f'User {email} cannot be found'}, 404
+        return  marshal(email_found, user_fields_get), 200
+
+    @user.doc('update_user')
+    @user.expect(user_fields_put)
+    @user.response(200, 'Success', response_fields)
+    @user.response(400, 'Input validation exception', response_fields)
+    @user.response(404, 'User not found', response_fields)
+    @user.response(409, 'Duplicate user', response_fields)
+    @user.doc(security='Bearer')
+    @common.api_token_authorization
+    def patch(self, email):
+        """ Update user """
+        data = api.payload
+        if not validators.email(email):
+            return { 'code': 400, 'message': f'Provided email address {data["email"]} is not a valid email address'}, 400
+        user_found = models.User.query.get(email)
+        if not user_found:
+            return {'code': 404, 'message': f'User {email} cannot be found'}, 404
+
+        if 'raw_password' in data:
+            user_found.set_password(data['raw_password'])
+        if 'comment' in data:
+            user_found.comment = data['comment']
+        if 'quota_bytes' in data:
+            user_found.quota_bytes = data['quota_bytes']
+        if 'global_admin' in data:
+            user_found.global_admin = data['global_admin']
+        if 'enabled' in data:
+            user_found.enabled = data['enabled']
+        if 'enable_imap' in data:
+            user_found.enable_imap = data['enable_imap']
+        if 'enable_pop' in data:
+            user_found.enable_pop = data['enable_pop']
+        if 'allow_spoofing' in data:
+            user_found.allow_spoofing = data['allow_spoofing']
+        if 'forward_enabled' in data:
+            user_found.forward_enabled = data['forward_enabled']
+        if 'forward_destination' in data:
+            user_found.forward_destination = data['forward_destination']
+        if 'forward_keep' in data:
+            user_found.forward_keep = data['forward_keep']
+        if 'reply_enabled' in data:
+            user_found.reply_enabled = data['reply_enabled']
+        if 'reply_subject' in data:
+            user_found.reply_subject = data['reply_subject']
+        if 'reply_body' in data:
+            user_found.reply_body = data['reply_body']
+        if 'reply_startdate' in data:
+            year, month, day = data['reply_startdate'].split('-')
+            date = datetime.datetime(int(year), int(month), int(day))
+            user_found.reply_startdate = date
+        if 'reply_enddate' in data:
+            year, month, day = data['reply_enddate'].split('-')
+            date = datetime.datetime(int(year), int(month), int(day))
+            user_found.reply_enddate = date
+        if 'displayed_name' in data:
+            user_found.displayed_name = data['displayed_name']
+        if 'spam_enabled' in data:
+            user_found.spam_enabled = data['spam_enabled']
+        if 'spam_mark_as_read' in data:
+            user_found.spam_mark_as_read = data['spam_mark_as_read']
+        if 'spam_threshold' in data:
+            user_found.spam_threshold = data['spam_threshold']
+        db.session.add(user_found)
+        db.session.commit()
+
+        return {'code': 200,'message': f'User {email} has been updated'}, 200
+
+
+    @user.doc('delete_user')
+    @user.response(200, 'Success', response_fields)
+    @user.response(400, 'Input validation exception', response_fields)
+    @user.response(404, 'User not found', response_fields)
+    @user.doc(security='Bearer')
+    @common.api_token_authorization
+    def delete(self, email):
+        """ Delete user """
+        if not validators.email(email):
+            return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400
+
+        email_found = models.User.query.filter_by(email=email).first()
+        if email_found is None:
+            return { 'code': 404, 'message': f'User {email} cannot be found'}, 404
+        db.session.delete(email_found)
+        db.session.commit()
+        return { 'code': 200, 'message': f'User {email} has been deleted'}, 200

core/admin/mailu/configuration.py

@@ -1,7 +1,6 @@
 import os
 
 from datetime import timedelta
-from socrate import system
 import ipaddress
 
 DEFAULT_CONFIG = {
@@ -18,7 +17,8 @@ DEFAULT_CONFIG = {
     'DOMAIN_REGISTRATION': False,
     'TEMPLATES_AUTO_RELOAD': True,
     'MEMORY_SESSIONS': False,
-    'FETCHMAIL_ENABLED': False,
+    'FETCHMAIL_ENABLED': True,
+    'MAILU_VERSION': 'unknown',
     # Database settings
     'DB_FLAVOR': None,
     'DB_USER': 'mailu',
@@ -40,9 +40,9 @@ DEFAULT_CONFIG = {
     'TLS_FLAVOR': 'cert',
     'INBOUND_TLS_ENFORCE': False,
     'DEFER_ON_TLS_ERROR': True,
-    'AUTH_RATELIMIT_IP': '60/hour',
+    'AUTH_RATELIMIT_IP': '5/hour',
     'AUTH_RATELIMIT_IP_V4_MASK': 24,
-    'AUTH_RATELIMIT_IP_V6_MASK': 56,
+    'AUTH_RATELIMIT_IP_V6_MASK': 48,
     'AUTH_RATELIMIT_USER': '100/day',
     'AUTH_RATELIMIT_EXEMPTION': '',
     'AUTH_RATELIMIT_EXEMPTION_LENGTH': 86400,
@@ -71,6 +71,9 @@ DEFAULT_CONFIG = {
     'LOGO_URL': None,
     'LOGO_BACKGROUND': None,
     # Advanced settings
+    'API': False,
+    'WEB_API': '/api',
+    'API_TOKEN': None,
     'LOG_LEVEL': 'WARNING',
     'SESSION_KEY_BITS': 128,
     'SESSION_TIMEOUT': 3600,
@@ -83,19 +86,8 @@ DEFAULT_CONFIG = {
     'PROXY_AUTH_WHITELIST': '',
     'PROXY_AUTH_HEADER': 'X-Auth-Email',
     'PROXY_AUTH_CREATE': False,
-    # Host settings
-    'HOST_IMAP': 'imap',
-    'HOST_LMTP': 'imap:2525',
-    'HOST_POP3': 'imap',
-    'HOST_SMTP': 'smtp',
-    'HOST_AUTHSMTP': 'smtp',
-    'HOST_ADMIN': 'admin',
-    'HOST_WEBMAIL': 'webmail',
-    'HOST_WEBDAV': 'webdav:5232',
-    'HOST_REDIS': 'redis',
-    'HOST_FRONT': 'front',
     'SUBNET': '192.168.203.0/24',
-    'SUBNET6': None
+    'SUBNET6': None,
 }
 
 class ConfigManager:
@@ -105,25 +97,12 @@ class ConfigManager:
     DB_TEMPLATES = {
         'sqlite': 'sqlite:////{SQLITE_DATABASE_FILE}',
         'postgresql': 'postgresql://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}',
-        'mysql': 'mysql+mysqlconnector://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}'
+        'mysql': 'mysql+mysqlconnector://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}',
     }
 
     def __init__(self):
         self.config = dict()
 
-    def get_host_address(self, name):
-        # if MYSERVICE_ADDRESS is defined, use this
-        if f'{name}_ADDRESS' in os.environ:
-            return os.environ.get(f'{name}_ADDRESS')
-        # otherwise use the host name and resolve it
-        return system.resolve_address(self.config[f'HOST_{name}'])
-
-    def resolve_hosts(self):
-        for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']:
-            self.config[f'{key}_ADDRESS'] = self.get_host_address(key)
-        if self.config['WEBMAIL'] != 'none':
-            self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL')
-
     def __get_env(self, key, value):
         key_file = key + "_FILE"
         if key_file in os.environ:
@@ -144,11 +123,14 @@ class ConfigManager:
         # get current app config
         self.config.update(app.config)
         # get environment variables
+        for key in os.environ:
+            if key.endswith('_ADDRESS'):
+                self.config[key] = os.environ[key]
+
         self.config.update({
             key: self.__coerce_value(self.__get_env(key, value))
             for key, value in DEFAULT_CONFIG.items()
         })
-        self.resolve_hosts()
 
         # automatically set the sqlalchemy string
         if self.config['DB_FLAVOR']:
@@ -165,6 +147,7 @@ class ConfigManager:
             self.config['SESSION_COOKIE_SECURE'] = self.config['TLS_FLAVOR'] != 'notls'
         self.config['SESSION_PERMANENT'] = True
         self.config['SESSION_TIMEOUT'] = int(self.config['SESSION_TIMEOUT'])
+        self.config['SESSION_KEY_BITS'] = int(self.config['SESSION_KEY_BITS'])
         self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME'])
         self.config['AUTH_RATELIMIT_IP_V4_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V4_MASK'])
         self.config['AUTH_RATELIMIT_IP_V6_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V6_MASK'])
@@ -175,7 +158,10 @@ class ConfigManager:
         self.config['HOSTNAME'] = hostnames[0]
         self.config['DEFAULT_SPAM_THRESHOLD'] = int(self.config['DEFAULT_SPAM_THRESHOLD'])
         self.config['PROXY_AUTH_WHITELIST'] = set(ipaddress.ip_network(cidr, False) for cidr in (cidr.strip() for cidr in self.config['PROXY_AUTH_WHITELIST'].split(',')) if cidr)
+        try:
+            self.config['MAILU_VERSION'] = open('/version', 'r').read()
+        except FileNotFoundError:
+            pass
 
         # update the app config
         app.config.update(self.config)
-

core/admin/mailu/dkim.py

@@ -2,20 +2,20 @@
 They are thus represented as ASCII armored PEM.
 """
 
-from OpenSSL import crypto
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
 
 
-def gen_key(key_type=crypto.TYPE_RSA, bits=2048):
+def gen_key(bits=2048):
     """ Generate and return a new RSA key.
     """
-    key = crypto.PKey()
-    key.generate_key(key_type, bits)
-    return crypto.dump_privatekey(crypto.FILETYPE_PEM, key)
+    k = rsa.generate_private_key(public_exponent=65537, key_size=bits)
+    return k.private_bytes(encoding=serialization.Encoding.PEM,format=serialization.PrivateFormat.PKCS8,encryption_algorithm=serialization.NoEncryption())
 
 
 def strip_key(pem):
     """ Return only the b64 part of the ASCII armored PEM.
     """
-    key = crypto.load_privatekey(crypto.FILETYPE_PEM, pem)
-    public_pem = crypto.dump_publickey(crypto.FILETYPE_PEM, key)
+    priv_key = serialization.load_pem_private_key(pem, password=None)
+    public_pem = priv_key.public_key().public_bytes(encoding=serialization.Encoding.PEM,format=serialization.PublicFormat.SubjectPublicKeyInfo)
     return public_pem.replace(b"\n", b"").split(b"-----")[2]

core/admin/mailu/internal/nginx.py

@@ -2,7 +2,6 @@ from mailu import models, utils
 from flask import current_app as app
 from socrate import system
 
-import re
 import urllib
 import ipaddress
 import sqlalchemy.exc
@@ -26,12 +25,14 @@ STATUSES = {
     }),
 }
 
+WEBMAIL_PORTS = ['10143', '10025']
+
 def check_credentials(user, password, ip, protocol=None, auth_port=None):
-    if not user or not user.enabled or (protocol == "imap" and not user.enable_imap) or (protocol == "pop3" and not user.enable_pop):
+    if not user or not user.enabled or (protocol == "imap" and not user.enable_imap and not auth_port in WEBMAIL_PORTS) or (protocol == "pop3" and not user.enable_pop):
         return False
     is_ok = False
     # webmails
-    if auth_port in ['10143', '10025'] and password.startswith('token-'):
+    if auth_port in WEBMAIL_PORTS and password.startswith('token-'):
         if utils.verify_temp_token(user.get_id(), password):
             is_ok = True
     # All tokens are 32 characters hex lowercase
@@ -126,20 +127,16 @@ def get_status(protocol, status):
     status, codes = STATUSES[status]
     return status, codes[protocol]
 
-def extract_host_port(host_and_port, default_port):
-    host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups()
-    return host, int(port) if port else default_port
-
 def get_server(protocol, authenticated=False):
     if protocol == "imap":
-        hostname, port = extract_host_port(app.config['IMAP_ADDRESS'], 143)
+        hostname, port = app.config['IMAP_ADDRESS'], 143
     elif protocol == "pop3":
-        hostname, port = extract_host_port(app.config['POP3_ADDRESS'], 110)
+        hostname, port = app.config['IMAP_ADDRESS'], 110
     elif protocol == "smtp":
         if authenticated:
-            hostname, port = extract_host_port(app.config['AUTHSMTP_ADDRESS'], 10025)
+            hostname, port = app.config['SMTP_ADDRESS'], 10025
         else:
-            hostname, port = extract_host_port(app.config['SMTP_ADDRESS'], 25)
+            hostname, port = app.config['SMTP_ADDRESS'], 25
     try:
         # test if hostname is already resolved to an ip address
         ipaddress.ip_address(hostname)

core/admin/mailu/internal/views/auth.py

@@ -33,8 +33,8 @@ def nginx_authentication():
     for key, value in headers.items():
         response.headers[key] = str(value)
     is_valid_user = False
+    username = response.headers.get('Auth-User', None)
     if response.headers.get("Auth-User-Exists") == "True":
-        username = response.headers["Auth-User"]
         if utils.limiter.should_rate_limit_user(username, client_ip):
             # FIXME could be done before handle_authentication()
             status, code = nginx.get_status(flask.request.headers['Auth-Protocol'], 'ratelimit')
@@ -50,7 +50,7 @@ def nginx_authentication():
     elif is_valid_user:
         utils.limiter.rate_limit_user(username, client_ip)
     elif not is_from_webmail:
-        utils.limiter.rate_limit_ip(client_ip)
+        utils.limiter.rate_limit_ip(client_ip, username)
     return response
 
 @internal.route("/auth/admin")
@@ -109,7 +109,7 @@ def basic_authentication():
                 utils.limiter.exempt_ip_from_ratelimits(client_ip)
                 return response
             # We failed check_credentials
-            utils.limiter.rate_limit_user(user_email, client_ip) if user else utils.limiter.rate_limit_ip(client_ip)
+            utils.limiter.rate_limit_user(user_email, client_ip) if user else utils.limiter.rate_limit_ip(client_ip, user_email)
     response = flask.Response(status=401)
     response.headers["WWW-Authenticate"] = 'Basic realm="Login Required"'
     return response

core/admin/mailu/internal/views/dovecot.py

@@ -17,7 +17,7 @@ def dovecot_passdb_dict(user_email):
     return flask.jsonify({
         "password": None,
         "nopassword": "Y",
-        "allow_nets": ",".join(allow_nets)
+        "allow_real_nets": ",".join(allow_nets)
     })
 
 @internal.route("/dovecot/userdb/")

core/admin/mailu/limiter.py

@@ -52,10 +52,13 @@ class LimitWraperFactory(object):
             app.logger.warn(f'Authentication attempt from {ip} has been rate-limited.')
         return is_rate_limited
 
-    def rate_limit_ip(self, ip):
-        limiter = self.get_limiter(app.config["AUTH_RATELIMIT_IP"], 'auth-ip')
+    def rate_limit_ip(self, ip, username=None):
+        limiter = self.get_limiter(app.config['AUTH_RATELIMIT_IP'], 'auth-ip')
         client_network = utils.extract_network_from_ip(ip)
         if self.is_subject_to_rate_limits(ip):
+            if username and self.storage.get(f'dedup-{client_network}-{username}') > 0:
+                return
+            self.storage.incr(f'dedup-{client_network}-{username}', limits.parse(app.config['AUTH_RATELIMIT_IP']).GRANULARITY.seconds, True)
             limiter.hit(client_network)
 
     def should_rate_limit_user(self, username, ip, device_cookie=None, device_cookie_name=None):

core/admin/mailu/models.py

@@ -75,7 +75,7 @@ class CommaSeparatedList(db.TypeDecorator):
     """ Stores a list as a comma-separated string, compatible with Postfix.
     """
 
-    impl = db.String
+    impl = db.String(255)
     cache_ok = True
     python_type = list
 
@@ -96,7 +96,7 @@ class JSONEncoded(db.TypeDecorator):
     """ Represents an immutable structure as a json-encoded string.
     """
 
-    impl = db.String
+    impl = db.String(255)
     cache_ok = True
     python_type = str
 
@@ -421,8 +421,7 @@ class Email(object):
         """ send an email to the address """
         try:
             f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}'
-            ip, port = app.config['HOST_LMTP'].rsplit(':')
-            with smtplib.LMTP(ip, port=port) as lmtp:
+            with smtplib.LMTP(host=app.config['IMAP_ADDRESS'], port=2525) as lmtp:
                 to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}'
                 msg = text.MIMEText(body)
                 msg['Subject'] = subject

core/admin/mailu/schemas.py

@@ -5,6 +5,7 @@ from copy import deepcopy
 from collections import Counter
 from datetime import timezone
 
+import inspect
 import json
 import logging
 import yaml
@@ -19,7 +20,7 @@ from marshmallow_sqlalchemy.fields import RelatedList
 
 from flask_marshmallow import Marshmallow
 
-from OpenSSL import crypto
+from cryptography.hazmat.primitives import serialization
 
 from pygments import highlight
 from pygments.token import Token
@@ -609,8 +610,8 @@ class DkimKeyField(fields.String):
 
         # check key validity
         try:
-            crypto.load_privatekey(crypto.FILETYPE_PEM, value)
-        except crypto.Error as exc:
+            serialization.load_pem_private_key(bytes(value, "ascii"), password=None)
+        except (UnicodeEncodeError, ValueError) as exc:
             raise ValidationError(f'invalid dkim key {bad_key!r}') from exc
         else:
             return value
@@ -669,20 +670,15 @@ class Storage:
 
     context = {}
 
-    def _bind(self, key, bind):
-        if bind is True:
-            return (self.__class__, key)
-        if isinstance(bind, str):
-            return (get_schema(self.recall(bind).__class__), key)
-        return (bind, key)
-
-    def store(self, key, value, bind=None):
+    def store(self, key, value):
         """ store value under key """
-        self.context.setdefault('_track', {})[self._bind(key, bind)]= value
+        key = f'{self.__class__.__name__}.{key}'
+        self.context.setdefault('_track', {})[key] = value
 
-    def recall(self, key, bind=None):
+    def recall(self, key):
         """ recall value from key """
-        return self.context['_track'][self._bind(key, bind)]
+        key = f'{self.__class__.__name__}.{key}'
+        return self.context['_track'][key]
 
 class BaseOpts(SQLAlchemyAutoSchemaOpts):
     """ Option class with sqla session
@@ -790,10 +786,16 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
             for key, value in data.items()
         }
 
-    def _call_and_store(self, *args, **kwargs):
-        """ track current parent field for pruning """
-        self.store('field', kwargs['field_name'], True)
-        return super()._call_and_store(*args, **kwargs)
+    def get_parent(self):
+        """ helper to determine parent of current object """
+        for x in inspect.stack():
+            loc = x[0].f_locals
+            if 'ret_d' in loc:
+                if isinstance(loc['self'], MailuSchema):
+                    return self.context.get('config'), loc['attr_name']
+                else:
+                    return loc['self'].get_instance(loc['ret_d']), loc['attr_name']
+        return None, None
 
     # this is only needed to work around the declared attr "email" primary key in model
     def get_instance(self, data):
@@ -803,9 +805,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
         if keys := getattr(self.Meta, 'primary_keys', None):
             filters = {key: data.get(key) for key in keys}
             if None not in filters.values():
-                res= self.session.query(self.opts.model).filter_by(**filters).first()
-                return res
-        res= super().get_instance(data)
+                try:
+                    res = self.session.query(self.opts.model).filter_by(**filters).first()
+                except sqlalchemy.exc.StatementError as exc:
+                    raise ValidationError(f'Invalid {keys[0]}: {data.get(keys[0])!r}', data.get(keys[0])) from exc
+                else:
+                    return res
+        res = super().get_instance(data)
         return res
 
     @pre_load(pass_many=True)
@@ -829,6 +835,10 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
         want_prune = []
         def patch(count, data):
 
+            # we only process objects here
+            if type(data) is not dict:
+                raise ValidationError(f'Invalid item. {self.Meta.model.__tablename__.title()} needs to be an object.', f'{data!r}')
+
             # don't allow __delete__ coming from input
             if '__delete__' in data:
                 raise ValidationError('Unknown field.', f'{count}.__delete__')
@@ -882,10 +892,10 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
         ]
 
         # remember if prune was requested for _prune_items@post_load
-        self.store('prune', bool(want_prune), True)
+        self.store('prune', bool(want_prune))
 
         # remember original items to stabilize password-changes in _add_instance@post_load
-        self.store('original', items, True)
+        self.store('original', items)
 
         return items
 
@@ -909,13 +919,15 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
         # stabilize import of auto-increment primary keys (not required),
         # by matching import data to existing items and setting primary key
         if not self._primary in data:
-            for item in getattr(self.recall('parent'), self.recall('field', 'parent')):
-                existing = self.dump(item, many=False)
-                this = existing.pop(self._primary)
-                if data == existing:
-                    instance = item
-                    data[self._primary] = this
-                    break
+            parent, field = self.get_parent()
+            if parent is not None:
+                for item in getattr(parent, field):
+                    existing = self.dump(item, many=False)
+                    this = existing.pop(self._primary)
+                    if data == existing:
+                        self.instance = item
+                        data[self._primary] = this
+                        break
 
         # try to load instance
         instance = self.instance or self.get_instance(data)
@@ -931,9 +943,6 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
         else:
 
             if self.context.get('update'):
-                # remember instance as parent for pruning siblings
-                if not self.Meta.sibling:
-                    self.store('parent', instance)
                 # delete instance from session when marked
                 if '__delete__' in data:
                     self.opts.sqla_session.delete(instance)
@@ -997,7 +1006,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
             return items
 
         # get prune flag from _patch_many@pre_load
-        want_prune = self.recall('prune', True)
+        want_prune = self.recall('prune')
 
         # prune: determine if existing items in db need to be added or marked for deletion
         add_items = False
@@ -1014,14 +1023,17 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
                 del_items = True
 
         if add_items or del_items:
-            existing = {item[self._primary] for item in items if self._primary in item}
-            for item in getattr(self.recall('parent'), self.recall('field', 'parent')):
-                key = getattr(item, self._primary)
-                if key not in existing:
-                    if add_items:
-                        items.append({self._primary: key})
-                    else:
-                        items.append({self._primary: key, '__delete__': '?'})
+            parent, field = self.get_parent()
+            if parent is not None:
+                existing = {item[self._primary] for item in items if self._primary in item}
+                for item in getattr(parent, field):
+                    key = getattr(item, self._primary)
+                    if key not in existing:
+                        if add_items:
+                            items.append({self._primary: key})
+                        else:
+                            if self.context.get('update'):
+                                self.opts.sqla_session.delete(self.instance or self.get_instance({self._primary: key}))
 
         return items
 
@@ -1042,7 +1054,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
         # did we hash a new plaintext password?
         original = None
         pkey = getattr(item, self._primary)
-        for data in self.recall('original', True):
+        for data in self.recall('original'):
             if 'hash_password' in data and data.get(self._primary) == pkey:
                 original = data['password']
                 break
@@ -1238,12 +1250,6 @@ class MailuSchema(Schema, Storage):
                 if field in fieldlist:
                     fieldlist[field] = fieldlist.pop(field)
 
-    def _call_and_store(self, *args, **kwargs):
-        """ track current parent and field for pruning """
-        self.store('field', kwargs['field_name'], True)
-        self.store('parent', self.context.get('config'))
-        return super()._call_and_store(*args, **kwargs)
-
     @pre_load
     def _clear_config(self, data, many, **kwargs): # pylint: disable=unused-argument
         """ create config object in context if missing

core/admin/mailu/sso/forms.py

@@ -5,7 +5,7 @@ import flask_wtf
 class LoginForm(flask_wtf.FlaskForm):
     class Meta:
         csrf = False
-    email = fields.StringField(_('E-mail'), [validators.Email(), validators.DataRequired()])
+    email = fields.StringField(_('E-mail'), [validators.Email(), validators.DataRequired()], render_kw={'autofocus': True})
     pw = fields.PasswordField(_('Password'), [validators.DataRequired()])
     pwned = fields.HiddenField(label='', default=-1)
     submitWebmail = fields.SubmitField(_('Sign in'))

core/admin/mailu/sso/views/base.py

@@ -47,7 +47,7 @@ def login():
                 flask.flash(msg, "error")
             return response
         else:
-            utils.limiter.rate_limit_user(username, client_ip, device_cookie, device_cookie_username) if models.User.get(username) else utils.limiter.rate_limit_ip(client_ip)
+            utils.limiter.rate_limit_user(username, client_ip, device_cookie, device_cookie_username) if models.User.get(username) else utils.limiter.rate_limit_ip(client_ip, username)
             flask.current_app.logger.warn(f'Login failed for {username} from {client_ip}.')
             flask.flash('Wrong e-mail or password', 'error')
     return flask.render_template('login.html', form=form, fields=fields)
@@ -57,8 +57,10 @@ def login():
 def logout():
     flask_login.logout_user()
     flask.session.destroy()
-    return flask.redirect(flask.url_for('.login'))
-
+    response = flask.redirect(flask.url_for('.login'))
+    for cookie in ['roundcube_sessauth', 'roundcube_sessid', 'smsession']:
+        response.set_cookie(cookie, 'empty', expires=0)
+    return response
 
 @sso.route('/proxy', methods=['GET'])
 @sso.route('/proxy/<target>', methods=['GET'])
@@ -94,6 +96,8 @@ def proxy(target='webmail'):
     user.set_password(secrets.token_urlsafe())
     models.db.session.add(user)
     models.db.session.commit()
+    flask.session.regenerate()
+    flask_login.login_user(user)
     user.send_welcome()
     flask.current_app.logger.info(f'Login succeeded by proxy created user: {user} from {client_ip} through {flask.request.remote_addr}.')
     return flask.redirect(app.config['WEB_ADMIN'] if target=='admin' else app.config['WEB_WEBMAIL'])

core/admin/mailu/translations/cs/LC_MESSAGES/messages.po

@@ -0,0 +1,733 @@
+# Czech translations for Mailu.io.
+# Copyright (C) 2023 S474N
+# This file is distributed under the same license as the PROJECT project.
+# S474N <translate@s474n.com>, 2023.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PROJECT VERSION\n"
+"Report-Msgid-Bugs-To: translate@s474n.com\n"
+"POT-Creation-Date: 2022-05-22 18:47+0200\n"
+"PO-Revision-Date: 2023-02-21 16:14+0100\n"
+"Last-Translator: S474N <translate@s474n.com>\n"
+"Language-Team: Czech\n"
+"Language: cs_CZ\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n>=2 && n<=4 ? 1 : 2);\n"
+"Generated-By: Babel 2.3.4\n"
+"X-Generator: Poedit 3.2.2\n"
+
+#: mailu/sso/forms.py:8 mailu/ui/forms.py:79
+msgid "E-mail"
+msgstr "E-mail"
+
+#: mailu/sso/forms.py:9 mailu/ui/forms.py:80 mailu/ui/forms.py:93
+#: mailu/ui/forms.py:112 mailu/ui/forms.py:166
+#: mailu/ui/templates/client.html:32 mailu/ui/templates/client.html:57
+msgid "Password"
+msgstr "Heslo"
+
+#: mailu/sso/forms.py:10 mailu/sso/forms.py:11 mailu/sso/templates/login.html:4
+#: mailu/ui/templates/sidebar.html:142
+msgid "Sign in"
+msgstr "Přihlásit se"
+
+#: mailu/sso/templates/base_sso.html:8 mailu/ui/templates/base.html:8
+msgid "Admin page for"
+msgstr "Admin stránka pro"
+
+#: mailu/sso/templates/base_sso.html:19 mailu/ui/templates/base.html:19
+msgid "toggle sidebar"
+msgstr "přepnout postranní panel"
+
+#: mailu/sso/templates/base_sso.html:37 mailu/ui/templates/base.html:37
+msgid "change language"
+msgstr "změnit jazyk"
+
+#: mailu/sso/templates/sidebar_sso.html:4 mailu/ui/templates/sidebar.html:94
+msgid "Go to"
+msgstr "Jít"
+
+#: mailu/sso/templates/sidebar_sso.html:9 mailu/ui/templates/client.html:4
+#: mailu/ui/templates/sidebar.html:50 mailu/ui/templates/sidebar.html:107
+msgid "Client setup"
+msgstr "Nastavení klienta"
+
+#: mailu/sso/templates/sidebar_sso.html:16 mailu/ui/templates/sidebar.html:114
+msgid "Website"
+msgstr "Webová stránka"
+
+#: mailu/sso/templates/sidebar_sso.html:22 mailu/ui/templates/sidebar.html:120
+msgid "Help"
+msgstr "Pomoc"
+
+#: mailu/sso/templates/sidebar_sso.html:35
+#: mailu/ui/templates/domain/signup.html:4 mailu/ui/templates/sidebar.html:127
+msgid "Register a domain"
+msgstr "Registrovat doménu"
+
+#: mailu/sso/templates/sidebar_sso.html:49 mailu/ui/forms.py:95
+#: mailu/ui/templates/sidebar.html:149 mailu/ui/templates/user/signup.html:4
+#: mailu/ui/templates/user/signup_domain.html:4
+msgid "Sign up"
+msgstr "Registrovat se"
+
+#: mailu/ui/forms.py:33 mailu/ui/forms.py:36
+msgid "Invalid email address."
+msgstr "Špatná mailová adresa."
+
+#: mailu/ui/forms.py:45
+msgid "Confirm"
+msgstr "Potvrdit"
+
+#: mailu/ui/forms.py:48 mailu/ui/forms.py:58
+#: mailu/ui/templates/domain/details.html:26
+#: mailu/ui/templates/domain/list.html:19 mailu/ui/templates/relay/list.html:18
+msgid "Domain name"
+msgstr "Název domény"
+
+#: mailu/ui/forms.py:49
+msgid "Maximum user count"
+msgstr "Maximální počet uživatelů"
+
+#: mailu/ui/forms.py:50
+msgid "Maximum alias count"
+msgstr "Maximální počet aliasů"
+
+#: mailu/ui/forms.py:51
+msgid "Maximum user quota"
+msgstr "Maximální uživatelská kvóta"
+
+#: mailu/ui/forms.py:52
+msgid "Enable sign-up"
+msgstr "Povolit registraci"
+
+#: mailu/ui/forms.py:53 mailu/ui/forms.py:74 mailu/ui/forms.py:86
+#: mailu/ui/forms.py:132 mailu/ui/forms.py:144
+#: mailu/ui/templates/alias/list.html:22 mailu/ui/templates/domain/list.html:22
+#: mailu/ui/templates/relay/list.html:20 mailu/ui/templates/token/list.html:20
+#: mailu/ui/templates/user/list.html:24
+msgid "Comment"
+msgstr "Komentář"
+
+#: mailu/ui/forms.py:54 mailu/ui/forms.py:68 mailu/ui/forms.py:75
+#: mailu/ui/forms.py:88 mailu/ui/forms.py:136 mailu/ui/forms.py:145
+msgid "Save"
+msgstr "Uložit"
+
+#: mailu/ui/forms.py:59
+msgid "Initial admin"
+msgstr "Hlavní admin"
+
+#: mailu/ui/forms.py:60
+msgid "Admin password"
+msgstr "Heslo admina"
+
+#: mailu/ui/forms.py:61 mailu/ui/forms.py:81 mailu/ui/forms.py:94
+msgid "Confirm password"
+msgstr "Potvrdit heslo"
+
+#: mailu/ui/forms.py:63
+msgid "Create"
+msgstr "Vytvořit"
+
+#: mailu/ui/forms.py:67
+msgid "Alternative name"
+msgstr "Alternativní jméno"
+
+#: mailu/ui/forms.py:72
+msgid "Relayed domain name"
+msgstr "Seznam předávaných domén"
+
+#: mailu/ui/forms.py:73 mailu/ui/templates/relay/list.html:19
+msgid "Remote host"
+msgstr "Vzdálený hostitel"
+
+#: mailu/ui/forms.py:82 mailu/ui/templates/user/list.html:23
+#: mailu/ui/templates/user/signup_domain.html:16
+msgid "Quota"
+msgstr "Kvóta"
+
+#: mailu/ui/forms.py:83
+msgid "Allow IMAP access"
+msgstr "Povolit přístup IMAP"
+
+#: mailu/ui/forms.py:84
+msgid "Allow POP3 access"
+msgstr "Povolit přístup POP3"
+
+#: mailu/ui/forms.py:85 mailu/ui/forms.py:101
+#: mailu/ui/templates/user/settings.html:15
+msgid "Displayed name"
+msgstr "Zobrazené jméno"
+
+#: mailu/ui/forms.py:87
+msgid "Enabled"
+msgstr "Povoleno"
+
+#: mailu/ui/forms.py:92
+msgid "Email address"
+msgstr "Emailová adresa"
+
+#: mailu/ui/forms.py:102
+msgid "Enable spam filter"
+msgstr "Povolit filtr spamu"
+
+#: mailu/ui/forms.py:103
+msgid "Enable marking spam mails as read"
+msgstr "Povolit označování spamových e-mailů jako přečtených"
+
+#: mailu/ui/forms.py:104
+msgid "Spam filter tolerance"
+msgstr "Tolerance spamového filtru"
+
+#: mailu/ui/forms.py:105
+msgid "Enable forwarding"
+msgstr "Povolit přeposílání"
+
+#: mailu/ui/forms.py:106
+msgid "Keep a copy of the emails"
+msgstr "Zachovat kopii e-mailů"
+
+#: mailu/ui/forms.py:107 mailu/ui/forms.py:143
+#: mailu/ui/templates/alias/list.html:21
+msgid "Destination"
+msgstr "Cíl"
+
+#: mailu/ui/forms.py:108
+msgid "Save settings"
+msgstr "Uložit nastavení"
+
+#: mailu/ui/forms.py:113
+msgid "Password check"
+msgstr "Kontrola hesla"
+
+#: mailu/ui/forms.py:114 mailu/ui/templates/sidebar.html:25
+msgid "Update password"
+msgstr "Aktualizovat heslo"
+
+#: mailu/ui/forms.py:118
+msgid "Enable automatic reply"
+msgstr "Povolit automatickou odpověď"
+
+#: mailu/ui/forms.py:119
+msgid "Reply subject"
+msgstr "Předmět odpovědi"
+
+#: mailu/ui/forms.py:120
+msgid "Reply body"
+msgstr "Tělo odpovědi"
+
+#: mailu/ui/forms.py:122
+msgid "Start of vacation"
+msgstr "Začátek dovolené"
+
+#: mailu/ui/forms.py:123
+msgid "End of vacation"
+msgstr "Konec dovolené"
+
+#: mailu/ui/forms.py:124
+msgid "Update"
+msgstr "Aktualizovat"
+
+#: mailu/ui/forms.py:129
+msgid "Your token (write it down, as it will never be displayed again)"
+msgstr "Váš token (zapište si ho, protože se již nikdy nezobrazí)"
+
+#: mailu/ui/forms.py:134 mailu/ui/templates/token/list.html:21
+msgid "Authorized IP"
+msgstr "Autorizovaná IP"
+
+#: mailu/ui/forms.py:140
+msgid "Alias"
+msgstr "Alias"
+
+#: mailu/ui/forms.py:142
+msgid "Use SQL LIKE Syntax (e.g. for catch-all aliases)"
+msgstr "Použít syntaxi jako SQL (např. pro doménové koše)"
+
+#: mailu/ui/forms.py:149
+msgid "Admin email"
+msgstr "Email admina"
+
+#: mailu/ui/forms.py:150 mailu/ui/forms.py:155 mailu/ui/forms.py:168
+msgid "Submit"
+msgstr "Poslat"
+
+#: mailu/ui/forms.py:154
+msgid "Manager email"
+msgstr "E-mail manažera"
+
+#: mailu/ui/forms.py:159
+msgid "Protocol"
+msgstr "Protokol"
+
+#: mailu/ui/forms.py:162
+msgid "Hostname or IP"
+msgstr "Hostitel nebo IP"
+
+#: mailu/ui/forms.py:163 mailu/ui/templates/client.html:20
+#: mailu/ui/templates/client.html:45
+msgid "TCP port"
+msgstr "TCP port"
+
+#: mailu/ui/forms.py:164
+msgid "Enable TLS"
+msgstr "Povolit TLS"
+
+#: mailu/ui/forms.py:165 mailu/ui/templates/client.html:28
+#: mailu/ui/templates/client.html:53 mailu/ui/templates/fetch/list.html:21
+msgid "Username"
+msgstr "Uživatelské jméno"
+
+#: mailu/ui/forms.py:167
+msgid "Keep emails on the server"
+msgstr "Zachovat e-maily na serveru"
+
+#: mailu/ui/forms.py:172
+msgid "Announcement subject"
+msgstr "Předmět oznámení"
+
+#: mailu/ui/forms.py:174
+msgid "Announcement body"
+msgstr "Tělo oznámení"
+
+#: mailu/ui/forms.py:176
+msgid "Send"
+msgstr "Poslat"
+
+#: mailu/ui/templates/announcement.html:4
+msgid "Public announcement"
+msgstr "Veřejné oznámení"
+
+#: mailu/ui/templates/antispam.html:4 mailu/ui/templates/sidebar.html:80
+#: mailu/ui/templates/user/settings.html:19
+msgid "Antispam"
+msgstr "Antispam"
+
+#: mailu/ui/templates/antispam.html:8
+msgid "RSPAMD status page"
+msgstr "Stavová stránka RSPAMD"
+
+#: mailu/ui/templates/client.html:8
+msgid "configure your email client"
+msgstr "nakonfigurovat e-mailového klienta"
+
+#: mailu/ui/templates/client.html:13
+msgid "Incoming mail"
+msgstr "Příchozí mail"
+
+#: mailu/ui/templates/client.html:16 mailu/ui/templates/client.html:41
+msgid "Mail protocol"
+msgstr "Poštovní protokol"
+
+#: mailu/ui/templates/client.html:24 mailu/ui/templates/client.html:49
+msgid "Server name"
+msgstr "Název serveru"
+
+#: mailu/ui/templates/client.html:38
+msgid "Outgoing mail"
+msgstr "Odchozí pošta"
+
+#: mailu/ui/templates/confirm.html:4
+msgid "Confirm action"
+msgstr "Potvrdit akci"
+
+#: mailu/ui/templates/confirm.html:13
+#, python-format
+msgid "You are about to %(action)s. Please confirm your action."
+msgstr "Chystáte se %(action)s. Potvrďte prosím vaši akci."
+
+#: mailu/ui/templates/docker-error.html:4
+msgid "Docker error"
+msgstr "Chyba Dockeru"
+
+#: mailu/ui/templates/docker-error.html:12
+msgid "An error occurred while talking to the Docker server."
+msgstr "Při komunikaci se serverem Docker došlo k chybě."
+
+#: mailu/ui/templates/macros.html:129
+msgid "copy to clipboard"
+msgstr "zkopírovat do schránky"
+
+#: mailu/ui/templates/sidebar.html:15
+msgid "My account"
+msgstr "Můj účet"
+
+#: mailu/ui/templates/sidebar.html:19 mailu/ui/templates/user/list.html:37
+msgid "Settings"
+msgstr "Nastavení"
+
+#: mailu/ui/templates/sidebar.html:31 mailu/ui/templates/user/list.html:38
+msgid "Auto-reply"
+msgstr "Automatická odpověď"
+
+#: mailu/ui/templates/fetch/list.html:4 mailu/ui/templates/sidebar.html:37
+#: mailu/ui/templates/user/list.html:39
+msgid "Fetched accounts"
+msgstr "Fetched účty"
+
+#: mailu/ui/templates/sidebar.html:43 mailu/ui/templates/token/list.html:4
+msgid "Authentication tokens"
+msgstr "Autentizační tokeny"
+
+#: mailu/ui/templates/sidebar.html:56
+msgid "Administration"
+msgstr "Administrace"
+
+#: mailu/ui/templates/sidebar.html:62
+msgid "Announcement"
+msgstr "Oznámení"
+
+#: mailu/ui/templates/sidebar.html:68
+msgid "Administrators"
+msgstr "Administrátoři"
+
+#: mailu/ui/templates/sidebar.html:74
+msgid "Relayed domains"
+msgstr "Relayované domény"
+
+#: mailu/ui/templates/sidebar.html:88
+msgid "Mail domains"
+msgstr "Poštovní domény"
+
+#: mailu/ui/templates/sidebar.html:99
+msgid "Webmail"
+msgstr "Webmail"
+
+#: mailu/ui/templates/sidebar.html:135
+msgid "Sign out"
+msgstr "Odhlásit se"
+
+#: mailu/ui/templates/working.html:4
+msgid "We are still working on this feature!"
+msgstr "Na této funkci stále pracujeme!"
+
+#: mailu/ui/templates/admin/create.html:4
+msgid "Add a global administrator"
+msgstr "Přidat globálního administrátora"
+
+#: mailu/ui/templates/admin/list.html:4
+msgid "Global administrators"
+msgstr "Globální administrátor"
+
+#: mailu/ui/templates/admin/list.html:9
+msgid "Add administrator"
+msgstr "Přidat administrátora"
+
+#: mailu/ui/templates/admin/list.html:17 mailu/ui/templates/alias/list.html:19
+#: mailu/ui/templates/alternative/list.html:19
+#: mailu/ui/templates/domain/list.html:17 mailu/ui/templates/fetch/list.html:19
+#: mailu/ui/templates/manager/list.html:19
+#: mailu/ui/templates/relay/list.html:17 mailu/ui/templates/token/list.html:19
+#: mailu/ui/templates/user/list.html:19
+msgid "Actions"
+msgstr "Akce"
+
+#: mailu/ui/templates/admin/list.html:18 mailu/ui/templates/alias/list.html:20
+#: mailu/ui/templates/manager/list.html:20 mailu/ui/templates/user/list.html:21
+msgid "Email"
+msgstr "Email"
+
+#: mailu/ui/templates/admin/list.html:25 mailu/ui/templates/alias/list.html:32
+#: mailu/ui/templates/alternative/list.html:29
+#: mailu/ui/templates/domain/list.html:34 mailu/ui/templates/fetch/list.html:34
+#: mailu/ui/templates/manager/list.html:27
+#: mailu/ui/templates/relay/list.html:30 mailu/ui/templates/token/list.html:30
+#: mailu/ui/templates/user/list.html:34
+msgid "Delete"
+msgstr "Vymazat"
+
+#: mailu/ui/templates/alias/create.html:4
+msgid "Create alias"
+msgstr "Vytvořit alias"
+
+#: mailu/ui/templates/alias/edit.html:4
+msgid "Edit alias"
+msgstr "Upravit alias"
+
+#: mailu/ui/templates/alias/list.html:4
+msgid "Alias list"
+msgstr "Seznam aliasů"
+
+#: mailu/ui/templates/alias/list.html:12
+msgid "Add alias"
+msgstr "Přidat alias"
+
+#: mailu/ui/templates/alias/list.html:23
+#: mailu/ui/templates/alternative/list.html:21
+#: mailu/ui/templates/domain/list.html:23 mailu/ui/templates/fetch/list.html:25
+#: mailu/ui/templates/relay/list.html:21 mailu/ui/templates/token/list.html:22
+#: mailu/ui/templates/user/list.html:25
+msgid "Created"
+msgstr "Vytvořeno"
+
+#: mailu/ui/templates/alias/list.html:24
+#: mailu/ui/templates/alternative/list.html:22
+#: mailu/ui/templates/domain/list.html:24 mailu/ui/templates/fetch/list.html:26
+#: mailu/ui/templates/relay/list.html:22 mailu/ui/templates/token/list.html:23
+#: mailu/ui/templates/user/list.html:26
+msgid "Last edit"
+msgstr "Poslední úprava"
+
+#: mailu/ui/templates/alias/list.html:31 mailu/ui/templates/domain/list.html:33
+#: mailu/ui/templates/fetch/list.html:33 mailu/ui/templates/relay/list.html:29
+#: mailu/ui/templates/user/list.html:33
+msgid "Edit"
+msgstr "Upravit"
+
+#: mailu/ui/templates/alternative/create.html:4
+msgid "Create alternative domain"
+msgstr "Vytvořit alternativní doménu"
+
+#: mailu/ui/templates/alternative/list.html:4
+msgid "Alternative domain list"
+msgstr "Seznam alternativních domén"
+
+#: mailu/ui/templates/alternative/list.html:12
+msgid "Add alternative"
+msgstr "Přidat alternativu"
+
+#: mailu/ui/templates/alternative/list.html:20
+msgid "Name"
+msgstr "Jméno"
+
+#: mailu/ui/templates/domain/create.html:4
+#: mailu/ui/templates/domain/list.html:9
+msgid "New domain"
+msgstr "Nová doména"
+
+#: mailu/ui/templates/domain/details.html:4
+msgid "Domain details"
+msgstr "Podrobnosti o doméně"
+
+#: mailu/ui/templates/domain/details.html:15
+msgid "Regenerate keys"
+msgstr "Obnovit klíče"
+
+#: mailu/ui/templates/domain/details.html:17
+msgid "Generate keys"
+msgstr "Generovat klíče"
+
+#: mailu/ui/templates/domain/details.html:30
+msgid "DNS MX entry"
+msgstr "Záznam DNS MX"
+
+#: mailu/ui/templates/domain/details.html:34
+msgid "DNS SPF entries"
+msgstr "Záznamy DNS SPF"
+
+#: mailu/ui/templates/domain/details.html:40
+msgid "DKIM public key"
+msgstr "Veřejný klíč DKIM"
+
+#: mailu/ui/templates/domain/details.html:44
+msgid "DNS DKIM entry"
+msgstr "Záznam DNS DKIM"
+
+#: mailu/ui/templates/domain/details.html:48
+msgid "DNS DMARC entry"
+msgstr "Záznam DNS DMARC"
+
+#: mailu/ui/templates/domain/details.html:58
+msgid "DNS TLSA entry"
+msgstr "Záznam DNS TLSA"
+
+#: mailu/ui/templates/domain/details.html:63
+msgid "DNS client auto-configuration entries"
+msgstr "Položky automatické konfigurace klienta DNS"
+
+#: mailu/ui/templates/domain/edit.html:4
+msgid "Edit domain"
+msgstr "Upravit doménu"
+
+#: mailu/ui/templates/domain/list.html:4
+msgid "Domain list"
+msgstr "Seznam domén"
+
+#: mailu/ui/templates/domain/list.html:18
+msgid "Manage"
+msgstr "Spravovat"
+
+#: mailu/ui/templates/domain/list.html:20
+msgid "Mailbox count"
+msgstr "Počet poštovních schránek"
+
+#: mailu/ui/templates/domain/list.html:21
+msgid "Alias count"
+msgstr "Počet aliasů"
+
+#: mailu/ui/templates/domain/list.html:31
+msgid "Details"
+msgstr "Podrobnosti"
+
+#: mailu/ui/templates/domain/list.html:38
+msgid "Users"
+msgstr "Uživatelů"
+
+#: mailu/ui/templates/domain/list.html:39
+msgid "Aliases"
+msgstr "Aliasů"
+
+#: mailu/ui/templates/domain/list.html:40
+msgid "Managers"
+msgstr "Manažerů"
+
+#: mailu/ui/templates/domain/list.html:42
+msgid "Alternatives"
+msgstr "Alternativ"
+
+#: mailu/ui/templates/domain/signup.html:13
+msgid ""
+"In order to register a new domain, you must first setup the\n"
+"    domain zone so that the domain <code>MX</code> points to this server"
+msgstr ""
+"Chcete-li zaregistrovat novou doménu, musíte nejprve nastavit\n"
+"     zónu domény tak, aby doménový <code>MX</code> záznam ukazovala na tento "
+"server"
+
+#: mailu/ui/templates/domain/signup.html:18
+msgid ""
+"If you do not know how to setup an <code>MX</code> record for your DNS "
+"zone,\n"
+"    please contact your DNS provider or administrator. Also, please wait a\n"
+"    couple minutes after the <code>MX</code> is set so the local server "
+"cache\n"
+"    expires."
+msgstr ""
+"Pokud nevíte, jak nastavit <code>MX</code> záznam pro zónu DNS,\n"
+"     kontaktujte svého poskytovatele DNS nebo správce. Také prosím počkejte "
+"a\n"
+"     několik minut po <code>MX</code> tak, aby vypršela  v mezipaměti "
+"místního\n"
+"     serveru."
+
+#: mailu/ui/templates/fetch/create.html:4
+msgid "Add a fetched account"
+msgstr "Přidejte fetched účet"
+
+#: mailu/ui/templates/fetch/edit.html:4
+msgid "Update a fetched account"
+msgstr "Aktualizujte fetched účet"
+
+#: mailu/ui/templates/fetch/list.html:12
+msgid "Add an account"
+msgstr "Přidat účet"
+
+#: mailu/ui/templates/fetch/list.html:20
+msgid "Endpoint"
+msgstr "Koncový bod"
+
+#: mailu/ui/templates/fetch/list.html:22
+msgid "Keep emails"
+msgstr "Zachovat emaily"
+
+#: mailu/ui/templates/fetch/list.html:23
+msgid "Last check"
+msgstr "Poslední kontrola"
+
+#: mailu/ui/templates/fetch/list.html:24
+msgid "Status"
+msgstr "Status"
+
+#: mailu/ui/templates/fetch/list.html:38
+msgid "yes"
+msgstr "ano"
+
+#: mailu/ui/templates/fetch/list.html:38
+msgid "no"
+msgstr "ne"
+
+#: mailu/ui/templates/manager/create.html:4
+msgid "Add a manager"
+msgstr "Přidat manažera"
+
+#: mailu/ui/templates/manager/list.html:4
+msgid "Manager list"
+msgstr "Seznam manažerů"
+
+#: mailu/ui/templates/manager/list.html:12
+msgid "Add manager"
+msgstr "Přidat manažera"
+
+#: mailu/ui/templates/relay/create.html:4
+msgid "New relay domain"
+msgstr "Nová relay doména"
+
+#: mailu/ui/templates/relay/edit.html:4
+msgid "Edit relayed domain"
+msgstr "Upravit relay doménu"
+
+#: mailu/ui/templates/relay/list.html:4
+msgid "Relayed domain list"
+msgstr "Seznam relay domén"
+
+#: mailu/ui/templates/relay/list.html:9
+msgid "New relayed domain"
+msgstr "Nová relay doména"
+
+#: mailu/ui/templates/token/create.html:4
+msgid "Create an authentication token"
+msgstr "Vytvořit ověřovací token"
+
+#: mailu/ui/templates/token/list.html:12
+msgid "New token"
+msgstr "Nový token"
+
+#: mailu/ui/templates/user/create.html:4
+msgid "New user"
+msgstr "Nový uživatel"
+
+#: mailu/ui/templates/user/create.html:15
+msgid "General"
+msgstr "Všeobecné"
+
+#: mailu/ui/templates/user/create.html:23
+msgid "Features and quotas"
+msgstr "Funkce a kvóty"
+
+#: mailu/ui/templates/user/edit.html:4
+msgid "Edit user"
+msgstr "Upravit uživatele"
+
+#: mailu/ui/templates/user/list.html:4
+msgid "User list"
+msgstr "Seznam uživatelů"
+
+#: mailu/ui/templates/user/list.html:12
+msgid "Add user"
+msgstr "Přidat uživatele"
+
+#: mailu/ui/templates/user/list.html:20 mailu/ui/templates/user/settings.html:4
+msgid "User settings"
+msgstr "Uživatelské nastavení"
+
+#: mailu/ui/templates/user/list.html:22
+msgid "Features"
+msgstr "Funkce"
+
+#: mailu/ui/templates/user/password.html:4
+msgid "Password update"
+msgstr "Aktualizace hesla"
+
+#: mailu/ui/templates/user/reply.html:4
+msgid "Automatic reply"
+msgstr "Automatická odpověď"
+
+#: mailu/ui/templates/user/settings.html:27
+msgid "Auto-forward"
+msgstr "Automatické přeposlání"
+
+#: mailu/ui/templates/user/signup_domain.html:8
+msgid "pick a domain for the new account"
+msgstr "vybrat doménu pro nový účet"
+
+#: mailu/ui/templates/user/signup_domain.html:14
+msgid "Domain"
+msgstr "Doména"
+
+#: mailu/ui/templates/user/signup_domain.html:15
+msgid "Available slots"
+msgstr "Dostupných slotů"

core/admin/mailu/ui/forms.py

@@ -37,7 +37,7 @@ class MultipleEmailAddressesVerify(object):
         self.message = message
 
     def __call__(self, form, field):
-        pattern = re.compile(r'^([_a-z0-9\-]+)(\.[_a-z0-9\-]+)*@([a-z0-9\-]{1,}\.)*([a-z]{1,})(,([_a-z0-9\-]+)(\.[_a-z0-9\-]+)*@([a-z0-9\-]{1,}\.)*([a-z]{2,}))*$')
+        pattern = re.compile(r'^([_a-z0-9\-\+]+)(\.[_a-z0-9\-\+]+)*@([a-z0-9\-]{1,}\.)*([a-z]{1,})(,([_a-z0-9\-\+]+)(\.[_a-z0-9\-\+]+)*@([a-z0-9\-]{1,}\.)*([a-z]{2,}))*$')
         if not pattern.match(field.data.replace(" ", "")):
             raise validators.ValidationError(self.message)
 
@@ -47,7 +47,7 @@ class MultipleFoldersVerify(object):
         self.message = message
 
     def __call__(self, form, field):
-        pattern = re.compile(r'^\w+(\s*,\s*\w+)*$')
+        pattern = re.compile(r'^[^,]+(,[^,]+)*$')
         if not pattern.match(field.data.replace(" ", "")):
             raise validators.ValidationError(self.message)
 

core/admin/mailu/ui/templates/admin/list.html

@@ -11,7 +11,7 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
+{%- call macros.table(order='[[1,"asc"]]') %}
 <thead>
   <tr>
     <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
@@ -22,6 +22,7 @@
   {%- for admin in admins %}
   <tr>
     <td>
+      <a href="{{ url_for('.user_edit', user_email=admin.email) }}" title="{% trans %}Edit{% endtrans %}"><i class="fas fa-pencil-alt"></i></a>&nbsp;
       <a href="{{ url_for('.admin_delete', admin=admin.email) }}" title="{% trans %}Delete{% endtrans %}"><i class="fa fa-trash"></i></a>
     </td>
     <td>{{ admin }}</td>

core/admin/mailu/ui/templates/alias/list.html

@@ -13,7 +13,7 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
+{%- call macros.table(order='[[1,"asc"]]') %}
 <thead>
   <tr>
     <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>

core/admin/mailu/ui/templates/alternative/list.html

@@ -13,7 +13,7 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
+{%- call macros.table(order='[[1,"asc"]]') %}
 <thead>
   <tr>
     <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>

core/admin/mailu/ui/templates/base.html

@@ -78,6 +78,11 @@
           <i class="fa fa-code-branch" aria-hidden="true"></i><span class="sr-only">fork</span>
           on <a href="https://github.com/Mailu/Mailu">Github</a>
         </span>
+        <div class="mailu-version">
+          <span class="fa-pull-right">
+            {{ config["MAILU_VERSION"] }}
+          </span>
+        </div>
       </footer>
     </div>
     <script src="{{ url_for('static', filename='vendor.js') }}"></script>

core/admin/mailu/ui/templates/client.html

@@ -21,7 +21,7 @@
     </tr>
     <tr>
       <th>{% trans %}Server name{% endtrans %}</th>
-      <td><pre class="pre-config border bg-light">{{ config["HOSTNAMES"] }}</pre></td>
+      <td><pre class="pre-config border bg-light">{{ config["HOSTNAME"] }}</pre></td>
     </tr>
     <tr>
       <th>{% trans %}Username{% endtrans %}</th>
@@ -46,7 +46,7 @@
     </tr>
     <tr>
       <th>{% trans %}Server name{% endtrans %}</th>
-      <td><pre class="pre-config border bg-light">{{ config["HOSTNAMES"] }}</pre></td>
+      <td><pre class="pre-config border bg-light">{{ config["HOSTNAME"] }}</pre></td>
     </tr>
     <tr>
       <th>{% trans %}Username{% endtrans %}</th>

core/admin/mailu/ui/templates/domain/create.html

@@ -10,7 +10,7 @@
   {{ form.hidden_tag() }}
   {{ macros.form_field(form.name) }}
   {{ macros.form_fields((form.max_users, form.max_aliases)) }}
-  {{ macros.form_field(form.max_quota_bytes, step=50*10**6, max=50*10**9, data_infinity="true",
+  {{ macros.form_field(form.max_quota_bytes, step=50*10**6, max=50*10**9, data_infinity="true", data_unit=10**9,
       prepend='<span class="input-group-text"><span id="max_quota_bytes_value"></span>&nbsp;GB</span>') }}
   {{ macros.form_field(form.signup_enabled) }}
   {{ macros.form_field(form.comment) }}

core/admin/mailu/ui/templates/domain/list.html

@@ -19,6 +19,7 @@
     <th>{% trans %}Domain name{% endtrans %}</th>
     <th>{% trans %}Mailbox count{% endtrans %}</th>
     <th>{% trans %}Alias count{% endtrans %}</th>
+    <th>{% trans %}Quota{% endtrans %}</th>
     <th>{% trans %}Comment{% endtrans %}</th>
     <th>{% trans %}Enable sign-up{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
@@ -46,6 +47,7 @@
     <td>{{ domain.name }}</td>
     <td data-order="{{ domain.users | count }}">{{ domain.users | count }} / {{ '∞' if domain.max_users == -1 else domain.max_users }}</td>
     <td data-order="{{ domain.aliases | count }}">{{ domain.aliases | count }} / {{ '∞' if domain.max_aliases == -1 else domain.max_aliases }}</td>
+    <td data-sort="{{ domain.max_quota_bytes }}">{{ (domain.max_quota_bytes | filesizeformat) if domain.max_quota_bytes else '∞' }}</td>
     <td>{{ domain.comment or '' }}</td>
     <td data-sort="{{ domain.signup_enabled }}">{% if domain.signup_enabled %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
     <td data-order="{{ domain.created_at or '0000-00-00' }}">{{ domain.created_at | format_date }}</td>

core/admin/mailu/ui/templates/fetch/list.html

@@ -13,7 +13,7 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
+{%- call macros.table(order='[[1,"asc"]]') %}
 <thead>
   <tr>
     <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>

core/admin/mailu/ui/templates/manager/list.html

@@ -13,7 +13,7 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table(order='[[2,"asc"]]') %}
+{%- call macros.table(order='[[1,"asc"]]') %}
 <thead>
   <tr>
     <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
@@ -24,6 +24,7 @@
 {%- for manager in domain.managers %}
 <tr>
   <td>
+    <a href="{{ url_for('.user_edit', user_email=manager.email) }}" title="{% trans %}Edit{% endtrans %}"><i class="fas fa-pencil-alt"></i></a>&nbsp;
     <a href="{{ url_for('.manager_delete', domain_name=domain.name, user_email=manager.email) }}" title="{% trans %}Delete{% endtrans %}"><i class="fa fa-trash"></i></a>
   </td>
   <td>{{ manager }}</td>

core/admin/mailu/ui/templates/sidebar.html

@@ -96,9 +96,9 @@
       <li class="nav-header text-uppercase text-primary" role="none">{% trans %}Go to{% endtrans %}</li>
       {%- if config["WEBMAIL"] != "none" and current_user.is_authenticated %}
       <li class="nav-item" role="none">
-        <a href="{{ config["WEB_WEBMAIL"] }}" target="_blank" class="nav-link" role="menuitem">
+        <a href="{{ config["WEB_WEBMAIL"] }}" class="nav-link" role="menuitem">
         <i class="nav-icon far fa-envelope"></i>
-        <p>{% trans %}Webmail{% endtrans %} <i class="fas fa-external-link-alt text-xs"></i></p>
+        <p>{% trans %}Webmail{% endtrans %}</p>
         </a>
       </li>
       {%- endif %}

core/admin/mailu/ui/templates/token/list.html

@@ -13,7 +13,7 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
+{%- call macros.table(order='[[1,"asc"]]') %}
 <thead>
   <tr>
     <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>

core/admin/mailu/ui/templates/user/create.html

@@ -21,7 +21,7 @@
   {%- endcall %}
 
   {%- call macros.card(_("Features and quotas"), theme="success") %}
-  {{ macros.form_field(form.quota_bytes, step=50*10**6, max=(max_quota_bytes or domain.max_quota_bytes or 50*10**9), data_infinity="true",
+  {{ macros.form_field(form.quota_bytes, step=50*10**6, max=(max_quota_bytes or domain.max_quota_bytes or 50*10**9), data_infinity="true", data_unit=10**9,
       prepend='<span class="input-group-text"><span id="quota_bytes_value"></span>&nbsp;GB</span>') }}
   {{ macros.form_field(form.enable_imap) }}
   {{ macros.form_field(form.enable_pop) }}

core/admin/mailu/ui/templates/user/list.html

@@ -13,7 +13,7 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
+{%- call macros.table(order='[[2,"asc"]]') %}
 <thead>
   <tr>
     <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
@@ -35,7 +35,9 @@
     <td>
       <a href="{{ url_for('.user_settings', user_email=user.email) }}" title="{% trans %}Settings{% endtrans %}"><i class="fa fa-wrench"></i></a>&nbsp;
       <a href="{{ url_for('.user_reply', user_email=user.email) }}" title="{% trans %}Auto-reply{% endtrans %}"><i class="fa fa-plane"></i></a>&nbsp;
+      {%- if config["FETCHMAIL_ENABLED"] -%}
       <a href="{{ url_for('.fetch_list', user_email=user.email) }}" title="{% trans %}Fetched accounts{% endtrans %}"><i class="fa fa-download"></i></a>&nbsp;
+      {%- endif -%}
     </td>
     <td>{{ user }}</td>
     <td data-sort="{{ user.allow_spoofing*4 + user.enable_imap*2 + user.enable_pop }}">

core/admin/migrations/versions/f4f0f89e0047_.py

@@ -21,5 +21,5 @@ def upgrade():
 
 def downgrade():
     with op.batch_alter_table('fetch') as batch:
-        batch.drop_column('fetch', 'folders')
-        batch.drop_column('fetch', 'scan')
+        batch.drop_column('folders')
+        batch.drop_column('scan')

core/admin/run_dev.sh

@@ -11,6 +11,7 @@ DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}"
 [[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}"
 DEV_ADMIN="${DEV_ADMIN:-admin@example.com}"
 DEV_PASSWORD="${DEV_PASSWORD:-letmein}"
+DEV_ARGS=( "$@" )
 
 ### MAIN
 
@@ -75,19 +76,23 @@ ENV \
     DEBUG_ASSETS="/app/static" \
     DEBUG_TB_INTERCEPT_REDIRECTS=False \
     \
-    IMAP_ADDRESS="127.0.0.1" \
-    POP3_ADDRESS="127.0.0.1" \
-    AUTHSMTP_ADDRESS="127.0.0.1" \
+    ADMIN_ADDRESS="127.0.0.1" \
+    FRONT_ADDRESS="127.0.0.1" \
     SMTP_ADDRESS="127.0.0.1" \
+    IMAP_ADDRESS="127.0.0.1" \
     REDIS_ADDRESS="127.0.0.1" \
-    WEBMAIL_ADDRESS="127.0.0.1"
+    ANTIVIRUS_ADDRESS="127.0.0.1" \
+    ANTISPAM_ADDRESS="127.0.0.1" \
+    WEBMAIL_ADDRESS="127.0.0.1" \
+    WEBDAV_ADDRESS="127.0.0.1"
 
 CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask --debug run --host=0.0.0.0 --port=8080"]
 EOF
 
 # build
 chmod -R u+rwX,go+rX .
-"${docker}" build --tag "${DEV_NAME}:latest" .
+echo Running: "${docker/*\/}" build --tag "${DEV_NAME}:latest" "${DEV_ARGS[@]}" .
+"${docker}" build --tag "${DEV_NAME}:latest" "${DEV_ARGS[@]}" .
 
 # gather volumes to map into container
 volumes=()
@@ -107,6 +112,7 @@ done
 cat <<EOF
 
 =============================================================================
+
 The "${DEV_NAME}" container was built using this configuration:
 
 DEV_NAME="${DEV_NAME}"
@@ -115,19 +121,34 @@ DEV_PROFILER="${DEV_PROFILER}"
 DEV_LISTEN="${DEV_LISTEN}"
 DEV_ADMIN="${DEV_ADMIN}"
 DEV_PASSWORD="${DEV_PASSWORD}"
-=============================================================================
+DEV_ARGS=( ${DEV_ARGS[*]} )
 
 =============================================================================
-You can start the container later using this commandline:
+
+You can start the container later using this command:
 
 ${docker/*\/} run --rm -it --name "${DEV_NAME}" --publish ${DEV_LISTEN}:8080$(printf " %q" "${volumes[@]}") "${DEV_NAME}"
-=============================================================================
 
 =============================================================================
+
+Enter the running container using this command:
+${docker/*\/} exec -it "${DEV_NAME}" /bin/bash
+
+=============================================================================
+
+To update requirements-prod.txt you can build (and test) using:
+${docker/*\/} build --tag "${DEV_NAME}:latest" --build-arg MAILU_DEPS=dev .
+
+And then fetch the new dependencies with:
+${docker/*\/} exec "${DEV_NAME}" pip freeze >$(realpath "${base}")/requirements-new.txt
+
+=============================================================================
+
 The Mailu UI can be found here: http://${DEV_LISTEN}/sso/login
 EOF
 [[ -z "${DEV_DB}" ]] && echo "You can log in with user ${DEV_ADMIN} and password ${DEV_PASSWORD}"
 cat <<EOF
+
 =============================================================================
 
 Starting mailu dev environment...

core/admin/start.py

@@ -2,16 +2,15 @@
 
 import os
 import logging as log
-from pwd import getpwnam
 import sys
+from socrate import system
 
 os.system("chown mailu:mailu -R /dkim")
 os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
-mailu_id = getpwnam('mailu')
-os.setgid(mailu_id.pw_gid)
-os.setuid(mailu_id.pw_uid)
+system.drop_privs_to('mailu')
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
+system.set_env(['SECRET'])
 
 os.system("flask mailu advertise")
 os.system("flask db upgrade")
@@ -53,12 +52,21 @@ def test_DNS():
 
 test_DNS()
 
-start_command="".join([
-    "gunicorn --threads ", str(os.cpu_count()),
-    " -b :80 ",
-    "--access-logfile - " if (log.root.level<=log.INFO) else "",
-    "--error-logfile - ",
-    "--preload ",
-    "'mailu:create_app()'"])
+cmdline = [
+	"gunicorn",
+	"--threads", f"{os.cpu_count()}",
+	# If SUBNET6 is defined, gunicorn must listen on IPv6 as well as IPv4
+	"-b", f"{'[::]' if os.environ.get('SUBNET6') else ''}:80",
+    "--logger-class mailu.Logger",
+    "--worker-tmp-dir /dev/shm",
+	"--error-logfile", "-",
+	"--preload"
+]
 
-os.system(start_command)
+# logging
+if log.root.level <= log.INFO:
+	cmdline.extend(["--access-logfile", "-"])
+
+cmdline.append("'mailu:create_app()'")
+
+os.system(" ".join(cmdline))

core/base/Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile-upstream:1.4.3
 
 # base system image (intermediate)
-ARG DISTRO=alpine:3.16.3
+ARG DISTRO=alpine:3.17.2
 FROM $DISTRO as system
 
 ENV TZ=Etc/UTC LANG=C.UTF-8
@@ -13,16 +13,9 @@ RUN set -euxo pipefail \
   ; addgroup -Sg ${MAILU_GID} mailu \
   ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \
   ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \
-  ; machine="$(uname -m)" \
-  ; ! [[ "${machine}" == x86_64 ]] \
+  ; ! [[ "$(uname -m)" == x86_64 ]] \
     || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0
 
-ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so
-ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
-ENV CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
-ENV CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2"
-ENV LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now"
-
 WORKDIR /app
 
 CMD /bin/bash
@@ -32,6 +25,7 @@ CMD /bin/bash
 FROM system as build
 
 ARG MAILU_DEPS=prod
+ARG SNUFFLEUPAGUS_VERSION=0.9.0
 
 ENV VIRTUAL_ENV=/app/venv
 
@@ -44,32 +38,31 @@ RUN set -euxo pipefail \
   ; apk del -r py3-pip \
   ; rm -f /tmp/*.pem
 
-ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
-
 COPY requirements-${MAILU_DEPS}.txt ./
 COPY libs/ libs/
 
-ARG SNUFFLEUPAGUS_VERSION=0.8.3
-ENV SNUFFLEUPAGUS_URL https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v$SNUFFLEUPAGUS_VERSION.tar.gz
+ENV \
+  PATH="${VIRTUAL_ENV}/bin:${PATH}" \
+  CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \
+  CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \
+  CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" \
+  LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now" \
+  SNUFFLEUPAGUS_URL="https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v${SNUFFLEUPAGUS_VERSION}.tar.gz"
 
 RUN set -euxo pipefail \
   ;  machine="$(uname -m)" \
   ; deps="build-base gcc libffi-dev python3-dev" \
   ; [[ "${machine}" != x86_64 ]] && \
-      deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \
+      deps="${deps} cargo git libretls-dev mariadb-connector-c-dev postgresql-dev" \
   ; apk add --virtual .build-deps ${deps} \
   ; [[ "${machine}" == armv7* ]] && \
       mkdir -p /root/.cargo/registry/index && \
       git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \
   ; pip install -r requirements-${MAILU_DEPS}.txt \
-  ; curl -sL  ${SNUFFLEUPAGUS_URL} | tar xz \
-  ; cd snuffleupagus-$SNUFFLEUPAGUS_VERSION \
+  ; curl -sL ${SNUFFLEUPAGUS_URL} | tar xz \
+  ; cd snuffleupagus-${SNUFFLEUPAGUS_VERSION} \
   ; rm -rf src/tests/*php7*/ src/tests/*session*/ src/tests/broken_configuration/ src/tests/*cookie* src/tests/upload_validation/ \
   ; apk add --virtual .build-deps php81-dev php81-cgi php81-simplexml php81-xml pcre-dev build-base php81-pear php81-openssl re2c \
-  ; ln -s /usr/bin/phpize81 /usr/bin/phpize \
-  ; ln -s /usr/bin/pecl81 /usr/bin/pecl \
-  ; ln -s /usr/bin/php-config81 /usr/bin/php-config \
-  ; ln -s /usr/bin/php81 /usr/bin/php \
   ; pecl install vld-beta \
   ; make -j $(grep -c processor /proc/cpuinfo) release \
   ; cp src/.libs/snuffleupagus.so /app \
@@ -80,7 +73,19 @@ FROM system
 
 COPY --from=build /app/venv/ /app/venv/
 COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/
-RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn
+RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn 'cap_net_bind_service=+ep' /usr/bin/python3.10
 
-ENV VIRTUAL_ENV=/app/venv
-ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+ENV \
+  VIRTUAL_ENV=/app/venv \
+  PATH="/app/venv/bin:${PATH}" \
+  LD_PRELOAD="/usr/lib/libhardened_malloc.so" \
+  ADMIN_ADDRESS="admin" \
+  FRONT_ADDRESS="front" \
+  SMTP_ADDRESS="smtp" \
+  IMAP_ADDRESS="imap" \
+  OLETOOLS_ADDRESS="oletools" \
+  REDIS_ADDRESS="redis" \
+  ANTIVIRUS_ADDRESS="antivirus" \
+  ANTISPAM_ADDRESS="antispam" \
+  WEBMAIL_ADDRESS="webmail" \
+  WEBDAV_ADDRESS="webdav"

core/base/libs/socrate/socrate/system.py

@@ -1,7 +1,9 @@
+import hmac
+import logging as log
+import os
+from pwd import getpwnam
 import socket
 import tenacity
-from os import environ
-import logging as log
 
 @tenacity.retry(stop=tenacity.stop_after_attempt(100),
                 wait=tenacity.wait_random(min=2, max=5))
@@ -15,24 +17,39 @@ def resolve_hostname(hostname):
         log.warn("Unable to lookup '%s': %s",hostname,e)
         raise e
 
+def _coerce_value(value):
+    if isinstance(value, str) and value.lower() in ('true','yes'):
+        return True
+    elif isinstance(value, str) and value.lower() in ('false', 'no'):
+        return False
+    return value
 
-def resolve_address(address):
-    """ This function is identical to ``resolve_hostname`` but also supports
-    resolving an address, i.e. including a port.
-    """
-    hostname, *rest = address.rsplit(":", 1)
-    ip_address = resolve_hostname(hostname)
-    if ":" in ip_address:
-        ip_address = "[{}]".format(ip_address)
-    return ip_address + "".join(":" + port for port in rest)
+def set_env(required_secrets=[]):
+    """ This will set all the environment variables and retains only the secrets we need """
+    secret_key = os.environ.get('SECRET_KEY')
+    if not secret_key:
+        try:
+            secret_key = open(os.environ.get("SECRET_KEY_FILE"), "r").read().strip()
+        except Exception as exc:
+            log.error(f"Can't read SECRET_KEY from file: {exc}")
+            raise exc
+    clean_env()
+    # derive the keys we need
+    for secret in required_secrets:
+        os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest()
 
+    return {
+            key: _coerce_value(os.environ.get(key, value))
+            for key, value in os.environ.items()
+           }
 
-def get_host_address_from_environment(name, default):
-    """ This function looks up an envionment variable ``{{ name }}_ADDRESS``.
-    If it's defined, it is returned unmodified. If it's undefined, an environment
-    variable ``HOST_{{ name }}`` is looked up and resolved to an ip address.
-    If this is also not defined, the default is resolved to an ip address.
-    """
-    if "{}_ADDRESS".format(name) in environ:
-        return environ.get("{}_ADDRESS".format(name))
-    return resolve_address(environ.get("HOST_{}".format(name), default))
+def clean_env():
+    """ remove all secret keys """
+    [os.environ.pop(key, None) for key in os.environ.keys() if key.endswith("_KEY")]
+
+def drop_privs_to(username='mailu'):
+    pwnam = getpwnam(username)
+    os.setgroups([])
+    os.setgid(pwnam.pw_gid)
+    os.setuid(pwnam.pw_uid)
+    os.environ['HOME'] = pwnam.pw_dir

core/base/libs/socrate/test.py

@@ -78,40 +78,5 @@ class TestSystem(unittest.TestCase):
             "2001:db8::f00"
         )
 
-
-    def test_resolve_address(self):
-        self.assertEqual(
-            system.resolve_address("1.2.3.4.sslip.io:80"),
-            "1.2.3.4:80"
-        )
-        self.assertEqual(
-            system.resolve_address("2001-db8--f00.sslip.io:80"),
-            "[2001:db8::f00]:80"
-        )
-
-    def test_get_host_address_from_environment(self):
-        if "TEST_ADDRESS" in os.environ:
-            del os.environ["TEST_ADDRESS"]
-        if "HOST_TEST" in os.environ:
-            del os.environ["HOST_TEST"]
-        # if nothing is set, the default must be resolved
-        self.assertEqual(
-            system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"),
-            "1.2.3.4:80"
-        )
-        # if HOST is set, the HOST must be resolved
-        os.environ['HOST_TEST']="1.2.3.5.sslip.io:80"
-        self.assertEqual(
-            system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"),
-            "1.2.3.5:80"
-        )
-        # if ADDRESS is set, the ADDRESS must be returned unresolved
-        os.environ['TEST_ADDRESS']="1.2.3.6.sslip.io:80"
-        self.assertEqual(
-            system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"),
-            "1.2.3.6.sslip.io:80"
-        )
-
-
 if __name__ == "__main__":
     unittest.main()

core/base/requirements-build.txt

@@ -1,3 +1,3 @@
-pip==22.3
-setuptools==65.5.0
-wheel==0.37.1
+pip==22.3.1
+setuptools==65.6.3
+wheel==0.38.4

core/base/requirements-dev.txt

@@ -15,6 +15,7 @@ Flask-DebugToolbar
 Flask-Login
 flask-marshmallow
 Flask-Migrate
+Flask-RESTX
 Flask-SQLAlchemy<3
 Flask-WTF
 gunicorn
@@ -27,7 +28,6 @@ mysql-connector-python==8.0.29
 passlib
 psycopg2-binary
 Pygments
-pyOpenSSL
 PyYAML
 redis
 SQLAlchemy
@@ -46,6 +46,10 @@ watchdog
 # core/postfix
 postfix-mta-sts-resolver
 
+# core/oletools
+python-magic
+oletools
+
 # optional/fetchmail
 requests
 

core/base/requirements-prod.txt

@@ -7,16 +7,18 @@ attrs==22.1.0
 Babel==2.11.0
 bcrypt==4.0.1
 blinker==1.5
-certifi==2022.9.24
+certifi==2022.12.7
 cffi==1.15.1
 charset-normalizer==2.1.1
 click==8.1.3
+colorclass==2.2.2
 cryptography==38.0.3
 decorator==5.1.1
 defusedxml==0.7.1
 Deprecated==1.2.13
 dnspython==2.2.1
 dominate==2.7.0
+easygui==0.98.3
 email-validator==1.3.0
 Flask==2.2.2
 Flask-Babel==2.0.0
@@ -25,6 +27,7 @@ Flask-DebugToolbar==0.13.1
 Flask-Login==0.6.2
 flask-marshmallow==0.14.0
 Flask-Migrate==3.1.0
+Flask-RESTX==1.0.5
 Flask-SQLAlchemy==2.5.1
 Flask-WTF==1.0.1
 frozenlist==1.3.1
@@ -40,20 +43,25 @@ Mako==1.2.3
 MarkupSafe==2.1.1
 marshmallow==3.18.0
 marshmallow-sqlalchemy==0.28.1
+msoffcrypto-tool==5.0.0
 multidict==6.0.2
 mysql-connector-python==8.0.29
+olefile==0.46
+oletools==0.60.1
 packaging==21.3
 passlib==1.7.4
+pcodedmp==1.2.6
 podop @ file:///app/libs/podop
 postfix-mta-sts-resolver==1.1.4
-protobuf==3.20.1
+protobuf==3.20.2
 psycopg2-binary==2.9.5
 pycares==4.2.2
 pycparser==2.21
 Pygments==2.13.0
 pyOpenSSL==22.1.0
-pyparsing==3.0.9
+pyparsing==2.4.7
 python-dateutil==2.8.2
+python-magic==0.4.27
 pytz==2022.6
 PyYAML==6.0
 Radicale==3.1.8

core/dovecot/Dockerfile

@@ -16,7 +16,7 @@ COPY start.py /
 RUN echo $VERSION >/version
 
 EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp
-HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready."
+HEALTHCHECK CMD echo PING|nc -w2 localhost 5001|grep "PONG"
 
 VOLUME ["/mail"]
 

core/dovecot/conf/dovecot.conf

@@ -7,6 +7,12 @@ postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
 hostname = {{ HOSTNAMES.split(",")[0] }}
 submission_host = {{ FRONT_ADDRESS }}
 
+default_internal_user = dovecot
+default_login_user = mail
+default_internal_group = dovecot
+
+haproxy_trusted_networks = {{ SUBNET }} {{ SUBNET6 }}
+
 ###############
 # Mailboxes
 ###############
@@ -80,18 +86,20 @@ userdb {
 }
 
 service auth {
-  user = dovecot
   unix_listener auth-userdb {
   }
 }
 
 service auth-worker {
   unix_listener auth-worker {
-    user = dovecot
-    group = mail
-    mode = 0660
   }
-  user = mail
+}
+
+service health-check {
+  executable = script -p health-check.sh
+  inet_listener health-check {
+      port = 5001
+  }
 }
 
 ###############
@@ -110,15 +118,16 @@ protocol pop3 {
 service imap-login {
   inet_listener imap {
     port = 143
+    haproxy = yes
   }
 }
 
 ###############
 # Delivery
 ###############
+recipient_delimiter = {{ RECIPIENT_DELIMITER }}
 protocol lmtp {
   mail_plugins = $mail_plugins sieve
-  recipient_delimiter = {{ RECIPIENT_DELIMITER }}
 }
 
 service lmtp {
@@ -133,10 +142,16 @@ service lmtp {
 service managesieve-login {
   inet_listener sieve {
     port = 4190
+    haproxy = yes
   }
 }
 
+protocol sieve {
+  ssl = no
+}
+
 service managesieve {
+  process_limit = 1024
 }
 
 plugin {
@@ -158,6 +173,9 @@ plugin {
   # Include the recipient in vacation replies so that DKIM applies
   sieve_vacation_send_from_recipient = yes
 
+  # Use To: header from original message becaus envelope has a SRS address
+  sieve_vacation_to_header_ignore_envelope = yes
+
   # extract spam score from headers
   sieve_spamtest_status_type = strlen
   sieve_spamtest_status_header = X-Spam-Level

core/dovecot/conf/ham.script

@@ -1,9 +1,8 @@
 #!/bin/bash
-{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %}
-RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\  -f1):{{ port }}"
+RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\  -f1):11334"
 if [[ $? -ne 0 ]]
 then
-	echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2
+	echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2
 	exit 1
 fi
 

core/dovecot/conf/spam.script

@@ -1,9 +1,8 @@
 #!/bin/bash
-{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %}
-RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\  -f1):{{ port }}"
+RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\  -f1):11334"
 if [[ $? -ne 0 ]]
 then
-	echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2
+	echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2
 	exit 1
 fi
 

core/dovecot/start.py

@@ -10,9 +10,10 @@ from podop import run_server
 from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 def start_podop():
-    os.setuid(8)
+    system.drop_privs_to('mail')
     url = "http://" + os.environ["ADMIN_ADDRESS"] + "/internal/dovecot/§"
     run_server(0, "dovecot", "/tmp/podop.socket", [
 		("quota", "url", url ),
@@ -21,10 +22,6 @@ def start_podop():
     ])
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-os.environ["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334")
-
 for dovecot_file in glob.glob("/conf/*.conf"):
     conf.jinja(dovecot_file, os.environ, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
 
@@ -35,7 +32,8 @@ for script_file in glob.glob("/conf/*.script"):
     os.chmod(out_file, 0o555)
 
 # Run Podop, then postfix
-multiprocessing.Process(target=start_podop).start()
 os.system("chown mail:mail /mail")
 os.system("chown -R mail:mail /var/lib/dovecot /conf")
+
+multiprocessing.Process(target=start_podop).start()
 os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])

core/nginx/Dockerfile

@@ -17,7 +17,8 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache certbot nginx nginx-mod-mail openssl
+  ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-stream nginx-mod-mail openssl \
+  ; rm /etc/nginx/conf.d/stream.conf
 
 COPY conf/ /conf/
 COPY --from=static /static/ /static/
@@ -25,7 +26,8 @@ COPY *.py /
 
 RUN echo $VERSION >/version
 
-EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp
+EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp
+# EXPOSE 10025/tcp 10143/tcp 14190/tcp
 HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://localhost/health
 
 VOLUME ["/certs", "/overrides"]

core/nginx/conf/nginx.conf

@@ -1,9 +1,11 @@
 # Basic configuration
 user nginx;
 worker_processes auto;
+pcre_jit on;
 error_log /dev/stderr notice;
 pid /var/run/nginx.pid;
 load_module "modules/ngx_mail_module.so";
+load_module "modules/ngx_stream_module.so";
 
 events {
     worker_connections 1024;
@@ -51,13 +53,15 @@ http {
     gzip_min_length 1024;
     # TODO: figure out how to server pre-compressed assets from admin container
 
-    {% if KUBERNETES_INGRESS != 'true' and TLS_FLAVOR in [ 'letsencrypt', 'cert' ] %}
+    {% if not KUBERNETES_INGRESS and TLS_FLAVOR in [ 'letsencrypt', 'cert' ] %}
     # Enable the proxy for certbot if the flavor is letsencrypt and not on kubernetes
     #
     server {
       # Listen over HTTP
       listen 80;
+{% if SUBNET6 %}
       listen [::]:80;
+{% endif %}
       {% if TLS_FLAVOR == 'letsencrypt' %}
       location ^~ /.well-known/acme-challenge/ {
           proxy_pass http://127.0.0.1:8008;
@@ -77,30 +81,34 @@ http {
       root /static;
       # Variables for proxifying
       set $admin {{ ADMIN_ADDRESS }};
-      set $antispam {{ ANTISPAM_WEBUI_ADDRESS }};
+      set $antispam {{ ANTISPAM_ADDRESS }}:11334;
       {% if WEBMAIL_ADDRESS %}
       set $webmail {{ WEBMAIL_ADDRESS }};
       {% endif %}
       {% if WEBDAV_ADDRESS %}
-      set $webdav {{ WEBDAV_ADDRESS }};
+      set $webdav {{ WEBDAV_ADDRESS }}:5232;
       {% endif %}
       client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }};
 
       # Listen on HTTP only in kubernetes or behind reverse proxy
-      {% if KUBERNETES_INGRESS == 'true' or TLS_FLAVOR in [ 'mail-letsencrypt', 'notls', 'mail' ] %}
+      {% if KUBERNETES_INGRESS or TLS_FLAVOR in [ 'mail-letsencrypt', 'notls', 'mail' ] %}
       listen 80;
+{% if SUBNET6 %}
       listen [::]:80;
+{% endif %}
       {% endif %}
 
       # Only enable HTTPS if TLS is enabled with no error and not on kubernetes
-      {% if KUBERNETES_INGRESS != 'true' and TLS and not TLS_ERROR %}
+      {% if not KUBERNETES_INGRESS and TLS and not TLS_ERROR %}
       listen 443 ssl http2;
+{% if SUBNET6 %}
       listen [::]:443 ssl http2;
+{% endif %}
 
       include /etc/nginx/tls.conf;
       ssl_stapling on;
       ssl_stapling_verify on;
-      ssl_session_cache shared:SSLHTTP:50m;
+      ssl_session_cache shared:SSLHTTP:3m;
       add_header Strict-Transport-Security 'max-age=31536000';
 
       {% if not TLS_FLAVOR in [ 'mail', 'mail-letsencrypt' ] %}
@@ -150,7 +158,7 @@ http {
       {% endif %}
 
       # If TLS is failing, prevent access to anything except certbot
-      {% if KUBERNETES_INGRESS != 'true' and TLS_ERROR and not (TLS_FLAVOR in [ 'mail-letsencrypt', 'mail' ]) %}
+      {% if not KUBERNETES_INGRESS and TLS_ERROR and not (TLS_FLAVOR in [ 'mail-letsencrypt', 'mail' ]) %}
       location / {
         return 403;
       }
@@ -158,7 +166,7 @@ http {
       include /overrides/*.conf;
 
       # Actual logic
-      {% if ADMIN == 'true' or WEBMAIL != 'none' %}
+      {% if ADMIN or WEBMAIL != 'none' %}
       location ~ ^/(sso|static)/ {
         include /etc/nginx/proxy.conf;
         proxy_pass http://$admin;
@@ -211,7 +219,7 @@ http {
         return 302 /sso/login;
       }
       {% endif %}
-      {% if ADMIN == 'true' %}
+      {% if ADMIN %}
        location {{ WEB_ADMIN }} {
          include /etc/nginx/proxy.conf;
          proxy_pass http://$admin;
@@ -244,6 +252,13 @@ http {
       {% endif %}
       {% endif %}
 
+      {% if API %}
+      location ~ {{ WEB_API or '/api' }} {
+        include /etc/nginx/proxy.conf;
+        proxy_pass http://$admin;
+      }
+      {% endif %}
+
       location /internal {
         internal;
 
@@ -275,6 +290,25 @@ http {
     include /etc/nginx/conf.d/*.conf;
 }
 
+stream {
+    log_format main '$remote_addr [$time_local] '
+    		'$protocol $status $bytes_sent $bytes_received '
+    		'$session_time "$upstream_addr" '
+    		'"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
+    access_log /dev/stdout main;
+
+    # managesieve
+    server {
+      listen 14190;
+      resolver {{ RESOLVER }} valid=30s;
+
+      proxy_connect_timeout 1s;
+      proxy_timeout 1m;
+      proxy_protocol on;
+      proxy_pass {{ IMAP_ADDRESS }}:4190;
+    }
+}
+
 mail {
     server_name {{ HOSTNAMES.split(",")[0] }};
     auth_http http://127.0.0.1:8000/auth/email;
@@ -284,7 +318,7 @@ mail {
 
     {% if TLS and not TLS_ERROR %}
     include /etc/nginx/tls.conf;
-    ssl_session_cache shared:SSLMAIL:50m;
+    ssl_session_cache shared:SSLMAIL:3m;
     {% endif %}
 
     # Advertise real capabilities of backends (postfix/dovecot)
@@ -306,12 +340,16 @@ mail {
       protocol imap;
       smtp_auth plain;
       auth_http_header Auth-Port 10143;
+      # ensure we talk HAPROXY protocol to the backends
+      proxy_protocol on;
     }
 
     # SMTP is always enabled, to avoid losing emails when TLS is failing
     server {
       listen 25;
+{% if SUBNET6 %}
       listen [::]:25;
+{% endif %}
       {% if TLS and not TLS_ERROR %}
       {% if TLS_FLAVOR in ['letsencrypt','mail-letsencrypt'] %}
       ssl_certificate /certs/letsencrypt/live/mailu/fullchain.pem;
@@ -333,29 +371,39 @@ mail {
     {% if not TLS_ERROR %}
     server {
       listen 143;
+{% if SUBNET6 %}
       listen [::]:143;
+{% endif %}
       {% if TLS %}
       starttls only;
       {% endif %}
       protocol imap;
       imap_auth plain;
       auth_http_header Auth-Port 143;
+      # ensure we talk HAPROXY protocol to the backends
+      proxy_protocol on;
     }
 
     server {
       listen 110;
+{% if SUBNET6 %}
       listen [::]:110;
+{% endif %}
       {% if TLS %}
       starttls only;
       {% endif %}
       protocol pop3;
       pop3_auth plain;
       auth_http_header Auth-Port 110;
+      # ensure we talk HAPROXY protocol to the backends
+      proxy_protocol on;
     }
 
     server {
       listen 587;
+{% if SUBNET6 %}
       listen [::]:587;
+{% endif %}
       {% if TLS %}
       starttls only;
       {% endif %}
@@ -367,7 +415,9 @@ mail {
     {% if TLS %}
     server {
       listen 465 ssl;
+{% if SUBNET6 %}
       listen [::]:465 ssl;
+{% endif %}
       protocol smtp;
       smtp_auth plain login;
       auth_http_header Auth-Port 465;
@@ -375,18 +425,26 @@ mail {
 
     server {
       listen 993 ssl;
+{% if SUBNET6 %}
       listen [::]:993 ssl;
+{% endif %}
       protocol imap;
       imap_auth plain;
       auth_http_header Auth-Port 993;
+      # ensure we talk HAPROXY protocol to the backends
+      proxy_protocol on;
     }
 
     server {
       listen 995 ssl;
+{% if SUBNET6 %}
       listen [::]:995 ssl;
+{% endif %}
       protocol pop3;
       pop3_auth plain;
       auth_http_header Auth-Port 995;
+      # ensure we talk HAPROXY protocol to the backends
+      proxy_protocol on;
     }
     {% endif %}
     {% endif %}

core/nginx/conf/tls.conf

@@ -6,7 +6,7 @@ ssl_certificate_key {{ TLS[3] }};
 ssl_trusted_certificate /etc/ssl/certs/ca-cert-ISRG_Root_X1.pem;
 {% endif %}
 ssl_session_timeout 1d;
-ssl_session_tickets off;
+ssl_session_tickets off; # this can be removed when we have nginx v1.23.2
 ssl_dhparam /conf/dhparam.pem;
 ssl_protocols TLSv1.2 TLSv1.3;
 ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;

core/nginx/config.py

@@ -5,8 +5,7 @@ import logging as log
 import sys
 from socrate import system, conf
 
-args = os.environ.copy()
-
+args = system.set_env()
 log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING"))
 
 args['TLS_PERMISSIVE'] = str(args.get('TLS_PERMISSIVE')).lower() not in ('false', 'no')
@@ -17,16 +16,9 @@ with open("/etc/resolv.conf") as handle:
     resolver = content[content.index("nameserver") + 1]
     args["RESOLVER"] = f"[{resolver}]" if ":" in resolver else resolver
 
-args["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-args["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334")
-if args["WEBMAIL"] != "none":
-    args["WEBMAIL_ADDRESS"] = system.get_host_address_from_environment("WEBMAIL", "webmail")
-if args["WEBDAV"] != "none":
-    args["WEBDAV_ADDRESS"] = system.get_host_address_from_environment("WEBDAV", "webdav:5232")
-
 # TLS configuration
-cert_name = os.getenv("TLS_CERT_FILENAME", default="cert.pem")
-keypair_name = os.getenv("TLS_KEYPAIR_FILENAME", default="key.pem")
+cert_name = args.get("TLS_CERT_FILENAME", "cert.pem")
+keypair_name = args.get("TLS_KEYPAIR_FILENAME", "key.pem")
 args["TLS"] = {
     "cert": ("/certs/%s" % cert_name, "/certs/%s" % keypair_name),
     "letsencrypt": ("/certs/letsencrypt/live/mailu/nginx-chain.pem",
@@ -44,7 +36,7 @@ def format_for_nginx(fullchain, output):
     split = '-----END CERTIFICATE-----\n'
     with open(fullchain, 'r') as pem:
         certs = [f'{cert}{split}' for cert in pem.read().split(split) if cert]
-    if len(certs)>2 and os.getenv('LETSENCRYPT_SHORTCHAIN'):
+    if len(certs)>2 and args.get('LETSENCRYPT_SHORTCHAIN'):
         del certs[-1]
     with open(output, 'w') as pem:
         pem.write(''.join(certs))

core/oletools/Dockerfile

@@ -0,0 +1,31 @@
+# syntax=docker/dockerfile-upstream:1.4.3
+
+# oletools image
+FROM base
+
+ARG VERSION=local
+LABEL version=$VERSION
+
+RUN set -euxo pipefail \
+  ; apk add --no-cache netcat-openbsd libmagic libffi \
+  ; curl -sLo olefy.py https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py \
+  ; chmod 755 olefy.py
+
+RUN echo $VERSION >/version
+
+HEALTHCHECK --start-period=60s CMD echo PING|nc -q1 127.0.0.1 11343|grep "PONG"
+#EXPOSE 11343/tcp
+
+USER nobody:nobody
+
+ENV \
+    OLEFY_BINDADDRESS="" \
+    OLEFY_BINDPORT="11343" \
+    OLEFY_OLEVBA_PATH="/app/venv/bin/olevba" \
+    OLEFY_PYTHON_PATH="/app/venv/bin/python3" \
+    OLEFY_TMPDIR="/dev/shm/" \
+    OLEFY_MINLENGTH="300" \
+    OLEFY_DEL_TMP="1" \
+    OLEFY_DEL_TMP_FAILED="1"
+
+CMD /app/olefy.py

core/postfix/Dockerfile

@@ -14,8 +14,8 @@ COPY start.py /
 
 RUN echo $VERSION >/version
 
-EXPOSE 25/tcp 10025/tcp
-HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix"
+#EXPOSE 25/tcp 10025/tcp
+HEALTHCHECK --start-period=350s CMD /usr/sbin/postfix status
 
 VOLUME ["/queue"]
 

core/postfix/conf/main.cf

@@ -14,7 +14,7 @@ queue_directory = /queue
 message_size_limit = {{ MESSAGE_SIZE_LIMIT }}
 
 # Relayed networks
-mynetworks = 127.0.0.1/32 [::1]/128 {{ SUBNET }}  {% if RELAYNETS %}{{ RELAYNETS.split(",") | join(' ') }}{% endif %}
+mynetworks = 127.0.0.1/32 {{ SUBNET }} {% if SUBNET6 %}[::1]/128 {{ "[{}]/{}".format(*SUBNET6.translate({91: None, 93: None}).split("/")) }}{% endif %} {% if RELAYNETS %}{{ RELAYNETS.split(",") | join(" ") }}{% endif %}
 
 # Empty alias list to override the configuration variable and disable NIS
 alias_maps =
@@ -22,6 +22,9 @@ alias_maps =
 # Podop configuration
 podop = socketmap:unix:/tmp/podop.socket:
 
+postscreen_upstream_proxy_protocol = haproxy
+compatibility_level=3.6
+
 # Only accept virtual emails
 mydestination =
 
@@ -37,10 +40,6 @@ smtp_sasl_tls_security_options = noanonymous
 # Recipient delimiter for extended addresses
 recipient_delimiter = {{ RECIPIENT_DELIMITER }}
 
-# Only the front server is allowed to perform xclient
-# In kubernetes and Docker swarm, such address cannot be determined using the hostname. Allow for the whole Mailu subnet instead.
-smtpd_authorized_xclient_hosts={{ SUBNET }}
-
 ###############
 # TLS
 ###############
@@ -81,7 +80,7 @@ virtual_mailbox_maps = ${podop}mailbox
 # Mails are transported if required, then forwarded to Dovecot for delivery
 relay_domains = ${podop}transport
 transport_maps = lmdb:/etc/postfix/transport.map, ${podop}transport
-virtual_transport = lmtp:inet:{{ LMTP_ADDRESS }}
+virtual_transport = lmtp:inet:{{ IMAP_ADDRESS }}:2525
 
 # Sender and recipient canonical maps, mostly for SRS
 sender_canonical_maps = ${podop}sendermap
@@ -122,11 +121,13 @@ smtpd_relay_restrictions =
 
 unverified_recipient_reject_reason = Address lookup failure
 
+smtpd_authorized_xclient_hosts={{ SUBNET }}{% if SUBNET6 %},{{ "[{}]/{}".format(*SUBNET6.translate({91: None, 93: None}).split("/")) }}{% endif %}
+
 ###############
 # Milter
 ###############
 
-smtpd_milters = inet:{{ ANTISPAM_MILTER_ADDRESS }}
+smtpd_milters = inet:{{ ANTISPAM_ADDRESS }}:11332
 milter_protocol = 6
 milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen}
 milter_default_action = tempfail

core/postfix/conf/master.cf

@@ -2,10 +2,10 @@
 #               (yes)   (yes)   (yes)   (never) (100)
 
 # Exposed SMTP service
-smtp      inet  n       -       n       -       -       smtpd
+smtp      inet  n       -       n       -       1       smtpd
 
 # Internal SMTP service
-10025     inet  n       -       n       -       -       smtpd
+10025     inet  n       -       n       -       1       smtpd
   -o smtpd_sasl_auth_enable=yes
   -o smtpd_discard_ehlo_keywords=pipelining
   -o smtpd_client_restrictions=$check_ratelimit,reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit
@@ -44,6 +44,7 @@ verify    unix  -       -       n       -       1       verify
 flush     unix  n       -       n       1000?   0       flush
 proxymap  unix  -       -       n       -       -       proxymap
 smtp      unix  -       -       n       -       -       smtp
+smtpd     pass  -       -       n       -       -       smtpd
 relay     unix  -       -       n       -       -       smtp
 error     unix  -       -       n       -       -       error
 retry     unix  -       -       n       -       -       error
@@ -52,4 +53,3 @@ lmtp      unix  -       -       n       -       -       lmtp
 anvil     unix  -       -       n       -       1       anvil
 scache    unix  -       -       n       -       1       scache
 postlog   unix-dgram n  -       n       -       1       postlogd
-

core/postfix/conf/rsyslog.conf

@@ -1,40 +1,43 @@
-# rsyslog configuration file
-#
-# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
-# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html
-# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
-
-
-#### Global directives ####
-
-# Sets the directory that rsyslog uses for work files.
-$WorkDirectory /var/lib/rsyslog
-
-# Sets default permissions for all log files.
-$FileOwner root
-$FileGroup adm
-$FileCreateMode 0640
-$DirCreateMode 0755
-$Umask 0022
-
-# Reduce repeating messages (default off).
-$RepeatedMsgReduction on
-
-
-#### Modules ####
-
-# Provides support for local system logging (e.g. via logger command).
-module(load="imuxsock")
-
-#### Rules ####
-
-# Discard messages from local test requests
-:msg, contains, "connect from localhost[127.0.0.1]"  ~
-
-{% if POSTFIX_LOG_FILE %}
-# Log mail logs to file
-mail.*							-{{POSTFIX_LOG_FILE}}
-{% endif %}
-
-# Log mail logs to stdout
-mail.*							-/dev/stdout
+# rsyslog configuration file
+#
+# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
+# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html
+# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
+
+
+#### Global directives ####
+
+# Sets the directory that rsyslog uses for work files.
+$WorkDirectory /var/lib/rsyslog
+
+# Sets default permissions for all log files.
+$FileOwner root
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+
+# Reduce repeating messages (default off).
+$RepeatedMsgReduction on
+
+
+#### Modules ####
+
+# Provides support for local system logging (e.g. via logger command).
+module(load="imuxsock")
+
+#### Rules ####
+
+# Discard messages from local test requests
+:msg, contains, "connect from localhost[127.0.0.1]"  ~
+:msg, contains, "connect from localhost[::1]"  ~
+:msg, contains, "haproxy read: short protocol header: QUIT"  ~
+:msg, contains, "discarding EHLO keywords: PIPELINING"  ~
+
+{% if POSTFIX_LOG_FILE %}
+# Log mail logs to file
+mail.*							-{{POSTFIX_LOG_FILE}}
+{% endif %}
+
+# Log mail logs to stdout
+mail.*							-/dev/stdout

core/postfix/start.py

@@ -9,13 +9,15 @@ import sys
 import re
 
 from podop import run_server
-from pwd import getpwnam
 from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
+
+os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid")
 
 def start_podop():
-    os.setuid(getpwnam('postfix').pw_uid)
+    system.drop_privs_to('postfix')
     os.makedirs('/dev/shm/postfix',mode=0o700, exist_ok=True)
     url = "http://" + os.environ["ADMIN_ADDRESS"] + "/internal/postfix/"
     # TODO: Remove verbosity setting from Podop?
@@ -33,7 +35,7 @@ def start_podop():
 
 def start_mta_sts_daemon():
     os.chmod("/root/", 0o755) # read access to /root/.netrc required
-    os.setuid(getpwnam('postfix').pw_uid)
+    system.drop_privs_to('postfix')
     from postfix_mta_sts_resolver import daemon
     daemon.main()
 
@@ -43,10 +45,6 @@ def is_valid_postconf_line(line):
 
 # Actual startup script
 os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True'
-os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-os.environ["ANTISPAM_MILTER_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_MILTER", "antispam:11332")
-os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
 os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local")
 os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "")
 

core/rspamd/Dockerfile

@@ -15,7 +15,7 @@ COPY start.py /
 
 RUN echo $VERSION >/version
 
-EXPOSE 11332/tcp 11334/tcp 11335/tcp
+#EXPOSE 11332/tcp 11334/tcp 11335/tcp
 HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost:11334/
 
 VOLUME ["/var/lib/rspamd"]

core/rspamd/conf/antivirus.conf

@@ -3,7 +3,7 @@ clamav {
   scan_mime_parts = true;
   symbol = "CLAM_VIRUS";
   type = "clamav";
-  servers = "{{ ANTIVIRUS_ADDRESS }}";
+  servers = "{{ ANTIVIRUS_ADDRESS }}:3310";
   {% if ANTIVIRUS_ACTION|default('discard') == 'reject' %}
   action = "reject"
   {% endif %}

core/rspamd/conf/composites.conf

@@ -0,0 +1,14 @@
+{% if SCAN_MACROS == 'True' %}
+OLETOOLS_MACRO_MRAPTOR {
+    expression = "(OLETOOLS_A & OLETOOLS_W) | (OLETOOLS_A & OLETOOLS_X) | (OLETOOLS_W & OLETOOLS_X)";
+    message = "Rejected (malicious macro - mraptor)";
+    policy = "leave";
+    score = 20.0;
+}
+OLETOOLS_MACRO_SUSPICIOUS {
+    expression = "OLETOOLS_FLAG | OLETOOLS_VBASTOMP | OLETOOLS_A";
+    message = "Rejected (malicious macro)";
+    policy = "leave";
+    score = 20.0;
+}
+{% endif %}

core/rspamd/conf/external_services.conf

@@ -0,0 +1,64 @@
+{% if SCAN_MACROS == 'True' %}
+oletools {
+  # default olefy settings
+  servers = "{{ OLETOOLS_ADDRESS }}:11343"
+
+  # needs to be set explicitly for Rspamd < 1.9.5
+  scan_mime_parts = true;
+  extended = true;
+  max_size = 3145728;
+  timeout = 20.0;
+  retransmits = 1;
+
+  patterns {
+    OLETOOLS_MACRO_FOUND= '^.....M..$';
+    OLETOOLS_AUTOEXEC   = '^A....M..$';
+    OLETOOLS_FLAG       = '^.....MS.$';
+    OLETOOLS_VBASTOMP   = '^VBA Stomping$';
+# see https://github.com/decalage2/oletools/blob/master/oletools/mraptor.py
+    OLETOOLS_A   = '(?i)\b(?:Auto(?:Exec|_?Open|_?Close|Exit|New)|Document(?:_?Open|_Close|_?BeforeClose|Change|_New)|NewDocument|Workbook(?:_Open|_Activate|_Close|_BeforeClose)|\w+_(?:Painted|Painting|GotFocus|LostFocus|MouseHover|Layout|Click|Change|Resize|BeforeNavigate2|BeforeScriptExecute|DocumentComplete|DownloadBegin|DownloadComplete|FileDownload|NavigateComplete2|NavigateError|ProgressChange|PropertyChange|SetSecureLockIcon|StatusTextChange|TitleChange|MouseMove|MouseEnter|MouseLeave|OnConnecting))\b|Auto_Ope\b';
+    OLETOOLS_W   = '(?i)\b(?:FileCopy|CopyFile|Kill|CreateTextFile|VirtualAlloc|RtlMoveMemory|URLDownloadToFileA?|AltStartupPath|WriteProcessMemory|ADODB\.Stream|WriteText|SaveToFile|SaveAs|SaveAsRTF|FileSaveAs|MkDir|RmDir|SaveSetting|SetAttr)\b|(?:\bOpen\b[^\n]+\b(?:Write|Append|Binary|Output|Random)\b)';
+    OLETOOLS_X   = '(?i)\b(?:Shell|CreateObject|GetObject|SendKeys|RUN|CALL|MacScript|FollowHyperlink|CreateThread|ShellExecuteA?|ExecuteExcel4Macro|EXEC|REGISTER|SetTimer)\b|(?:\bDeclare\b[^\n]+\bLib\b)';
+  }
+
+  # mime-part regex matching in content-type or filename
+  mime_parts_filter_regex {
+    #UNKNOWN = "application\/octet-stream";
+    DOC2 = "application\/msword";
+    DOC3 = "application\/vnd\.ms-word.*";
+    XLS = "application\/vnd\.ms-excel.*";
+    PPT = "application\/vnd\.ms-powerpoint.*";
+    GENERIC = "application\/vnd\.openxmlformats-officedocument.*";
+  }
+  # mime-part filename extension matching (no regex)
+  mime_parts_filter_ext {
+    doc = "doc";
+    dot = "dot";
+    docx = "docx";
+    dotx = "dotx";
+    docm = "docm";
+    dotm = "dotm";
+    xls = "xls";
+    xlt = "xlt";
+    xla = "xla";
+    xlsx = "xlsx";
+    xltx = "xltx";
+    xlsm = "xlsm";
+    xltm = "xltm";
+    xlam = "xlam";
+    xlsb = "xlsb";
+    ppt = "ppt";
+    pot = "pot";
+    pps = "pps";
+    ppa = "ppa";
+    pptx = "pptx";
+    potx = "potx";
+    ppsx = "ppsx";
+    ppam = "ppam";
+    pptm = "pptm";
+    potm = "potm";
+    ppsm = "ppsm";
+    slk = "slk";
+  }
+}
+{% endif %}

core/rspamd/conf/external_services_group.conf

@@ -0,0 +1,40 @@
+{% if SCAN_MACROS == 'True' %}
+# local.d/external_services_group.conf
+
+description = "Oletools content rules";
+symbols = {
+  "OLETOOLS" {
+    weight = 1.0;
+    description = "OLETOOLS found a Macro";
+    one_shot = true;
+  },
+  "OLETOOLS_MACRO_FOUND" {
+    weight = 0.0;
+    one_shot = true;
+  },
+  "OLETOOLS_AUTOEXEC" {
+    weight = 0.0;
+    one_shot = true;
+  },
+  "OLETOOLS_FLAG" {
+    weight = 0.0;
+    one_shot = true;
+  },
+  "OLETOOLS_VBASTOMP" {
+    weight = 0.0;
+    one_shot = true;
+  },
+  "OLETOOLS_A" {
+    weight = 0.0;
+    one_shot = true;
+  },
+  "OLETOOLS_W" {
+    weight = 0.0;
+    one_shot = true;
+  },
+  "OLETOOLS_X" {
+    weight = 0.0;
+    one_shot = true;
+  },
+}
+{% endif %}

core/rspamd/conf/forbidden_file_extension.map

@@ -0,0 +1,68 @@
+ace
+ade
+adp
+apk
+appx
+appxbundle
+arj
+bat
+bin
+cab
+chm
+class
+cmd
+com
+cpl
+diagcab
+diagcfg
+diagpack
+dll
+ex
+ex_
+exe
+hlp
+hta
+img
+ins
+iso
+isp
+jar
+jnlp
+js
+jse
+lib
+lnk
+lzh
+mde
+msc
+msi
+msix
+msixbundle
+msp
+mst
+msu
+nsh
+ocx
+ovl
+pif
+ps1
+r01
+r14
+r18
+r25
+scr
+sct
+shb
+shs
+sys
+vb
+vbe
+vbs
+vbscript
+vdl
+vhd
+vxd
+wsc
+wsf
+wsh
+xll

core/rspamd/conf/force_actions.conf

@@ -1,17 +1,17 @@
 rules {
   ANTISPOOF_NOAUTH {
     action = "reject";
-    expression = "!MAILLIST & ((IS_LOCAL_DOMAIN_E & MISSING_FROM) | (IS_LOCAL_DOMAIN_H & (R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA)))";
+    expression = "!IS_LOCALLY_GENERATED & !MAILLIST & ((IS_LOCAL_DOMAIN_E & MISSING_FROM) | (IS_LOCAL_DOMAIN_H & (R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA)))";
     message = "Rejected (anti-spoofing: noauth). Please setup DMARC with DKIM or SPF if you want to send emails from your domain from other servers.";
   }
   ANTISPOOF_DMARC_ENFORCE_LOCAL {
     action = "reject";
-    expression = "!MAILLIST & (IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE | DMARC_NA)";
+    expression = "!IS_LOCALLY_GENERATED & !MAILLIST & (IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE | DMARC_NA)";
     message = "Rejected (anti-spoofing: DMARC compliance is enforced for local domains, regardless of the policy setting)";
   }
   ANTISPOOF_AUTH_FAILED {
     action = "reject";
-    expression = "!MAILLIST & BLACKLIST_ANTISPOOF";
+    expression = "!IS_LOCALLY_GENERATED & !MAILLIST & BLACKLIST_ANTISPOOF";
     message = "Rejected (anti-spoofing: auth-failed)";
   }
 }

core/rspamd/conf/local_subnet.map

@@ -0,0 +1,2 @@
+{{ SUBNET }}
+{{ SUBNET6 }}

core/rspamd/conf/multimap.conf

@@ -9,3 +9,21 @@ IS_LOCAL_DOMAIN_E {
   selector = "from('smtp'):domain";
   map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains";
 }
+
+IS_LOCALLY_GENERATED {
+  type = "ip"
+  map = ["/etc/rspamd/local.d/local_subnet.map"];
+}
+
+FORBIDDEN_FILE_EXTENSION {
+    type = "filename";
+    filter = "extension";
+    map = [
+    "/etc/rspamd/local.d/forbidden_file_extension.map",
+    ];
+    prefilter = true;
+    action = "reject";
+    symbol = "FORBIDDEN_FILE_EXTENSION";
+    description = "List of forbidden file extensions";
+    message = "Forbidden attachment extension";
+}

core/rspamd/conf/options.inc

@@ -1,3 +1 @@
-{% if RELAYNETS %}
-local_networks = [{{ RELAYNETS }}];
-{% endif %}
+local_networks = [{{ SUBNET }}{% if SUBNET6 %}, {{ SUBNET6 }}{% endif %}{% if RELAYNETS %}, {{ RELAYNETS }}{% endif %}];

core/rspamd/conf/worker-controller.inc

@@ -2,3 +2,6 @@ type = "controller";
 bind_socket = "*:11334";
 password = "mailu";
 secure_ip = "{{ SUBNET }}";
+{%- if SUBNET6 %}
+secure_ip = "{{ SUBNET6 }}";
+{%- endif %}

core/rspamd/start.py

@@ -6,18 +6,13 @@ import logging as log
 import requests
 import sys
 import time
-from socrate import system, conf
+from socrate import system,conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 # Actual startup script
 
-os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-
-if os.environ.get("ANTIVIRUS") == 'clamav':
-    os.environ["ANTIVIRUS_ADDRESS"] = system.get_host_address_from_environment("ANTIVIRUS", "antivirus:3310")
-
 for rspamd_file in glob.glob("/conf/*"):
     conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
 

design/mailu-directory-structure.md

@@ -88,7 +88,7 @@ If RFC issue #1222 is accepted, Dovecot will need read-only access to the certif
 
 - Path: `/mailu/data/`
 
-Database files, like SQLite or PostgreSQL files. Databases don't perform well on network filesystems as they depend heavily on file locking and full controll on the database files. Making it unfit for concurrent access from multiple hosts. This directory should always live on a local filesystem. This makes it only usable in `docker-compose` deployments. Usage of this directory should be avoided in Kubernetes and Docker Swarm deployments. Some services will need to be improved to allow for this.
+Database files, like SQLite or PostgreSQL files. Databases don't perform well on network filesystems as they depend heavily on file locking and full controll on the database files. Making it unfit for concurrent access from multiple hosts. This directory should always live on a local filesystem. This makes it only usable in `docker compose` deployments. Usage of this directory should be avoided in Kubernetes and Docker Swarm deployments. Some services will need to be improved to allow for this.
 
 #### admin data
 
@@ -138,7 +138,7 @@ In the old situation, Maildir indexes are stored on the same volume. However, th
 
 - Path: `/mailu/local` (new)
 
-Persistent storage not suitable for replication. In `docker-compose` deployments it lives inside `/mailu` and in replicated deployments it should live somewhere on the local host machine.
+Persistent storage not suitable for replication. In `docker compose` deployments it lives inside `/mailu` and in replicated deployments it should live somewhere on the local host machine.
 
 #### Mailqueue
 
@@ -185,7 +185,7 @@ The final layout of the Mailu filesystem will look like:
 Where in replicated environments:
 
 - `/mailu/config/`: should be a small, low performant and shared filesystem.
-- `/mailu/data`: should be avoided. More work will need to be done to configure external DB servers for relevant services. Ideally, this directory should only exist on docker-compose deployments.
+- `/mailu/data`: should be avoided. More work will need to be done to configure external DB servers for relevant services. Ideally, this directory should only exist on docker compose deployments.
 - `/mailu/local/`: Should exist only on local file systems of worker nodes.
 - `/mailu/mail`: A distributed filesystem with sufficient performance and storage requirements to hold and process all user mailboxes. Ideally only Maildir without indexes.
 

docs/antispam.rst

@@ -125,12 +125,12 @@ The following steps have to be taken to configure an additional symbol (rule) th
      #This file is LIVE reloaded by rspamd. Any changes are EFFECTIVE IMMEDIATELY.
      dummy.com
 
-3. Reload Rspamd by stopping the Rspamd container and starting the Rspamd container again. Example for docker-compose setup:
+3. Reload Rspamd by stopping the Rspamd container and starting the Rspamd container again. Example for docker compose setup:
 
    .. code-block:: bash
 
-     docker-compose scale antispam=0
-     docker-compose scale antispam=1
+     docker compose scale antispam=0
+     docker compose scale antispam=1
 
 4. (Optional) Check if the custom symbol is loaded. To access the Rspamd webgui, log in the Mailu administration web interface with a user that is an administrator and go to Antispam. In Rspamd webgui go to tab Symbols. Change the group drop-down box to local_bl. The following additional rule will be listed.
 
@@ -155,3 +155,31 @@ For more information on using the multimap filter see the official `multimap doc
 .. _`1438`: https://github.com/Mailu/Mailu/issues/1438
 .. _`1167`: https://github.com/Mailu/Mailu/issues/1167
 .. _`1566`: https://github.com/Mailu/Mailu/issues/1566
+
+Can I change the list of authorized file attachments?
+-----------------------------------------------------
+
+Mailu rejects emails with file attachements it deems to be "executable" or otherwise dangerous. If you would like to tweak the block list, you can do so using the following commands:
+
+   .. code-block:: bash
+
+     docker compose exec antispam cat /etc/rspamd/local.d/forbidden_file_extension.map > overrides/rspamd/forbidden_file_extension.map
+     docker compose restart antispam
+
+Now the file `overrides/rspamd/forbidden_file_extension.map` can be edited, to make changes to the forbidden file extensions list.
+For the changes to take effect, rspamd must be restarted.
+
+Mailu rejects emails with documents attached containing some macros. How can I fix it?
+--------------------------------------------------------------------------------------
+
+If configured to do so, Mailu uses a lightweight tool called `mraptor from oletools`_ to scan documents containing macros. By default only macros deemed potentially harmful are blocked, but there may be false positives. If you want to change the default behaviour, you may need to override the ``/etc/rspamd/local.d/composites.conf`` file in the antispam container. The following commands may be useful:
+
+   .. code-block:: bash
+
+     docker compose exec antispam cat /etc/rspamd/local.d/composites.conf > overrides/rspamd/composites.conf
+     docker compose restart antispam
+
+Now the file `overrides/rspamd/composites.conf` can be edited, to override the mraptor configuration in rspamd.
+For the changes to take effect, rspamd must be restarted.
+
+.. _`mraptor from oletools`: https://github.com/decalage2/oletools/wiki/mraptor

docs/api.rst

@@ -0,0 +1,33 @@
+Mailu RESTful API
+=================
+
+Mailu offers a RESTful API for changing the Mailu configuration.
+Anything that can be configured via the Mailu web administration interface,
+can also be configured via the API.
+
+The Mailu API can be configured via the setup utility (setup.mailu.io).
+It can also be manually configured via mailu.env:
+
+* ``API`` - Expose the API interface (value: true, false)
+* ``WEB_API`` - Path to the API interface
+* ``API_TOKEN`` - API token for authentication
+
+For more information refer to the detailed descriptions in the
+:ref:`configuration reference <advanced_settings>`.
+
+
+Swagger.json
+------------
+
+The swagger.json file can be retrieved via: https://myserver/api/v1/swagger.json
+(WEB_API=/api)
+The swagger.json file can be consumed in programs such as Postman for generating all API calls.
+
+
+In-built SwaggerUI
+------------------
+The Mailu API comes with an in-built SwaggerUI. It is a web client that allows
+anyone to visualize and interact with the Mailu API.
+
+Assuming ``/api`` is configured as value for ``WEB_API``, it
+is accessible via the URL: https://myserver/api/

docs/cli.rst

@@ -1,7 +1,7 @@
 Mailu command line
 ==================
 
-Managing users and aliases can be done from CLI using commands:
+Managing domains, users and aliases can be done from CLI using the commands:
 
 * alias
 * alias-delete
@@ -19,7 +19,7 @@ alias
 
 .. code-block:: bash
 
-  docker-compose exec admin flask mailu alias foo example.net "mail1@example.com,mail2@example.com"
+  docker compose exec admin flask mailu alias foo example.net "mail1@example.com,mail2@example.com"
 
 
 alias-delete
@@ -27,7 +27,7 @@ alias-delete
 
 .. code-block:: bash
 
-  docker-compose exec admin flask mailu alias-delete foo@example.net
+  docker compose exec admin flask mailu alias-delete foo@example.net
 
 
 domain
@@ -35,7 +35,7 @@ domain
 
 .. code-block:: bash
 
-  docker-compose exec admin flask mailu domain example.net
+  docker compose exec admin flask mailu domain example.net
 
 
 password
@@ -43,7 +43,7 @@ password
 
 .. code-block:: bash
 
-  docker-compose exec admin flask mailu password myuser example.net 'password123'
+  docker compose exec admin flask mailu password myuser example.net 'password123'
 
 
 user
@@ -51,7 +51,7 @@ user
 
 .. code-block:: bash
 
-  docker-compose exec admin flask mailu user myuser example.net 'password123'
+  docker compose exec admin flask mailu user myuser example.net 'password123'
 
 
 user-import
@@ -61,7 +61,7 @@ primary difference with simple `user` command is that password is being imported
 
 .. code-block:: bash
 
-  docker-compose run --rm admin flask mailu user-import myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' 'SHA512-CRYPT'
+  docker compose run --rm admin flask mailu user-import myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' 'SHA512-CRYPT'
 
 
 user-delete
@@ -73,7 +73,7 @@ Add the flag `-r` to really delete the user after you have deleted user-data man
 
 .. code-block:: bash
 
-  docker-compose exec admin flask mailu user-delete foo@example.net
+  docker compose exec admin flask mailu user-delete foo@example.net
 
 
 config-update
@@ -83,7 +83,7 @@ The sole purpose of this command is for importing users/aliases in bulk and sync
 
 .. code-block:: bash
 
-  cat mail-config.yml | docker-compose exec -T admin flask mailu config-update --delete-objects
+  cat mail-config.yml | docker compose exec -T admin flask mailu config-update --delete-objects
 
 where mail-config.yml looks like:
 
@@ -137,7 +137,7 @@ The purpose of this command is to export the complete configuration in YAML or J
 
 .. code-block:: bash
 
-  $ docker-compose exec admin flask mailu config-export --help
+  $ docker compose exec -T admin flask mailu config-export --help
 
  Usage: flask mailu config-export [OPTIONS] [FILTER]...
 
@@ -162,11 +162,11 @@ Attributes explicitly specified in filters are automatically exported: there is
 
 .. code-block:: bash
 
-  $ docker-compose exec admin flask mailu config-export --output mail-config.yml
+  $ docker compose exec admin flask mailu config-export --output mail-config.yml
 
-  $ docker-compose exec admin flask mailu config-export domain.dns_mx domain.dns_spf
+  $ docker compose exec -T admin flask mailu config-export domain.dns_mx domain.dns_spf
 
-  $ docker-compose exec admin flask mailu config-export user.spam_threshold
+  $ docker compose exec -T admin flask mailu config-export user.email user.spam_threshold
 
 config-import
 -------------
@@ -175,7 +175,7 @@ This command imports configuration data from an external YAML or JSON source.
 
 .. code-block:: bash
 
-  $ docker-compose exec admin flask mailu config-import --help
+  $ docker compose exec -T admin flask mailu config-import --help
 
  Usage: flask mailu config-import [OPTIONS] [FILENAME|-]
 
@@ -190,11 +190,11 @@ This command imports configuration data from an external YAML or JSON source.
    -n, --dry-run   Perform a trial run with no changes made.
    -?, -h, --help  Show this message and exit.
 
-The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec instead:
+To pass stdin correctly you have to use the `-T` option:
 
 .. code-block:: bash
 
-  docker exec -i $(docker-compose ps -q admin) flask mailu config-import -nv < mail-config.yml
+  docker compose exec -T admin flask mailu config-import -nv < mail-config.yml
 
 mail-config.yml contains the configuration and looks like this:
 

docs/compose/.env

@@ -138,7 +138,7 @@ WEBSITE=https://mailu.io
 # Log driver for front service. Possible values:
 # json-file (default)
 # journald (On systemd platforms, useful for Fail2Ban integration)
-# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker compose log` for front!)
 LOG_DRIVER=json-file
 
 # Docker-compose project name, this will prepended to containers names.

docs/compose/requirements.rst

@@ -91,7 +91,7 @@ The Docker website is full of `detailed instructions`_
 about setting up a proper Docker install. Default configuration should be
 suited for Mailu.
 
-Additionally, you must install ``docker-compose`` by following the instructions
+Additionally, you must install ``docker compose`` v2 by following the instructions
 from the `Docker website`_ if you plan on using the Compose flavor. Compose is a
 management tool for Docker, especially suited for multiple containers systems
 like Mailu.
@@ -105,24 +105,34 @@ Once everything is setup, you should be able to run the following commands
 .. code-block:: bash
 
   $ docker version
-  Client:
-   Version:      1.11.2
-   API version:  1.23
-   Go version:   go1.6.2
-   Git commit:   b9f10c9
-   Built:        Sun Jun  5 23:17:55 2016
-   OS/Arch:      linux/amd64
+    Client: Docker Engine - Community
+     Version:           20.10.22
+     API version:       1.41
+     Go version:        go1.18.9
+     Git commit:        3a2c30b
+     Built:             Thu Dec 15 22:27:03 2022
+     OS/Arch:           linux/arm64
+     Context:           default
+     Experimental:      true
+    
+    Server: Docker Engine - Community
+     Engine:
+      Version:          20.10.22
+      API version:      1.41 (minimum version 1.12)
+      Go version:       go1.18.9
+      Git commit:       42c8b31
+      Built:            Thu Dec 15 22:25:25 2022
+      OS/Arch:          linux/arm64
+      Experimental:     false
+     containerd:
+      Version:          1.6.14
+      GitCommit:        9ba4b250366a5ddde94bb7c9d1def331423aa323
+     runc:
+      Version:          1.1.4
+      GitCommit:        v1.1.4-0-g5fd4c4d
+     docker-init:
+      Version:          0.19.0
+      GitCommit:        de40ad0
 
-  Server:
-   Version:      1.11.1
-   API version:  1.23
-   Go version:   go1.6.2
-   Git commit:   5604cbe
-   Built:        Mon May  2 00:06:51 2016
-   OS/Arch:      linux/amd64
-
-  $ docker-compose version
-  docker-compose version 1.7.1, build 6c29830
-  docker-py version: 1.8.1
-  CPython version: 3.5.1
-  OpenSSL version: OpenSSL 1.0.2h  3 May 2016
+  $ docker compose version
+    Docker Compose version v2.14.1

docs/compose/setup.rst

@@ -95,7 +95,7 @@ You may now start Mailu. Move the to the Mailu directory and run:
 
 .. code-block:: bash
 
-  docker-compose up -d
+  docker compose up -d
 
 Finally, you need an admin user account. 
 
@@ -106,7 +106,7 @@ Else, if you don't go with the automatic way, you need to manually create the ad
 
 .. code-block:: bash
 
-  docker-compose exec admin flask mailu admin me example.net 'password'
+  docker compose exec admin flask mailu admin me example.net 'password'
 
 This will create a user named ``me@example.net`` with password ``password`` and administration privileges.
 Connect to the Web admin interface and change the password to a strong one.

docs/configuration.rst

@@ -37,14 +37,15 @@ The ``POSTMASTER`` is the local part of the postmaster email address. It is
 recommended to setup a generic value and later configure a mail alias for that
 address.
 
-The ``WILDCARD_SENDERS`` setting is a comma delimited list of user email addresses 
+The ``WILDCARD_SENDERS`` setting is a comma delimited list of user email addresses
 that are allowed to send emails from any existing address (spoofing the sender).
 
-The ``AUTH_RATELIMIT_IP`` (default: 60/hour) holds a security setting for fighting
-attackers that waste server resources by trying to guess user passwords (typically
-using a password spraying attack). The value defines the limit of authentication
-attempts that will be processed on non-existing accounts for a specific IP subnet
-(as defined in ``AUTH_RATELIMIT_IP_V4_MASK`` and ``AUTH_RATELIMIT_IP_V6_MASK`` below).
+The ``AUTH_RATELIMIT_IP`` (default: 5/hour) holds a security setting for fighting
+attackers that attempt a password spraying attack. The value defines the limit of
+authentication attempts that will be processed on **distinct** non-existing
+accounts for a specific IP subnet as defined in
+``AUTH_RATELIMIT_IP_V4_MASK`` (default: /24) and
+``AUTH_RATELIMIT_IP_V6_MASK`` (default: /48).
 
 The ``AUTH_RATELIMIT_USER`` (default: 100/day) holds a security setting for fighting
 attackers that attempt to guess a user's password (typically using a password
@@ -100,12 +101,19 @@ by setting ``INBOUND_TLS_ENFORCE`` to ``True``. Please note that this is forbidd
 internet facing hosts according to e.g. `RFC 3207`_ , because this prevents MTAs without STARTTLS
 support or e.g. mismatching TLS versions to deliver emails to Mailu.
 
+The ``SCAN_MACROS`` (default: True) setting controls whether Mailu will endavour
+to reject emails containing documents with malicious macros. Under the hood, it uses
+`mraptor from oletools`_ to determine whether a macro is malicious or not.
+
+.. _`mraptor from oletools`: https://github.com/decalage2/oletools/wiki/mraptor
+
 .. _`RFC 3207`: https://tools.ietf.org/html/rfc3207
 
 .. _fetchmail:
 
-When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is enabled in the admin interface.
-The container itself still needs to be deployed manually. ``FETCHMAIL_ENABLED`` defaults to ``True``.
+When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is enabled and
+shown in the admin interface. The container itself still needs to be deployed manually.
+``FETCHMAIL_ENABLED`` defaults to ``True``.
 
 The ``FETCHMAIL_DELAY`` is a delay (in seconds) for the fetchmail service to
 go and fetch new email if available. Do not use too short delays if you do not
@@ -134,13 +142,15 @@ Web settings
 
 - ``WEB_WEBMAIL`` contains the path to the Web email client.
 
+- ``WEB_API`` contains the path to the RESTful API.
+
 - ``WEBROOT_REDIRECT`` redirects all non-found queries to the set path.
-  An empty ``WEBROOT_REDIRECT`` value disables redirecting and enables 
+  An empty ``WEBROOT_REDIRECT`` value disables redirecting and enables
   classic behavior of a 404 result when not found.
-  Alternatively, ``WEBROOT_REDIRECT`` can be set to ``none`` if you 
+  Alternatively, ``WEBROOT_REDIRECT`` can be set to ``none`` if you
   are using an Nginx override for ``location /``.
 
-All three options need a leading slash (``/``) to work.
+All four options need a leading slash (``/``) to work.
 
   .. note:: ``WEBROOT_REDIRECT`` has to point to a valid path on the webserver.
     This means it cannot point to any services which are not enabled.
@@ -150,11 +160,11 @@ Both ``SITENAME`` and ``WEBSITE`` are customization options for the panel menu
 in the admin interface, while ``SITENAME`` is a customization option for
 every Web interface.
 
-- ``LOGO_BACKGROUND`` sets a custom background colour for the brand logo 
-  in the top left of the main admin interface.
+- ``LOGO_BACKGROUND`` sets a custom background colour for the brand logo
+  in the topleft of the main admin interface.
   For a list of colour codes refer to this page of `w3schools`_.
 
-- ``LOGO_URL`` sets a URL for a custom logo. This logo replaces the Mailu 
+- ``LOGO_URL`` sets a URL for a custom logo. This logo replaces the Mailu
   logo in the topleft of the main admin interface.
 
 .. _`w3schools`: https://www.w3schools.com/cssref/css_colors.asp
@@ -178,7 +188,7 @@ To have the account created automatically, you just need to define a few environ
   - ``ifmissing``: creates a new admin account when the admin account does not exist.
   - ``update``: creates a new admin account when it does not exist, or update the password of an existing admin account.
 
-Note: It is recommended to set ``INITIAL_ADMIN_MODE`` to either ``update`` or ``ifmissing``. Leaving it with the 
+Note: It is recommended to set ``INITIAL_ADMIN_MODE`` to either ``update`` or ``ifmissing``. Leaving it with the
 default value will cause an error when the system is restarted.
 
 An example:
@@ -192,23 +202,27 @@ An example:
 
 Depending on your particular deployment you most probably will want to change the default.
 
-.. _advanced_cfg:
+.. _advanced_settings:
 
 Advanced settings
 -----------------
 
-The ``CREDENTIAL_ROUNDS`` (default: 12) setting is the number of rounds used by the 
-password hashing scheme. The number of rounds can be reduced in case faster 
-authentication is needed or increased when additional protection is desired. 
-Keep in mind that this is a mitigation against offline attacks on password hashes, 
+The ``API_TOKEN`` (default: None) configures the authentication token.
+This token must be passed as request header to the API as authentication token.
+This is a mandatory setting for using the RESTful API.
+
+The ``CREDENTIAL_ROUNDS`` (default: 12) setting is the number of rounds used by the
+password hashing scheme. The number of rounds can be reduced in case faster
+authentication is needed or increased when additional protection is desired.
+Keep in mind that this is a mitigation against offline attacks on password hashes,
 aiming to prevent credential stuffing (due to password re-use) on other systems.
 
-The ``SESSION_COOKIE_SECURE`` (default: True) setting controls the secure flag on 
-the cookies of the administrative interface. It should only be turned off if you 
+The ``SESSION_COOKIE_SECURE`` (default: True) setting controls the secure flag on
+the cookies of the administrative interface. It should only be turned off if you
 intend to access it over plain HTTP.
 
-``SESSION_TIMEOUT`` (default: 3600) is the maximum amount of time in seconds between 
-requests before a session is invalidated. ``PERMANENT_SESSION_LIFETIME`` (default: 108000) 
+``SESSION_TIMEOUT`` (default: 3600) is the maximum amount of time in seconds between
+requests before a session is invalidated. ``PERMANENT_SESSION_LIFETIME`` (default: 108000)
 is the maximum amount of time in seconds a session can be kept alive for if it hasn't timed-out.
 
 The ``LOG_LEVEL`` setting is used by the python start-up scripts as a logging threshold.
@@ -218,8 +232,8 @@ See the `python docs`_ for more information.
 
 .. _`python docs`: https://docs.python.org/3.6/library/logging.html#logging-levels
 
-The ``LETSENCRYPT_SHORTCHAIN`` (default: False) setting controls whether we send the 
-ISRG Root X1 certificate in TLS handshakes. This is required for `android handsets older than 7.1.1` 
+The ``LETSENCRYPT_SHORTCHAIN`` (default: False) setting controls whether we send the
+ISRG Root X1 certificate in TLS handshakes. This is required for `android handsets older than 7.1.1`
 but slows down the performance of modern devices.
 
 .. _`android handsets older than 7.1.1`: https://community.letsencrypt.org/t/production-chain-changes/150739
@@ -228,11 +242,11 @@ The ``TLS_PERMISSIVE`` (default: true) setting controls whether ciphers and prot
 
 .. _reverse_proxy_headers:
 
-The ``REAL_IP_HEADER`` (default: unset) and ``REAL_IP_FROM`` (default: unset) settings 
-controls whether HTTP headers such as ``X-Forwarded-For`` or ``X-Real-IP`` should be trusted. 
-The former should be the name of the HTTP header to extract the client IP address from and the 
-later a comma separated list of IP addresses designating which proxies to trust. 
-If you are using Mailu behind a reverse proxy, you should set both. Setting the former without 
+The ``REAL_IP_HEADER`` (default: unset) and ``REAL_IP_FROM`` (default: unset) settings
+controls whether HTTP headers such as ``X-Forwarded-For`` or ``X-Real-IP`` should be trusted.
+The former should be the name of the HTTP header to extract the client IP address from and the
+later a comma separated list of IP addresses designating which proxies to trust.
+If you are using Mailu behind a reverse proxy, you should set both. Setting the former without
 the later introduces a security vulnerability allowing a potential attacker to spoof his source address.
 
 The ``TZ`` sets the timezone Mailu will use. The timezone naming convention usually uses a ``Region/City`` format. See `TZ database name`_  for a list of valid timezones This defaults to ``Etc/UTC``. Warning: if you are observing different timestamps in your log files you should change your hosts timezone to UTC instead of changing TZ to your local timezone. Using UTC allows easy log correlation with remote MTAs.
@@ -249,32 +263,22 @@ virus mails during SMTP dialogue, so the sender will receive a reject message.
 Infrastructure settings
 -----------------------
 
-Various environment variables ``HOST_*`` can be used to run Mailu containers
+Various environment variables ``*_ADDRESS`` can be used to run Mailu containers
 separately from a supported orchestrator. It is used by the various components
-to find the location of the other containers it depends on. They can contain an
-optional port number. Those variables are:
+to find the location of the other containers it depends on. Those variables are:
 
-- ``HOST_IMAP``: the container that is running the IMAP server (default: ``imap``, port 143)
-- ``HOST_LMTP``: the container that is running the LMTP server (default: ``imap:2525``)
-- ``HOST_HOSTIMAP``: the container that is running the IMAP server for the webmail (default: ``imap``, port 10143)
-- ``HOST_POP3``: the container that is running the POP3 server (default: ``imap``, port 110)
-- ``HOST_SMTP``: the container that is running the SMTP server (default: ``smtp``, port 25)
-- ``HOST_AUTHSMTP``: the container that is running the authenticated SMTP server for the webnmail (default: ``smtp``, port 10025)
-- ``HOST_ADMIN``: the container that is running the admin interface (default: ``admin``)
-- ``HOST_ANTISPAM_MILTER``: the container that is running the antispam milter service (default: ``antispam:11332``)
-- ``HOST_ANTISPAM_WEBUI``: the container that is running the antispam webui service (default: ``antispam:11334``)
-- ``HOST_ANTIVIRUS``: the container that is running the antivirus service (default: ``antivirus:3310``)
-- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
-- ``HOST_WEBDAV``: the container that is running the webdav server (default: ``webdav:5232``)
-- ``HOST_REDIS``: the container that is running the redis daemon (default: ``redis``)
-- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
+- ``ADMIN_ADDRESS``
+- ``ANTISPAM_ADDRESS``
+- ``ANTIVIRUS_ADDRESS``
+- ``FRONT_ADDRESS``
+- ``IMAP_ADDRESS``
+- ``REDIS_ADDRESS``
+- ``SMTP_ADDRESS``
+- ``WEBDAV_ADDRESS``
+- ``WEBMAIL_ADDRESS``
 
-The startup scripts will resolve ``HOST_*`` to their IP addresses and store the result in ``*_ADDRESS`` for further use.
-
-Alternatively, ``*_ADDRESS`` can directly be set. In this case, the values of ``*_ADDRESS`` is kept and not
-resolved. This can be used to rely on DNS based service discovery with changing services IP addresses.
-When using ``*_ADDRESS``, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to
-resolve the hostnames.
+These are used for DNS based service discovery with possibly changing services IP addresses.
+``*_ADDRESS`` values must be fully qualified domain names without port numbers.
 
 .. _db_settings:
 
@@ -352,15 +356,15 @@ Mail log settings
 
 By default, all services log directly to stdout/stderr. Logs can be collected by any docker log processing solution.
 
-Postfix writes the logs to a syslog server which logs to stdout. This is used to filter 
-out messages from the healthcheck. In some situations, a separate mail log is required 
-(e.g. for legal reasons). The syslog server can be configured to write log files to a volume. 
+Postfix writes the logs to a syslog server which logs to stdout. This is used to filter
+out messages from the healthcheck. In some situations, a separate mail log is required
+(e.g. for legal reasons). The syslog server can be configured to write log files to a volume.
 It can be configured with the following option:
 
 - ``POSTFIX_LOG_FILE``: The file to log the mail log to. When enabled, the syslog server will also log to stdout.
 
-When ``POSTFIX_LOG_FILE`` is enabled, the logrotate program will automatically rotate the 
-logs every week and keep 52 logs. To override the logrotate configuration, create the file logrotate.conf 
+When ``POSTFIX_LOG_FILE`` is enabled, the logrotate program will automatically rotate the
+logs every week and keep 52 logs. To override the logrotate configuration, create the file logrotate.conf
 with the desired configuration in the :ref:`Postfix overrides folder<override-label>`.
 
 

docs/contributors/environment.rst

@@ -153,25 +153,25 @@ After that you can run:
 
 .. code-block:: bash
 
-  docker-compose up -d
+  docker compose up -d
 
 If you wish to run commands inside a container, simply run (example):
 
 .. code-block:: bash
 
-  docker-compose exec admin ls -lah /
+  docker compose exec admin ls -lah /
 
 Or if you wish to start a shell for debugging:
 
 .. code-block:: bash
 
-  docker-compose exec admin sh
+  docker compose exec admin sh
 
 Finally, if you need to install packages inside the containers for debugging:
 
 .. code-block:: bash
 
-  docker-compose exec admin apk add --no-cache package-name
+  docker compose exec admin apk add --no-cache package-name
 
 Reviewing
 ---------
@@ -215,8 +215,8 @@ For example, to test PR #500 against master, reviewers can use:
 
   export DOCKER_ORG="mailuci"
   export MAILU_VERSION="pr-500"
-  docker-compose pull
-  docker-compose up -d
+  docker compose pull
+  docker compose up -d
 
 You can now test the PR. Play around. See if (external) mails work. Check for whatever functionality the PR is
 trying to fix. When happy, you can approve the PR. When running into failures, mark the review as

docs/database.rst

@@ -15,11 +15,11 @@ This means it is not possible to switch the database back-end used by roundcube
 
 To switch to a different database back-end:
 
-1. Run config-export to export the configuration. E.g.  `docker-compose exec admin flask mailu config-export --secrets --output mail-config.yml`
+1. Run config-export to export the configuration. E.g.  `docker compose exec admin flask mailu config-export --secrets --output mail-config.yml`
 2. Set up your new database server. Refer to the subsequent sections for tips for creating the database.
 3. Modify the database settings (DB_*) in mailu.env. Refer to the :ref:`configuration guide (link) <db_settings>` for the exact settings.
 4. Start your Mailu deployment.
-5. Run config-import to import the configuration. E.g. `docker exec -i $(docker-compose ps -q admin) flask mailu config-import -v < mail-config.yml`
+5. Run config-import to import the configuration. E.g. `docker exec -i $(docker compose ps -q admin) flask mailu config-import -v < mail-config.yml`
 
 Mailu has now been switched to the new database back-end. The Mailu configuration has also been migrated.
 
@@ -114,22 +114,22 @@ Prepare the environment. Mailu must not be in use. Only the database container.
 
 1. Open a terminal.
 2. `cd /mailu`
-3. `docker-compose -p mailu down`
-4. `docker-compose -p mailu up -d database`
+3. `docker compose -p mailu down`
+4. `docker compose -p mailu up -d database`
 
 Create the dump SQL file for recreating the database.
 
-1. `docker-compose -p mailu exec database /bin/bash`
+1. `docker compose -p mailu exec database /bin/bash`
 2. `pg_dump -h database -p 5432 -U mailu > /backup/backup_db.sql`
 3. Enter the password. See the value of DB_PW in mailu.env.
 4. `exit`
 5. The dump is saved to /mailu/data/psql_backup/backup_db.sql.
-6. `docker-compose -p mailu down`
+6. `docker compose -p mailu down`
 
 Prepare the new PostgreSQL deployment.
 
 1. `mkdir -p /mailu/data/external_psql/pgdata`
-2. Create the file docker-compose-postgresql.yml with the following contents:
+2. Create the file docker compose-postgresql.yml with the following contents:
 
 .. code-block:: docker
 
@@ -147,12 +147,12 @@ Prepare the new PostgreSQL deployment.
          - "/mailu/data/psql_backup:/dump"
 
 
-3. `docker-compose -f docker-compose-postgresql.yml up -d`
-4. `docker-compose -f docker-compose-postgresql.yml exec database /bin/bash`
+3. `docker compose -f docker compose-postgresql.yml up -d`
+4. `docker compose -f docker compose-postgresql.yml exec database /bin/bash`
 5. `cat /dump/backup_db.sql | psql -h localhost -p 5432 -U mailu`
 6. `exit`
-7. `docker-compose -f docker-compose-postgresql.yml down`
-8. Remove the file docker-compose-postgresql.yml.
+7. `docker compose -f docker compose-postgresql.yml down`
+8. Remove the file docker compose-postgresql.yml.
 
 The new PostgreSQL deployment has the dump loaded now. Now it is time to modify Mailu to use the official PostgreSQL docker image.
 
@@ -199,7 +199,7 @@ to
 
 Mailu is now configured to use the official PostgreSQL docker image. Bring your new deployment online
 
-1. `docker-compose -p mailu up -d`
+1. `docker compose -p mailu up -d`
 
 Optionally you can remove left-over files which were used by the old database:
 

docs/demo.rst

@@ -16,12 +16,13 @@ If you find actual bugs when using the demo server, please report these!
 Functionality
 -------------
 
-- The server is reset every day at 3am, UTC.
+- The server is reset every day at 3am, 12pm, 8pm UTC.
 - You can send mail from any client to the server.
   However, the SMTP server is made incapable of relaying the e-mail to the destination server.
   As such, the mail will never arrive. This is to prevent abuse of the server.
 - The server is capable of receiving mail for any configured domains.
 - The server exposes IMAP, POP3 and SMTP as usual for connection with mail clients such as Thunderbird.
+- The RESTful API is enabled.
 - The containers have limited (throttled) CPU, this means it can respond slow during heavy operations.
 - The containers have limited memory available and will be killed when exceeded.
   This is to prevent people from doing nasty things to the server as a whole.
@@ -35,6 +36,8 @@ Connecting to the server
  * Admin UI : https://test.mailu.io/admin/
  * Admin login : ``admin@test.mailu.io``
  * Admin password : ``letmein``
+ * RESTful API: https://test.mailu.io/api
+ * API token: ``Bearer APITokenForMailu``
 
 Adding domains
 --------------

docs/faq.rst

@@ -145,51 +145,43 @@ Your mail service will be reachable for IMAP, POP3, SMTP and Webmail at the addr
 How to make IPv6 work?
 ``````````````````````
 
-Docker currently does not expose the IPv6 ports properly, as it does not interface with ``ip6tables``.
-Lets start with quoting everything that's wrong:
+Docker IPv6 interfacing with ``ip6tables``, which is required for proper IPv6 support, is currently considered experimental.
 
-  Unfortunately, initially Docker was not created with IPv6 in mind.
-  It was added later and, while it has come a long way, is still not as usable as one would want.
-  Much discussion is still going on as to how IPv6 should be used in a containerized world;
-  See the various GitHub issues linked below:
+Although the supposed way to enable IPv6 would be to give each container a publicly routable address, docker's IPv6 support
+uses NAT to pass outside connections to the containers.
 
-  - Giving each container a publicly routable address means all ports (even unexposed / unpublished ports) are suddenly
-    reachable by everyone, if no additional filtering is done
-    (`docker/docker#21614 <https://github.com/docker/docker/issues/21614>`_)
-  - By default, each container gets a random IPv6, making it impossible to do properly do DNS;
-    the alternative is to assign a specific IPv6 address to each container,
-    still an administrative hassle (`docker/docker#13481 <https://github.com/docker/docker/issues/13481>`_)
-  - Published ports won't work on IPv6, unless you have the userland proxy enabled
-    (which, for now, is enabled by default in Docker)
-  - The userland proxy, however, seems to be on its way out
-    (`docker/docker#14856 <https://github.com/docker/docker/issues/14856>`_) and has various issues, like:
+Currently we recommend to use `docker-ipv6nat` by `Robert Klarenbeek <https://github.com/robbertkl>` instead of docker's
+experimental support.
 
-    - It can use a lot of RAM (`docker/docker#11185 <https://github.com/docker/docker/issues/11185>`_)
-    - Source IP addresses are rewritten, making it completely unusable for many purposes, e.g. mail servers
-      (`docker/docker#17666 <https://github.com/docker/docker/issues/17666>`_),
-      (`docker/libnetwork#1099 <https://github.com/docker/libnetwork/issues/1099>`_).
+Before enabling IPv6 you **MUST** disable the userland-proxy in your ``/etc/docker/daemon.json`` to not create an Open Relay!
 
-  -- `Robbert Klarenbeek <https://github.com/robbertkl>`_ (docker-ipv6nat author)
+.. code-block:: json
 
-Okay, but I still want to use IPv6! Can I just use the installers IPv6 checkbox? **NO, YOU SHOULD NOT DO THAT!** Why you ask?
-Mailu has its own trusted IPv4 network, every container inside this network can use e.g. the SMTP container without further
-authentication. If you enabled IPv6 inside the setup assistant (and fixed the ports to also be exposed on IPv6) Docker will
-still rewrite any incoming IPv6 requests to an IPv4 address, *which is located inside the trusted network*. Therefore any
-incoming connection to the SMTP container will bypass the authentication stage by the front container regardless of your
-settings and causes an Open Relay. And you really don't want this!
+  {
+      "userland-proxy": false
+  }
 
-So, how to make it work? Well, by using `docker-ipv6nat`_! This nifty container will set up ``ip6tables``,
-just as Docker would do for IPv4. We know that NAT-ing is not advised in IPv6,
-however exposing all containers to public network neither. The choice is ultimately yous.
+You can enable `docker-ipv6nat` like this:
 
-Mailu `setup utility`_ generates a safe IPv6 ULA subnet by default. So when you run the following command,
-Mailu will start to function on IPv6:
+  docker run -d --name ipv6nat --privileged --network host --restart unless-stopped -v /var/run/docker.sock:/var/run/docker.sock:ro -v /lib/modules:/lib/modules:ro robbertkl/ipv6nat
 
-.. code-block:: bash
+If you want to try docker's experimental IPv6 support, it can be enabled like this:
 
-  docker run -d --restart=always -v /var/run/docker.sock:/var/run/docker.sock:ro --privileged --net=host robbertkl/ipv6nat
+.. code-block:: json
+
+  {
+      "userland-proxy": false,
+      "ipv6": true,
+      "experimental": true,
+      "fixed-cidr-v6": "fd00:1234:abcd::/48",
+      "ip6tables": true
+  }
+
+and enabling the IPv6 checkbox in the `setup utility`_.
+
+This setup however is not officially supported, and might result in unforeseen issues.
+With bad misconfiguration you might even cause your instance to become an Open Relay, you have been warned!
 
-.. _`docker-ipv6nat`: https://github.com/robbertkl/docker-ipv6nat
 .. _`setup utility`: https://setup.mailu.io
 
 How does Mailu scale up?
@@ -449,8 +441,8 @@ down and up again. A container restart is not sufficient.
 
 .. code-block:: bash
 
-  docker-compose down && \
-  docker-compose up -d
+  docker compose down && \
+  docker compose up -d
 
 *Issue reference:* `615`_.
 
@@ -527,8 +519,8 @@ to check the logs.
 
 .. code-block:: bash
 
-  docker-compose logs front | less -R
-  docker-compose exec front less /var/log/letsencrypt/letsencrypt.log
+  docker compose logs front | less -R
+  docker compose exec front less /var/log/letsencrypt/letsencrypt.log
 
 Common problems:
 
@@ -586,20 +578,21 @@ down brute force attacks. The same applies to login attempts via the single sign
 
 We *do* provide a possibility to export the logs from the ``front`` service and ``Admin`` service to the host.
 The ``front`` container logs failed logon attempts on SMTP, IMAP and POP3.
-The ``Admin``container logs failed logon attempt on the single sign on page.
+The ``Admin`` container logs failed logon attempt on the single sign on page.
 For this you need to set ``LOG_DRIVER=journald`` or ``syslog``, depending on the log
 manager of the host. You will need to setup the proper Regex in the Fail2Ban configuration.
 Below an example how to do so.
 
 If you use a reverse proxy in front of Mailu, it is vital to set the environment variables REAL_IP_HEADER and REAL_IP_FROM.
 Without these environment variables, Mailu will not trust the remote client IP passed on by the reverse proxy and as a result your reverse proxy will be banned.
-See the :ref:`[configuration reference <reverse_proxy_headers>` for more information.
+
+See the :ref:`configuration reference <reverse_proxy_headers>` for more information.
 
 
 Assuming you have a working Fail2Ban installation on the host running your Docker containers,
 follow these steps:
 
-1. In the mailu docker-compose set the logging driver of the front container to journald; and set the tag to mailu-front
+1. In the mailu docker compose set the logging driver of the front container to journald; and set the tag to mailu-front
 
 .. code-block:: bash
 
@@ -608,32 +601,57 @@ follow these steps:
     options:
       tag: mailu-front
 
-2. Add the /etc/fail2ban/filter.d/bad-auth.conf
+2. Add the /etc/fail2ban/filter.d/bad-auth-bots.conf
 
 .. code-block:: bash
 
   # Fail2Ban configuration file
   [Definition]
-  failregex = .* client login failed: .+ client:\ <HOST>
+  failregex = ^\s?\S+ mailu\-front\[\d+\]: \S+ \S+ \[info\] \d+#\d+: \*\d+ client login failed: \"AUTH not supported\" while in http auth state, client: <HOST>, server:
   ignoreregex =
   journalmatch = CONTAINER_TAG=mailu-front
 
-3. Add the /etc/fail2ban/jail.d/bad-auth.conf
+3. Add the /etc/fail2ban/jail.d/bad-auth-bots.conf
 
 .. code-block:: bash
 
-  [bad-auth]
+  [bad-auth-bots]
   enabled = true
   backend = systemd
-  filter = bad-auth
+  filter = bad-auth-bots
   bantime = 604800
-  findtime = 300
-  maxretry = 10
-  action = docker-action
+  findtime = 600
+  maxretry = 5
+  action = docker-action-net
 
-The above will block flagged IPs for a week, you can of course change it to you needs.
+The above will block flagged IPs for a week, you can of course change it to your needs.
 
-4. In the mailu docker-compose set the logging driver of the Admin container to journald; and set the tag to mailu-admin
+4.  Add the following to /etc/fail2ban/action.d/docker-action-net.conf
+
+IMPORTANT: You have to install ipset on the host system, eg. `apt-get install ipset` on a Debian/Ubuntu system.
+
+See ipset homepage for details on ipset, https://ipset.netfilter.org/.
+
+.. code-block:: bash
+
+  [Definition]
+
+  actionstart = ipset --create f2b-bad-auth-bots nethash
+                iptables -I DOCKER-USER -m set --match-set f2b-bad-auth-bots src -p tcp -m tcp --dport 25 -j DROP
+
+  actionstop = iptables -D DOCKER-USER -m set --match-set f2b-bad-auth-bots src -p tcp -m tcp --dport 25 -j DROP
+               ipset --destroy f2b-bad-auth-bots
+
+
+  actionban = ipset add -exist f2b-bad-auth-bots <ip>/24
+
+  actionunban = ipset del -exist f2b-bad-auth-bots <ip>/24
+
+Using DOCKER-USER chain ensures that the blocked IPs are processed in the correct order with Docker. See more in: https://docs.docker.com/network/iptables/.
+
+Please note that the provided example will block the subnet from sending any email to the Mailu instance.
+
+5. In the mailu docker-compose set the logging driver of the Admin container to journald; and set the tag to mailu-admin
 
 .. code-block:: bash
 
@@ -642,70 +660,38 @@ The above will block flagged IPs for a week, you can of course change it to you
     options:
       tag: mailu-admin
 
-5. Add the /etc/fail2ban/filter.d/bad-auth-sso.conf
+6. Add the /etc/fail2ban/filter.d/bad-auth.conf
 
 .. code-block:: bash
 
   # Fail2Ban configuration file
   [Definition]
-  failregex = .* Login failed for .+ from <HOST>.
+  failregex = : Authentication attempt from <HOST> has been rate-limited\.$
   ignoreregex =
   journalmatch = CONTAINER_TAG=mailu-admin
 
-6. Add the /etc/fail2ban/jail.d/bad-auth-sso.conf
+7. Add the /etc/fail2ban/jail.d/bad-auth.conf
 
 .. code-block:: bash
 
-  [bad-auth-sso]
+  [bad-auth]
   enabled = true
   backend = systemd
-  filter = bad-auth-sso
+  filter = bad-auth
   bantime = 604800
-  findtime = 300
-  maxretry = 10
+  findtime = 900
+  maxretry = 15
   action = docker-action
 
-The above will block flagged IPs for a week, you can of course change it to you needs.
+The above will block flagged IPs for a week, you can of course change it to your needs.
 
-7. Add the /etc/fail2ban/action.d/docker-action.conf
-
-Option 1: Use plain iptables
+8.  Add the following to /etc/fail2ban/action.d/docker-action.conf
 
 .. code-block:: bash
 
   [Definition]
 
-  actionstart = iptables -N f2b-bad-auth
-                iptables -A f2b-bad-auth -j RETURN
-                iptables -I DOCKER-USER -j f2b-bad-auth
-
-  actionstop = iptables -D DOCKER-USER -j f2b-bad-auth
-               iptables -F f2b-bad-auth
-               iptables -X f2b-bad-auth
-
-  actioncheck = iptables -n -L DOCKER-USER | grep -q 'f2b-bad-auth[ \t]'
-
-  actionban = iptables -I f2b-bad-auth 1 -s <ip> -j DROP
-
-  actionunban = iptables -D f2b-bad-auth -s <ip> -j DROP
-
-Using DOCKER-USER chain ensures that the blocked IPs are processed in the correct order with Docker. See more in: https://docs.docker.com/network/iptables/
-
-Option 2:  Use ipset together with iptables
-IMPORTANT: You have to install ipset on the host system, eg. `apt-get install ipset` on a Debian/Ubuntu system.
-
-See ipset homepage for details on ipset, https://ipset.netfilter.org/.
-
-ipset and iptables provide one big advantage over just using iptables: This setup reduces the overall iptable rules.
-There is just one rule for the bad authentications and the IPs are within the ipset.
-Specially in larger setups with a high amount of brute force attacks this comes in handy.
-Using iptables with ipset might reduce the system load in such attacks significantly.
-
-.. code-block:: bash
-
-  [Definition]
-
-  actionstart = actionstart = ipset --create f2b-bad-auth iphash
+  actionstart = ipset --create f2b-bad-auth iphash
                 iptables -I DOCKER-USER -m set --match-set f2b-bad-auth src -j DROP
 
   actionstop = iptables -D DOCKER-USER -m set --match-set f2b-bad-auth src -j DROP
@@ -718,7 +704,7 @@ Using iptables with ipset might reduce the system load in such attacks significa
 
 Using DOCKER-USER chain ensures that the blocked IPs are processed in the correct order with Docker. See more in: https://docs.docker.com/network/iptables/
 
-1. Configure and restart the Fail2Ban service
+9. Configure and restart the Fail2Ban service
 
 Make sure Fail2Ban is started after the Docker service by adding a partial override which appends this to the existing configuration.
 
@@ -813,7 +799,7 @@ In many cases, Docker Compose will complain about the yaml syntax because it is
 Unless your distribution has proper up-to-date packages for Compose, we strongly advise that you install it either:
 
  - from the Docker-CE repositories along with Docker CE itself,
- - from PyPI using `pip install docker-compose` or
+ - from PyPI using `pip install docker compose` or
  - from Github by downloading it directly.
 
 Detailed instructions can be found at https://docs.docker.com/compose/install/

docs/index.rst

@@ -28,7 +28,7 @@ Main features include:
 - **Web access**, multiple Webmails and administration interface
 - **User features**, aliases, auto-reply, auto-forward, fetched accounts
 - **Admin features**, global admins, announcements, per-domain delegation, quotas
-- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, Snuffleupagus
+- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, [Snuffleupagus](https://github.com/jvoisin/snuffleupagus/), block malicious attachments
 - **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing
 - **Freedom**, all FOSS components, no tracker included
 
@@ -70,6 +70,7 @@ the version of Mailu that you are running.
     webadministration
     antispam
     cli
+    api
 
 .. toctree::
     :maxdepth: 2

docs/maintain.rst

@@ -14,9 +14,9 @@ simply pull the latest images and recreate the containers :
 
 .. code-block:: bash
 
-  docker-compose pull
-  docker-compose down
-  docker-compose up -d
+  docker compose pull
+  docker compose down
+  docker compose up -d
 
 Monitoring the mail server
 --------------------------
@@ -25,7 +25,7 @@ Logs are managed by Docker directly. You can easily read your logs using:
 
 .. code-block:: bash
 
-  docker-compose logs
+  docker compose logs
 
 Docker is able to forward logs to multiple log engines. Read the following documentation for details: https://docs.docker.com/engine/admin/logging/overview/.
 

docs/releases.rst

@@ -382,9 +382,9 @@ For this upgrade it is necessary to bring the project down and up, due to networ
 
 .. code-block:: bash
 
-  docker-compose pull
-  docker-compose down --remove-orphans
-  docker-compose up -d
+  docker compose pull
+  docker compose down --remove-orphans
+  docker compose up -d
 
 After everything runs successfully, ``/mailu/certs/dhparam.pem`` is no longer needed and can be deleted.
 It's included in the Mailu distribution by default now. Also the old ``.env`` can be deleted.
@@ -441,8 +441,8 @@ were removed (e.g. rmilter):
 
 .. code-block:: bash
 
-  docker-compose pull
-  docker-compose up -d --remove-orphans
+  docker compose pull
+  docker compose up -d --remove-orphans
 
 If you experience problems when upgrading, feel free to post issues and contact
 us on our chat channel for emergency support.

docs/requirements.txt

@@ -1,5 +1,5 @@
-recommonmark
-Sphinx
-sphinx-autobuild
-sphinx-rtd-theme
+recommonmark==0.7.1
+Sphinx==5.2.0
+sphinx-autobuild==2021.3.14
+sphinx-rtd-theme==1.0.0
 docutils==0.16

docs/reverse.rst

@@ -1,17 +1,17 @@
 Using an external reverse proxy
 ===============================
 
-One of Mailu's use cases is as part of a larger services platform, where maybe 
+One of Mailu's use cases is as part of a larger services platform, where maybe
 other Web services are available than just Mailu Webmail and Admin interfaces.
 
-In such a configuration, one would usually run a frontend reverse proxy to serve all 
-Web contents based on criteria like the requested hostname (virtual hosts) 
-and/or the requested path. 
+In such a configuration, one would usually run a frontend reverse proxy to serve all
+Web contents based on criteria like the requested hostname (virtual hosts)
+and/or the requested path.
 
-The Mailu Admin Web frontend is disabled in the default setup for security reasons, 
-it is however expected that most users will enable it at some point. Also, due 
-to the Docker Compose configuration structure, it is impossible for us to facilitate 
-disabling the Web frontend with a configuration variable. This guide was written to 
+The Mailu Admin Web frontend is disabled in the default setup for security reasons,
+it is however expected that most users will enable it at some point. Also, due
+to the Docker Compose configuration structure, it is impossible for us to facilitate
+disabling the Web frontend with a configuration variable. This guide was written to
 help users setup such an architecture.
 
 There are basically three options, from the most to the least recommended one:
@@ -22,13 +22,13 @@ There are basically three options, from the most to the least recommended one:
 
 All options will require that you modify the ``docker-compose.yml`` and ``mailu.env`` file.
 
-Mailu must also be configured with the information what header is used by the reverse proxy for passing the remote client IP. 
+Mailu must also be configured with the information what header is used by the reverse proxy for passing the remote client IP.
 This is configured in the mailu.env file. See the :ref:`configuration reference <reverse_proxy_headers>` for more information.
 
 Have Mailu Web frontend listen locally
 --------------------------------------
 
-The simplest and safest option is to modify the port forwards for Mailu Web frontend and have your own frontend point there. 
+The simplest and safest option is to modify the port forwards for Mailu Web frontend and have your own frontend point there.
 For instance, in the ``front`` section of Mailu ``docker-compose.yml``, use local ports 8080 and 8443 respectively for HTTP and HTTPS:
 
 .. code-block:: yaml
@@ -45,7 +45,7 @@ For instance, in the ``front`` section of Mailu ``docker-compose.yml``, use loca
     volumes:
       - "$ROOT/certs:/certs"
 
-Then on your own frontend, point to these local ports. In practice, you only need to point to the HTTPS port 
+Then on your own frontend, point to these local ports. In practice, you only need to point to the HTTPS port
 (as the HTTP port simply redirects there). Here is an example Nginx configuration:
 
 .. code-block:: nginx
@@ -68,19 +68,19 @@ Then on your own frontend, point to these local ports. In practice, you only nee
   #mailu.env file
   REAL_IP_HEADER=X-Real-IP
   REAL_IP_FROM=x.x.x.x,y.y.y.y.y
-  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. 
-  
-Because the admin interface is served as ``/admin``, the Webmail as ``/webmail``, the single sign on page as ``/sso``, webdav as ``/webdav``, the client-autoconfiguration and the static files endpoint as ``/static``, you may also want to use a single virtual host and serve other applications (still Nginx):
+  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu.
+
+Because the admin interface is served as ``/admin``, the RESTful API as ``/api``, the Webmail as ``/webmail``, the single sign on page as ``/sso``, webdav as ``/webdav``, the client-autoconfiguration and the static files endpoint as ``/static``, you may also want to use a single virtual host and serve other applications (still Nginx):
 
 .. code-block:: nginx
 
   server {
     # [...] here goes your standard configuration
 
-    location ~* ^/(admin|sso|static|webdav|webmail|(apple\.)?mobileconfig|(\.well\-known/autoconfig/)?mail/|Autodiscover/Autodiscover) {
+    location ~* ^/(admin|api|sso|static|webdav|webmail|(apple\.)?mobileconfig|(\.well\-known/autoconfig/)?mail/|Autodiscover/Autodiscover) {
       proxy_set_header Host $host;
       proxy_set_header X-Real-IP $remote_addr;
-      proxy_pass https://localhost:8443;     
+      proxy_pass https://localhost:8443;
     }
 
     location /main_app {
@@ -103,13 +103,13 @@ Because the admin interface is served as ``/admin``, the Webmail as ``/webmail``
 .. note:: Please don’t add a ``/`` at the end of the location pattern or all your redirects will fail with 404 because the ``/`` would be missing, and you would have to add it manually to move on
 
 .. code-block:: docker
-  
+
   #mailu.env file
   REAL_IP_HEADER=X-Real-IP
   REAL_IP_FROM=x.x.x.x,y.y.y.y.y
-  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. 
+  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu.
 
-Finally, you might want to serve the admin interface on a separate virtual host but not expose the admin container 
+Finally, you might want to serve the admin interface on a separate virtual host but not expose the admin container
 directly (have your own HTTPS virtual hosts on top of Mailu, one public for the Webmail and one internal for administration for instance).
 
 Here is an example configuration :
@@ -147,7 +147,7 @@ Here is an example configuration :
   #mailu.env file
   REAL_IP_HEADER=X-Real-IP
   REAL_IP_FROM=x.x.x.x,y.y.y.y.y
-  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. 
+  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu.
 
 Depending on how you access the front server, you might want to add a ``proxy_redirect`` directive to your ``location`` blocks:
 
@@ -166,8 +166,8 @@ Traefik as reverse proxy
 As such, many may wish to integrate Mailu into a system which already uses Traefik as its sole ingress/reverse-proxy.
 
 As the ``mailu/front`` container uses Nginx not only for ``HTTP`` forwarding, but also for the mail-protocols like ``SMTP``, ``IMAP``, etc
-, we need to keep this container around even when using another ``HTTP`` reverse-proxy. Furthermore, Traefik is neither able to 
-forward non-HTTP, nor can it easily forward HTTPS-to-HTTPS. 
+, we need to keep this container around even when using another ``HTTP`` reverse-proxy. Furthermore, Traefik is neither able to
+forward non-HTTP, nor can it easily forward HTTPS-to-HTTPS.
 
 This, however, means 3 things:
 
@@ -175,9 +175,9 @@ This, however, means 3 things:
 - ``mailu/front`` is not exposed to the outside world on ``HTTP``
 - ``mailu/front`` still needs ``SSL`` certificates (here, we assume ``letsencrypt``) for a well-behaved mail service
 
-This makes the setup with Traefik a bit harder: Traefik saves its certificates in a proprietary *JSON* file, which is not readable 
-by Nginx in the ``front``-container. To solve this, your ``acme.json`` needs to be exposed to the host or a ``docker-volume``. 
-It will then be read by a script in another container, which will dump the certificates as ``PEM`` files, readable for 
+This makes the setup with Traefik a bit harder: Traefik saves its certificates in a proprietary *JSON* file, which is not readable
+by Nginx in the ``front``-container. To solve this, your ``acme.json`` needs to be exposed to the host or a ``docker-volume``.
+It will then be read by a script in another container, which will dump the certificates as ``PEM`` files, readable for
 Nginx. The ``front`` container will automatically reload Nginx whenever these certificates change.
 
 To set this up, first set ``TLS_FLAVOR=mail`` in your ``.env``. This tells ``mailu/front`` not to try to request certificates using ``letsencrypt``,
@@ -194,20 +194,20 @@ Add the respective Traefik labels for your domain/configuration, like
 
 .. note:: Please don’t forget to add ``TRAEFIK_DOMAIN=[...]`` TO YOUR ``.env``
 
-If your Traefik is configured to automatically request certificates from *letsencrypt*, then you’ll have a certificate 
-for ``mail.your.example.com`` now. However, ``mail.your.example.com`` might only be the location where you want the Mailu web-interfaces 
+If your Traefik is configured to automatically request certificates from *letsencrypt*, then you’ll have a certificate
+for ``mail.your.example.com`` now. However, ``mail.your.example.com`` might only be the location where you want the Mailu web-interfaces
 to live — your mail should be sent/received from ``your.example.com``, and this is the ``DOMAIN`` in your ``.env``?
 To support that use-case, Traefik can request ``SANs`` for your domain. The configuration for this will depend on your Traefik version.
 
-Mailu must also be configured with the information what header is used by the reverse proxy for passing the remote 
+Mailu must also be configured with the information what header is used by the reverse proxy for passing the remote
 client IP.  This is configured in mailu.env:
 
 .. code-block:: docker
-  
+
   #mailu.env file
   REAL_IP_HEADER=X-Real-Ip
   REAL_IP_FROM=x.x.x.x,y.y.y.y.y
-  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. 
+  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu.
 
 For more information see the :ref:`configuration reference <reverse_proxy_headers>` for more information.
 
@@ -235,7 +235,7 @@ Add the appropriate labels for your domain(s) to the ``front`` container in ``do
 
 Of course, be sure to define the Certificate Resolver ``foo`` in the static configuration as well.
 
-Alternatively, you can define SANs in the Traefik static configuration using routers, or in the static configuration using entrypoints. 
+Alternatively, you can define SANs in the Traefik static configuration using routers, or in the static configuration using entrypoints.
 Refer to the Traefik documentation for more details.
 
 .. _`Traefik`: https://traefik.io/

docs/setup.rst

@@ -67,7 +67,7 @@ Make sure that you test properly before going live!
 - Try to send an email to an external service
 - On the external service, verify that DKIM and SPF are listed as passing
 - Try to receive an email from an external service
-- Check the logs (``docker-compose logs -f servicenamehere``) to look for
+- Check the logs (``docker compose logs -f servicenamehere``) to look for
   warnings or errors
 - Use an open relay checker like `mxtoolbox`_
   to ensure you're not contributing to the spam problem on the internet.

docs/webadministration.rst

@@ -162,6 +162,7 @@ You can add a fetched account by clicking on the `Add an account` button on the
 * Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled.
 
 Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``.
+Make sure ``FETCHMAIL_ENABLED`` is set to ``true`` in ``mailu.env`` to enable fetching and showing fetchmail in the admin interface.
 
 
 Authentication tokens

optional/clamav/Dockerfile

@@ -14,7 +14,7 @@ COPY start.py /
 
 RUN echo $VERSION >/version
 
-EXPOSE 3310/tcp
+#EXPOSE 3310/tcp
 HEALTHCHECK --start-period=350s CMD echo PING|nc localhost 3310|grep "PONG"
 
 VOLUME ["/data"]

optional/fetchmail/fetchmail.py

@@ -7,7 +7,6 @@ from pwd import getpwnam
 import tempfile
 import shlex
 import subprocess
-import re
 import requests
 from socrate import system
 import sys
@@ -34,11 +33,6 @@ poll "{host}" proto {protocol}  port {port}
 """
 
 
-def extract_host_port(host_and_port, default_port):
-    host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups()
-    return host, int(port) if port else default_port
-
-
 def escape_rc_string(arg):
     return "".join("\\x%2x" % ord(char) for char in arg)
 
@@ -54,20 +48,7 @@ def fetchmail(fetchmailrc):
 
 def run(debug):
     try:
-        os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
-        os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
         fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
-        smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
-        if smtpport is None:
-            smtphostport = smtphost
-        else:
-            smtphostport = "%s/%d" % (smtphost, smtpport)
-        os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
-        lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
-        if lmtpport is None:
-            lmtphostport = lmtphost
-        else:
-            lmtphostport = "%s/%d" % (lmtphost, lmtpport)
         for fetch in fetches:
             fetchmailrc = ""
             options = "options antispam 501, 504, 550, 553, 554"
@@ -79,7 +60,7 @@ def run(debug):
                 protocol=fetch["protocol"],
                 host=escape_rc_string(fetch["host"]),
                 port=fetch["port"],
-                smtphost=smtphostport if fetch['scan'] else lmtphostport,
+                smtphost=f'{os.environ["SMTP_ADDRESS"]}' if fetch['scan'] else f'{os.environ["IMAP_ADDRESS"]}/2525',
                 username=escape_rc_string(fetch["username"]),
                 password=escape_rc_string(fetch["password"]),
                 options=options,
@@ -116,16 +97,16 @@ if __name__ == "__main__":
     os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
     os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
     os.chmod("/data/fetchids", 0o700)
-    os.setgid(id_fetchmail.pw_gid)
-    os.setuid(id_fetchmail.pw_uid)
+    system.drop_privs_to('fetchmail')
+    config = system.set_env()
     while True:
-        delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
+        delay = int(os.environ.get('FETCHMAIL_DELAY', 60))
         print("Sleeping for {} seconds".format(delay))
         time.sleep(delay)
 
-        if not os.environ.get("FETCHMAIL_ENABLED", 'True') in ('True', 'true'):
+        if not config.get('FETCHMAIL_ENABLED', True):
             print("Fetchmail disabled, skipping...")
             continue
 
-        run(os.environ.get("DEBUG", None) == "True")
+        run(config.get('DEBUG', False))
         sys.stdout.flush()

optional/radicale/Dockerfile

@@ -10,7 +10,7 @@ COPY radicale.conf /
 
 RUN echo $VERSION >/version
 
-EXPOSE 5232/tcp
+#EXPOSE 5232/tcp
 HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1
 
 VOLUME ["/data"]