From 5dfccdafe9e4bc62dbcbab166a8f62b256af8af2 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 11:11:23 +0200 Subject: [PATCH 001/181] fixed some minor typos, removed unused variable --- core/admin/mailu/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index bbc00f2d..6c9a2ab1 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -65,10 +65,10 @@ class CommaSeparatedList(db.TypeDecorator): def process_bind_param(self, value, dialect): if type(value) is not list: - raise TypeError("Shoud be a list") + raise TypeError("Should be a list") for item in value: if "," in item: - raise ValueError("No item should contain a comma") + raise ValueError("Item must not contain a comma") return ",".join(value) def process_result_value(self, value, dialect): @@ -76,7 +76,7 @@ class CommaSeparatedList(db.TypeDecorator): class JSONEncoded(db.TypeDecorator): - """Represents an immutable structure as a json-encoded string. + """ Represents an immutable structure as a json-encoded string. """ impl = db.String @@ -172,7 +172,7 @@ class Domain(Base): str(rset).split()[-1][:-1] in hostnames for rset in dns.resolver.query(self.name, 'MX') ) - except Exception as e: + except Exception: return False def __str__(self): @@ -503,7 +503,7 @@ class Token(Base): class Fetch(Base): - """ A fetched account is a repote POP/IMAP account fetched into a local + """ A fetched account is a remote POP/IMAP account fetched into a local account. """ __tablename__ = "fetch" From c26ddd3c68b41b35395850a9157d61b83655ac0a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 11:19:01 +0200 Subject: [PATCH 002/181] fixed user's destination property self.forward_destination is a list (and not string) --- core/admin/mailu/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 6c9a2ab1..0a447758 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -346,10 +346,10 @@ class User(Base, Email): @property def destination(self): if self.forward_enabled: - result = self.forward_destination + result = list(self.forward_destination) if self.forward_keep: - result += ',' + self.email - return result + result.append(self.email) + return ','.join(result) else: return self.email From 5c0efe82cf96f73a8c15e130051f0ccbe91a4c5a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 11:27:38 +0200 Subject: [PATCH 003/181] implemented config_update and config_dump enhanced data model with to_dict and from_dict methods added config_dump function to manage command config_update now uses new data model methods --- core/admin/mailu/manage.py | 263 ++++++++++++++++---------------- core/admin/mailu/models.py | 301 ++++++++++++++++++++++++++++++++++++- 2 files changed, 431 insertions(+), 133 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 62f214d3..bf0148df 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -8,6 +8,8 @@ import os import socket import uuid import click +import yaml +import sys db = models.db @@ -169,146 +171,147 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() +yaml_sections = [ + ('domains', models.Domain), + ('relays', models.Relay), + ('users', models.User), + ('aliases', models.Alias), +# ('config', models.Config), +] + @mailu.command() -@click.option('-v', '--verbose') -@click.option('-d', '--delete-objects') +@click.option('-v', '--verbose', is_flag=True) +@click.option('-d', '--delete-objects', is_flag=True) +@click.option('-n', '--dry-run', is_flag=True) @flask_cli.with_appcontext -def config_update(verbose=False, delete_objects=False): +def config_update(verbose=False, delete_objects=False, dry_run=False, file=None): """sync configuration with data from YAML-formatted stdin""" - import yaml - import sys - new_config = yaml.safe_load(sys.stdin) - # print new_config - domains = new_config.get('domains', []) - tracked_domains = set() - for domain_config in domains: - if verbose: - print(str(domain_config)) - domain_name = domain_config['name'] - max_users = domain_config.get('max_users', -1) - max_aliases = domain_config.get('max_aliases', -1) - max_quota_bytes = domain_config.get('max_quota_bytes', 0) - tracked_domains.add(domain_name) - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name, - max_users=max_users, - max_aliases=max_aliases, - max_quota_bytes=max_quota_bytes) - db.session.add(domain) - print("Added " + str(domain_config)) - else: - domain.max_users = max_users - domain.max_aliases = max_aliases - domain.max_quota_bytes = max_quota_bytes - db.session.add(domain) - print("Updated " + str(domain_config)) - users = new_config.get('users', []) - tracked_users = set() - user_optional_params = ('comment', 'quota_bytes', 'global_admin', - 'enable_imap', 'enable_pop', 'forward_enabled', - 'forward_destination', 'reply_enabled', - 'reply_subject', 'reply_body', 'displayed_name', - 'spam_enabled', 'email', 'spam_threshold') - for user_config in users: - if verbose: - print(str(user_config)) - localpart = user_config['localpart'] - domain_name = user_config['domain'] - password_hash = user_config.get('password_hash', None) - hash_scheme = user_config.get('hash_scheme', None) - domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) - optional_params = {} - for k in user_optional_params: - if k in user_config: - optional_params[k] = user_config[k] - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User.query.get(email) - tracked_users.add(email) - tracked_domains.add(domain_name) - if not user: - user = models.User( - localpart=localpart, - domain=domain, - **optional_params - ) - else: - for k in optional_params: - setattr(user, k, optional_params[k]) - user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) - db.session.add(user) + out = (lambda *args: print('(DRY RUN)', *args)) if dry_run else print - aliases = new_config.get('aliases', []) - tracked_aliases = set() - for alias_config in aliases: - if verbose: - print(str(alias_config)) - localpart = alias_config['localpart'] - domain_name = alias_config['domain'] - if type(alias_config['destination']) is str: - destination = alias_config['destination'].split(',') - else: - destination = alias_config['destination'] - wildcard = alias_config.get('wildcard', False) - domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - alias = models.Alias.query.get(email) - tracked_aliases.add(email) - tracked_domains.add(domain_name) - if not alias: - alias = models.Alias( - localpart=localpart, - domain=domain, - wildcard=wildcard, - destination=destination, - email=email - ) - else: - alias.destination = destination - alias.wildcard = wildcard - db.session.add(alias) + try: + new_config = yaml.safe_load(sys.stdin) + except (yaml.scanner.ScannerError, yaml.parser.ParserError) as reason: + out(f'[ERROR] Invalid yaml: {reason}') + sys.exit(1) + else: + if type(new_config) is str: + out(f'[ERROR] Invalid yaml: {new_config!r}') + sys.exit(1) + elif new_config is None or not len(new_config): + out('[ERROR] Empty yaml: Please pipe yaml into stdin') + sys.exit(1) - db.session.commit() + error = False + tracked = {} + for section, model in yaml_sections: - managers = new_config.get('managers', []) - # tracked_managers=set() - for manager_config in managers: - if verbose: - print(str(manager_config)) - domain_name = manager_config['domain'] - user_name = manager_config['user'] - domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) - if manageruser not in domain.managers: - domain.managers.append(manageruser) - db.session.add(domain) + items = new_config.get(section) + if items is None: + if delete_objects: + out(f'[ERROR] Invalid yaml: Section "{section}" is missing') + error = True + break + else: + continue - db.session.commit() + del new_config[section] + if type(items) is not list: + out(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}') + error = True + break + elif not items: + continue + + # create items + for data in items: + + if verbose: + out(f'Handling {model.__table__} data: {data!r}') + + try: + changed = model.from_dict(data, delete_objects) + except Exception as reason: + out(f'[ERROR] {reason.args[0]} in data: {data}') + error = True + break + + for item, created in changed: + + if created is True: + # flush newly created item + db.session.add(item) + db.session.flush() + if verbose: + out(f'Added {item!r}: {item.to_dict()}') + else: + out(f'Added {item!r}') + + elif len(created): + # modified instance + if verbose: + for key, old, new in created: + out(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}') + else: + out(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}') + + # track primary key of all items + tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()])) + + if error: + break + + # on error: stop early + if error: + out('An error occured. Not committing changes.') + db.session.rollback() + sys.exit(1) + + # are there sections left in new_config? + if new_config: + out(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}') + error = True + + # test for conflicting domains + domains = set() + for model, items in tracked.items(): + if model in (models.Domain, models.Alternative, models.Relay): + if domains & items: + for domain in domains & items: + out(f'[ERROR] Duplicate domain name used: {domain}') + error = True + domains.update(items) + + # delete items not tracked if delete_objects: - for user in db.session.query(models.User).all(): - if not (user.email in tracked_users): - if verbose: - print("Deleting user: " + str(user.email)) - db.session.delete(user) - for alias in db.session.query(models.Alias).all(): - if not (alias.email in tracked_aliases): - if verbose: - print("Deleting alias: " + str(alias.email)) - db.session.delete(alias) - for domain in db.session.query(models.Domain).all(): - if not (domain.name in tracked_domains): - if verbose: - print("Deleting domain: " + str(domain.name)) - db.session.delete(domain) - db.session.commit() + for model, items in tracked.items(): + for item in model.query.all(): + if not item._dict_pval() in items: + out(f'Deleted {item!r} {item}') + db.session.delete(item) + + # don't commit when running dry + if dry_run: + db.session.rollback() + else: + db.session.commit() + + +@mailu.command() +@click.option('-v', '--verbose', is_flag=True) +@click.option('-s', '--secrets', is_flag=True) +@flask_cli.with_appcontext +def config_dump(verbose=False, secrets=False): + """dump configuration as YAML-formatted data to stdout""" + + config = {} + for section, model in yaml_sections: + dump = [item.to_dict(verbose, secrets) for item in model.query.all()] + if len(dump): + config[section] = dump + + yaml.dump(config, sys.stdout, default_flow_style=False, allow_unicode=True) @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 0a447758..fde4d6f1 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -5,6 +5,7 @@ from passlib import context, hash from datetime import datetime, date from email.mime import text from flask import current_app as app +from textwrap import wrap import flask_sqlalchemy import sqlalchemy @@ -15,6 +16,8 @@ import glob import smtplib import idna import dns +import json +import itertools db = flask_sqlalchemy.SQLAlchemy() @@ -32,6 +35,7 @@ class IdnaDomain(db.TypeDecorator): def process_result_value(self, value, dialect): return idna.decode(value) + python_type = str class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) @@ -56,6 +60,7 @@ class IdnaEmail(db.TypeDecorator): idna.decode(domain_name), ) + python_type = str class CommaSeparatedList(db.TypeDecorator): """ Stores a list as a comma-separated string, compatible with Postfix. @@ -74,6 +79,7 @@ class CommaSeparatedList(db.TypeDecorator): def process_result_value(self, value, dialect): return list(filter(bool, value.split(","))) if value else [] + python_type = list class JSONEncoded(db.TypeDecorator): """ Represents an immutable structure as a json-encoded string. @@ -87,6 +93,7 @@ class JSONEncoded(db.TypeDecorator): def process_result_value(self, value, dialect): return json.loads(value) if value else None + python_type = str class Base(db.Model): """ Base class for all models @@ -105,6 +112,219 @@ class Base(db.Model): updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) comment = db.Column(db.String(255), nullable=True) + @classmethod + def _dict_pkey(model): + return model.__mapper__.primary_key[0].name + + def _dict_pval(self): + return getattr(self, self._dict_pkey()) + + def to_dict(self, full=False, include_secrets=False, recursed=False, hide=None): + """ Return a dictionary representation of this model. + """ + + if recursed and not getattr(self, '_dict_recurse', False): + return str(self) + + hide = set(hide or []) | {'created_at', 'updated_at'} + if hasattr(self, '_dict_hide'): + hide |= self._dict_hide + + secret = set() + if not include_secrets and hasattr(self, '_dict_secret'): + secret |= self._dict_secret + + convert = getattr(self, '_dict_output', {}) + + res = {} + + for key in itertools.chain(self.__table__.columns.keys(), getattr(self, '_dict_show', [])): + if key in hide: + continue + if key in self.__table__.columns: + default = self.__table__.columns[key].default + if isinstance(default, sqlalchemy.sql.schema.ColumnDefault): + default = default.arg + else: + default = None + value = getattr(self, key) + if full or ((default or value) and value != default): + if key in secret: + value = '' + elif value is not None and key in convert: + value = convert[key](value) + res[key] = value + + for key in self.__mapper__.relationships.keys(): + if key in hide: + continue + if self.__mapper__.relationships[key].uselist: + items = getattr(self, key) + if self.__mapper__.relationships[key].query_class is not None: + if hasattr(items, 'all'): + items = items.all() + if full or len(items): + if key in secret: + res[key] = '' + else: + res[key] = [item.to_dict(full, include_secrets, True) for item in items] + else: + value = getattr(self, key) + if full or value is not None: + if key in secret: + res[key] = '' + else: + res[key] = value.to_dict(full, include_secrets, True) + + return res + + @classmethod + def from_dict(model, data, delete=False): + + changed = [] + + pkey = model._dict_pkey() + + # handle "primary key" only + if type(data) is not dict: + data = {pkey: data} + + # modify input data + if hasattr(model, '_dict_input'): + try: + model._dict_input(data) + except Exception as reason: + raise ValueError(f'{reason}', model, None, data) + + # check for primary key (if not recursed) + if not getattr(model, '_dict_recurse', False): + if not pkey in data: + raise KeyError(f'primary key {model.__table__}.{pkey} is missing', model, pkey, data) + + # check data keys and values + for key, value in data.items(): + + # check key + if not hasattr(model, key): + raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data) + + # check value type + col = model.__mapper__.columns.get(key) + if col is not None: + if not type(value) is col.type.python_type: + raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) + else: + rel = model.__mapper__.relationships.get(key) + if rel is None: + itype = getattr(model, '_dict_types', {}).get(key) + if itype is not None: + if type(value) is not itype: + raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) + else: + raise NotImplementedError(f'type not defined for {model.__table__}.{key}') + + # handle relationships + if key in model.__mapper__.relationships: + rel_model = model.__mapper__.relationships[key].argument + if not isinstance(rel_model, sqlalchemy.orm.Mapper): + add = rel_model.from_dict(value, delete) + assert len(add) == 1 + item, updated = add[0] + changed.append((item, updated)) + data[key] = item + + # create or update item? + item = model.query.get(data[pkey]) if pkey in data else None + if item is None: + # create item + + # check for mandatory keys + missing = getattr(model, '_dict_mandatory', set()) - set(data.keys()) + if missing: + raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {model.__table__} missing', model, missing, data) + + changed.append((model(**data), True)) + + else: + # update item + + updated = [] + for key, value in data.items(): + + # skip primary key + if key == pkey: + continue + + if key in model.__mapper__.relationships: + # update relationship + rel_model = model.__mapper__.relationships[key].argument + if isinstance(rel_model, sqlalchemy.orm.Mapper): + rel_model = rel_model.class_ + # add (and create) referenced items + cur = getattr(item, key) + old = sorted(cur, key=lambda i:id(i)) + new = [] + for rel_data in value: + # get or create related item + add = rel_model.from_dict(rel_data, delete) + assert len(add) == 1 + rel_item, rel_updated = add[0] + changed.append((rel_item, rel_updated)) + if rel_item not in cur: + cur.append(rel_item) + new.append(rel_item) + + # delete referenced items missing in yaml + rel_pkey = rel_model._dict_pkey() + new_data = list([i.to_dict(True, True, True, [rel_pkey]) for i in new]) + for rel_item in old: + if rel_item not in new: + # check if item with same data exists to stabilze import without primary key + rel_data = rel_item.to_dict(True, True, True, [rel_pkey]) + try: + same_idx = new_data.index(rel_data) + except ValueError: + same = None + else: + same = new[same_idx] + + if same is None: + # delete items missing in new + if delete: + cur.remove(rel_item) + else: + new.append(rel_item) + else: + # swap found item with same data with newly created item + new.append(rel_item) + new_data.append(rel_data) + new.remove(same) + del new_data[same_idx] + for i, (ch_item, ch_update) in enumerate(changed): + if ch_item is same: + changed[i] = (rel_item, []) + db.session.flush() + db.session.delete(ch_item) + break + + # remember changes + new = sorted(new, key=lambda i:id(i)) + if new != old: + updated.append((key, old, new)) + + else: + # update key + old = getattr(item, key) + if type(old) is list and not delete: + value = old + value + if value != old: + updated.append((key, old, value)) + setattr(item, key, value) + + changed.append((item, updated)) + + return changed + # Many-to-many association table for domain managers managers = db.Table('manager', Base.metadata, @@ -126,6 +346,27 @@ class Domain(Base): """ __tablename__ = "domain" + _dict_hide = {'users', 'managers', 'aliases'} + _dict_show = {'dkim_key'} + _dict_secret = {'dkim_key'} + _dict_types = {'dkim_key': bytes} + _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} + @staticmethod + def _dict_input(data): + key = data.get('dkim_key') + if key is not None: + key = data['dkim_key'] + if type(key) is list: + key = ''.join(key) + if type(key) is str: + key = ''.join(key.strip().split()) + if key.startswith('-----BEGIN PRIVATE KEY-----'): + key = key[25:] + if key.endswith('-----END PRIVATE KEY-----'): + key = key[:-23] + key = '\n'.join(wrap(key, 64)) + data['dkim_key'] = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, backref=db.backref('manager_of'), lazy='dynamic') @@ -208,6 +449,8 @@ class Relay(Base): __tablename__ = "relay" + _dict_mandatory = {'smtp'} + name = db.Column(IdnaDomain, primary_key=True, nullable=False) smtp = db.Column(db.String(80), nullable=True) @@ -221,6 +464,16 @@ class Email(object): localpart = db.Column(db.String(80), nullable=False) + @staticmethod + def _dict_input(data): + if 'email' in data: + if 'localpart' in data or 'domain' in data: + raise ValueError('ambigous key email and localpart/domain') + elif type(data['email']) is str: + data['localpart'], data['domain'] = data['email'].rsplit('@', 1) + else: + data['email'] = f"{data['localpart']}@{data['domain']}" + @declarative.declared_attr def domain_name(cls): return db.Column(IdnaDomain, db.ForeignKey(Domain.name), @@ -306,6 +559,28 @@ class User(Base, Email): """ __tablename__ = "user" + _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} + _dict_mandatory = {'localpart', 'domain', 'password'} + @classmethod + def _dict_input(cls, data): + Email._dict_input(data) + # handle password + if 'password' in data: + if 'password_hash' in data or 'hash_scheme' in data: + raise ValueError('ambigous key password and password_hash/hash_scheme') + # check (hashed) password + password = data['password'] + if password.startswith('{') and '}' in password: + scheme = password[1:password.index('}')] + if scheme not in cls.scheme_dict: + raise ValueError(f'invalid password scheme {scheme!r}') + else: + raise ValueError(f'invalid hashed password {password!r}') + elif 'password_hash' in data and 'hash_scheme' in data: + if data['hash_scheme'] not in cls.scheme_dict: + raise ValueError(f'invalid password scheme {scheme!r}') + data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) @@ -431,6 +706,14 @@ class Alias(Base, Email): """ __tablename__ = "alias" + _dict_hide = {'domain_name', 'domain', 'localpart'} + @staticmethod + def _dict_input(data): + # handle comma delimited string for backwards compability + dst = data.get('destination') + if type(dst) is str: + data['destination'] = list([adr.strip() for adr in dst.split(',')]) + domain = db.relationship(Domain, backref=db.backref('aliases', cascade='all, delete-orphan')) wildcard = db.Column(db.Boolean(), nullable=False, default=False) @@ -484,6 +767,10 @@ class Token(Base): """ __tablename__ = "token" + _dict_recurse = True + _dict_hide = {'user', 'user_email'} + _dict_mandatory = {'password'} + id = db.Column(db.Integer(), primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) @@ -499,7 +786,7 @@ class Token(Base): self.password = hash.sha256_crypt.using(rounds=1000).hash(password) def __str__(self): - return self.comment + return self.comment or self.ip class Fetch(Base): @@ -508,6 +795,11 @@ class Fetch(Base): """ __tablename__ = "fetch" + _dict_recurse = True + _dict_hide = {'user_email', 'user', 'last_check', 'error'} + _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} + _dict_secret = {'password'} + id = db.Column(db.Integer(), primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) @@ -516,9 +808,12 @@ class Fetch(Base): protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) host = db.Column(db.String(255), nullable=False) port = db.Column(db.Integer(), nullable=False) - tls = db.Column(db.Boolean(), nullable=False) + tls = db.Column(db.Boolean(), nullable=False, default=False) username = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False) - keep = db.Column(db.Boolean(), nullable=False) + keep = db.Column(db.Boolean(), nullable=False, default=False) last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) + + def __str__(self): + return f'{self.protocol}{"s" if self.tls else ""}://{self.username}@{self.host}:{self.port}' From 190e7a709b6e3bb134ba8086e54796f14d42a05c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 23:14:27 +0200 Subject: [PATCH 004/181] renamed config-dump option --verbose to --full --- core/admin/mailu/manage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index bf0148df..f74d24b5 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -299,15 +299,15 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @mailu.command() -@click.option('-v', '--verbose', is_flag=True) +@click.option('-f', '--full', is_flag=True) @click.option('-s', '--secrets', is_flag=True) @flask_cli.with_appcontext -def config_dump(verbose=False, secrets=False): +def config_dump(full=False, secrets=False): """dump configuration as YAML-formatted data to stdout""" config = {} for section, model in yaml_sections: - dump = [item.to_dict(verbose, secrets) for item in model.query.all()] + dump = [item.to_dict(full, secrets) for item in model.query.all()] if len(dump): config[section] = dump From 69ccf791d27dabfe1b8943d837923d5b2ba78d72 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 23:16:37 +0200 Subject: [PATCH 005/181] fixed data import via from_dict - stabilized CommaSeparatedList by sorting values - CommaSeparatedList can now handle list and set input - from_dict now handles mapped keys - from_dict now handles null values - class Domain: handle dkim-key None correctly - class User: delete obsolete keys after converting - class Alias: now uses Email._dict_input --- core/admin/mailu/models.py | 205 ++++++++++++++++++++----------------- 1 file changed, 113 insertions(+), 92 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index fde4d6f1..cbafc6a4 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -69,12 +69,12 @@ class CommaSeparatedList(db.TypeDecorator): impl = db.String def process_bind_param(self, value, dialect): - if type(value) is not list: + if not isinstance(value, (list, set)): raise TypeError("Should be a list") for item in value: if "," in item: raise ValueError("Item must not contain a comma") - return ",".join(value) + return ",".join(sorted(value)) def process_result_value(self, value, dialect): return list(filter(bool, value.split(","))) if value else [] @@ -205,13 +205,13 @@ class Base(db.Model): for key, value in data.items(): # check key - if not hasattr(model, key): + if not hasattr(model, key) and not key in model.__mapper__.relationships: raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data) # check value type col = model.__mapper__.columns.get(key) if col is not None: - if not type(value) is col.type.python_type: + if not ((value is None and col.nullable) or (type(value) is col.type.python_type)): raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) else: rel = model.__mapper__.relationships.get(key) @@ -229,99 +229,115 @@ class Base(db.Model): if not isinstance(rel_model, sqlalchemy.orm.Mapper): add = rel_model.from_dict(value, delete) assert len(add) == 1 - item, updated = add[0] - changed.append((item, updated)) - data[key] = item + rel_item, updated = add[0] + changed.append((rel_item, updated)) + data[key] = rel_item - # create or update item? + # create item if necessary + created = False item = model.query.get(data[pkey]) if pkey in data else None if item is None: - # create item # check for mandatory keys missing = getattr(model, '_dict_mandatory', set()) - set(data.keys()) if missing: raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {model.__table__} missing', model, missing, data) - changed.append((model(**data), True)) - - else: - # update item - - updated = [] - for key, value in data.items(): - - # skip primary key - if key == pkey: - continue - + # remove mapped relationships from data + mapped = {} + for key in list(data.keys()): if key in model.__mapper__.relationships: - # update relationship - rel_model = model.__mapper__.relationships[key].argument - if isinstance(rel_model, sqlalchemy.orm.Mapper): - rel_model = rel_model.class_ - # add (and create) referenced items - cur = getattr(item, key) - old = sorted(cur, key=lambda i:id(i)) - new = [] - for rel_data in value: - # get or create related item - add = rel_model.from_dict(rel_data, delete) - assert len(add) == 1 - rel_item, rel_updated = add[0] - changed.append((rel_item, rel_updated)) - if rel_item not in cur: - cur.append(rel_item) - new.append(rel_item) + if isinstance(model.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): + mapped[key] = data[key] + del data[key] - # delete referenced items missing in yaml - rel_pkey = rel_model._dict_pkey() - new_data = list([i.to_dict(True, True, True, [rel_pkey]) for i in new]) - for rel_item in old: - if rel_item not in new: - # check if item with same data exists to stabilze import without primary key - rel_data = rel_item.to_dict(True, True, True, [rel_pkey]) - try: - same_idx = new_data.index(rel_data) - except ValueError: - same = None - else: - same = new[same_idx] + # create new item + item = model(**data) + created = True - if same is None: - # delete items missing in new - if delete: - cur.remove(rel_item) - else: - new.append(rel_item) + # and update mapped relationships (below) + data = mapped + + # update item + updated = [] + for key, value in data.items(): + + # skip primary key + if key == pkey: + continue + + if key in model.__mapper__.relationships: + # update relationship + rel_model = model.__mapper__.relationships[key].argument + if isinstance(rel_model, sqlalchemy.orm.Mapper): + rel_model = rel_model.class_ + # add (and create) referenced items + cur = getattr(item, key) + old = sorted(cur, key=lambda i:id(i)) + new = [] + for rel_data in value: + # get or create related item + add = rel_model.from_dict(rel_data, delete) + assert len(add) == 1 + rel_item, rel_updated = add[0] + changed.append((rel_item, rel_updated)) + if rel_item not in cur: + cur.append(rel_item) + new.append(rel_item) + + # delete referenced items missing in yaml + rel_pkey = rel_model._dict_pkey() + new_data = list([i.to_dict(True, True, True, [rel_pkey]) for i in new]) + for rel_item in old: + if rel_item not in new: + # check if item with same data exists to stabilze import without primary key + rel_data = rel_item.to_dict(True, True, True, [rel_pkey]) + try: + same_idx = new_data.index(rel_data) + except ValueError: + same = None + else: + same = new[same_idx] + + if same is None: + # delete items missing in new + if delete: + cur.remove(rel_item) else: - # swap found item with same data with newly created item new.append(rel_item) - new_data.append(rel_data) - new.remove(same) - del new_data[same_idx] - for i, (ch_item, ch_update) in enumerate(changed): - if ch_item is same: - changed[i] = (rel_item, []) - db.session.flush() - db.session.delete(ch_item) - break + else: + # swap found item with same data with newly created item + new.append(rel_item) + new_data.append(rel_data) + new.remove(same) + del new_data[same_idx] + for i, (ch_item, ch_update) in enumerate(changed): + if ch_item is same: + changed[i] = (rel_item, []) + db.session.flush() + db.session.delete(ch_item) + break - # remember changes - new = sorted(new, key=lambda i:id(i)) - if new != old: - updated.append((key, old, new)) + # remember changes + new = sorted(new, key=lambda i:id(i)) + if new != old: + updated.append((key, old, new)) - else: - # update key - old = getattr(item, key) - if type(old) is list and not delete: - value = old + value - if value != old: - updated.append((key, old, value)) - setattr(item, key, value) + else: + # update key + old = getattr(item, key) + if type(old) is list: + # deduplicate list value + assert type(value) is list + value = set(value) + old = set(old) + if not delete: + value = old | value + if value != old: + updated.append((key, old, value)) + setattr(item, key, value) - changed.append((item, updated)) + changed.append((item, created if created else updated)) return changed @@ -353,19 +369,21 @@ class Domain(Base): _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} @staticmethod def _dict_input(data): - key = data.get('dkim_key') - if key is not None: + if 'dkim_key' in data: key = data['dkim_key'] - if type(key) is list: - key = ''.join(key) - if type(key) is str: - key = ''.join(key.strip().split()) - if key.startswith('-----BEGIN PRIVATE KEY-----'): - key = key[25:] - if key.endswith('-----END PRIVATE KEY-----'): - key = key[:-23] - key = '\n'.join(wrap(key, 64)) - data['dkim_key'] = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + if key is None: + del data['dkim_key'] + else: + if type(key) is list: + key = ''.join(key) + if type(key) is str: + key = ''.join(key.strip().split()) + if key.startswith('-----BEGIN PRIVATE KEY-----'): + key = key[25:] + if key.endswith('-----END PRIVATE KEY-----'): + key = key[:-23] + key = '\n'.join(wrap(key, 64)) + data['dkim_key'] = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, @@ -580,6 +598,8 @@ class User(Base, Email): if data['hash_scheme'] not in cls.scheme_dict: raise ValueError(f'invalid password scheme {scheme!r}') data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + del data['hash_scheme'] + del data['password_hash'] domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) @@ -709,6 +729,7 @@ class Alias(Base, Email): _dict_hide = {'domain_name', 'domain', 'localpart'} @staticmethod def _dict_input(data): + Email._dict_input(data) # handle comma delimited string for backwards compability dst = data.get('destination') if type(dst) is str: From 3a5a15a5e6326cee38b018fcad6d05ab92c0485a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 23:23:03 +0200 Subject: [PATCH 006/181] updated documentation and changelog added some documentation for cli commands config-dump and config-update --- CHANGELOG.md | 2 + docs/cli.rst | 127 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 98 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ccde4bc..1c20d9b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ deprecated before 1.8.0, you can switch to an external database server by then. - Features: use HTTP/1.1 for proxyied connections ([#1070](https://github.com/Mailu/Mailu/issues/1070)) - Features: Update Rainloop to 1.13.0 ([#1071](https://github.com/Mailu/Mailu/issues/1071)) - Features: Use python package socrate instead of Mailustart ([#1082](https://github.com/Mailu/Mailu/issues/1082)) +- Features: Added cli command config-dump ([#1377](https://github.com/Mailu/Mailu/issues/1377)) - Bugfixes: Use ldez/traefik-certs-dumper in our certificate dumper to have a more robust solution ([#820](https://github.com/Mailu/Mailu/issues/820)) - Bugfixes: Make aliases optionally case-insensitive: After attempting to resolve an alias in its preserved case, also attempt to match it case-insensitively ([#867](https://github.com/Mailu/Mailu/issues/867)) - Bugfixes: Fix HOST_* variable usage ([#884](https://github.com/Mailu/Mailu/issues/884)) @@ -47,6 +48,7 @@ deprecated before 1.8.0, you can switch to an external database server by then. - Enhancement: Create an Authentication Token with IPv6 address restriction ([#829](https://github.com/Mailu/Mailu/issues/829)) - Enhancement: Automatically create admin user on container startup if given appropriate environment variables - Enhancement: Missing wildcard option in alias flask command ([#869](https://github.com/Mailu/Mailu/issues/869)) +- Enhancement: Cli command config-update now updates all objects and parameters ([#1377](https://github.com/Mailu/Mailu/issues/1377)) v1.6.0 - 2019-01-18 ------------------- diff --git a/docs/cli.rst b/docs/cli.rst index a9cff41c..7476e676 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -10,6 +10,7 @@ Managing users and aliases can be done from CLI using commands: * user * user-import * user-delete +* config-dump * config-update alias @@ -62,60 +63,124 @@ primary difference with simple `user` command is that password is being imported docker-compose run --rm admin flask mailu user-import myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' 'SHA512-CRYPT' user-delete ------------- +----------- .. code-block:: bash docker-compose exec admin flask mailu user-delete foo@example.net +config-dump +----------- + +The purpose of this command is to dump domain-, relay-, alias- and user-configuration to a YAML template. +If you want to export non-hashed secrets you have to add the ``--secrets`` option. +Only non-default options are dumped. If you want to dump all options use ``--full``. + +.. code-block:: bash + + docker-compose exec admin flask mailu config-dump > mail-config.yml + config-update ------------- -The sole purpose of this command is for importing users/aliases in bulk and synchronizing DB entries with external YAML template: +The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML template. .. code-block:: bash - cat mail-config.yml | docker-compose exec -T admin flask mailu config-update --delete-objects + docker exec -i $(docker-compose ps -q admin) flask mailu config-update -nvd < mail-config.yml -where mail-config.yml looks like: +*(The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec)* -.. code-block:: bash +mail-config.yml looks like: + +.. code-block:: yaml + + domains: + - name: example.com + alternatives: + - alternative.example.com users: - - localpart: foo - domain: example.com + - email: foo@example.com password_hash: klkjhumnzxcjkajahsdqweqqwr hash_scheme: MD5-CRYPT aliases: - - localpart: alias1 - domain: example.com + - email: alias1@example.com destination: "user1@example.com,user2@example.com" -without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. + relays: + - name: relay.example.com + comment: test + smtp: mx.example.com -Users ------ +You can use ``--dry-run`` to test your YAML without omitting any changes to the database. +With ``--verbose`` config-update will show exactly what it changes in the database. +Without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. -following are additional parameters that could be defined for users: +This is a complete YAML template with all additional parameters that could be defined: -* comment -* quota_bytes -* global_admin -* enable_imap -* enable_pop -* forward_enabled -* forward_destination -* reply_enabled -* reply_subject -* reply_body -* displayed_name -* spam_enabled -* spam_threshold +.. code-block:: yaml -Alias ------ + aliases: + - email: email@example.com + comment: '' + destination: + - address@example.com + wildcard: false + + domains: + - name: example.com + alternatives: + - alternative.tld + comment: '' + dkim_key: '' + max_aliases: -1 + max_quota_bytes: 0 + max_users: -1 + signup_enabled: false + + relays: + - name: relay.example.com + comment: '' + smtp: mx.example.com + + users: + - email: postmaster@example.com + comment: '' + displayed_name: 'Postmaster' + enable_imap: true + enable_pop: false + enabled: true + fetches: + - id: 1 + comment: 'test fetch' + username: fetch-user + host: other.example.com + password: 'secret' + port: 993 + protocol: imap + tls: true + keep: true + forward_destination: + - address@remote.example.com + forward_enabled: true + forward_keep: true + global_admin: true + manager_of: + - example.com + password: '{BLF-CRYPT}$2b$12$...' + quota_bytes: 1000000000 + reply_body: '' + reply_enabled: false + reply_enddate: 2999-12-31 + reply_startdate: 1900-01-01 + reply_subject: '' + spam_enabled: true + spam_threshold: 80 + tokens: + - id: 1 + comment: email-client + ip: 192.168.1.1 + password: '$5$rounds=1000$...' -additional fields: - -* wildcard From 0cf91f35a45b7272a1b9b5cb0d8760ab4cb0bdfe Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 27 Aug 2020 16:08:15 +0200 Subject: [PATCH 007/181] moved change log entry to towncrier --- CHANGELOG.md | 2 -- towncrier/newsfragments/1604.feature | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/1604.feature diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c20d9b2..7ccde4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,6 @@ deprecated before 1.8.0, you can switch to an external database server by then. - Features: use HTTP/1.1 for proxyied connections ([#1070](https://github.com/Mailu/Mailu/issues/1070)) - Features: Update Rainloop to 1.13.0 ([#1071](https://github.com/Mailu/Mailu/issues/1071)) - Features: Use python package socrate instead of Mailustart ([#1082](https://github.com/Mailu/Mailu/issues/1082)) -- Features: Added cli command config-dump ([#1377](https://github.com/Mailu/Mailu/issues/1377)) - Bugfixes: Use ldez/traefik-certs-dumper in our certificate dumper to have a more robust solution ([#820](https://github.com/Mailu/Mailu/issues/820)) - Bugfixes: Make aliases optionally case-insensitive: After attempting to resolve an alias in its preserved case, also attempt to match it case-insensitively ([#867](https://github.com/Mailu/Mailu/issues/867)) - Bugfixes: Fix HOST_* variable usage ([#884](https://github.com/Mailu/Mailu/issues/884)) @@ -48,7 +47,6 @@ deprecated before 1.8.0, you can switch to an external database server by then. - Enhancement: Create an Authentication Token with IPv6 address restriction ([#829](https://github.com/Mailu/Mailu/issues/829)) - Enhancement: Automatically create admin user on container startup if given appropriate environment variables - Enhancement: Missing wildcard option in alias flask command ([#869](https://github.com/Mailu/Mailu/issues/869)) -- Enhancement: Cli command config-update now updates all objects and parameters ([#1377](https://github.com/Mailu/Mailu/issues/1377)) v1.6.0 - 2019-01-18 ------------------- diff --git a/towncrier/newsfragments/1604.feature b/towncrier/newsfragments/1604.feature new file mode 100644 index 00000000..06ee0beb --- /dev/null +++ b/towncrier/newsfragments/1604.feature @@ -0,0 +1 @@ +Added cli command config-dump and enhanced config-update From 85de70212904755a5bc6ef756cfe615dec7ff0bb Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 27 Aug 2020 16:10:53 +0200 Subject: [PATCH 008/181] small typo. Change 'omitting' to 'commiting' --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 7476e676..64a4b17a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -114,7 +114,7 @@ mail-config.yml looks like: comment: test smtp: mx.example.com -You can use ``--dry-run`` to test your YAML without omitting any changes to the database. +You can use ``--dry-run`` to test your YAML without comitting any changes to the database. With ``--verbose`` config-update will show exactly what it changes in the database. Without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. From ffbeabeb6f685fa3939bbc1ef1266ef8c7ae511c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 27 Aug 2020 22:02:20 +0200 Subject: [PATCH 009/181] updated test to use --verbose flag --verbose (or -v) is now a flag and not an option --- tests/compose/core/02_forward_test.sh | 4 ++-- tests/compose/core/03_alias_test.sh | 4 ++-- tests/compose/core/04_reply_test.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/compose/core/02_forward_test.sh b/tests/compose/core/02_forward_test.sh index 595820cf..651e027c 100755 --- a/tests/compose/core/02_forward_test.sh +++ b/tests/compose/core/02_forward_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -10,7 +10,7 @@ EOF python3 tests/forward_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" diff --git a/tests/compose/core/03_alias_test.sh b/tests/compose/core/03_alias_test.sh index dce1918a..2d40903a 100755 --- a/tests/compose/core/03_alias_test.sh +++ b/tests/compose/core/03_alias_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose aliases: - localpart: alltheusers domain: mailu.io @@ -7,6 +7,6 @@ EOF python3 tests/alias_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose aliases: [] EOF diff --git a/tests/compose/core/04_reply_test.sh b/tests/compose/core/04_reply_test.sh index 83c114f6..7615a0f8 100755 --- a/tests/compose/core/04_reply_test.sh +++ b/tests/compose/core/04_reply_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -11,7 +11,7 @@ EOF python3 tests/reply_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" From 9d2327b0f1b62de4f57e0c4367434572ae85c46a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Sep 2020 12:32:51 +0200 Subject: [PATCH 010/181] add space for more human readable indentation add a newline before main sections add some spaces to indent --- core/admin/mailu/manage.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index f74d24b5..c36f45c0 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -305,13 +305,23 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) def config_dump(full=False, secrets=False): """dump configuration as YAML-formatted data to stdout""" + class spacedDumper(yaml.Dumper): + + def write_line_break(self, data=None): + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + config = {} for section, model in yaml_sections: dump = [item.to_dict(full, secrets) for item in model.query.all()] if len(dump): config[section] = dump - yaml.dump(config, sys.stdout, default_flow_style=False, allow_unicode=True) + yaml.dump(config, sys.stdout, Dumper=spacedDumper, default_flow_style=False, allow_unicode=True) @mailu.command() From 8e14aa80ee2ea6546bd659a03813db44bd9f0024 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Sep 2020 12:57:40 +0200 Subject: [PATCH 011/181] documented options and added help text --- core/admin/mailu/manage.py | 10 +++++----- docs/cli.rst | 40 +++++++++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index c36f45c0..b25ab568 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -180,9 +180,9 @@ yaml_sections = [ ] @mailu.command() -@click.option('-v', '--verbose', is_flag=True) -@click.option('-d', '--delete-objects', is_flag=True) -@click.option('-n', '--dry-run', is_flag=True) +@click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') +@click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') +@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') @flask_cli.with_appcontext def config_update(verbose=False, delete_objects=False, dry_run=False, file=None): """sync configuration with data from YAML-formatted stdin""" @@ -299,8 +299,8 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @mailu.command() -@click.option('-f', '--full', is_flag=True) -@click.option('-s', '--secrets', is_flag=True) +@click.option('-f', '--full', is_flag=True, help='Include default attributes') +@click.option('-s', '--secrets', is_flag=True, help='Include secrets (plain-text / not hashed)') @flask_cli.with_appcontext def config_dump(full=False, secrets=False): """dump configuration as YAML-formatted data to stdout""" diff --git a/docs/cli.rst b/docs/cli.rst index 64a4b17a..0036e504 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -73,8 +73,22 @@ config-dump ----------- The purpose of this command is to dump domain-, relay-, alias- and user-configuration to a YAML template. -If you want to export non-hashed secrets you have to add the ``--secrets`` option. -Only non-default options are dumped. If you want to dump all options use ``--full``. + +.. code-block:: bash + + # docker-compose exec admin flask mailu config-dump --help + + Usage: flask mailu config-dump [OPTIONS] + + dump configuration as YAML-formatted data to stdout + + Options: + -f, --full Include default attributes + -s, --secrets Include secrets (plain-text / not hashed) + --help Show this message and exit. + +If you want to export secrets (plain-text / not hashed) you have to add the ``--secrets`` option. +Only non-default attributes are dumped. If you want to dump all attributes use ``--full``. .. code-block:: bash @@ -85,13 +99,29 @@ config-update The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML template. +.. code-block:: bash + + # docker-compose exec admin flask mailu config-update --help + + Usage: flask mailu config-update [OPTIONS] + + sync configuration with data from YAML-formatted stdin + + Options: + -v, --verbose Increase verbosity + -d, --delete-objects Remove objects not included in yaml + -n, --dry-run Perform a trial run with no changes made + --help Show this message and exit. + + +The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec instead: + .. code-block:: bash docker exec -i $(docker-compose ps -q admin) flask mailu config-update -nvd < mail-config.yml -*(The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec)* -mail-config.yml looks like: +mail-config.yml looks like this: .. code-block:: yaml @@ -116,7 +146,7 @@ mail-config.yml looks like: You can use ``--dry-run`` to test your YAML without comitting any changes to the database. With ``--verbose`` config-update will show exactly what it changes in the database. -Without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. +Without ``--delete-object`` option config-update will only add/update changed values but will *not* remove any entries missing in provided YAML input. This is a complete YAML template with all additional parameters that could be defined: From 5e32447f07a2148cc1800a49ce7c9c7a19049d31 Mon Sep 17 00:00:00 2001 From: Jon Wilson Date: Mon, 21 Sep 2020 15:06:43 +0100 Subject: [PATCH 012/181] Change unbound logfile to the empty string This is defined to send log messages to stderr, which is what we want - fixes #1536 ("Could not open logfile /dev/stdout: Permission denied") --- optional/unbound/unbound.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optional/unbound/unbound.conf b/optional/unbound/unbound.conf index 8abd4325..6c8fc64d 100644 --- a/optional/unbound/unbound.conf +++ b/optional/unbound/unbound.conf @@ -2,7 +2,7 @@ server: verbosity: 1 interface: 0.0.0.0 interface: ::0 - logfile: /dev/stdout + logfile: "" do-ip4: yes do-ip6: yes do-udp: yes From ef71bc04cb27f6ed0779584afcc8690fbf20048a Mon Sep 17 00:00:00 2001 From: Patryk Tech Date: Thu, 1 Oct 2020 13:51:06 +0300 Subject: [PATCH 013/181] Update docs/reverse.rst with Traefik v2+ info --- docs/reverse.rst | 41 ++++++++++++++++++++++++++++++-- towncrier/newsfragments/1503.doc | 1 + 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/1503.doc diff --git a/docs/reverse.rst b/docs/reverse.rst index 9a7a7dc0..f3b3e7bd 100644 --- a/docs/reverse.rst +++ b/docs/reverse.rst @@ -154,7 +154,40 @@ Add the respective Traefik labels for your domain/configuration, like If your Traefik is configured to automatically request certificates from *letsencrypt*, then you’ll have a certificate for ``mail.your.doma.in`` now. However, ``mail.your.doma.in`` might only be the location where you want the Mailu web-interfaces to live — your mail should be sent/received from ``your.doma.in``, and this is the ``DOMAIN`` in your ``.env``? -To support that use-case, Traefik can request ``SANs`` for your domain. Lets add something like +To support that use-case, Traefik can request ``SANs`` for your domain. The configuration for this will depend on your Traefik version. + +---- + +Traefik 2.x using labels configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add the appropriate labels for your domain(s) to the ``front`` container in ``docker-compose.yml``. + +.. code-block:: yaml + + services: + front: + labels: + # Enable TLS + - "traefik.http.routers.mailu-secure.tls" + # Your main domain + - "traefik.http.routers.mailu-secure.tls.domains[0].main=your.doma.in" + # Optional SANs for your main domain + - "traefik.http.routers.mailu-secure.tls.domains[0].sans=mail.your.doma.in,webmail.your.doma.in,smtp.your.doma.in" + # Optionally add other domains + - "traefik.http.routers.mailu-secure.tls.domains[1].main=mail.other.doma.in" + - "traefik.http.routers.mailu-secure.tls.domains[1].sans=mail2.other.doma.in,mail3.other.doma.in" + # Your ACME certificate resolver + - "traefik.http.routers.mailu-secure.tls.certResolver=foo" + +Of course, be sure to define the Certificate Resolver ``foo`` in the static configuration as well. + +Alternatively, you can define SANs in the Traefik static configuration using routers, or in the static configuration using entrypoints. Refer to the Traefik documentation for more details. + +Traefik 1.x with TOML configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Lets add something like .. code-block:: yaml @@ -163,7 +196,11 @@ To support that use-case, Traefik can request ``SANs`` for your domain. Lets add main = "your.doma.in" # this is the same as $TRAEFIK_DOMAIN! sans = ["mail.your.doma.in", "webmail.your.doma.in", "smtp.your.doma.in"] -to your ``traefik.toml``. You might need to clear your ``acme.json``, if a certificate for one of these domains already exists. +to your ``traefik.toml``. + +---- + +You might need to clear your ``acme.json``, if a certificate for one of these domains already exists. You will need some solution which dumps the certificates in ``acme.json``, so you can include them in the ``mailu/front`` container. One such example is ``mailu/traefik-certdumper``, which has been adapted for use in Mailu. You can add it to your ``docker-compose.yml`` like: diff --git a/towncrier/newsfragments/1503.doc b/towncrier/newsfragments/1503.doc new file mode 100644 index 00000000..9c59feb7 --- /dev/null +++ b/towncrier/newsfragments/1503.doc @@ -0,0 +1 @@ +Add documentation for Traefik 2 in Reverse Proxy \ No newline at end of file From e7caff9811512dc47bddcd9aa11ce983254e7c40 Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Mon, 5 Oct 2020 15:13:07 +0100 Subject: [PATCH 014/181] Add ability to set no WEBROOT_REDIRECT to Nginx Adds a 'none' env option to WEBROOT_REDIRECT so that no `location /` configuration is written to nginx.conf. This is useful for setting up Mailu and Mailman where we override the root to proxy to the mailing list server instead. Without this change the nginx container will not start, or for 1.7 users can set their WEBMAIL_PATH to / with no webmail to get the same results. This fix means that future users don't have to choose between webmail and a root override and makes the configuration intention clear. --- core/nginx/conf/nginx.conf | 2 +- docs/configuration.rst | 15 ++++++++++----- docs/faq.rst | 2 ++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 8f6eaa0d..f672c7a3 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -115,7 +115,7 @@ http { include /overrides/*.conf; # Actual logic - {% if WEB_WEBMAIL != '/' %} + {% if WEB_WEBMAIL != '/' and WEBROOT_REDIRECT != 'none' %} location / { {% if WEBROOT_REDIRECT %} try_files $uri {{ WEBROOT_REDIRECT }}; diff --git a/docs/configuration.rst b/docs/configuration.rst index 5ff3546a..c2c55190 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -92,14 +92,19 @@ the localpart for DMARC rua and ruf email addresses. Full-text search is enabled for IMAP is enabled by default. This feature can be disabled (e.g. for performance reasons) by setting the optional variable ``FULL_TEXT_SEARCH`` to ``off``. +.. _web_settings: + Web settings ------------ -The ``WEB_ADMIN`` contains the path to the main admin interface, while -``WEB_WEBMAIL`` contains the path to the Web email client. -The ``WEBROOT_REDIRECT`` redirects all non-found queries to the set path. -An empty ``WEBROOT_REDIRECT`` value disables redirecting and enables classic -behavior of a 404 result when not found. +- ``WEB_ADMIN`` contains the path to the main admin interface + +- ``WEB_WEBMAIL`` contains the path to the Web email client. + +- ``WEBROOT_REDIRECT`` redirects all non-found queries to the set path. + An empty ``WEBROOT_REDIRECT`` value disables redirecting and enables classic behavior of a 404 result when not found. + Alternatively, ``WEBROOT_REDIRECT`` can be set to ``none`` if you are using an Nginx override for ``location /``. + All three options need a leading slash (``/``) to work. .. note:: ``WEBROOT_REDIRECT`` has to point to a valid path on the webserver. diff --git a/docs/faq.rst b/docs/faq.rst index b292cd05..296cd59f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -262,6 +262,8 @@ correct syntax. The following file names will be taken as override configuration - `Nginx`_ - All ``*.conf`` files in the ``nginx`` sub-directory; - `Rspamd`_ - All files in the ``rspamd`` sub-directory. +To override the root location (``/``) in Nginx ``WEBROOT_REDIRECT`` needs to be set to ``none`` in the env file (see :ref:`web settings `). + *Issue reference:* `206`_, `1368`_. I want to integrate Nextcloud 15 (and newer) with Mailu From 45cdcbdab9635e2b63d8862898187e7a0c9b13b5 Mon Sep 17 00:00:00 2001 From: Ciprian Pascu Date: Sun, 18 Oct 2020 19:27:50 +0300 Subject: [PATCH 015/181] Update front.yaml To match the other kubernetes objects' version --- docs/kubernetes/mailu/front.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kubernetes/mailu/front.yaml b/docs/kubernetes/mailu/front.yaml index a1d5acb2..2fba1026 100644 --- a/docs/kubernetes/mailu/front.yaml +++ b/docs/kubernetes/mailu/front.yaml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: DaemonSet metadata: name: mailu-front From acc728109bc94e9d996985c582cdfc96ba8ecde9 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:13 +0200 Subject: [PATCH 016/181] validate dkim keys and allow removal --- core/admin/mailu/models.py | 41 +++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index cbafc6a4..0dc82bf4 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -218,7 +218,7 @@ class Base(db.Model): if rel is None: itype = getattr(model, '_dict_types', {}).get(key) if itype is not None: - if type(value) is not itype: + if not isinstance(value, itype): raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) else: raise NotImplementedError(f'type not defined for {model.__table__}.{key}') @@ -365,25 +365,34 @@ class Domain(Base): _dict_hide = {'users', 'managers', 'aliases'} _dict_show = {'dkim_key'} _dict_secret = {'dkim_key'} - _dict_types = {'dkim_key': bytes} + _dict_types = {'dkim_key': (bytes, type(None))} _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} @staticmethod def _dict_input(data): if 'dkim_key' in data: key = data['dkim_key'] - if key is None: - del data['dkim_key'] - else: + if key is not None: if type(key) is list: key = ''.join(key) if type(key) is str: - key = ''.join(key.strip().split()) - if key.startswith('-----BEGIN PRIVATE KEY-----'): - key = key[25:] - if key.endswith('-----END PRIVATE KEY-----'): - key = key[:-23] - key = '\n'.join(wrap(key, 64)) - data['dkim_key'] = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + key = ''.join(key.strip().split()) # removes all whitespace + if key: + m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) + if m is not None: + key = key[m.end():] + m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) + if m is not None: + key = key[:m.start()] + key = '\n'.join(wrap(key, 64)) + key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + try: + dkim.strip_key(key) + except: + raise ValueError('invalid dkim key') + else: + data['dkim_key'] = key + else: + data['dkim_key'] = None name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, @@ -405,8 +414,12 @@ class Domain(Base): def dkim_key(self, value): file_path = app.config["DKIM_PATH"].format( domain=self.name, selector=app.config["DKIM_SELECTOR"]) - with open(file_path, "wb") as handle: - handle.write(value) + if value is None: + if os.path.exists(file_path): + os.unlink(file_path) + else: + with open(file_path, "wb") as handle: + handle.write(value) @property def dkim_publickey(self): From c46f9328f700b12cd5dda4fbe0c270d54e584ae2 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:26 +0200 Subject: [PATCH 017/181] also dump dkim_publickey. allow key generation. --- core/admin/mailu/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 0dc82bf4..a9cc8d16 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -363,7 +363,7 @@ class Domain(Base): __tablename__ = "domain" _dict_hide = {'users', 'managers', 'aliases'} - _dict_show = {'dkim_key'} + _dict_show = {'dkim_key', 'dkim_publickey'} _dict_secret = {'dkim_key'} _dict_types = {'dkim_key': (bytes, type(None))} _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} @@ -376,7 +376,9 @@ class Domain(Base): key = ''.join(key) if type(key) is str: key = ''.join(key.strip().split()) # removes all whitespace - if key: + if key == 'generate': + data['dkim_key'] = dkim.gen_key() + elif key: m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) if m is not None: key = key[m.end():] @@ -390,7 +392,7 @@ class Domain(Base): except: raise ValueError('invalid dkim key') else: - data['dkim_key'] = key + data['dkim_key'] = key else: data['dkim_key'] = None From 500967b2f585bca74a53b15a49986293f16e1750 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:29 +0200 Subject: [PATCH 018/181] ignore dkim_publickey when updating config --- core/admin/mailu/models.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a9cc8d16..6c82baa0 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -202,13 +202,14 @@ class Base(db.Model): raise KeyError(f'primary key {model.__table__}.{pkey} is missing', model, pkey, data) # check data keys and values - for key, value in data.items(): + for key in list(data.keys()): # check key if not hasattr(model, key) and not key in model.__mapper__.relationships: raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data) # check value type + value = data[key] col = model.__mapper__.columns.get(key) if col is not None: if not ((value is None and col.nullable) or (type(value) is col.type.python_type)): @@ -218,7 +219,10 @@ class Base(db.Model): if rel is None: itype = getattr(model, '_dict_types', {}).get(key) if itype is not None: - if not isinstance(value, itype): + if not itype: # empty tuple => ignore value + del data[key] + continue + elif not isinstance(value, itype): raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) else: raise NotImplementedError(f'type not defined for {model.__table__}.{key}') @@ -365,7 +369,7 @@ class Domain(Base): _dict_hide = {'users', 'managers', 'aliases'} _dict_show = {'dkim_key', 'dkim_publickey'} _dict_secret = {'dkim_key'} - _dict_types = {'dkim_key': (bytes, type(None))} + _dict_types = {'dkim_key': (bytes, type(None)), 'dkim_publickey': tuple()} _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} @staticmethod def _dict_input(data): From 2a5c46c890bffac107d00f13cf6acbb9047e59e8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:31 +0200 Subject: [PATCH 019/181] Allow to dump only selected sections --- core/admin/mailu/manage.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index b25ab568..44be509c 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -300,10 +300,13 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @mailu.command() @click.option('-f', '--full', is_flag=True, help='Include default attributes') -@click.option('-s', '--secrets', is_flag=True, help='Include secrets (plain-text / not hashed)') +@click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)') +@click.argument('sections', nargs=-1) @flask_cli.with_appcontext -def config_dump(full=False, secrets=False): - """dump configuration as YAML-formatted data to stdout""" +def config_dump(full=False, secrets=False, sections=None): + """dump configuration as YAML-formatted data to stdout + valid SECTIONS are: domains, relays, users, aliases + """ class spacedDumper(yaml.Dumper): @@ -315,11 +318,19 @@ def config_dump(full=False, secrets=False): def increase_indent(self, flow=False, indentless=False): return super().increase_indent(flow, False) + if sections: + check = dict(yaml_sections) + for section in sections: + if section not in check: + print(f'[ERROR] Invalid section: {section}') + return 1 + config = {} for section, model in yaml_sections: - dump = [item.to_dict(full, secrets) for item in model.query.all()] - if len(dump): - config[section] = dump + if not sections or section in sections: + dump = [item.to_dict(full, secrets) for item in model.query.all()] + if len(dump): + config[section] = dump yaml.dump(config, sys.stdout, Dumper=spacedDumper, default_flow_style=False, allow_unicode=True) From adc9c70c3e5b0f2f5c37ae298fd92d5d903c6d94 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:32 +0200 Subject: [PATCH 020/181] added dump option to dump dns data of domains --- core/admin/mailu/manage.py | 14 +++++--- core/admin/mailu/models.py | 72 ++++++++++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 44be509c..42d5b5f7 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -301,11 +301,13 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @mailu.command() @click.option('-f', '--full', is_flag=True, help='Include default attributes') @click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)') +@click.option('-d', '--dns', is_flag=True, help='Include dns records') @click.argument('sections', nargs=-1) @flask_cli.with_appcontext -def config_dump(full=False, secrets=False, sections=None): - """dump configuration as YAML-formatted data to stdout - valid SECTIONS are: domains, relays, users, aliases +def config_dump(full=False, secrets=False, dns=False, sections=None): + """dump configuration as YAML-formatted data to stdout + + SECTIONS can be: domains, relays, users, aliases """ class spacedDumper(yaml.Dumper): @@ -325,10 +327,14 @@ def config_dump(full=False, secrets=False, sections=None): print(f'[ERROR] Invalid section: {section}') return 1 + extra = [] + if dns: + extra.append('dns') + config = {} for section, model in yaml_sections: if not sections or section in sections: - dump = [item.to_dict(full, secrets) for item in model.query.all()] + dump = [item.to_dict(full, secrets, extra) for item in model.query.all()] if len(dump): config[section] = dump diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 6c82baa0..79ce49ab 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -119,7 +119,7 @@ class Base(db.Model): def _dict_pval(self): return getattr(self, self._dict_pkey()) - def to_dict(self, full=False, include_secrets=False, recursed=False, hide=None): + def to_dict(self, full=False, include_secrets=False, include_extra=None, recursed=False, hide=None): """ Return a dictionary representation of this model. """ @@ -136,9 +136,17 @@ class Base(db.Model): convert = getattr(self, '_dict_output', {}) + extra_keys = getattr(self, '_dict_extra', {}) + if include_extra is None: + include_extra = [] + res = {} - for key in itertools.chain(self.__table__.columns.keys(), getattr(self, '_dict_show', [])): + for key in itertools.chain( + self.__table__.columns.keys(), + getattr(self, '_dict_show', []), + *[extra_keys.get(extra, []) for extra in include_extra] + ): if key in hide: continue if key in self.__table__.columns: @@ -167,14 +175,14 @@ class Base(db.Model): if key in secret: res[key] = '' else: - res[key] = [item.to_dict(full, include_secrets, True) for item in items] + res[key] = [item.to_dict(full, include_secrets, include_extra, True) for item in items] else: value = getattr(self, key) if full or value is not None: if key in secret: res[key] = '' else: - res[key] = value.to_dict(full, include_secrets, True) + res[key] = value.to_dict(full, include_secrets, include_extra, True) return res @@ -219,7 +227,7 @@ class Base(db.Model): if rel is None: itype = getattr(model, '_dict_types', {}).get(key) if itype is not None: - if not itype: # empty tuple => ignore value + if itype is False: # ignore value del data[key] continue elif not isinstance(value, itype): @@ -291,11 +299,11 @@ class Base(db.Model): # delete referenced items missing in yaml rel_pkey = rel_model._dict_pkey() - new_data = list([i.to_dict(True, True, True, [rel_pkey]) for i in new]) + new_data = list([i.to_dict(True, True, None, True, [rel_pkey]) for i in new]) for rel_item in old: if rel_item not in new: # check if item with same data exists to stabilze import without primary key - rel_data = rel_item.to_dict(True, True, True, [rel_pkey]) + rel_data = rel_item.to_dict(True, True, None, True, [rel_pkey]) try: same_idx = new_data.index(rel_data) except ValueError: @@ -367,10 +375,18 @@ class Domain(Base): __tablename__ = "domain" _dict_hide = {'users', 'managers', 'aliases'} - _dict_show = {'dkim_key', 'dkim_publickey'} + _dict_show = {'dkim_key'} + _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} _dict_secret = {'dkim_key'} - _dict_types = {'dkim_key': (bytes, type(None)), 'dkim_publickey': tuple()} - _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} + _dict_types = { + 'dkim_key': (bytes, type(None)), + 'dkim_publickey': False, + 'dns_mx': False, + 'dns_spf': False, + 'dns_dkim': False, + 'dns_dmarc': False, + } + _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} @staticmethod def _dict_input(data): if 'dkim_key' in data: @@ -408,18 +424,46 @@ class Domain(Base): max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + def _dkim_file(self): + return app.config["DKIM_PATH"].format( + domain=self.name, selector=app.config["DKIM_SELECTOR"]) + + @property + def dns_mx(self): + hostname = app.config['HOSTNAMES'].split(',')[0] + return f'{self.name}. 600 IN MX 10 {hostname}.' + + @property + def dns_spf(self): + hostname = app.config['HOSTNAMES'].split(',')[0] + return f'{self.name}. 600 IN TXT "v=spf1 mx a:{hostname} ~all"' + + @property + def dns_dkim(self): + if os.path.exists(self._dkim_file()): + selector = app.config['DKIM_SELECTOR'] + return f'{selector}._domainkey.{self.name}. 600 IN TXT "v=DKIM1; k=rsa; p={self.dkim_publickey}"' + + @property + def dns_dmarc(self): + if os.path.exists(self._dkim_file()): + domain = app.config['DOMAIN'] + rua = app.config['DMARC_RUA'] + rua = f' rua=mailto:{rua}@{domain};' if rua else '' + ruf = app.config['DMARC_RUF'] + ruf = f' ruf=mailto:{ruf}@{domain};' if ruf else '' + return f'_dmarc.{self.name}. 600 IN TXT "v=DMARC1; p=reject;{rua}{ruf} adkim=s; aspf=s"' + @property def dkim_key(self): - file_path = app.config["DKIM_PATH"].format( - domain=self.name, selector=app.config["DKIM_SELECTOR"]) + file_path = self._dkim_file() if os.path.exists(file_path): with open(file_path, "rb") as handle: return handle.read() @dkim_key.setter def dkim_key(self, value): - file_path = app.config["DKIM_PATH"].format( - domain=self.name, selector=app.config["DKIM_SELECTOR"]) + file_path = self._dkim_file() if value is None: if os.path.exists(file_path): os.unlink(file_path) From 0a907a744e1027f8e2b79b6fce54fc03e27b69f8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:32:08 +0200 Subject: [PATCH 021/181] updated documentation for config-dump --- docs/cli.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 0036e504..1b2ed14f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -78,22 +78,29 @@ The purpose of this command is to dump domain-, relay-, alias- and user-configur # docker-compose exec admin flask mailu config-dump --help - Usage: flask mailu config-dump [OPTIONS] + Usage: flask mailu config-dump [OPTIONS] [SECTIONS]... dump configuration as YAML-formatted data to stdout + SECTIONS can be: domains, relays, users, aliases + Options: -f, --full Include default attributes - -s, --secrets Include secrets (plain-text / not hashed) + -s, --secrets Include secrets (dkim-key, plain-text / not hashed) + -d, --dns Include dns records --help Show this message and exit. -If you want to export secrets (plain-text / not hashed) you have to add the ``--secrets`` option. +If you want to export secrets (dkim-keys, plain-text / not hashed) you have to add the ``--secrets`` option. Only non-default attributes are dumped. If you want to dump all attributes use ``--full``. +To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. +Unless you specify some sections all sections are dumped by default. .. code-block:: bash docker-compose exec admin flask mailu config-dump > mail-config.yml + docker-compose exec admin flask mailu config-dump --dns domains + config-update ------------- From 32f6a23a95de6d5ee2d64eecc452469967289fdf Mon Sep 17 00:00:00 2001 From: cbachert Date: Fri, 30 Oct 2020 17:12:34 +0000 Subject: [PATCH 022/181] Remove rspamd unused env var from start script Environment variable FRONT_ADDRESS is unused in rspamd FRONT_ADDRESS references were removed with commit 8172f3e in PR #727 --- core/rspamd/start.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/rspamd/start.py b/core/rspamd/start.py index bde708f2..e2e72bcb 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -10,7 +10,6 @@ log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) # Actual startup script -os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis") if os.environ.get("ANTIVIRUS") == 'clamav': From 98a6ffb497e0e368808614219c6b099dcc828c0b Mon Sep 17 00:00:00 2001 From: lub Date: Thu, 17 Sep 2020 19:33:55 +0200 Subject: [PATCH 023/181] add compression via xz and lz4 --- core/dovecot/conf/dovecot.conf | 2 +- docs/compose/.env | 2 +- setup/flavors/compose/mailu.env | 2 +- tests/compose/core/mailu.env | 2 +- tests/compose/fetchmail/mailu.env | 2 +- tests/compose/filters/mailu.env | 2 +- tests/compose/rainloop/mailu.env | 2 +- tests/compose/roundcube/mailu.env | 2 +- tests/compose/webdav/mailu.env | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 81811cdb..ab5cb43a 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -50,7 +50,7 @@ plugin { fts_autoindex_exclude = \Trash {% endif %} - {% if COMPRESSION in [ 'gz', 'bz2' ] %} + {% if COMPRESSION in [ 'gz', 'bz2', 'xz', 'lz4' ] %} zlib_save = {{ COMPRESSION }} {% endif %} diff --git a/docs/compose/.env b/docs/compose/.env index 7f91c270..b4a8b218 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -97,7 +97,7 @@ WELCOME_SUBJECT=Welcome to your new email account WELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly! # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 44452e36..04148b40 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -86,7 +86,7 @@ WELCOME_SUBJECT={{ welcome_subject or 'Welcome to your new email account' }} WELCOME_BODY={{ welcome_body or 'Welcome to your new email account, if you can read this, then it is configured properly!' }} # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION={{ compression }} # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL={{ compression_level }} diff --git a/tests/compose/core/mailu.env b/tests/compose/core/mailu.env index b13e57c5..edea6a5c 100644 --- a/tests/compose/core/mailu.env +++ b/tests/compose/core/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/fetchmail/mailu.env b/tests/compose/fetchmail/mailu.env index 636a09a9..4a53ec46 100644 --- a/tests/compose/fetchmail/mailu.env +++ b/tests/compose/fetchmail/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/filters/mailu.env b/tests/compose/filters/mailu.env index b6d5ca8f..1106deb0 100644 --- a/tests/compose/filters/mailu.env +++ b/tests/compose/filters/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/rainloop/mailu.env b/tests/compose/rainloop/mailu.env index 9c31c8bb..d02b98f2 100644 --- a/tests/compose/rainloop/mailu.env +++ b/tests/compose/rainloop/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/roundcube/mailu.env b/tests/compose/roundcube/mailu.env index dc503268..e1005487 100644 --- a/tests/compose/roundcube/mailu.env +++ b/tests/compose/roundcube/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/webdav/mailu.env b/tests/compose/webdav/mailu.env index 90fb25b1..58b9810a 100644 --- a/tests/compose/webdav/mailu.env +++ b/tests/compose/webdav/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= From 2606ace1dfddcc32c00b640a95300b9a0c820635 Mon Sep 17 00:00:00 2001 From: lub Date: Sat, 21 Nov 2020 12:39:42 +0100 Subject: [PATCH 024/181] add changelog for #1694 --- towncrier/newsfragments/1694.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/1694.feature diff --git a/towncrier/newsfragments/1694.feature b/towncrier/newsfragments/1694.feature new file mode 100644 index 00000000..41548707 --- /dev/null +++ b/towncrier/newsfragments/1694.feature @@ -0,0 +1 @@ +Support configuring xz and lz4 compression for dovecot. From 0051b93077530c9950973ce2a50fea725a2b110e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 16 Dec 2020 22:39:50 +0100 Subject: [PATCH 025/181] removed unused variable --- core/admin/mailu/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 79ce49ab..ab9a8329 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -323,7 +323,7 @@ class Base(db.Model): new_data.append(rel_data) new.remove(same) del new_data[same_idx] - for i, (ch_item, ch_update) in enumerate(changed): + for i, (ch_item, _) in enumerate(changed): if ch_item is same: changed[i] = (rel_item, []) db.session.flush() From 3064a1dcff573f6dea99a90cc5b545bc6c3d2670 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 20 Dec 2020 23:38:55 +0100 Subject: [PATCH 026/181] removed call to (undefined) cli --- core/admin/mailu/manage.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 42d5b5f7..f70c5c85 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -416,7 +416,3 @@ def setmanager(domain_name, user_name='manager'): domain.managers.append(manageruser) db.session.add(domain) db.session.commit() - - -if __name__ == '__main__': - cli() From 0a594aaa2ca9ca1964d03e914e636a7be5296e10 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 20 Dec 2020 23:45:27 +0100 Subject: [PATCH 027/181] cosmetic changes --- core/admin/mailu/models.py | 83 ++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index ab9a8329..a752000e 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -30,7 +30,7 @@ class IdnaDomain(db.TypeDecorator): impl = db.String(80) def process_bind_param(self, value, dialect): - return idna.encode(value).decode("ascii").lower() + return idna.encode(value).decode('ascii').lower() def process_result_value(self, value, dialect): return idna.decode(value) @@ -46,7 +46,7 @@ class IdnaEmail(db.TypeDecorator): def process_bind_param(self, value, dialect): try: localpart, domain_name = value.split('@') - return "{0}@{1}".format( + return '{0}@{1}'.format( localpart, idna.encode(domain_name).decode('ascii'), ).lower() @@ -55,7 +55,7 @@ class IdnaEmail(db.TypeDecorator): def process_result_value(self, value, dialect): localpart, domain_name = value.split('@') - return "{0}@{1}".format( + return '{0}@{1}'.format( localpart, idna.decode(domain_name), ) @@ -70,14 +70,14 @@ class CommaSeparatedList(db.TypeDecorator): def process_bind_param(self, value, dialect): if not isinstance(value, (list, set)): - raise TypeError("Should be a list") + raise TypeError('Should be a list') for item in value: - if "," in item: - raise ValueError("Item must not contain a comma") - return ",".join(sorted(value)) + if ',' in item: + raise ValueError('Item must not contain a comma') + return ','.join(sorted(value)) def process_result_value(self, value, dialect): - return list(filter(bool, value.split(","))) if value else [] + return list(filter(bool, value.split(','))) if value else [] python_type = list @@ -103,8 +103,8 @@ class Base(db.Model): metadata = sqlalchemy.schema.MetaData( naming_convention={ - "fk": "%(table_name)s_%(column_0_name)s_fkey", - "pk": "%(table_name)s_pkey" + 'fk': '%(table_name)s_%(column_0_name)s_fkey', + 'pk': '%(table_name)s_pkey' } ) @@ -227,7 +227,7 @@ class Base(db.Model): if rel is None: itype = getattr(model, '_dict_types', {}).get(key) if itype is not None: - if itype is False: # ignore value + if itype is False: # ignore value. TODO: emit warning? del data[key] continue elif not isinstance(value, itype): @@ -372,7 +372,8 @@ class Config(Base): class Domain(Base): """ A DNS domain that has mail addresses associated to it. """ - __tablename__ = "domain" + + __tablename__ = 'domain' _dict_hide = {'users', 'managers', 'aliases'} _dict_show = {'dkim_key'} @@ -425,8 +426,8 @@ class Domain(Base): signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) def _dkim_file(self): - return app.config["DKIM_PATH"].format( - domain=self.name, selector=app.config["DKIM_SELECTOR"]) + return app.config['DKIM_PATH'].format( + domain=self.name, selector=app.config['DKIM_SELECTOR']) @property def dns_mx(self): @@ -458,7 +459,7 @@ class Domain(Base): def dkim_key(self): file_path = self._dkim_file() if os.path.exists(file_path): - with open(file_path, "rb") as handle: + with open(file_path, 'rb') as handle: return handle.read() @dkim_key.setter @@ -468,14 +469,14 @@ class Domain(Base): if os.path.exists(file_path): os.unlink(file_path) else: - with open(file_path, "wb") as handle: + with open(file_path, 'wb') as handle: handle.write(value) @property def dkim_publickey(self): dkim_key = self.dkim_key if dkim_key: - return dkim.strip_key(self.dkim_key).decode("utf8") + return dkim.strip_key(self.dkim_key).decode('utf8') def generate_dkim_key(self): self.dkim_key = dkim.gen_key() @@ -512,7 +513,7 @@ class Alternative(Base): The name "domain alias" was avoided to prevent some confusion. """ - __tablename__ = "alternative" + __tablename__ = 'alternative' name = db.Column(IdnaDomain, primary_key=True, nullable=False) domain_name = db.Column(IdnaDomain, db.ForeignKey(Domain.name)) @@ -528,7 +529,7 @@ class Relay(Base): The domain is either relayed publicly or through a specified SMTP host. """ - __tablename__ = "relay" + __tablename__ = 'relay' _dict_mandatory = {'smtp'} @@ -553,7 +554,7 @@ class Email(object): elif type(data['email']) is str: data['localpart'], data['domain'] = data['email'].rsplit('@', 1) else: - data['email'] = f"{data['localpart']}@{data['domain']}" + data['email'] = f'{data["localpart"]}@{data["domain"]}' @declarative.declared_attr def domain_name(cls): @@ -565,9 +566,9 @@ class Email(object): # especially when the mail server is reading the database. @declarative.declared_attr def email(cls): - updater = lambda context: "{0}@{1}".format( - context.current_parameters["localpart"], - context.current_parameters["domain_name"], + updater = lambda context: '{0}@{1}'.format( + context.current_parameters['localpart'], + context.current_parameters['domain_name'], ) return db.Column(IdnaEmail, primary_key=True, nullable=False, @@ -576,12 +577,12 @@ class Email(object): def sendmail(self, subject, body): """ Send an email to the address. """ - from_address = "{0}@{1}".format( + from_address = '{0}@{1}'.format( app.config['POSTMASTER'], idna.encode(app.config['DOMAIN']).decode('ascii'), ) with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: - to_address = "{0}@{1}".format( + to_address = '{0}@{1}'.format( self.localpart, idna.encode(self.domain_name).decode('ascii'), ) @@ -638,7 +639,8 @@ class Email(object): class User(Base, Email): """ A user is an email address that has a password to access a mailbox. """ - __tablename__ = "user" + + __tablename__ = 'user' _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} _dict_mandatory = {'localpart', 'domain', 'password'} @@ -689,7 +691,7 @@ class User(Base, Email): default=date(2999, 12, 31)) # Settings - displayed_name = db.Column(db.String(160), nullable=False, default="") + displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) spam_threshold = db.Column(db.Integer(), nullable=False, default=80) @@ -720,12 +722,12 @@ class User(Base, Email): self.reply_enddate > now ) - scheme_dict = {'PBKDF2': "pbkdf2_sha512", - 'BLF-CRYPT': "bcrypt", - 'SHA512-CRYPT': "sha512_crypt", - 'SHA256-CRYPT': "sha256_crypt", - 'MD5-CRYPT': "md5_crypt", - 'CRYPT': "des_crypt"} + scheme_dict = {'PBKDF2': 'pbkdf2_sha512', + 'BLF-CRYPT': 'bcrypt', + 'SHA512-CRYPT': 'sha512_crypt', + 'SHA256-CRYPT': 'sha256_crypt', + 'MD5-CRYPT': 'md5_crypt', + 'CRYPT': 'des_crypt'} def get_password_context(self): return context.CryptContext( @@ -745,7 +747,7 @@ class User(Base, Email): def set_password(self, password, hash_scheme=None, raw=False): """Set password for user with specified encryption scheme - @password: plain text password to encrypt (if raw == True the hash itself) + @password: plain text password to encrypt (if raw == True the hash itself) """ if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] @@ -787,7 +789,8 @@ class User(Base, Email): class Alias(Base, Email): """ An alias is an email address that redirects to some destination. """ - __tablename__ = "alias" + + __tablename__ = 'alias' _dict_hide = {'domain_name', 'domain', 'localpart'} @staticmethod @@ -813,7 +816,7 @@ class Alias(Base, Email): cls.localpart == localpart ), sqlalchemy.and_( cls.wildcard == True, - sqlalchemy.bindparam("l", localpart).like(cls.localpart) + sqlalchemy.bindparam('l', localpart).like(cls.localpart) ) ) ) @@ -828,7 +831,7 @@ class Alias(Base, Email): sqlalchemy.func.lower(cls.localpart) == localpart_lower ), sqlalchemy.and_( cls.wildcard == True, - sqlalchemy.bindparam("l", localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) + sqlalchemy.bindparam('l', localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) ) ) ) @@ -849,7 +852,8 @@ class Alias(Base, Email): class Token(Base): """ A token is an application password for a given user. """ - __tablename__ = "token" + + __tablename__ = 'token' _dict_recurse = True _dict_hide = {'user', 'user_email'} @@ -877,7 +881,8 @@ class Fetch(Base): """ A fetched account is a remote POP/IMAP account fetched into a local account. """ - __tablename__ = "fetch" + + __tablename__ = 'fetch' _dict_recurse = True _dict_hide = {'user_email', 'user', 'last_check', 'error'} From 815f47667bbd940c136175f73b1b10a8157c9075 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 20 Dec 2020 23:49:42 +0100 Subject: [PATCH 028/181] update dkim-key on commit only --- core/admin/mailu/models.py | 42 ++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a752000e..d38c0ad9 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -369,6 +369,21 @@ class Config(Base): value = db.Column(JSONEncoded) +@sqlalchemy.event.listens_for(db.session, 'after_commit') +def store_dkim_key(session): + """ Store DKIM key on commit + """ + + for obj in session.identity_map.values(): + if isinstance(obj, Domain): + if obj._dkim_key_changed: + file_path = obj._dkim_file() + if obj._dkim_key: + with open(file_path, 'wb') as handle: + handle.write(obj._dkim_key) + elif os.path.exists(file_path): + os.unlink(file_path) + class Domain(Base): """ A DNS domain that has mail addresses associated to it. """ @@ -424,6 +439,9 @@ class Domain(Base): max_aliases = db.Column(db.Integer, nullable=False, default=-1) max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + + _dkim_key = None + _dkim_key_changed = False def _dkim_file(self): return app.config['DKIM_PATH'].format( @@ -457,26 +475,28 @@ class Domain(Base): @property def dkim_key(self): - file_path = self._dkim_file() - if os.path.exists(file_path): - with open(file_path, 'rb') as handle: - return handle.read() + if self._dkim_key is None: + file_path = self._dkim_file() + if os.path.exists(file_path): + with open(file_path, 'rb') as handle: + self._dkim_key = handle.read() + else: + self._dkim_key = b'' + return self._dkim_key if self._dkim_key else None @dkim_key.setter def dkim_key(self, value): - file_path = self._dkim_file() + old_key = self.dkim_key if value is None: - if os.path.exists(file_path): - os.unlink(file_path) - else: - with open(file_path, 'wb') as handle: - handle.write(value) + value = b'' + self._dkim_key_changed = value != old_key + self._dkim_key = value @property def dkim_publickey(self): dkim_key = self.dkim_key if dkim_key: - return dkim.strip_key(self.dkim_key).decode('utf8') + return dkim.strip_key(dkim_key).decode('utf8') def generate_dkim_key(self): self.dkim_key = dkim.gen_key() From 3b35180b41ee298735946d3fbc3bab34f686ccbd Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 20 Dec 2020 23:50:26 +0100 Subject: [PATCH 029/181] cosmetic changes --- core/admin/mailu/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index d38c0ad9..13ebce60 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -70,7 +70,7 @@ class CommaSeparatedList(db.TypeDecorator): def process_bind_param(self, value, dialect): if not isinstance(value, (list, set)): - raise TypeError('Should be a list') + raise TypeError('Must be a list') for item in value: if ',' in item: raise ValueError('Item must not contain a comma') @@ -792,9 +792,9 @@ class User(Base, Email): return emails def send_welcome(self): - if app.config["WELCOME"]: - self.sendmail(app.config["WELCOME_SUBJECT"], - app.config["WELCOME_BODY"]) + if app.config['WELCOME']: + self.sendmail(app.config['WELCOME_SUBJECT'], + app.config['WELCOME_BODY']) @classmethod def get(cls, email): From a1a527f201c25ab553ea3d41f37c70a390abe340 Mon Sep 17 00:00:00 2001 From: Grace <30454698+c4lliope@users.noreply.github.com> Date: Mon, 4 Jan 2021 02:37:48 -0500 Subject: [PATCH 030/181] Warn people off of the documentation's K8s recipe. Based on a discussion on Matrix (2020.12.19), the helm charts are the way to go. --- docs/kubernetes/mailu/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/kubernetes/mailu/index.rst b/docs/kubernetes/mailu/index.rst index 5d3502a7..0af3942e 100644 --- a/docs/kubernetes/mailu/index.rst +++ b/docs/kubernetes/mailu/index.rst @@ -3,6 +3,10 @@ Kubernetes setup ================ +> Hold up! +> These instructions are not recommended for setting up Mailu in a production Kubernetes environment. +> Please see [the Helm Chart documentation](https://github.com/Mailu/helm-charts/blob/master/mailu/README.md). + Prequisites ----------- From 7229c89de1e1aec152f275d6e55a2077a810e40e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 6 Jan 2021 16:31:03 +0100 Subject: [PATCH 031/181] ConfigManager should not replace app.config Updated ConfigManager to only modify app.config and not replace it. Swagger does not play well, when app.config is not a real dict and it is not necessary to keep ConfigManager around after init. Also added "API" flag to config (default: disabled). --- core/admin/mailu/configuration.py | 48 +++++++++---------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 2cf6a478..a4d3c069 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -51,6 +51,7 @@ DEFAULT_CONFIG = { 'WEBMAIL': 'none', 'RECAPTCHA_PUBLIC_KEY': '', 'RECAPTCHA_PRIVATE_KEY': '', + 'API': False, # Advanced settings 'PASSWORD_SCHEME': 'PBKDF2', 'LOG_LEVEL': 'WARNING', @@ -71,7 +72,7 @@ DEFAULT_CONFIG = { 'POD_ADDRESS_RANGE': None } -class ConfigManager(dict): +class ConfigManager: """ Naive configuration manager that uses environment only """ @@ -86,19 +87,16 @@ class ConfigManager(dict): def get_host_address(self, name): # if MYSERVICE_ADDRESS is defined, use this - if '{}_ADDRESS'.format(name) in os.environ: - return os.environ.get('{}_ADDRESS'.format(name)) + if f'{name}_ADDRESS' in os.environ: + return os.environ.get(f'{name}_ADDRESS') # otherwise use the host name and resolve it - return system.resolve_address(self.config['HOST_{}'.format(name)]) + return system.resolve_address(self.config[f'HOST_{name}']) def resolve_hosts(self): - self.config["IMAP_ADDRESS"] = self.get_host_address("IMAP") - self.config["POP3_ADDRESS"] = self.get_host_address("POP3") - self.config["AUTHSMTP_ADDRESS"] = self.get_host_address("AUTHSMTP") - self.config["SMTP_ADDRESS"] = self.get_host_address("SMTP") - self.config["REDIS_ADDRESS"] = self.get_host_address("REDIS") - if self.config["WEBMAIL"] != "none": - self.config["WEBMAIL_ADDRESS"] = self.get_host_address("WEBMAIL") + for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']: + self.config[f'{key}_ADDRESS'] = self.get_host_address(key) + if self.config['WEBMAIL'] != 'none': + self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL') def __coerce_value(self, value): if isinstance(value, str) and value.lower() in ('true','yes'): @@ -108,6 +106,7 @@ class ConfigManager(dict): return value def init_app(self, app): + # get current app config self.config.update(app.config) # get environment variables self.config.update({ @@ -121,27 +120,8 @@ class ConfigManager(dict): template = self.DB_TEMPLATES[self.config['DB_FLAVOR']] self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config) - self.config['RATELIMIT_STORAGE_URL'] = 'redis://{0}/2'.format(self.config['REDIS_ADDRESS']) - self.config['QUOTA_STORAGE_URL'] = 'redis://{0}/1'.format(self.config['REDIS_ADDRESS']) - # update the app config itself - app.config = self + self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2' + self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1' - def setdefault(self, key, value): - if key not in self.config: - self.config[key] = value - return self.config[key] - - def get(self, *args): - return self.config.get(*args) - - def keys(self): - return self.config.keys() - - def __getitem__(self, key): - return self.config.get(key) - - def __setitem__(self, key, value): - self.config[key] = value - - def __contains__(self, key): - return key in self.config + # update the app config + app.config.update(self.config) From 4c258f5a6b8ba082f016cf186884b42e8a3d8549 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 6 Jan 2021 16:45:55 +0100 Subject: [PATCH 032/181] cosmetic changes & make linter happy renamed single letter variables (m => match) renamed classmethod arguments to cls (model) removed shadowing of variables (hash, context) shortened unneeded lambda functions (id) converted type ... is to isinstance(...) removed unneded imports (flask) --- core/admin/mailu/manage.py | 33 ++++++------ core/admin/mailu/models.py | 107 ++++++++++++++++++------------------- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index f70c5c85..c07ca2b7 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,9 +1,8 @@ from mailu import models from flask import current_app as app -from flask import cli as flask_cli +from flask.cli import FlaskGroup, with_appcontext -import flask import os import socket import uuid @@ -15,14 +14,14 @@ import sys db = models.db -@click.group() -def mailu(cls=flask_cli.FlaskGroup): +@click.group(cls=FlaskGroup) +def mailu(): """ Mailu command line """ @mailu.command() -@flask_cli.with_appcontext +@with_appcontext def advertise(): """ Advertise this server against statistic services. """ @@ -45,7 +44,7 @@ def advertise(): @click.argument('domain_name') @click.argument('password') @click.option('-m', '--mode') -@flask_cli.with_appcontext +@with_appcontext def admin(localpart, domain_name, password, mode='create'): """ Create an admin user 'mode' can be: @@ -89,7 +88,7 @@ def admin(localpart, domain_name, password, mode='create'): @click.argument('domain_name') @click.argument('password') @click.argument('hash_scheme', required=False) -@flask_cli.with_appcontext +@with_appcontext def user(localpart, domain_name, password, hash_scheme=None): """ Create a user """ @@ -114,7 +113,7 @@ def user(localpart, domain_name, password, hash_scheme=None): @click.argument('domain_name') @click.argument('password') @click.argument('hash_scheme', required=False) -@flask_cli.with_appcontext +@with_appcontext def password(localpart, domain_name, password, hash_scheme=None): """ Change the password of an user """ @@ -134,7 +133,7 @@ def password(localpart, domain_name, password, hash_scheme=None): @click.option('-u', '--max-users') @click.option('-a', '--max-aliases') @click.option('-q', '--max-quota-bytes') -@flask_cli.with_appcontext +@with_appcontext def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): """ Create a domain """ @@ -151,7 +150,7 @@ def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): @click.argument('domain_name') @click.argument('password_hash') @click.argument('hash_scheme') -@flask_cli.with_appcontext +@with_appcontext def user_import(localpart, domain_name, password_hash, hash_scheme = None): """ Import a user along with password hash. """ @@ -183,7 +182,7 @@ yaml_sections = [ @click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') @click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') -@flask_cli.with_appcontext +@with_appcontext def config_update(verbose=False, delete_objects=False, dry_run=False, file=None): """sync configuration with data from YAML-formatted stdin""" @@ -303,7 +302,7 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)') @click.option('-d', '--dns', is_flag=True, help='Include dns records') @click.argument('sections', nargs=-1) -@flask_cli.with_appcontext +@with_appcontext def config_dump(full=False, secrets=False, dns=False, sections=None): """dump configuration as YAML-formatted data to stdout @@ -343,7 +342,7 @@ def config_dump(full=False, secrets=False, dns=False, sections=None): @mailu.command() @click.argument('email') -@flask_cli.with_appcontext +@with_appcontext def user_delete(email): """delete user""" user = models.User.query.get(email) @@ -354,7 +353,7 @@ def user_delete(email): @mailu.command() @click.argument('email') -@flask_cli.with_appcontext +@with_appcontext def alias_delete(email): """delete alias""" alias = models.Alias.query.get(email) @@ -368,7 +367,7 @@ def alias_delete(email): @click.argument('domain_name') @click.argument('destination') @click.option('-w', '--wildcard', is_flag=True) -@flask_cli.with_appcontext +@with_appcontext def alias(localpart, domain_name, destination, wildcard=False): """ Create an alias """ @@ -392,7 +391,7 @@ def alias(localpart, domain_name, destination, wildcard=False): @click.argument('max_users') @click.argument('max_aliases') @click.argument('max_quota_bytes') -@flask_cli.with_appcontext +@with_appcontext def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): """ Set domain limits """ @@ -407,7 +406,7 @@ def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): @mailu.command() @click.argument('domain_name') @click.argument('user_name') -@flask_cli.with_appcontext +@with_appcontext def setmanager(domain_name, user_name='manager'): """ Make a user manager of a domain """ diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 13ebce60..3bf92244 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,7 +1,6 @@ from mailu import dkim from sqlalchemy.ext import declarative -from passlib import context, hash from datetime import datetime, date from email.mime import text from flask import current_app as app @@ -12,6 +11,7 @@ import sqlalchemy import re import time import os +import passlib import glob import smtplib import idna @@ -113,8 +113,8 @@ class Base(db.Model): comment = db.Column(db.String(255), nullable=True) @classmethod - def _dict_pkey(model): - return model.__mapper__.primary_key[0].name + def _dict_pkey(cls): + return cls.__mapper__.primary_key[0].name def _dict_pval(self): return getattr(self, self._dict_pkey()) @@ -187,57 +187,57 @@ class Base(db.Model): return res @classmethod - def from_dict(model, data, delete=False): + def from_dict(cls, data, delete=False): changed = [] - pkey = model._dict_pkey() + pkey = cls._dict_pkey() # handle "primary key" only - if type(data) is not dict: + if isinstance(data, dict): data = {pkey: data} # modify input data - if hasattr(model, '_dict_input'): + if hasattr(cls, '_dict_input'): try: - model._dict_input(data) + cls._dict_input(data) except Exception as reason: - raise ValueError(f'{reason}', model, None, data) + raise ValueError(f'{reason}', cls, None, data) # check for primary key (if not recursed) - if not getattr(model, '_dict_recurse', False): + if not getattr(cls, '_dict_recurse', False): if not pkey in data: - raise KeyError(f'primary key {model.__table__}.{pkey} is missing', model, pkey, data) + raise KeyError(f'primary key {cls.__table__}.{pkey} is missing', cls, pkey, data) # check data keys and values for key in list(data.keys()): # check key - if not hasattr(model, key) and not key in model.__mapper__.relationships: - raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data) + if not hasattr(cls, key) and not key in cls.__mapper__.relationships: + raise KeyError(f'unknown key {cls.__table__}.{key}', cls, key, data) # check value type value = data[key] - col = model.__mapper__.columns.get(key) + col = cls.__mapper__.columns.get(key) if col is not None: - if not ((value is None and col.nullable) or (type(value) is col.type.python_type)): - raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) + if not ((value is None and col.nullable) or (isinstance(value, col.type.python_type))): + raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) else: - rel = model.__mapper__.relationships.get(key) + rel = cls.__mapper__.relationships.get(key) if rel is None: - itype = getattr(model, '_dict_types', {}).get(key) + itype = getattr(cls, '_dict_types', {}).get(key) if itype is not None: if itype is False: # ignore value. TODO: emit warning? del data[key] continue elif not isinstance(value, itype): - raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) + raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) else: - raise NotImplementedError(f'type not defined for {model.__table__}.{key}') + raise NotImplementedError(f'type not defined for {cls.__table__}.{key}') # handle relationships - if key in model.__mapper__.relationships: - rel_model = model.__mapper__.relationships[key].argument + if key in cls.__mapper__.relationships: + rel_model = cls.__mapper__.relationships[key].argument if not isinstance(rel_model, sqlalchemy.orm.Mapper): add = rel_model.from_dict(value, delete) assert len(add) == 1 @@ -247,24 +247,24 @@ class Base(db.Model): # create item if necessary created = False - item = model.query.get(data[pkey]) if pkey in data else None + item = cls.query.get(data[pkey]) if pkey in data else None if item is None: # check for mandatory keys - missing = getattr(model, '_dict_mandatory', set()) - set(data.keys()) + missing = getattr(cls, '_dict_mandatory', set()) - set(data.keys()) if missing: - raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {model.__table__} missing', model, missing, data) + raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {cls.__table__} missing', cls, missing, data) # remove mapped relationships from data mapped = {} for key in list(data.keys()): - if key in model.__mapper__.relationships: - if isinstance(model.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): + if key in cls.__mapper__.relationships: + if isinstance(cls.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): mapped[key] = data[key] del data[key] # create new item - item = model(**data) + item = cls(**data) created = True # and update mapped relationships (below) @@ -278,14 +278,14 @@ class Base(db.Model): if key == pkey: continue - if key in model.__mapper__.relationships: + if key in cls.__mapper__.relationships: # update relationship - rel_model = model.__mapper__.relationships[key].argument + rel_model = cls.__mapper__.relationships[key].argument if isinstance(rel_model, sqlalchemy.orm.Mapper): rel_model = rel_model.class_ # add (and create) referenced items cur = getattr(item, key) - old = sorted(cur, key=lambda i:id(i)) + old = sorted(cur, key=id) new = [] for rel_data in value: # get or create related item @@ -331,16 +331,16 @@ class Base(db.Model): break # remember changes - new = sorted(new, key=lambda i:id(i)) + new = sorted(new, key=id) if new != old: updated.append((key, old, new)) else: # update key old = getattr(item, key) - if type(old) is list: + if isinstance(old, list): # deduplicate list value - assert type(value) is list + assert isinstance(value, list) value = set(value) old = set(old) if not delete: @@ -408,19 +408,19 @@ class Domain(Base): if 'dkim_key' in data: key = data['dkim_key'] if key is not None: - if type(key) is list: + if isinstance(key, list): key = ''.join(key) - if type(key) is str: + if isinstance(key, str): key = ''.join(key.strip().split()) # removes all whitespace if key == 'generate': data['dkim_key'] = dkim.gen_key() elif key: - m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) - if m is not None: - key = key[m.end():] - m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) - if m is not None: - key = key[:m.start()] + match = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) + if match is not None: + key = key[match.end():] + match = re.search('-----END (RSA )?PRIVATE KEY-----$', key) + if match is not None: + key = key[:match.start()] key = '\n'.join(wrap(key, 64)) key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') try: @@ -428,7 +428,7 @@ class Domain(Base): except: raise ValueError('invalid dkim key') else: - data['dkim_key'] = key + data['dkim_key'] = key else: data['dkim_key'] = None @@ -505,8 +505,7 @@ class Domain(Base): for email in self.users + self.aliases: if email.localpart == localpart: return True - else: - return False + return False def check_mx(self): try: @@ -519,7 +518,7 @@ class Domain(Base): return False def __str__(self): - return self.name + return str(self.name) def __eq__(self, other): try: @@ -541,7 +540,7 @@ class Alternative(Base): backref=db.backref('alternatives', cascade='all, delete-orphan')) def __str__(self): - return self.name + return str(self.name) class Relay(Base): @@ -557,7 +556,7 @@ class Relay(Base): smtp = db.Column(db.String(80), nullable=True) def __str__(self): - return self.name + return str(self.name) class Email(object): @@ -571,7 +570,7 @@ class Email(object): if 'email' in data: if 'localpart' in data or 'domain' in data: raise ValueError('ambigous key email and localpart/domain') - elif type(data['email']) is str: + elif isinstance(data['email'], str): data['localpart'], data['domain'] = data['email'].rsplit('@', 1) else: data['email'] = f'{data["localpart"]}@{data["domain"]}' @@ -653,7 +652,7 @@ class Email(object): return pure_alias.destination def __str__(self): - return self.email + return str(self.email) class User(Base, Email): @@ -750,7 +749,7 @@ class User(Base, Email): 'CRYPT': 'des_crypt'} def get_password_context(self): - return context.CryptContext( + return passlib.context.CryptContext( schemes=self.scheme_dict.values(), default=self.scheme_dict[app.config['PASSWORD_SCHEME']], ) @@ -818,7 +817,7 @@ class Alias(Base, Email): Email._dict_input(data) # handle comma delimited string for backwards compability dst = data.get('destination') - if type(dst) is str: + if isinstance(dst, str): data['destination'] = list([adr.strip() for adr in dst.split(',')]) domain = db.relationship(Domain, @@ -888,10 +887,10 @@ class Token(Base): ip = db.Column(db.String(255)) def check_password(self, password): - return hash.sha256_crypt.verify(password, self.password) + return passlib.hash.sha256_crypt.verify(password, self.password) def set_password(self, password): - self.password = hash.sha256_crypt.using(rounds=1000).hash(password) + self.password = passlib.hash.sha256_crypt.using(rounds=1000).hash(password) def __str__(self): return self.comment or self.ip From 6629aa3ff824c57b1c752fdf7a7e7ea87423bc5e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 6 Jan 2021 17:05:21 +0100 Subject: [PATCH 033/181] first try at api using flask-restx & marshmallow --- core/admin/mailu/__init__.py | 6 +- core/admin/mailu/api/__init__.py | 38 +++++ core/admin/mailu/api/common.py | 8 + core/admin/mailu/api/v1/__init__.py | 27 ++++ core/admin/mailu/api/v1/domains.py | 183 +++++++++++++++++++++++ core/admin/mailu/manage.py | 33 +++-- core/admin/mailu/schemas.py | 222 ++++++++++++++++++++++++++++ 7 files changed, 502 insertions(+), 15 deletions(-) create mode 100644 core/admin/mailu/api/__init__.py create mode 100644 core/admin/mailu/api/common.py create mode 100644 core/admin/mailu/api/v1/__init__.py create mode 100644 core/admin/mailu/api/v1/domains.py create mode 100644 core/admin/mailu/schemas.py diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 4de3e580..9ab90add 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -41,15 +41,17 @@ def create_app_from_config(config): ) # Import views - from mailu import ui, internal + from mailu import ui, internal, api app.register_blueprint(ui.ui, url_prefix='/ui') app.register_blueprint(internal.internal, url_prefix='/internal') + if app.config.get('API'): + api.register(app) return app def create_app(): - """ Create a new application based on the config module + """ Create a new application based on the config module """ config = configuration.ConfigManager() return create_app_from_config(config) diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py new file mode 100644 index 00000000..6e7d6386 --- /dev/null +++ b/core/admin/mailu/api/__init__.py @@ -0,0 +1,38 @@ +from flask import redirect, url_for + +# import api version(s) +from . import v1 + +# api +ROOT='/api' +ACTIVE=v1 + +# patch url for swaggerui static assets +from flask_restx.apidoc import apidoc +apidoc.static_url_path = f'{ROOT}/swaggerui' + +def register(app): + + # register api bluprint(s) + app.register_blueprint(v1.blueprint, url_prefix=f'{ROOT}/v{int(v1.VERSION)}') + + # add redirect to current api version + @app.route(f'{ROOT}/') + def redir(): + return redirect(url_for(f'{ACTIVE.blueprint.name}.root')) + + # swagger ui config + app.config.SWAGGER_UI_DOC_EXPANSION = 'list' + app.config.SWAGGER_UI_OPERATION_ID = True + app.config.SWAGGER_UI_REQUEST_DURATION = True + + # TODO: remove patch of static assets for debugging + import os + if 'STATIC_ASSETS' in os.environ: + app.blueprints['ui'].static_folder = os.environ['STATIC_ASSETS'] + +# TODO: authentication via username + password +# TODO: authentication via api token +# TODO: api access for all users (via token) +# TODO: use permissions from "manager_of" +# TODO: switch to marshmallow, as parser is deprecated. use flask_accepts? diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py new file mode 100644 index 00000000..700835ac --- /dev/null +++ b/core/admin/mailu/api/common.py @@ -0,0 +1,8 @@ +from .. import models + +def fqdn_in_use(*names): + for name in names: + for model in models.Domain, models.Alternative, models.Relay: + if model.query.get(name): + return model + return None diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py new file mode 100644 index 00000000..c6de6fa4 --- /dev/null +++ b/core/admin/mailu/api/v1/__init__.py @@ -0,0 +1,27 @@ +from flask import Blueprint +from flask_restx import Api, fields + +VERSION = 1.0 + +blueprint = Blueprint(f'api_v{int(VERSION)}', __name__) + +api = Api( + blueprint, version=f'{VERSION:.1f}', + title='Mailu API', default_label='Mailu', + validate=True +) + +response_fields = api.model('Response', { + 'code': fields.Integer, + 'message': fields.String, +}) + +error_fields = api.model('Error', { + 'errors': fields.Nested(api.model('Error_Key', { + 'key': fields.String, + 'message':fields.String + })), + 'message': fields.String, +}) + +from . import domains diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py new file mode 100644 index 00000000..7afd26b9 --- /dev/null +++ b/core/admin/mailu/api/v1/domains.py @@ -0,0 +1,183 @@ +from flask_restx import Resource, fields, abort + +from . import api, response_fields, error_fields +from .. import common +from ... import models + +db = models.db + +dom = api.namespace('domain', description='Domain operations') +alt = api.namespace('alternative', description='Alternative operations') + +domain_fields = api.model('Domain', { + 'name': fields.String(description='FQDN', example='example.com', required=True), + 'comment': fields.String(description='a comment'), + 'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1), + 'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1), + 'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0), + 'signup_enabled': fields.Boolean(description='allow signup'), +# 'dkim_key': fields.String, + 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example.com')), +}) +# TODO - name ist required on creation but immutable on change +# TODO - name and alteranatives need to be checked to be a fqdn (regex) + +domain_parser = api.parser() +domain_parser.add_argument('max_users', type=int, help='maximum number of users') +# TODO ... add more (or use marshmallow) + +alternative_fields = api.model('Domain', { + 'name': fields.String(description='alternative FQDN', example='example.com', required=True), + 'domain': fields.String(description='domain FQDN', example='example.com', required=True), + 'dkim_key': fields.String, +}) +# TODO: domain and name are not always required and can't be changed + + +@dom.route('') +class Domains(Resource): + + @dom.doc('list_domain') + @dom.marshal_with(domain_fields, as_list=True, skip_none=True, mask=['dkim_key']) + def get(self): + """ List domains """ + return models.Domain.query.all() + + @dom.doc('create_domain') + @dom.expect(domain_fields) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(409, 'Duplicate domain name', error_fields) + def post(self): + """ Create a new domain """ + data = api.payload + if common.fqdn_in_use(data['name']): + abort(409, f'Duplicate domain name {data["name"]!r}', errors={ + 'name': data['name'], + }) + for item, created in models.Domain.from_dict(data): + if not created: + abort(409, f'Duplicate domain name {item.name!r}', errors={ + 'alternatives': item.name, + }) + db.session.add(item) + db.session.commit() + +@dom.route('/') +class Domain(Resource): + + @dom.doc('get_domain') + @dom.response(200, 'Success', domain_fields) + @dom.response(404, 'Domain not found') + @dom.marshal_with(domain_fields) + def get(self, name): + """ Find domain by name """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + return domain + + @dom.doc('update_domain') + @dom.expect(domain_fields) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(404, 'Domain not found') + def put(self, name): + """ Update an existing domain """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + data = api.payload + data['name'] = name + for item, created in models.Domain.from_dict(data): + if created is True: + db.session.add(item) + db.session.commit() + + @dom.doc('modify_domain') + @dom.expect(domain_parser) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(404, 'Domain not found') + def post(self, name=None): + """ Updates domain with form data """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + data = dict(domain_parser.parse_args()) + data['name'] = name + for item, created in models.Domain.from_dict(data): + if created is True: + db.session.add(item) + # TODO: flush? + db.session.commit() + + @dom.doc('delete_domain') + @dom.response(200, 'Success', response_fields) + @dom.response(404, 'Domain not found') + def delete(self, name=None): + """ Delete domain """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + db.session.delete(domain) + db.session.commit() + + +# @dom.route('//alternative') +# @alt.route('') +# class Alternatives(Resource): + +# @alt.doc('alternatives_list') +# @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=['dkim_key']) +# def get(self, name=None): +# """ List alternatives (of domain) """ +# if name is None: +# return models.Alternative.query.all() +# else: +# return models.Alternative.query.filter_by(domain_name = name).all() + +# @alt.doc('alternative_create') +# @alt.expect(alternative_fields) +# @alt.response(200, 'Success', response_fields) +# @alt.response(400, 'Input validation exception', error_fields) +# @alt.response(404, 'Domain not found') +# @alt.response(409, 'Duplicate domain name', error_fields) +# def post(self, name=None): +# """ Create new alternative (for domain) """ +# # abort(501) +# data = api.payload +# if name is not None: +# data['name'] = name +# domain = models.Domain.query.get(name) +# if not domain: +# abort(404) +# if common.fqdn_in_use(data['name']): +# abort(409, f'Duplicate domain name {data["name"]!r}', errors={ +# 'name': data['name'], +# }) +# for item, created in models.Alternative.from_dict(data): +# # TODO: handle creation of domain +# if not created: +# abort(409, f'Duplicate domain name {item.name!r}', errors={ +# 'alternatives': item.name, +# }) +# # db.session.add(item) +# # db.session.commit() + +# @dom.route('//alternative/') +# @alt.route('/') +# class Alternative(Resource): +# def get(self, name, alt=None): +# """ Find alternative (of domain) """ +# abort(501) +# def put(self, name, alt=None): +# """ Update alternative (of domain) """ +# abort(501) +# def post(self, name, alt=None): +# """ Update alternative (of domain) with form data """ +# abort(501) +# def delete(self, name, alt=None): +# """ Delete alternative (for domain) """ +# abort(501) + diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index c07ca2b7..569c161c 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,4 +1,5 @@ from mailu import models +from .schemas import schemas from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext @@ -320,24 +321,30 @@ def config_dump(full=False, secrets=False, dns=False, sections=None): return super().increase_indent(flow, False) if sections: - check = dict(yaml_sections) for section in sections: - if section not in check: + if section not in schemas: print(f'[ERROR] Invalid section: {section}') return 1 + else: + sections = sorted(schemas.keys()) - extra = [] - if dns: - extra.append('dns') +# TODO: create World Schema and dump only this with Word.dumps ? - config = {} - for section, model in yaml_sections: - if not sections or section in sections: - dump = [item.to_dict(full, secrets, extra) for item in model.query.all()] - if len(dump): - config[section] = dump - - yaml.dump(config, sys.stdout, Dumper=spacedDumper, default_flow_style=False, allow_unicode=True) + for section in sections: + schema = schemas[section](many=True) + schema.context.update({ + 'full': full, + 'secrets': secrets, + 'dns': dns, + }) + yaml.dump( + {section: schema.dump(schema.Meta.model.query.all())}, + sys.stdout, + Dumper=spacedDumper, + default_flow_style=False, + allow_unicode=True + ) + sys.stdout.write('\n') @mailu.command() diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py new file mode 100644 index 00000000..20aa98c9 --- /dev/null +++ b/core/admin/mailu/schemas.py @@ -0,0 +1,222 @@ +import marshmallow +import flask_marshmallow + +from . import models + + +ma = flask_marshmallow.Marshmallow() + + +class BaseSchema(ma.SQLAlchemyAutoSchema): + + SKIP_IF = { + 'comment': {'', None} + } + + @marshmallow.post_dump + def remove_skip_values(self, data, many, **kwargs): + print(repr(data), self.context) + return { + key: value for key, value in data.items() + if key not in self.SKIP_IF or value not in self.SKIP_IF[key] + } + +class BaseMeta: + exclude = ['created_at', 'updated_at'] + + +class DomainSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.Domain + + # _dict_hide = {'users', 'managers', 'aliases'} + # _dict_show = {'dkim_key'} + # _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} + # _dict_secret = {'dkim_key'} + # _dict_types = { + # 'dkim_key': (bytes, type(None)), + # 'dkim_publickey': False, + # 'dns_mx': False, + # 'dns_spf': False, + # 'dns_dkim': False, + # 'dns_dmarc': False, + # } + # _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} + # @staticmethod + # def _dict_input(data): + # if 'dkim_key' in data: + # key = data['dkim_key'] + # if key is not None: + # if type(key) is list: + # key = ''.join(key) + # if type(key) is str: + # key = ''.join(key.strip().split()) # removes all whitespace + # if key == 'generate': + # data['dkim_key'] = dkim.gen_key() + # elif key: + # m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) + # if m is not None: + # key = key[m.end():] + # m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) + # if m is not None: + # key = key[:m.start()] + # key = '\n'.join(wrap(key, 64)) + # key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + # try: + # dkim.strip_key(key) + # except: + # raise ValueError('invalid dkim key') + # else: + # data['dkim_key'] = key + # else: + # data['dkim_key'] = None + + # name = db.Column(IdnaDomain, primary_key=True, nullable=False) + # managers = db.relationship('User', secondary=managers, + # backref=db.backref('manager_of'), lazy='dynamic') + # max_users = db.Column(db.Integer, nullable=False, default=-1) + # max_aliases = db.Column(db.Integer, nullable=False, default=-1) + # max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) + # signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + + +class UserSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.User + + # _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} + # _dict_mandatory = {'localpart', 'domain', 'password'} + # @classmethod + # def _dict_input(cls, data): + # Email._dict_input(data) + # # handle password + # if 'password' in data: + # if 'password_hash' in data or 'hash_scheme' in data: + # raise ValueError('ambigous key password and password_hash/hash_scheme') + # # check (hashed) password + # password = data['password'] + # if password.startswith('{') and '}' in password: + # scheme = password[1:password.index('}')] + # if scheme not in cls.scheme_dict: + # raise ValueError(f'invalid password scheme {scheme!r}') + # else: + # raise ValueError(f'invalid hashed password {password!r}') + # elif 'password_hash' in data and 'hash_scheme' in data: + # if data['hash_scheme'] not in cls.scheme_dict: + # raise ValueError(f'invalid password scheme {scheme!r}') + # data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + # del data['hash_scheme'] + # del data['password_hash'] + + # domain = db.relationship(Domain, + # backref=db.backref('users', cascade='all, delete-orphan')) + # password = db.Column(db.String(255), nullable=False) + # quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) + # quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) + # global_admin = db.Column(db.Boolean(), nullable=False, default=False) + # enabled = db.Column(db.Boolean(), nullable=False, default=True) + + # # Features + # enable_imap = db.Column(db.Boolean(), nullable=False, default=True) + # enable_pop = db.Column(db.Boolean(), nullable=False, default=True) + + # # Filters + # forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) + # forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) + # forward_keep = db.Column(db.Boolean(), nullable=False, default=True) + # reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) + # reply_subject = db.Column(db.String(255), nullable=True, default=None) + # reply_body = db.Column(db.Text(), nullable=True, default=None) + # reply_startdate = db.Column(db.Date, nullable=False, + # default=date(1900, 1, 1)) + # reply_enddate = db.Column(db.Date, nullable=False, + # default=date(2999, 12, 31)) + + # # Settings + # displayed_name = db.Column(db.String(160), nullable=False, default='') + # spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) + # spam_threshold = db.Column(db.Integer(), nullable=False, default=80) + +class AliasSchema(BaseSchema): + class Meta(BaseMeta): + model = models.Alias + exclude = BaseMeta.exclude + ['localpart'] + # TODO look for good way to exclude secrets, unverbose and defaults + + # _dict_hide = {'domain_name', 'domain', 'localpart'} + # @staticmethod + # def _dict_input(data): + # Email._dict_input(data) + # # handle comma delimited string for backwards compability + # dst = data.get('destination') + # if type(dst) is str: + # data['destination'] = list([adr.strip() for adr in dst.split(',')]) + + +class TokenSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.Token + + # _dict_recurse = True + # _dict_hide = {'user', 'user_email'} + # _dict_mandatory = {'password'} + + # id = db.Column(db.Integer(), primary_key=True) + # user_email = db.Column(db.String(255), db.ForeignKey(User.email), + # nullable=False) + # user = db.relationship(User, + # backref=db.backref('tokens', cascade='all, delete-orphan')) + # password = db.Column(db.String(255), nullable=False) + # ip = db.Column(db.String(255)) + + +class FetchSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.Fetch + + # _dict_recurse = True + # _dict_hide = {'user_email', 'user', 'last_check', 'error'} + # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} + # _dict_secret = {'password'} + + # id = db.Column(db.Integer(), primary_key=True) + # user_email = db.Column(db.String(255), db.ForeignKey(User.email), + # nullable=False) + # user = db.relationship(User, + # backref=db.backref('fetches', cascade='all, delete-orphan')) + # protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) + # host = db.Column(db.String(255), nullable=False) + # port = db.Column(db.Integer(), nullable=False) + # tls = db.Column(db.Boolean(), nullable=False, default=False) + # username = db.Column(db.String(255), nullable=False) + # password = db.Column(db.String(255), nullable=False) + # keep = db.Column(db.Boolean(), nullable=False, default=False) + # last_check = db.Column(db.DateTime, nullable=True) + # error = db.Column(db.String(1023), nullable=True) + + +class ConfigSchema(ma.SQLAlchemySchema): + class Meta: + model = models.Config +# created_at = ma.auto_field(dump_only=True) +# updated_at = ma.auto_field(dump_only=True) + comment = ma.auto_field() + name = ma.auto_field(required=True) + value = ma.auto_field(required=True) + +class RelaySchema(BaseSchema): + class Meta(BaseMeta): + model = models.Relay +# created_at = ma.auto_field(dump_only=True) +# updated_at = ma.auto_field(dump_only=True) +# comment = ma.auto_field() +# name = ma.auto_field(required=True) +# smtp = ma.auto_field(required=True) + +schemas = { + 'domains': DomainSchema, + 'relays': RelaySchema, + 'users': UserSchema, + 'aliases': AliasSchema, +# 'config': ConfigSchema, +} From b3f8dacdadcc562c34205fb9c8e2fe95c90394e8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 8 Jan 2021 14:17:28 +0100 Subject: [PATCH 034/181] add docstrings and make linter happy --- core/admin/mailu/api/__init__.py | 13 ++++++++++--- core/admin/mailu/api/common.py | 6 ++++++ core/admin/mailu/api/v1/__init__.py | 5 +++++ core/admin/mailu/api/v1/domains.py | 24 ++++++++++++++---------- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py index 6e7d6386..a4d8689f 100644 --- a/core/admin/mailu/api/__init__.py +++ b/core/admin/mailu/api/__init__.py @@ -1,4 +1,11 @@ +""" +Mailu API +""" + +import os + from flask import redirect, url_for +from flask_restx.apidoc import apidoc # import api version(s) from . import v1 @@ -8,10 +15,11 @@ ROOT='/api' ACTIVE=v1 # patch url for swaggerui static assets -from flask_restx.apidoc import apidoc apidoc.static_url_path = f'{ROOT}/swaggerui' def register(app): + """ Register api blueprint in flask app + """ # register api bluprint(s) app.register_blueprint(v1.blueprint, url_prefix=f'{ROOT}/v{int(v1.VERSION)}') @@ -26,8 +34,7 @@ def register(app): app.config.SWAGGER_UI_OPERATION_ID = True app.config.SWAGGER_UI_REQUEST_DURATION = True - # TODO: remove patch of static assets for debugging - import os + # TODO: remove patch of static asset location if 'STATIC_ASSETS' in os.environ: app.blueprints['ui'].static_folder = os.environ['STATIC_ASSETS'] diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py index 700835ac..000bb2d6 100644 --- a/core/admin/mailu/api/common.py +++ b/core/admin/mailu/api/common.py @@ -1,6 +1,12 @@ +""" +Common functions for all API versions +""" + from .. import models def fqdn_in_use(*names): + """ Checks if fqdn is used + """ for name in names: for model in models.Domain, models.Alternative, models.Relay: if model.query.get(name): diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py index c6de6fa4..c4379074 100644 --- a/core/admin/mailu/api/v1/__init__.py +++ b/core/admin/mailu/api/v1/__init__.py @@ -1,3 +1,7 @@ +""" +API Blueprint +""" + from flask import Blueprint from flask_restx import Api, fields @@ -24,4 +28,5 @@ error_fields = api.model('Error', { 'message': fields.String, }) +# import api namespaces (below field defs to avoid circular reference) from . import domains diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py index 7afd26b9..82ed1fde 100644 --- a/core/admin/mailu/api/v1/domains.py +++ b/core/admin/mailu/api/v1/domains.py @@ -1,3 +1,7 @@ +""" +API: domain +""" + from flask_restx import Resource, fields, abort from . import api, response_fields, error_fields @@ -9,6 +13,7 @@ db = models.db dom = api.namespace('domain', description='Domain operations') alt = api.namespace('alternative', description='Alternative operations') +# TODO: use marshmallow domain_fields = api.model('Domain', { 'name': fields.String(description='FQDN', example='example.com', required=True), 'comment': fields.String(description='a comment'), @@ -19,12 +24,12 @@ domain_fields = api.model('Domain', { # 'dkim_key': fields.String, 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example.com')), }) -# TODO - name ist required on creation but immutable on change -# TODO - name and alteranatives need to be checked to be a fqdn (regex) +# TODO: name ist required on creation but immutable on change +# TODO: name and alteranatives need to be checked to be a fqdn (regex) domain_parser = api.parser() domain_parser.add_argument('max_users', type=int, help='maximum number of users') -# TODO ... add more (or use marshmallow) +# TODO: ... add more (or use marshmallow) alternative_fields = api.model('Domain', { 'name': fields.String(description='alternative FQDN', example='example.com', required=True), @@ -65,7 +70,7 @@ class Domains(Resource): @dom.route('/') class Domain(Resource): - + @dom.doc('get_domain') @dom.response(200, 'Success', domain_fields) @dom.response(404, 'Domain not found') @@ -76,7 +81,7 @@ class Domain(Resource): if not domain: abort(404) return domain - + @dom.doc('update_domain') @dom.expect(domain_fields) @dom.response(200, 'Success', response_fields) @@ -109,7 +114,7 @@ class Domain(Resource): for item, created in models.Domain.from_dict(data): if created is True: db.session.add(item) - # TODO: flush? + # TODO: flush? db.session.commit() @dom.doc('delete_domain') @@ -123,11 +128,11 @@ class Domain(Resource): db.session.delete(domain) db.session.commit() - + # @dom.route('//alternative') # @alt.route('') # class Alternatives(Resource): - + # @alt.doc('alternatives_list') # @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=['dkim_key']) # def get(self, name=None): @@ -136,7 +141,7 @@ class Domain(Resource): # return models.Alternative.query.all() # else: # return models.Alternative.query.filter_by(domain_name = name).all() - + # @alt.doc('alternative_create') # @alt.expect(alternative_fields) # @alt.response(200, 'Success', response_fields) @@ -180,4 +185,3 @@ class Domain(Resource): # def delete(self, name, alt=None): # """ Delete alternative (for domain) """ # abort(501) - From 82cf0d843f70584e7d0779bb58088e295193c63b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 8 Jan 2021 14:22:11 +0100 Subject: [PATCH 035/181] fix sqlalchemy column definitions --- core/admin/mailu/models.py | 45 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 3bf92244..625d1fd6 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -369,6 +369,7 @@ class Config(Base): value = db.Column(JSONEncoded) +# TODO: use sqlalchemy.event.listen() on a store method of object? @sqlalchemy.event.listens_for(db.session, 'after_commit') def store_dkim_key(session): """ Store DKIM key on commit @@ -437,8 +438,8 @@ class Domain(Base): backref=db.backref('manager_of'), lazy='dynamic') max_users = db.Column(db.Integer, nullable=False, default=-1) max_aliases = db.Column(db.Integer, nullable=False, default=-1) - max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) - signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + max_quota_bytes = db.Column(db.BigInteger, nullable=False, default=0) + signup_enabled = db.Column(db.Boolean, nullable=False, default=False) _dkim_key = None _dkim_key_changed = False @@ -688,22 +689,22 @@ class User(Base, Email): domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) - quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) - global_admin = db.Column(db.Boolean(), nullable=False, default=False) - enabled = db.Column(db.Boolean(), nullable=False, default=True) + quota_bytes = db.Column(db.BigInteger, nullable=False, default=10**9) + quota_bytes_used = db.Column(db.BigInteger, nullable=False, default=0) + global_admin = db.Column(db.Boolean, nullable=False, default=False) + enabled = db.Column(db.Boolean, nullable=False, default=True) # Features - enable_imap = db.Column(db.Boolean(), nullable=False, default=True) - enable_pop = db.Column(db.Boolean(), nullable=False, default=True) + enable_imap = db.Column(db.Boolean, nullable=False, default=True) + enable_pop = db.Column(db.Boolean, nullable=False, default=True) # Filters - forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) - forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) - forward_keep = db.Column(db.Boolean(), nullable=False, default=True) - reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) + forward_enabled = db.Column(db.Boolean, nullable=False, default=False) + forward_destination = db.Column(CommaSeparatedList, nullable=True, default=list) + forward_keep = db.Column(db.Boolean, nullable=False, default=True) + reply_enabled = db.Column(db.Boolean, nullable=False, default=False) reply_subject = db.Column(db.String(255), nullable=True, default=None) - reply_body = db.Column(db.Text(), nullable=True, default=None) + reply_body = db.Column(db.Text, nullable=True, default=None) reply_startdate = db.Column(db.Date, nullable=False, default=date(1900, 1, 1)) reply_enddate = db.Column(db.Date, nullable=False, @@ -711,8 +712,8 @@ class User(Base, Email): # Settings displayed_name = db.Column(db.String(160), nullable=False, default='') - spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) - spam_threshold = db.Column(db.Integer(), nullable=False, default=80) + spam_enabled = db.Column(db.Boolean, nullable=False, default=True) + spam_threshold = db.Column(db.Integer, nullable=False, default=80) # Flask-login attributes is_authenticated = True @@ -822,8 +823,8 @@ class Alias(Base, Email): domain = db.relationship(Domain, backref=db.backref('aliases', cascade='all, delete-orphan')) - wildcard = db.Column(db.Boolean(), nullable=False, default=False) - destination = db.Column(CommaSeparatedList, nullable=False, default=[]) + wildcard = db.Column(db.Boolean, nullable=False, default=False) + destination = db.Column(CommaSeparatedList, nullable=False, default=list) @classmethod def resolve(cls, localpart, domain_name): @@ -878,7 +879,7 @@ class Token(Base): _dict_hide = {'user', 'user_email'} _dict_mandatory = {'password'} - id = db.Column(db.Integer(), primary_key=True) + id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) user = db.relationship(User, @@ -908,18 +909,18 @@ class Fetch(Base): _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} _dict_secret = {'password'} - id = db.Column(db.Integer(), primary_key=True) + id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) user = db.relationship(User, backref=db.backref('fetches', cascade='all, delete-orphan')) protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) host = db.Column(db.String(255), nullable=False) - port = db.Column(db.Integer(), nullable=False) - tls = db.Column(db.Boolean(), nullable=False, default=False) + port = db.Column(db.Integer, nullable=False) + tls = db.Column(db.Boolean, nullable=False, default=False) username = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False) - keep = db.Column(db.Boolean(), nullable=False, default=False) + keep = db.Column(db.Boolean, nullable=False, default=False) last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) From dc42d375e262d9d1d4f34364dbd68c50e5fd452f Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 8 Jan 2021 14:22:59 +0100 Subject: [PATCH 036/181] added filtering of keys and default value --- core/admin/mailu/schemas.py | 88 +++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 20aa98c9..c2a3d03d 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1,4 +1,5 @@ import marshmallow +import sqlalchemy import flask_marshmallow from . import models @@ -6,27 +7,51 @@ from . import models ma = flask_marshmallow.Marshmallow() - +import collections class BaseSchema(ma.SQLAlchemyAutoSchema): - SKIP_IF = { - 'comment': {'', None} - } + class Meta: + base_hide_always = {'created_at', 'updated_at'} + base_hide_secrets = set() + base_hide_by_value = { +# 'comment': {'', None} + } @marshmallow.post_dump def remove_skip_values(self, data, many, **kwargs): - print(repr(data), self.context) +# print(repr(data), self.context) + + # always hide + hide_by_key = self.Meta.base_hide_always | set(getattr(self.Meta, 'hide_always', ())) + + # hide secrets + if not self.context.get('secrets'): + hide_by_key |= self.Meta.base_hide_secrets + hide_by_key |= set(getattr(self.Meta, 'hide_secrets', ())) + + # hide by value + hide_by_value = self.Meta.base_hide_by_value | getattr(self.Meta, 'hide_by_value', {}) + + # hide defaults + if not self.context.get('full'): + for column in self.Meta.model.__table__.columns: +# print(column.name, column.default.arg if isinstance(column.default, sqlalchemy.sql.schema.ColumnDefault) else column.default) +# alias.destiantion has default [] - is this okay. how to check it? + if column.name not in hide_by_key: + hide_by_value.setdefault(column.name, set()).add(None if column.default is None else column.default.arg) + return { key: value for key, value in data.items() - if key not in self.SKIP_IF or value not in self.SKIP_IF[key] + if + not isinstance(value, collections.Hashable) + or( + key not in hide_by_key + and + (key not in hide_by_value or value not in hide_by_value[key])) } -class BaseMeta: - exclude = ['created_at', 'updated_at'] - - -class DomainSchema(ma.SQLAlchemyAutoSchema): - class Meta: +class DomainSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.Domain # _dict_hide = {'users', 'managers', 'aliases'} @@ -80,8 +105,8 @@ class DomainSchema(ma.SQLAlchemyAutoSchema): # signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) -class UserSchema(ma.SQLAlchemyAutoSchema): - class Meta: +class UserSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.User # _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} @@ -138,12 +163,14 @@ class UserSchema(ma.SQLAlchemyAutoSchema): # spam_threshold = db.Column(db.Integer(), nullable=False, default=80) class AliasSchema(BaseSchema): - class Meta(BaseMeta): + class Meta(BaseSchema.Meta): model = models.Alias - exclude = BaseMeta.exclude + ['localpart'] - # TODO look for good way to exclude secrets, unverbose and defaults + hide_always = {'localpart'} + hide_secrets = {'wildcard'} + hide_by_value = { + 'destination': set([]) # always hide empty lists?! + } - # _dict_hide = {'domain_name', 'domain', 'localpart'} # @staticmethod # def _dict_input(data): # Email._dict_input(data) @@ -153,8 +180,8 @@ class AliasSchema(BaseSchema): # data['destination'] = list([adr.strip() for adr in dst.split(',')]) -class TokenSchema(ma.SQLAlchemyAutoSchema): - class Meta: +class TokenSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.Token # _dict_recurse = True @@ -170,8 +197,8 @@ class TokenSchema(ma.SQLAlchemyAutoSchema): # ip = db.Column(db.String(255)) -class FetchSchema(ma.SQLAlchemyAutoSchema): - class Meta: +class FetchSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.Fetch # _dict_recurse = True @@ -195,23 +222,18 @@ class FetchSchema(ma.SQLAlchemyAutoSchema): # error = db.Column(db.String(1023), nullable=True) -class ConfigSchema(ma.SQLAlchemySchema): - class Meta: +class ConfigSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.Config -# created_at = ma.auto_field(dump_only=True) -# updated_at = ma.auto_field(dump_only=True) - comment = ma.auto_field() +# TODO: how to mark keys as "required" while unserializing (in certain use cases/API) name = ma.auto_field(required=True) value = ma.auto_field(required=True) + class RelaySchema(BaseSchema): - class Meta(BaseMeta): + class Meta(BaseSchema.Meta): model = models.Relay -# created_at = ma.auto_field(dump_only=True) -# updated_at = ma.auto_field(dump_only=True) -# comment = ma.auto_field() -# name = ma.auto_field(required=True) -# smtp = ma.auto_field(required=True) + schemas = { 'domains': DomainSchema, From 7413f9b7b42d8797f5de6203260761f2561859f0 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 13 Jan 2021 00:05:43 +0100 Subject: [PATCH 037/181] config_dump now using marshmallow --- core/admin/mailu/manage.py | 46 +--- core/admin/mailu/models.py | 4 +- core/admin/mailu/schemas.py | 498 ++++++++++++++++++++++++------------ 3 files changed, 346 insertions(+), 202 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 569c161c..a67abdf7 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,5 +1,5 @@ from mailu import models -from .schemas import schemas +from .schemas import MailuConfig, MailuSchema from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext @@ -310,41 +310,17 @@ def config_dump(full=False, secrets=False, dns=False, sections=None): SECTIONS can be: domains, relays, users, aliases """ - class spacedDumper(yaml.Dumper): + try: + config = MailuConfig(sections) + except ValueError as reason: + print(f'[ERROR] {reason}') + return 1 - def write_line_break(self, data=None): - super().write_line_break(data) - if len(self.indents) == 1: - super().write_line_break() - - def increase_indent(self, flow=False, indentless=False): - return super().increase_indent(flow, False) - - if sections: - for section in sections: - if section not in schemas: - print(f'[ERROR] Invalid section: {section}') - return 1 - else: - sections = sorted(schemas.keys()) - -# TODO: create World Schema and dump only this with Word.dumps ? - - for section in sections: - schema = schemas[section](many=True) - schema.context.update({ - 'full': full, - 'secrets': secrets, - 'dns': dns, - }) - yaml.dump( - {section: schema.dump(schema.Meta.model.query.all())}, - sys.stdout, - Dumper=spacedDumper, - default_flow_style=False, - allow_unicode=True - ) - sys.stdout.write('\n') + MailuSchema(context={ + 'full': full, + 'secrets': secrets, + 'dns': dns, + }).dumps(config, sys.stdout) @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 625d1fd6..8b77b011 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -110,7 +110,7 @@ class Base(db.Model): created_at = db.Column(db.Date, nullable=False, default=date.today) updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) - comment = db.Column(db.String(255), nullable=True) + comment = db.Column(db.String(255), nullable=True, default='') @classmethod def _dict_pkey(cls): @@ -171,7 +171,7 @@ class Base(db.Model): if self.__mapper__.relationships[key].query_class is not None: if hasattr(items, 'all'): items = items.all() - if full or len(items): + if full or items: if key in secret: res[key] = '' else: diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index c2a3d03d..bd9d228a 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1,63 +1,252 @@ -import marshmallow -import sqlalchemy -import flask_marshmallow +""" +Mailu marshmallow schema +""" -from . import models +from textwrap import wrap + +import re +import yaml + +from marshmallow import post_dump, fields, Schema +from flask_marshmallow import Marshmallow +from OpenSSL import crypto + +from . import models, dkim -ma = flask_marshmallow.Marshmallow() +ma = Marshmallow() +# TODO: +# how to mark keys as "required" while unserializing (in certain use cases/API)? +# - fields withoud default => required +# - fields which are the primary key => unchangeable when updating + + +### yaml render module ### + +class RenderYAML: + """ Marshmallow YAML Render Module + """ + + class SpacedDumper(yaml.Dumper): + """ YAML Dumper to add a newline between main sections + and double the indent used + """ + + def write_line_break(self, data=None): + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + @staticmethod + def _update_dict(dict1, dict2): + """ sets missing keys in dict1 to values of dict2 + """ + for key, value in dict2.items(): + if key not in dict1: + dict1[key] = value + + _load_defaults = {} + @classmethod + def loads(cls, *args, **kwargs): + """ load yaml data from string + """ + cls._update_dict(kwargs, cls._load_defaults) + return yaml.load(*args, **kwargs) + + _dump_defaults = { + 'Dumper': SpacedDumper, + 'default_flow_style': False, + 'allow_unicode': True, + } + @classmethod + def dumps(cls, *args, **kwargs): + """ dump yaml data to string + """ + cls._update_dict(kwargs, cls._dump_defaults) + return yaml.dump(*args, **kwargs) + + +### field definitions ### + +class LazyString(fields.String): + """ Field that serializes a "false" value to the empty string + """ + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize None to the empty string + """ + return value if value else '' + + +class CommaSeparatedList(fields.Raw): + """ Field that deserializes a string containing comma-separated values to + a list of strings + """ + # TODO: implement this + + +class DkimKey(fields.String): + """ Field that serializes a dkim key to a list of strings (lines) and + deserializes a string or list of strings. + """ + + _clean_re = re.compile( + r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)', + flags=re.UNICODE + ) + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize dkim key to a list of strings (lines) + """ + + # map empty string and None to None + if not value: + return None + + # return list of key lines without header/footer + return value.decode('utf-8').strip().split('\n')[1:-1] + + def _deserialize(self, value, attr, data, **kwargs): + """ deserialize a string or list of strings to dkim key data + with verification + """ + + # convert list to str + if isinstance(value, list): + value = ''.join(value) + + # only strings are allowed + if not isinstance(value, str): + raise TypeError(f'invalid type: {type(value).__name__!r}') + + # clean value (remove whitespace and header/footer) + value = self._clean_re.sub('', value.strip()) + + # map empty string/list to None + if not value: + return None + + # handle special value 'generate' + elif value == 'generate': + return dkim.gen_key() + + # wrap value into valid pem layout and check validity + value = ( + '-----BEGIN PRIVATE KEY-----\n' + + '\n'.join(wrap(value, 64)) + + '\n-----END PRIVATE KEY-----\n' + ).encode('ascii') + try: + crypto.load_privatekey(crypto.FILETYPE_PEM, value) + except crypto.Error as exc: + raise ValueError('invalid dkim key') from exc + else: + return value + + +### schema definitions ### -import collections class BaseSchema(ma.SQLAlchemyAutoSchema): + """ Marshmallow base schema with custom exclude logic + and option to hide sqla defaults + """ class Meta: - base_hide_always = {'created_at', 'updated_at'} - base_hide_secrets = set() - base_hide_by_value = { -# 'comment': {'', None} - } + """ Schema config """ + model = None - @marshmallow.post_dump - def remove_skip_values(self, data, many, **kwargs): -# print(repr(data), self.context) + def __init__(self, *args, **kwargs): - # always hide - hide_by_key = self.Meta.base_hide_always | set(getattr(self.Meta, 'hide_always', ())) + # get and remove config from kwargs + context = kwargs.get('context', {}) - # hide secrets - if not self.context.get('secrets'): - hide_by_key |= self.Meta.base_hide_secrets - hide_by_key |= set(getattr(self.Meta, 'hide_secrets', ())) + # compile excludes + exclude = set(kwargs.get('exclude', [])) - # hide by value - hide_by_value = self.Meta.base_hide_by_value | getattr(self.Meta, 'hide_by_value', {}) + # always exclude + exclude.update({'created_at', 'updated_at'}) - # hide defaults - if not self.context.get('full'): + # add include_by_context + if context is not None: + for ctx, what in getattr(self.Meta, 'include_by_context', {}).items(): + if not context.get(ctx): + exclude |= set(what) + + # update excludes + kwargs['exclude'] = exclude + + # exclude_by_value + self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {}) + + # exclude default values + if not context.get('full'): for column in self.Meta.model.__table__.columns: -# print(column.name, column.default.arg if isinstance(column.default, sqlalchemy.sql.schema.ColumnDefault) else column.default) -# alias.destiantion has default [] - is this okay. how to check it? - if column.name not in hide_by_key: - hide_by_value.setdefault(column.name, set()).add(None if column.default is None else column.default.arg) + if column.name not in exclude: + self._exclude_by_value.setdefault(column.name, []).append( + None if column.default is None else column.default.arg + ) + # hide by context + self._hide_by_context = set() + if context is not None: + for ctx, what in getattr(self.Meta, 'hide_by_context', {}).items(): + if not context.get(ctx): + self._hide_by_context |= set(what) + + # init SQLAlchemyAutoSchema + super().__init__(*args, **kwargs) + + @post_dump + def _remove_skip_values(self, data, many, **kwargs): # pylint: disable=unused-argument + + if not self._exclude_by_value and not self._hide_by_context: + return data + + full = self.context.get('full') return { - key: value for key, value in data.items() - if - not isinstance(value, collections.Hashable) - or( - key not in hide_by_key - and - (key not in hide_by_value or value not in hide_by_value[key])) + key: '' if key in self._hide_by_context else value + for key, value in data.items() + if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] } + # TODO: remove LazyString and fix model definition (comment should not be nullable) + comment = LazyString() + class DomainSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Domain model """ + class Meta: + """ Schema config """ model = models.Domain + include_relationships = True + #include_fk = True + exclude = ['users', 'managers', 'aliases'] + + include_by_context = { + 'dns': {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}, + } + hide_by_context = { + 'secrets': {'dkim_key'}, + } + exclude_by_value = { + 'alternatives': [[]], + 'dkim_key': [None], + 'dkim_publickey': [None], + 'dns_mx': [None], + 'dns_spf': [None], + 'dns_dkim': [None], + 'dns_dmarc': [None], + } + + dkim_key = DkimKey() + dkim_publickey = fields.String(dump_only=True) + dns_mx = fields.String(dump_only=True) + dns_spf = fields.String(dump_only=True) + dns_dkim = fields.String(dump_only=True) + dns_dmarc = fields.String(dump_only=True) - # _dict_hide = {'users', 'managers', 'aliases'} - # _dict_show = {'dkim_key'} - # _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} - # _dict_secret = {'dkim_key'} # _dict_types = { # 'dkim_key': (bytes, type(None)), # 'dkim_publickey': False, @@ -66,50 +255,62 @@ class DomainSchema(BaseSchema): # 'dns_dkim': False, # 'dns_dmarc': False, # } - # _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} - # @staticmethod - # def _dict_input(data): - # if 'dkim_key' in data: - # key = data['dkim_key'] - # if key is not None: - # if type(key) is list: - # key = ''.join(key) - # if type(key) is str: - # key = ''.join(key.strip().split()) # removes all whitespace - # if key == 'generate': - # data['dkim_key'] = dkim.gen_key() - # elif key: - # m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) - # if m is not None: - # key = key[m.end():] - # m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) - # if m is not None: - # key = key[:m.start()] - # key = '\n'.join(wrap(key, 64)) - # key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') - # try: - # dkim.strip_key(key) - # except: - # raise ValueError('invalid dkim key') - # else: - # data['dkim_key'] = key - # else: - # data['dkim_key'] = None - # name = db.Column(IdnaDomain, primary_key=True, nullable=False) - # managers = db.relationship('User', secondary=managers, - # backref=db.backref('manager_of'), lazy='dynamic') - # max_users = db.Column(db.Integer, nullable=False, default=-1) - # max_aliases = db.Column(db.Integer, nullable=False, default=-1) - # max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) - # signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + +class TokenSchema(BaseSchema): + """ Marshmallow schema for Token model """ + class Meta: + """ Schema config """ + model = models.Token + + # _dict_recurse = True + # _dict_hide = {'user', 'user_email'} + # _dict_mandatory = {'password'} + + # id = db.Column(db.Integer(), primary_key=True) + # user_email = db.Column(db.String(255), db.ForeignKey(User.email), + # nullable=False) + # user = db.relationship(User, + # backref=db.backref('tokens', cascade='all, delete-orphan')) + # password = db.Column(db.String(255), nullable=False) + # ip = db.Column(db.String(255)) + + +class FetchSchema(BaseSchema): + """ Marshmallow schema for Fetch model """ + class Meta: + """ Schema config """ + model = models.Fetch + include_by_context = { + 'full': {'last_check', 'error'}, + } + hide_by_context = { + 'secrets': {'password'}, + } + +# TODO: What about mandatory keys? + # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} class UserSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for User model """ + class Meta: + """ Schema config """ model = models.User + include_relationships = True + exclude = ['localpart', 'domain', 'quota_bytes_used'] - # _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} + exclude_by_value = { + 'forward_destination': [[]], + 'tokens': [[]], + 'reply_enddate': ['2999-12-31'], + 'reply_startdate': ['1900-01-01'], + } + + tokens = fields.Nested(TokenSchema, many=True) + fetches = fields.Nested(FetchSchema, many=True) + +# TODO: deserialize password/password_hash! What about mandatory keys? # _dict_mandatory = {'localpart', 'domain', 'password'} # @classmethod # def _dict_input(cls, data): @@ -133,44 +334,19 @@ class UserSchema(BaseSchema): # del data['hash_scheme'] # del data['password_hash'] - # domain = db.relationship(Domain, - # backref=db.backref('users', cascade='all, delete-orphan')) - # password = db.Column(db.String(255), nullable=False) - # quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) - # quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) - # global_admin = db.Column(db.Boolean(), nullable=False, default=False) - # enabled = db.Column(db.Boolean(), nullable=False, default=True) - - # # Features - # enable_imap = db.Column(db.Boolean(), nullable=False, default=True) - # enable_pop = db.Column(db.Boolean(), nullable=False, default=True) - - # # Filters - # forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) - # forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) - # forward_keep = db.Column(db.Boolean(), nullable=False, default=True) - # reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) - # reply_subject = db.Column(db.String(255), nullable=True, default=None) - # reply_body = db.Column(db.Text(), nullable=True, default=None) - # reply_startdate = db.Column(db.Date, nullable=False, - # default=date(1900, 1, 1)) - # reply_enddate = db.Column(db.Date, nullable=False, - # default=date(2999, 12, 31)) - - # # Settings - # displayed_name = db.Column(db.String(160), nullable=False, default='') - # spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) - # spam_threshold = db.Column(db.Integer(), nullable=False, default=80) class AliasSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Alias model """ + class Meta: + """ Schema config """ model = models.Alias - hide_always = {'localpart'} - hide_secrets = {'wildcard'} - hide_by_value = { - 'destination': set([]) # always hide empty lists?! + exclude = ['localpart'] + + exclude_by_value = { + 'destination': [[]], } +# TODO: deserialize destination! # @staticmethod # def _dict_input(data): # Email._dict_input(data) @@ -180,65 +356,57 @@ class AliasSchema(BaseSchema): # data['destination'] = list([adr.strip() for adr in dst.split(',')]) -class TokenSchema(BaseSchema): - class Meta(BaseSchema.Meta): - model = models.Token - - # _dict_recurse = True - # _dict_hide = {'user', 'user_email'} - # _dict_mandatory = {'password'} - - # id = db.Column(db.Integer(), primary_key=True) - # user_email = db.Column(db.String(255), db.ForeignKey(User.email), - # nullable=False) - # user = db.relationship(User, - # backref=db.backref('tokens', cascade='all, delete-orphan')) - # password = db.Column(db.String(255), nullable=False) - # ip = db.Column(db.String(255)) - - -class FetchSchema(BaseSchema): - class Meta(BaseSchema.Meta): - model = models.Fetch - - # _dict_recurse = True - # _dict_hide = {'user_email', 'user', 'last_check', 'error'} - # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} - # _dict_secret = {'password'} - - # id = db.Column(db.Integer(), primary_key=True) - # user_email = db.Column(db.String(255), db.ForeignKey(User.email), - # nullable=False) - # user = db.relationship(User, - # backref=db.backref('fetches', cascade='all, delete-orphan')) - # protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) - # host = db.Column(db.String(255), nullable=False) - # port = db.Column(db.Integer(), nullable=False) - # tls = db.Column(db.Boolean(), nullable=False, default=False) - # username = db.Column(db.String(255), nullable=False) - # password = db.Column(db.String(255), nullable=False) - # keep = db.Column(db.Boolean(), nullable=False, default=False) - # last_check = db.Column(db.DateTime, nullable=True) - # error = db.Column(db.String(1023), nullable=True) - - class ConfigSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Config model """ + class Meta: + """ Schema config """ model = models.Config -# TODO: how to mark keys as "required" while unserializing (in certain use cases/API) - name = ma.auto_field(required=True) - value = ma.auto_field(required=True) class RelaySchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Relay model """ + class Meta: + """ Schema config """ model = models.Relay -schemas = { - 'domains': DomainSchema, - 'relays': RelaySchema, - 'users': UserSchema, - 'aliases': AliasSchema, -# 'config': ConfigSchema, -} +class MailuSchema(Schema): + """ Marshmallow schema for Mailu config """ + class Meta: + """ Schema config """ + render_module = RenderYAML + domains = fields.Nested(DomainSchema, many=True) + relays = fields.Nested(RelaySchema, many=True) + users = fields.Nested(UserSchema, many=True) + aliases = fields.Nested(AliasSchema, many=True) + config = fields.Nested(ConfigSchema, many=True) + + +### config class ### + +class MailuConfig: + """ Class which joins whole Mailu config for dumping + """ + + _models = { + 'domains': models.Domain, + 'relays': models.Relay, + 'users': models.User, + 'aliases': models.Alias, +# 'config': models.Config, + } + + def __init__(self, sections): + if sections: + for section in sections: + if section not in self._models: + raise ValueError(f'Unknown section: {section!r}') + self._sections = set(sections) + else: + self._sections = set(self._models.keys()) + + def __getattr__(self, section): + if section in self._sections: + return self._models[section].query.all() + else: + raise AttributeError From c24bff1c1be3980892fddb13a139b8142bb71f96 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 14 Jan 2021 01:11:04 +0100 Subject: [PATCH 038/181] added config_import using marshmallow --- core/admin/mailu/api/__init__.py | 2 +- core/admin/mailu/manage.py | 314 ++++++++------- core/admin/mailu/models.py | 650 +++++++++++++++---------------- core/admin/mailu/schemas.py | 238 +++++------ 4 files changed, 611 insertions(+), 593 deletions(-) diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py index a4d8689f..dee43036 100644 --- a/core/admin/mailu/api/__init__.py +++ b/core/admin/mailu/api/__init__.py @@ -26,7 +26,7 @@ def register(app): # add redirect to current api version @app.route(f'{ROOT}/') - def redir(): + def _redirect_to_active_api(): return redirect(url_for(f'{ACTIVE.blueprint.name}.root')) # swagger ui config diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index a67abdf7..6103f904 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,21 +1,25 @@ -from mailu import models -from .schemas import MailuConfig, MailuSchema - -from flask import current_app as app -from flask.cli import FlaskGroup, with_appcontext +""" Mailu command line interface +""" +import sys import os import socket import uuid + import click -import yaml -import sys + +from flask import current_app as app +from flask.cli import FlaskGroup, with_appcontext +from marshmallow.exceptions import ValidationError + +from . import models +from .schemas import MailuSchema db = models.db -@click.group(cls=FlaskGroup) +@click.group(cls=FlaskGroup, context_settings={'help_option_names': ['-?', '-h', '--help']}) def mailu(): """ Mailu command line """ @@ -26,17 +30,17 @@ def mailu(): def advertise(): """ Advertise this server against statistic services. """ - if os.path.isfile(app.config["INSTANCE_ID_PATH"]): - with open(app.config["INSTANCE_ID_PATH"], "r") as handle: + if os.path.isfile(app.config['INSTANCE_ID_PATH']): + with open(app.config['INSTANCE_ID_PATH'], 'r') as handle: instance_id = handle.read() else: instance_id = str(uuid.uuid4()) - with open(app.config["INSTANCE_ID_PATH"], "w") as handle: + with open(app.config['INSTANCE_ID_PATH'], 'w') as handle: handle.write(instance_id) - if not app.config["DISABLE_STATISTICS"]: + if not app.config['DISABLE_STATISTICS']: try: - socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id)) - except: + socket.gethostbyname(app.config['STATS_ENDPOINT'].format(instance_id)) + except OSError: pass @@ -171,156 +175,196 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() -yaml_sections = [ - ('domains', models.Domain), - ('relays', models.Relay), - ('users', models.User), - ('aliases', models.Alias), -# ('config', models.Config), -] +# @mailu.command() +# @click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') +# @click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') +# @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') +# @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) +# @with_appcontext +# def config_update(verbose=False, delete_objects=False, dry_run=False, source=None): +# """ Update configuration with data from YAML-formatted input +# """ + + # try: + # new_config = yaml.safe_load(source) + # except (yaml.scanner.ScannerError, yaml.parser.ParserError) as exc: + # print(f'[ERROR] Invalid yaml: {exc}') + # sys.exit(1) + # else: + # if isinstance(new_config, str): + # print(f'[ERROR] Invalid yaml: {new_config!r}') + # sys.exit(1) + # elif new_config is None or not new_config: + # print('[ERROR] Empty yaml: Please pipe yaml into stdin') + # sys.exit(1) + + # error = False + # tracked = {} + # for section, model in yaml_sections: + + # items = new_config.get(section) + # if items is None: + # if delete_objects: + # print(f'[ERROR] Invalid yaml: Section "{section}" is missing') + # error = True + # break + # else: + # continue + + # del new_config[section] + + # if not isinstance(items, list): + # print(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}') + # error = True + # break + # elif not items: + # continue + + # # create items + # for data in items: + + # if verbose: + # print(f'Handling {model.__table__} data: {data!r}') + + # try: + # changed = model.from_dict(data, delete_objects) + # except Exception as exc: + # print(f'[ERROR] {exc.args[0]} in data: {data}') + # error = True + # break + + # for item, created in changed: + + # if created is True: + # # flush newly created item + # db.session.add(item) + # db.session.flush() + # if verbose: + # print(f'Added {item!r}: {item.to_dict()}') + # else: + # print(f'Added {item!r}') + + # elif created: + # # modified instance + # if verbose: + # for key, old, new in created: + # print(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}') + # else: + # print(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}') + + # # track primary key of all items + # tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()])) + + # if error: + # break + + # # on error: stop early + # if error: + # print('[ERROR] An error occured. Not committing changes.') + # db.session.rollback() + # sys.exit(1) + + # # are there sections left in new_config? + # if new_config: + # print(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}') + # error = True + + # # test for conflicting domains + # domains = set() + # for model, items in tracked.items(): + # if model in (models.Domain, models.Alternative, models.Relay): + # if domains & items: + # for fqdn in domains & items: + # print(f'[ERROR] Duplicate domain name used: {fqdn}') + # error = True + # domains.update(items) + + # # delete items not tracked + # if delete_objects: + # for model, items in tracked.items(): + # for item in model.query.all(): + # if not item._dict_pval() in items: + # print(f'Deleted {item!r} {item}') + # db.session.delete(item) + + # # don't commit when running dry + # if dry_run: + # print('Dry run. Not commiting changes.') + # db.session.rollback() + # else: + # db.session.commit() + + +SECTIONS = {'domains', 'relays', 'users', 'aliases'} + @mailu.command() @click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') -@click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') +@click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext -def config_update(verbose=False, delete_objects=False, dry_run=False, file=None): - """sync configuration with data from YAML-formatted stdin""" +def config_import(verbose=False, dry_run=False, source=None): + """ Import configuration YAML + """ - out = (lambda *args: print('(DRY RUN)', *args)) if dry_run else print + context = { + 'verbose': verbose, # TODO: use callback function to be verbose? + 'import': True, + } try: - new_config = yaml.safe_load(sys.stdin) - except (yaml.scanner.ScannerError, yaml.parser.ParserError) as reason: - out(f'[ERROR] Invalid yaml: {reason}') + config = MailuSchema(context=context).loads(source) + except ValidationError as exc: + print(f'[ERROR] {exc}') + # TODO: show nice errors + from pprint import pprint + pprint(exc.messages) sys.exit(1) else: - if type(new_config) is str: - out(f'[ERROR] Invalid yaml: {new_config!r}') - sys.exit(1) - elif new_config is None or not len(new_config): - out('[ERROR] Empty yaml: Please pipe yaml into stdin') - sys.exit(1) - - error = False - tracked = {} - for section, model in yaml_sections: - - items = new_config.get(section) - if items is None: - if delete_objects: - out(f'[ERROR] Invalid yaml: Section "{section}" is missing') - error = True - break - else: - continue - - del new_config[section] - - if type(items) is not list: - out(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}') - error = True - break - elif not items: - continue - - # create items - for data in items: - - if verbose: - out(f'Handling {model.__table__} data: {data!r}') - - try: - changed = model.from_dict(data, delete_objects) - except Exception as reason: - out(f'[ERROR] {reason.args[0]} in data: {data}') - error = True - break - - for item, created in changed: - - if created is True: - # flush newly created item - db.session.add(item) - db.session.flush() - if verbose: - out(f'Added {item!r}: {item.to_dict()}') - else: - out(f'Added {item!r}') - - elif len(created): - # modified instance - if verbose: - for key, old, new in created: - out(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}') - else: - out(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}') - - # track primary key of all items - tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()])) - - if error: - break - - # on error: stop early - if error: - out('An error occured. Not committing changes.') - db.session.rollback() - sys.exit(1) - - # are there sections left in new_config? - if new_config: - out(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}') - error = True - - # test for conflicting domains - domains = set() - for model, items in tracked.items(): - if model in (models.Domain, models.Alternative, models.Relay): - if domains & items: - for domain in domains & items: - out(f'[ERROR] Duplicate domain name used: {domain}') - error = True - domains.update(items) - - # delete items not tracked - if delete_objects: - for model, items in tracked.items(): - for item in model.query.all(): - if not item._dict_pval() in items: - out(f'Deleted {item!r} {item}') - db.session.delete(item) + print(config) + print(MailuSchema().dumps(config)) + # TODO: does not commit yet. + # TODO: delete other entries? # don't commit when running dry - if dry_run: + if True: #dry_run: + print('Dry run. Not commiting changes.') db.session.rollback() else: db.session.commit() @mailu.command() -@click.option('-f', '--full', is_flag=True, help='Include default attributes') -@click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)') +@click.option('-f', '--full', is_flag=True, help='Include attributes with default value') +@click.option('-s', '--secrets', is_flag=True, + help='Include secret attributes (dkim-key, passwords)') @click.option('-d', '--dns', is_flag=True, help='Include dns records') +@click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), + help='save yaml to file') @click.argument('sections', nargs=-1) @with_appcontext -def config_dump(full=False, secrets=False, dns=False, sections=None): - """dump configuration as YAML-formatted data to stdout +def config_dump(full=False, secrets=False, dns=False, output=None, sections=None): + """ Dump configuration as YAML to stdout or file SECTIONS can be: domains, relays, users, aliases """ - try: - config = MailuConfig(sections) - except ValueError as reason: - print(f'[ERROR] {reason}') - return 1 + if sections: + for section in sections: + if section not in SECTIONS: + print(f'[ERROR] Unknown section: {section!r}') + sys.exit(1) + sections = set(sections) + else: + sections = SECTIONS - MailuSchema(context={ + context={ 'full': full, 'secrets': secrets, 'dns': dns, - }).dumps(config, sys.stdout) + } + + MailuSchema(only=sections, context=context).dumps(models.MailuConfig(), output) @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 8b77b011..73d05801 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,23 +1,26 @@ -from mailu import dkim +""" Mailu config storage model +""" -from sqlalchemy.ext import declarative -from datetime import datetime, date +import re +import os +import smtplib +import json + +from datetime import date from email.mime import text -from flask import current_app as app -from textwrap import wrap import flask_sqlalchemy import sqlalchemy -import re -import time -import os import passlib -import glob -import smtplib import idna import dns -import json -import itertools + +from flask import current_app as app +from sqlalchemy.ext import declarative +from sqlalchemy.inspection import inspect +from werkzeug.utils import cached_property + +from . import dkim db = flask_sqlalchemy.SQLAlchemy() @@ -30,9 +33,11 @@ class IdnaDomain(db.TypeDecorator): impl = db.String(80) def process_bind_param(self, value, dialect): + """ encode unicode domain name to punycode """ return idna.encode(value).decode('ascii').lower() def process_result_value(self, value, dialect): + """ decode punycode domain name to unicode """ return idna.decode(value) python_type = str @@ -44,6 +49,7 @@ class IdnaEmail(db.TypeDecorator): impl = db.String(255) def process_bind_param(self, value, dialect): + """ encode unicode domain part of email address to punycode """ try: localpart, domain_name = value.split('@') return '{0}@{1}'.format( @@ -54,6 +60,7 @@ class IdnaEmail(db.TypeDecorator): pass def process_result_value(self, value, dialect): + """ decode punycode domain part of email to unicode """ localpart, domain_name = value.split('@') return '{0}@{1}'.format( localpart, @@ -69,14 +76,16 @@ class CommaSeparatedList(db.TypeDecorator): impl = db.String def process_bind_param(self, value, dialect): - if not isinstance(value, (list, set)): - raise TypeError('Must be a list') + """ join list of items to comma separated string """ + if not isinstance(value, (list, tuple, set)): + raise TypeError('Must be a list of strings') for item in value: if ',' in item: raise ValueError('Item must not contain a comma') return ','.join(sorted(value)) def process_result_value(self, value, dialect): + """ split comma separated string to list """ return list(filter(bool, value.split(','))) if value else [] python_type = list @@ -88,9 +97,11 @@ class JSONEncoded(db.TypeDecorator): impl = db.String def process_bind_param(self, value, dialect): + """ encode data as json """ return json.dumps(value) if value else None def process_result_value(self, value, dialect): + """ decode json to data """ return json.loads(value) if value else None python_type = str @@ -112,246 +123,172 @@ class Base(db.Model): updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) comment = db.Column(db.String(255), nullable=True, default='') - @classmethod - def _dict_pkey(cls): - return cls.__mapper__.primary_key[0].name + # @classmethod + # def from_dict(cls, data, delete=False): - def _dict_pval(self): - return getattr(self, self._dict_pkey()) + # changed = [] - def to_dict(self, full=False, include_secrets=False, include_extra=None, recursed=False, hide=None): - """ Return a dictionary representation of this model. - """ + # pkey = cls._dict_pkey() - if recursed and not getattr(self, '_dict_recurse', False): - return str(self) + # # handle "primary key" only + # if not isinstance(data, dict): + # data = {pkey: data} - hide = set(hide or []) | {'created_at', 'updated_at'} - if hasattr(self, '_dict_hide'): - hide |= self._dict_hide + # # modify input data + # if hasattr(cls, '_dict_input'): + # try: + # cls._dict_input(data) + # except Exception as exc: + # raise ValueError(f'{exc}', cls, None, data) from exc - secret = set() - if not include_secrets and hasattr(self, '_dict_secret'): - secret |= self._dict_secret + # # check for primary key (if not recursed) + # if not getattr(cls, '_dict_recurse', False): + # if not pkey in data: + # raise KeyError(f'primary key {cls.__table__}.{pkey} is missing', cls, pkey, data) - convert = getattr(self, '_dict_output', {}) + # # check data keys and values + # for key in list(data.keys()): - extra_keys = getattr(self, '_dict_extra', {}) - if include_extra is None: - include_extra = [] + # # check key + # if not hasattr(cls, key) and not key in cls.__mapper__.relationships: + # raise KeyError(f'unknown key {cls.__table__}.{key}', cls, key, data) - res = {} + # # check value type + # value = data[key] + # col = cls.__mapper__.columns.get(key) + # if col is not None: + # if not ((value is None and col.nullable) or (isinstance(value, col.type.python_type))): + # raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) + # else: + # rel = cls.__mapper__.relationships.get(key) + # if rel is None: + # itype = getattr(cls, '_dict_types', {}).get(key) + # if itype is not None: + # if itype is False: # ignore value. TODO: emit warning? + # del data[key] + # continue + # elif not isinstance(value, itype): + # raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) + # else: + # raise NotImplementedError(f'type not defined for {cls.__table__}.{key}') - for key in itertools.chain( - self.__table__.columns.keys(), - getattr(self, '_dict_show', []), - *[extra_keys.get(extra, []) for extra in include_extra] - ): - if key in hide: - continue - if key in self.__table__.columns: - default = self.__table__.columns[key].default - if isinstance(default, sqlalchemy.sql.schema.ColumnDefault): - default = default.arg - else: - default = None - value = getattr(self, key) - if full or ((default or value) and value != default): - if key in secret: - value = '' - elif value is not None and key in convert: - value = convert[key](value) - res[key] = value + # # handle relationships + # if key in cls.__mapper__.relationships: + # rel_model = cls.__mapper__.relationships[key].argument + # if not isinstance(rel_model, sqlalchemy.orm.Mapper): + # add = rel_model.from_dict(value, delete) + # assert len(add) == 1 + # rel_item, updated = add[0] + # changed.append((rel_item, updated)) + # data[key] = rel_item - for key in self.__mapper__.relationships.keys(): - if key in hide: - continue - if self.__mapper__.relationships[key].uselist: - items = getattr(self, key) - if self.__mapper__.relationships[key].query_class is not None: - if hasattr(items, 'all'): - items = items.all() - if full or items: - if key in secret: - res[key] = '' - else: - res[key] = [item.to_dict(full, include_secrets, include_extra, True) for item in items] - else: - value = getattr(self, key) - if full or value is not None: - if key in secret: - res[key] = '' - else: - res[key] = value.to_dict(full, include_secrets, include_extra, True) + # # create item if necessary + # created = False + # item = cls.query.get(data[pkey]) if pkey in data else None + # if item is None: - return res + # # check for mandatory keys + # missing = getattr(cls, '_dict_mandatory', set()) - set(data.keys()) + # if missing: + # raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {cls.__table__} missing', cls, missing, data) - @classmethod - def from_dict(cls, data, delete=False): + # # remove mapped relationships from data + # mapped = {} + # for key in list(data.keys()): + # if key in cls.__mapper__.relationships: + # if isinstance(cls.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): + # mapped[key] = data[key] + # del data[key] - changed = [] + # # create new item + # item = cls(**data) + # created = True - pkey = cls._dict_pkey() + # # and update mapped relationships (below) + # data = mapped - # handle "primary key" only - if isinstance(data, dict): - data = {pkey: data} + # # update item + # updated = [] + # for key, value in data.items(): - # modify input data - if hasattr(cls, '_dict_input'): - try: - cls._dict_input(data) - except Exception as reason: - raise ValueError(f'{reason}', cls, None, data) + # # skip primary key + # if key == pkey: + # continue - # check for primary key (if not recursed) - if not getattr(cls, '_dict_recurse', False): - if not pkey in data: - raise KeyError(f'primary key {cls.__table__}.{pkey} is missing', cls, pkey, data) + # if key in cls.__mapper__.relationships: + # # update relationship + # rel_model = cls.__mapper__.relationships[key].argument + # if isinstance(rel_model, sqlalchemy.orm.Mapper): + # rel_model = rel_model.class_ + # # add (and create) referenced items + # cur = getattr(item, key) + # old = sorted(cur, key=id) + # new = [] + # for rel_data in value: + # # get or create related item + # add = rel_model.from_dict(rel_data, delete) + # assert len(add) == 1 + # rel_item, rel_updated = add[0] + # changed.append((rel_item, rel_updated)) + # if rel_item not in cur: + # cur.append(rel_item) + # new.append(rel_item) - # check data keys and values - for key in list(data.keys()): + # # delete referenced items missing in yaml + # rel_pkey = rel_model._dict_pkey() + # new_data = list([i.to_dict(True, True, None, True, [rel_pkey]) for i in new]) + # for rel_item in old: + # if rel_item not in new: + # # check if item with same data exists to stabilze import without primary key + # rel_data = rel_item.to_dict(True, True, None, True, [rel_pkey]) + # try: + # same_idx = new_data.index(rel_data) + # except ValueError: + # same = None + # else: + # same = new[same_idx] - # check key - if not hasattr(cls, key) and not key in cls.__mapper__.relationships: - raise KeyError(f'unknown key {cls.__table__}.{key}', cls, key, data) + # if same is None: + # # delete items missing in new + # if delete: + # cur.remove(rel_item) + # else: + # new.append(rel_item) + # else: + # # swap found item with same data with newly created item + # new.append(rel_item) + # new_data.append(rel_data) + # new.remove(same) + # del new_data[same_idx] + # for i, (ch_item, _) in enumerate(changed): + # if ch_item is same: + # changed[i] = (rel_item, []) + # db.session.flush() + # db.session.delete(ch_item) + # break - # check value type - value = data[key] - col = cls.__mapper__.columns.get(key) - if col is not None: - if not ((value is None and col.nullable) or (isinstance(value, col.type.python_type))): - raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) - else: - rel = cls.__mapper__.relationships.get(key) - if rel is None: - itype = getattr(cls, '_dict_types', {}).get(key) - if itype is not None: - if itype is False: # ignore value. TODO: emit warning? - del data[key] - continue - elif not isinstance(value, itype): - raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) - else: - raise NotImplementedError(f'type not defined for {cls.__table__}.{key}') + # # remember changes + # new = sorted(new, key=id) + # if new != old: + # updated.append((key, old, new)) - # handle relationships - if key in cls.__mapper__.relationships: - rel_model = cls.__mapper__.relationships[key].argument - if not isinstance(rel_model, sqlalchemy.orm.Mapper): - add = rel_model.from_dict(value, delete) - assert len(add) == 1 - rel_item, updated = add[0] - changed.append((rel_item, updated)) - data[key] = rel_item + # else: + # # update key + # old = getattr(item, key) + # if isinstance(old, list): + # # deduplicate list value + # assert isinstance(value, list) + # value = set(value) + # old = set(old) + # if not delete: + # value = old | value + # if value != old: + # updated.append((key, old, value)) + # setattr(item, key, value) - # create item if necessary - created = False - item = cls.query.get(data[pkey]) if pkey in data else None - if item is None: + # changed.append((item, created if created else updated)) - # check for mandatory keys - missing = getattr(cls, '_dict_mandatory', set()) - set(data.keys()) - if missing: - raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {cls.__table__} missing', cls, missing, data) - - # remove mapped relationships from data - mapped = {} - for key in list(data.keys()): - if key in cls.__mapper__.relationships: - if isinstance(cls.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): - mapped[key] = data[key] - del data[key] - - # create new item - item = cls(**data) - created = True - - # and update mapped relationships (below) - data = mapped - - # update item - updated = [] - for key, value in data.items(): - - # skip primary key - if key == pkey: - continue - - if key in cls.__mapper__.relationships: - # update relationship - rel_model = cls.__mapper__.relationships[key].argument - if isinstance(rel_model, sqlalchemy.orm.Mapper): - rel_model = rel_model.class_ - # add (and create) referenced items - cur = getattr(item, key) - old = sorted(cur, key=id) - new = [] - for rel_data in value: - # get or create related item - add = rel_model.from_dict(rel_data, delete) - assert len(add) == 1 - rel_item, rel_updated = add[0] - changed.append((rel_item, rel_updated)) - if rel_item not in cur: - cur.append(rel_item) - new.append(rel_item) - - # delete referenced items missing in yaml - rel_pkey = rel_model._dict_pkey() - new_data = list([i.to_dict(True, True, None, True, [rel_pkey]) for i in new]) - for rel_item in old: - if rel_item not in new: - # check if item with same data exists to stabilze import without primary key - rel_data = rel_item.to_dict(True, True, None, True, [rel_pkey]) - try: - same_idx = new_data.index(rel_data) - except ValueError: - same = None - else: - same = new[same_idx] - - if same is None: - # delete items missing in new - if delete: - cur.remove(rel_item) - else: - new.append(rel_item) - else: - # swap found item with same data with newly created item - new.append(rel_item) - new_data.append(rel_data) - new.remove(same) - del new_data[same_idx] - for i, (ch_item, _) in enumerate(changed): - if ch_item is same: - changed[i] = (rel_item, []) - db.session.flush() - db.session.delete(ch_item) - break - - # remember changes - new = sorted(new, key=id) - if new != old: - updated.append((key, old, new)) - - else: - # update key - old = getattr(item, key) - if isinstance(old, list): - # deduplicate list value - assert isinstance(value, list) - value = set(value) - old = set(old) - if not delete: - value = old | value - if value != old: - updated.append((key, old, value)) - setattr(item, key, value) - - changed.append((item, created if created else updated)) - - return changed + # return changed # Many-to-many association table for domain managers @@ -391,48 +328,6 @@ class Domain(Base): __tablename__ = 'domain' - _dict_hide = {'users', 'managers', 'aliases'} - _dict_show = {'dkim_key'} - _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} - _dict_secret = {'dkim_key'} - _dict_types = { - 'dkim_key': (bytes, type(None)), - 'dkim_publickey': False, - 'dns_mx': False, - 'dns_spf': False, - 'dns_dkim': False, - 'dns_dmarc': False, - } - _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} - @staticmethod - def _dict_input(data): - if 'dkim_key' in data: - key = data['dkim_key'] - if key is not None: - if isinstance(key, list): - key = ''.join(key) - if isinstance(key, str): - key = ''.join(key.strip().split()) # removes all whitespace - if key == 'generate': - data['dkim_key'] = dkim.gen_key() - elif key: - match = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) - if match is not None: - key = key[match.end():] - match = re.search('-----END (RSA )?PRIVATE KEY-----$', key) - if match is not None: - key = key[:match.start()] - key = '\n'.join(wrap(key, 64)) - key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') - try: - dkim.strip_key(key) - except: - raise ValueError('invalid dkim key') - else: - data['dkim_key'] = key - else: - data['dkim_key'] = None - name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, backref=db.backref('manager_of'), lazy='dynamic') @@ -440,7 +335,7 @@ class Domain(Base): max_aliases = db.Column(db.Integer, nullable=False, default=-1) max_quota_bytes = db.Column(db.BigInteger, nullable=False, default=0) signup_enabled = db.Column(db.Boolean, nullable=False, default=False) - + _dkim_key = None _dkim_key_changed = False @@ -452,17 +347,20 @@ class Domain(Base): def dns_mx(self): hostname = app.config['HOSTNAMES'].split(',')[0] return f'{self.name}. 600 IN MX 10 {hostname}.' - + @property def dns_spf(self): hostname = app.config['HOSTNAMES'].split(',')[0] return f'{self.name}. 600 IN TXT "v=spf1 mx a:{hostname} ~all"' - + @property def dns_dkim(self): if os.path.exists(self._dkim_file()): selector = app.config['DKIM_SELECTOR'] - return f'{selector}._domainkey.{self.name}. 600 IN TXT "v=DKIM1; k=rsa; p={self.dkim_publickey}"' + return ( + f'{selector}._domainkey.{self.name}. 600 IN TXT' + f'"v=DKIM1; k=rsa; p={self.dkim_publickey}"' + ) @property def dns_dmarc(self): @@ -473,7 +371,7 @@ class Domain(Base): ruf = app.config['DMARC_RUF'] ruf = f' ruf=mailto:{ruf}@{domain};' if ruf else '' return f'_dmarc.{self.name}. 600 IN TXT "v=DMARC1; p=reject;{rua}{ruf} adkim=s; aspf=s"' - + @property def dkim_key(self): if self._dkim_key is None: @@ -525,7 +423,11 @@ class Domain(Base): try: return self.name == other.name except AttributeError: - return False + return NotImplemented + + def __hash__(self): + return hash(str(self.name)) + class Alternative(Base): @@ -551,8 +453,6 @@ class Relay(Base): __tablename__ = 'relay' - _dict_mandatory = {'smtp'} - name = db.Column(IdnaDomain, primary_key=True, nullable=False) smtp = db.Column(db.String(80), nullable=True) @@ -566,18 +466,8 @@ class Email(object): localpart = db.Column(db.String(80), nullable=False) - @staticmethod - def _dict_input(data): - if 'email' in data: - if 'localpart' in data or 'domain' in data: - raise ValueError('ambigous key email and localpart/domain') - elif isinstance(data['email'], str): - data['localpart'], data['domain'] = data['email'].rsplit('@', 1) - else: - data['email'] = f'{data["localpart"]}@{data["domain"]}' - @declarative.declared_attr - def domain_name(cls): + def domain_name(self): return db.Column(IdnaDomain, db.ForeignKey(Domain.name), nullable=False, default=IdnaDomain) @@ -585,7 +475,7 @@ class Email(object): # It is however very useful for quick lookups without joining tables, # especially when the mail server is reading the database. @declarative.declared_attr - def email(cls): + def email(self): updater = lambda context: '{0}@{1}'.format( context.current_parameters['localpart'], context.current_parameters['domain_name'], @@ -662,30 +552,6 @@ class User(Base, Email): __tablename__ = 'user' - _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} - _dict_mandatory = {'localpart', 'domain', 'password'} - @classmethod - def _dict_input(cls, data): - Email._dict_input(data) - # handle password - if 'password' in data: - if 'password_hash' in data or 'hash_scheme' in data: - raise ValueError('ambigous key password and password_hash/hash_scheme') - # check (hashed) password - password = data['password'] - if password.startswith('{') and '}' in password: - scheme = password[1:password.index('}')] - if scheme not in cls.scheme_dict: - raise ValueError(f'invalid password scheme {scheme!r}') - else: - raise ValueError(f'invalid hashed password {password!r}') - elif 'password_hash' in data and 'hash_scheme' in data: - if data['hash_scheme'] not in cls.scheme_dict: - raise ValueError(f'invalid password scheme {scheme!r}') - data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] - del data['hash_scheme'] - del data['password_hash'] - domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) @@ -775,7 +641,8 @@ class User(Base, Email): if raw: self.password = '{'+hash_scheme+'}' + password else: - self.password = '{'+hash_scheme+'}' + self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme]) + self.password = '{'+hash_scheme+'}' + \ + self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme]) def get_managed_domains(self): if self.global_admin: @@ -812,15 +679,6 @@ class Alias(Base, Email): __tablename__ = 'alias' - _dict_hide = {'domain_name', 'domain', 'localpart'} - @staticmethod - def _dict_input(data): - Email._dict_input(data) - # handle comma delimited string for backwards compability - dst = data.get('destination') - if isinstance(dst, str): - data['destination'] = list([adr.strip() for adr in dst.split(',')]) - domain = db.relationship(Domain, backref=db.backref('aliases', cascade='all, delete-orphan')) wildcard = db.Column(db.Boolean, nullable=False, default=False) @@ -832,10 +690,10 @@ class Alias(Base, Email): sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( sqlalchemy.and_( - cls.wildcard == False, + cls.wildcard is False, cls.localpart == localpart ), sqlalchemy.and_( - cls.wildcard == True, + cls.wildcard is True, sqlalchemy.bindparam('l', localpart).like(cls.localpart) ) ) @@ -847,10 +705,10 @@ class Alias(Base, Email): sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( sqlalchemy.and_( - cls.wildcard == False, + cls.wildcard is False, sqlalchemy.func.lower(cls.localpart) == localpart_lower ), sqlalchemy.and_( - cls.wildcard == True, + cls.wildcard is True, sqlalchemy.bindparam('l', localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) ) ) @@ -875,10 +733,6 @@ class Token(Base): __tablename__ = 'token' - _dict_recurse = True - _dict_hide = {'user', 'user_email'} - _dict_mandatory = {'password'} - id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) @@ -904,11 +758,6 @@ class Fetch(Base): __tablename__ = 'fetch' - _dict_recurse = True - _dict_hide = {'user_email', 'user', 'last_check', 'error'} - _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} - _dict_secret = {'password'} - id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) @@ -926,3 +775,124 @@ class Fetch(Base): def __str__(self): return f'{self.protocol}{"s" if self.tls else ""}://{self.username}@{self.host}:{self.port}' + + +class MailuConfig: + """ Class which joins whole Mailu config for dumping + and loading + """ + + # TODO: add sqlalchemy session updating (.add & .del) + class MailuCollection: + """ Provides dict- and list-like access to all instances + of a sqlalchemy model + """ + + def __init__(self, model : db.Model): + self._model = model + + @cached_property + def _items(self): + return { + inspect(item).identity: item + for item in self._model.query.all() + } + + def __len__(self): + return len(self._items) + + def __iter__(self): + return iter(self._items.values()) + + def __getitem__(self, key): + return self._items[key] + + def __setitem__(self, key, item): + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items[key] = item + + def __delitem__(self, key): + del self._items[key] + + def append(self, item): + """ list-like append """ + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + key = inspect(item).identity + if key in self._items: + raise ValueError(f'item {key!r} already present in collection') + self._items[key] = item + + def extend(self, items): + """ list-like extend """ + add = {} + for item in items: + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + key = inspect(item).identity + if key in self._items: + raise ValueError(f'item {key!r} already present in collection') + add[key] = item + self._items.update(add) + + def pop(self, *args): + """ list-like (no args) and dict-like (1 or 2 args) pop """ + if args: + if len(args) > 2: + raise TypeError(f'pop expected at most 2 arguments, got {len(args)}') + return self._items.pop(*args) + else: + return self._items.popitem()[1] + + def popitem(self): + """ dict-like popitem """ + return self._items.popitem() + + def remove(self, item): + """ list-like remove """ + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + key = inspect(item).identity + if not key in self._items: + raise ValueError(f'item {key!r} not found in collection') + del self._items[key] + + def clear(self): + """ dict-like clear """ + while True: + try: + self.pop() + except IndexError: + break + + def update(self, items): + """ dict-like update """ + for key, item in items: + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + if key in self._items: + raise ValueError(f'item {key!r} already present in collection') + + def setdefault(self, key, item=None): + """ dict-like setdefault """ + if key in self._items: + return self._items[key] + if item is None: + return None + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items[key] = item + return item + + domains = MailuCollection(Domain) + relays = MailuCollection(Relay) + users = MailuCollection(User) + aliases = MailuCollection(Alias) + config = MailuCollection(Config) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index bd9d228a..fc08b67c 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1,13 +1,15 @@ +""" Mailu marshmallow fields and schema """ -Mailu marshmallow schema -""" + +import re from textwrap import wrap -import re import yaml -from marshmallow import post_dump, fields, Schema +from marshmallow import pre_load, post_dump, fields, Schema +from marshmallow.exceptions import ValidationError +from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts from flask_marshmallow import Marshmallow from OpenSSL import crypto @@ -15,9 +17,9 @@ from . import models, dkim ma = Marshmallow() -# TODO: -# how to mark keys as "required" while unserializing (in certain use cases/API)? -# - fields withoud default => required + +# TODO: how and where to mark keys as "required" while unserializing (on commandline, in api)? +# - fields without default => required # - fields which are the primary key => unchangeable when updating @@ -41,7 +43,7 @@ class RenderYAML: return super().increase_indent(flow, False) @staticmethod - def _update_dict(dict1, dict2): + def _update_items(dict1, dict2): """ sets missing keys in dict1 to values of dict2 """ for key, value in dict2.items(): @@ -53,8 +55,8 @@ class RenderYAML: def loads(cls, *args, **kwargs): """ load yaml data from string """ - cls._update_dict(kwargs, cls._load_defaults) - return yaml.load(*args, **kwargs) + cls._update_items(kwargs, cls._load_defaults) + return yaml.safe_load(*args, **kwargs) _dump_defaults = { 'Dumper': SpacedDumper, @@ -65,13 +67,33 @@ class RenderYAML: def dumps(cls, *args, **kwargs): """ dump yaml data to string """ - cls._update_dict(kwargs, cls._dump_defaults) + cls._update_items(kwargs, cls._dump_defaults) return yaml.dump(*args, **kwargs) +### functions ### + +def handle_email(data): + """ merge separate localpart and domain to email + """ + + localpart = 'localpart' in data + domain = 'domain' in data + + if 'email' in data: + if localpart or domain: + raise ValidationError('duplicate email and localpart/domain') + elif localpart and domain: + data['email'] = f'{data["localpart"]}@{data["domain"]}' + elif localpart or domain: + raise ValidationError('incomplete localpart/domain') + + return data + + ### field definitions ### -class LazyString(fields.String): +class LazyStringField(fields.String): """ Field that serializes a "false" value to the empty string """ @@ -81,14 +103,27 @@ class LazyString(fields.String): return value if value else '' -class CommaSeparatedList(fields.Raw): +class CommaSeparatedListField(fields.Raw): """ Field that deserializes a string containing comma-separated values to a list of strings """ - # TODO: implement this + + def _deserialize(self, value, attr, data, **kwargs): + """ deserialize comma separated string to list of strings + """ + + # empty + if not value: + return [] + + # split string + if isinstance(value, str): + return list([item.strip() for item in value.split(',') if item.strip()]) + else: + return value -class DkimKey(fields.String): +class DkimKeyField(fields.String): """ Field that serializes a dkim key to a list of strings (lines) and deserializes a string or list of strings. """ @@ -120,7 +155,7 @@ class DkimKey(fields.String): # only strings are allowed if not isinstance(value, str): - raise TypeError(f'invalid type: {type(value).__name__!r}') + raise ValidationError(f'invalid type {type(value).__name__!r}') # clean value (remove whitespace and header/footer) value = self._clean_re.sub('', value.strip()) @@ -133,6 +168,11 @@ class DkimKey(fields.String): elif value == 'generate': return dkim.gen_key() + # remember some keydata for error message + keydata = value + if len(keydata) > 40: + keydata = keydata[:25] + '...' + keydata[-10:] + # wrap value into valid pem layout and check validity value = ( '-----BEGIN PRIVATE KEY-----\n' + @@ -142,26 +182,37 @@ class DkimKey(fields.String): try: crypto.load_privatekey(crypto.FILETYPE_PEM, value) except crypto.Error as exc: - raise ValueError('invalid dkim key') from exc + raise ValidationError(f'invalid dkim key {keydata!r}') from exc else: return value -### schema definitions ### +### base definitions ### + +class BaseOpts(SQLAlchemyAutoSchemaOpts): + """ Option class with sqla session + """ + def __init__(self, meta, ordered=False): + if not hasattr(meta, 'sqla_session'): + meta.sqla_session = models.db.session + super(BaseOpts, self).__init__(meta, ordered=ordered) class BaseSchema(ma.SQLAlchemyAutoSchema): """ Marshmallow base schema with custom exclude logic and option to hide sqla defaults """ + OPTIONS_CLASS = BaseOpts + class Meta: """ Schema config """ model = None def __init__(self, *args, **kwargs): - # get and remove config from kwargs + # context? context = kwargs.get('context', {}) + flags = set([key for key, value in context.items() if value is True]) # compile excludes exclude = set(kwargs.get('exclude', [])) @@ -171,8 +222,8 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # add include_by_context if context is not None: - for ctx, what in getattr(self.Meta, 'include_by_context', {}).items(): - if not context.get(ctx): + for need, what in getattr(self.Meta, 'include_by_context', {}).items(): + if not flags & set(need): exclude |= set(what) # update excludes @@ -192,8 +243,8 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # hide by context self._hide_by_context = set() if context is not None: - for ctx, what in getattr(self.Meta, 'hide_by_context', {}).items(): - if not context.get(ctx): + for need, what in getattr(self.Meta, 'hide_by_context', {}).items(): + if not flags & set(need): self._hide_by_context |= set(what) # init SQLAlchemyAutoSchema @@ -212,23 +263,26 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] } - # TODO: remove LazyString and fix model definition (comment should not be nullable) - comment = LazyString() + # TODO: remove LazyString and change model (IMHO comment should not be nullable) + comment = LazyStringField() + + +### schema definitions ### class DomainSchema(BaseSchema): """ Marshmallow schema for Domain model """ class Meta: """ Schema config """ model = models.Domain + load_instance = True include_relationships = True - #include_fk = True exclude = ['users', 'managers', 'aliases'] include_by_context = { - 'dns': {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}, + ('dns',): {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}, } hide_by_context = { - 'secrets': {'dkim_key'}, + ('secrets',): {'dkim_key'}, } exclude_by_value = { 'alternatives': [[]], @@ -240,40 +294,20 @@ class DomainSchema(BaseSchema): 'dns_dmarc': [None], } - dkim_key = DkimKey() + dkim_key = DkimKeyField(allow_none=True) dkim_publickey = fields.String(dump_only=True) dns_mx = fields.String(dump_only=True) dns_spf = fields.String(dump_only=True) dns_dkim = fields.String(dump_only=True) dns_dmarc = fields.String(dump_only=True) - # _dict_types = { - # 'dkim_key': (bytes, type(None)), - # 'dkim_publickey': False, - # 'dns_mx': False, - # 'dns_spf': False, - # 'dns_dkim': False, - # 'dns_dmarc': False, - # } - class TokenSchema(BaseSchema): """ Marshmallow schema for Token model """ class Meta: """ Schema config """ model = models.Token - - # _dict_recurse = True - # _dict_hide = {'user', 'user_email'} - # _dict_mandatory = {'password'} - - # id = db.Column(db.Integer(), primary_key=True) - # user_email = db.Column(db.String(255), db.ForeignKey(User.email), - # nullable=False) - # user = db.relationship(User, - # backref=db.backref('tokens', cascade='all, delete-orphan')) - # password = db.Column(db.String(255), nullable=False) - # ip = db.Column(db.String(255)) + load_instance = True class FetchSchema(BaseSchema): @@ -281,58 +315,57 @@ class FetchSchema(BaseSchema): class Meta: """ Schema config """ model = models.Fetch + load_instance = True include_by_context = { - 'full': {'last_check', 'error'}, + ('full', 'import'): {'last_check', 'error'}, } hide_by_context = { - 'secrets': {'password'}, + ('secrets',): {'password'}, } -# TODO: What about mandatory keys? - # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} - class UserSchema(BaseSchema): """ Marshmallow schema for User model """ class Meta: """ Schema config """ model = models.User + load_instance = True include_relationships = True exclude = ['localpart', 'domain', 'quota_bytes_used'] exclude_by_value = { 'forward_destination': [[]], 'tokens': [[]], + 'manager_of': [[]], 'reply_enddate': ['2999-12-31'], 'reply_startdate': ['1900-01-01'], } + @pre_load + def _handle_password(self, data, many, **kwargs): # pylint: disable=unused-argument + data = handle_email(data) + if 'password' in data: + if 'password_hash' in data or 'hash_scheme' in data: + raise ValidationError('ambigous key password and password_hash/hash_scheme') + # check (hashed) password + password = data['password'] + if password.startswith('{') and '}' in password: + scheme = password[1:password.index('}')] + if scheme not in self.Meta.model.scheme_dict: + raise ValidationError(f'invalid password scheme {scheme!r}') + else: + raise ValidationError(f'invalid hashed password {password!r}') + elif 'password_hash' in data and 'hash_scheme' in data: + if data['hash_scheme'] not in self.Meta.model.scheme_dict: + raise ValidationError(f'invalid password scheme {scheme!r}') + data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + del data['hash_scheme'] + del data['password_hash'] + return data + tokens = fields.Nested(TokenSchema, many=True) fetches = fields.Nested(FetchSchema, many=True) -# TODO: deserialize password/password_hash! What about mandatory keys? - # _dict_mandatory = {'localpart', 'domain', 'password'} - # @classmethod - # def _dict_input(cls, data): - # Email._dict_input(data) - # # handle password - # if 'password' in data: - # if 'password_hash' in data or 'hash_scheme' in data: - # raise ValueError('ambigous key password and password_hash/hash_scheme') - # # check (hashed) password - # password = data['password'] - # if password.startswith('{') and '}' in password: - # scheme = password[1:password.index('}')] - # if scheme not in cls.scheme_dict: - # raise ValueError(f'invalid password scheme {scheme!r}') - # else: - # raise ValueError(f'invalid hashed password {password!r}') - # elif 'password_hash' in data and 'hash_scheme' in data: - # if data['hash_scheme'] not in cls.scheme_dict: - # raise ValueError(f'invalid password scheme {scheme!r}') - # data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] - # del data['hash_scheme'] - # del data['password_hash'] class AliasSchema(BaseSchema): @@ -340,20 +373,18 @@ class AliasSchema(BaseSchema): class Meta: """ Schema config """ model = models.Alias + load_instance = True exclude = ['localpart'] exclude_by_value = { 'destination': [[]], } -# TODO: deserialize destination! - # @staticmethod - # def _dict_input(data): - # Email._dict_input(data) - # # handle comma delimited string for backwards compability - # dst = data.get('destination') - # if type(dst) is str: - # data['destination'] = list([adr.strip() for adr in dst.split(',')]) + @pre_load + def _handle_password(self, data, many, **kwargs): # pylint: disable=unused-argument + return handle_email(data) + + destination = CommaSeparatedListField() class ConfigSchema(BaseSchema): @@ -361,6 +392,7 @@ class ConfigSchema(BaseSchema): class Meta: """ Schema config """ model = models.Config + load_instance = True class RelaySchema(BaseSchema): @@ -368,45 +400,17 @@ class RelaySchema(BaseSchema): class Meta: """ Schema config """ model = models.Relay + load_instance = True class MailuSchema(Schema): - """ Marshmallow schema for Mailu config """ + """ Marshmallow schema for complete Mailu config """ class Meta: """ Schema config """ render_module = RenderYAML + domains = fields.Nested(DomainSchema, many=True) relays = fields.Nested(RelaySchema, many=True) users = fields.Nested(UserSchema, many=True) aliases = fields.Nested(AliasSchema, many=True) config = fields.Nested(ConfigSchema, many=True) - - -### config class ### - -class MailuConfig: - """ Class which joins whole Mailu config for dumping - """ - - _models = { - 'domains': models.Domain, - 'relays': models.Relay, - 'users': models.User, - 'aliases': models.Alias, -# 'config': models.Config, - } - - def __init__(self, sections): - if sections: - for section in sections: - if section not in self._models: - raise ValueError(f'Unknown section: {section!r}') - self._sections = set(sections) - else: - self._sections = set(self._models.keys()) - - def __getattr__(self, section): - if section in self._sections: - return self._models[section].query.all() - else: - raise AttributeError From 31a903f95959e4234f21b9c5e059a73d37d8ec46 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 15 Jan 2021 13:45:36 +0100 Subject: [PATCH 039/181] revived & renamed config-fns. cosmetics. - revived original config-update function for backwards compability - renamed config-dump to config-export to be in line with config-import - converted '*'.format(*) to f-strings - converted string-concatenation to f-strings --- core/admin/mailu/manage.py | 310 +++++++++++++++++++++---------------- 1 file changed, 176 insertions(+), 134 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 6103f904..80a73230 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -7,6 +7,7 @@ import socket import uuid import click +import yaml from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext @@ -64,7 +65,7 @@ def admin(localpart, domain_name, password, mode='create'): user = None if mode == 'ifmissing' or mode == 'update': - email = '{}@{}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' user = models.User.query.get(email) if user and mode == 'ifmissing': @@ -122,14 +123,14 @@ def user(localpart, domain_name, password, hash_scheme=None): def password(localpart, domain_name, password, hash_scheme=None): """ Change the password of an user """ - email = '{0}@{1}'.format(localpart, domain_name) - user = models.User.query.get(email) + email = f'{localpart}@{domain_name}' + user = models.User.query.get(email) if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] if user: user.set_password(password, hash_scheme=hash_scheme) else: - print("User " + email + " not found.") + print(f'User {email} not found.') db.session.commit() @@ -157,7 +158,7 @@ def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): @click.argument('hash_scheme') @with_appcontext def user_import(localpart, domain_name, password_hash, hash_scheme = None): - """ Import a user along with password hash. + """ Import a user along with password hash """ if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] @@ -175,124 +176,146 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() -# @mailu.command() -# @click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') -# @click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') -# @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') -# @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) -# @with_appcontext -# def config_update(verbose=False, delete_objects=False, dry_run=False, source=None): -# """ Update configuration with data from YAML-formatted input -# """ +# TODO: remove this deprecated function +@mailu.command() +@click.option('-v', '--verbose') +@click.option('-d', '--delete-objects') +@with_appcontext +def config_update(verbose=False, delete_objects=False): + """ Sync configuration with data from YAML (deprecated) + """ + new_config = yaml.safe_load(sys.stdin) + # print new_config + domains = new_config.get('domains', []) + tracked_domains = set() + for domain_config in domains: + if verbose: + print(str(domain_config)) + domain_name = domain_config['name'] + max_users = domain_config.get('max_users', -1) + max_aliases = domain_config.get('max_aliases', -1) + max_quota_bytes = domain_config.get('max_quota_bytes', 0) + tracked_domains.add(domain_name) + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name, + max_users=max_users, + max_aliases=max_aliases, + max_quota_bytes=max_quota_bytes) + db.session.add(domain) + print(f'Added {domain_config}') + else: + domain.max_users = max_users + domain.max_aliases = max_aliases + domain.max_quota_bytes = max_quota_bytes + db.session.add(domain) + print(f'Updated {domain_config}') - # try: - # new_config = yaml.safe_load(source) - # except (yaml.scanner.ScannerError, yaml.parser.ParserError) as exc: - # print(f'[ERROR] Invalid yaml: {exc}') - # sys.exit(1) - # else: - # if isinstance(new_config, str): - # print(f'[ERROR] Invalid yaml: {new_config!r}') - # sys.exit(1) - # elif new_config is None or not new_config: - # print('[ERROR] Empty yaml: Please pipe yaml into stdin') - # sys.exit(1) + users = new_config.get('users', []) + tracked_users = set() + user_optional_params = ('comment', 'quota_bytes', 'global_admin', + 'enable_imap', 'enable_pop', 'forward_enabled', + 'forward_destination', 'reply_enabled', + 'reply_subject', 'reply_body', 'displayed_name', + 'spam_enabled', 'email', 'spam_threshold') + for user_config in users: + if verbose: + print(str(user_config)) + localpart = user_config['localpart'] + domain_name = user_config['domain'] + password_hash = user_config.get('password_hash', None) + hash_scheme = user_config.get('hash_scheme', None) + domain = models.Domain.query.get(domain_name) + email = f'{localpart}@{domain_name}' + optional_params = {} + for k in user_optional_params: + if k in user_config: + optional_params[k] = user_config[k] + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User.query.get(email) + tracked_users.add(email) + tracked_domains.add(domain_name) + if not user: + user = models.User( + localpart=localpart, + domain=domain, + **optional_params + ) + else: + for k in optional_params: + setattr(user, k, optional_params[k]) + user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) + db.session.add(user) - # error = False - # tracked = {} - # for section, model in yaml_sections: + aliases = new_config.get('aliases', []) + tracked_aliases = set() + for alias_config in aliases: + if verbose: + print(str(alias_config)) + localpart = alias_config['localpart'] + domain_name = alias_config['domain'] + if isinstance(alias_config['destination'], str): + destination = alias_config['destination'].split(',') + else: + destination = alias_config['destination'] + wildcard = alias_config.get('wildcard', False) + domain = models.Domain.query.get(domain_name) + email = f'{localpart}@{domain_name}' + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + alias = models.Alias.query.get(email) + tracked_aliases.add(email) + tracked_domains.add(domain_name) + if not alias: + alias = models.Alias( + localpart=localpart, + domain=domain, + wildcard=wildcard, + destination=destination, + email=email + ) + else: + alias.destination = destination + alias.wildcard = wildcard + db.session.add(alias) - # items = new_config.get(section) - # if items is None: - # if delete_objects: - # print(f'[ERROR] Invalid yaml: Section "{section}" is missing') - # error = True - # break - # else: - # continue + db.session.commit() - # del new_config[section] + managers = new_config.get('managers', []) + # tracked_managers=set() + for manager_config in managers: + if verbose: + print(str(manager_config)) + domain_name = manager_config['domain'] + user_name = manager_config['user'] + domain = models.Domain.query.get(domain_name) + manageruser = models.User.query.get(f'{user_name}@{domain_name}') + if manageruser not in domain.managers: + domain.managers.append(manageruser) + db.session.add(domain) - # if not isinstance(items, list): - # print(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}') - # error = True - # break - # elif not items: - # continue + db.session.commit() - # # create items - # for data in items: - - # if verbose: - # print(f'Handling {model.__table__} data: {data!r}') - - # try: - # changed = model.from_dict(data, delete_objects) - # except Exception as exc: - # print(f'[ERROR] {exc.args[0]} in data: {data}') - # error = True - # break - - # for item, created in changed: - - # if created is True: - # # flush newly created item - # db.session.add(item) - # db.session.flush() - # if verbose: - # print(f'Added {item!r}: {item.to_dict()}') - # else: - # print(f'Added {item!r}') - - # elif created: - # # modified instance - # if verbose: - # for key, old, new in created: - # print(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}') - # else: - # print(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}') - - # # track primary key of all items - # tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()])) - - # if error: - # break - - # # on error: stop early - # if error: - # print('[ERROR] An error occured. Not committing changes.') - # db.session.rollback() - # sys.exit(1) - - # # are there sections left in new_config? - # if new_config: - # print(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}') - # error = True - - # # test for conflicting domains - # domains = set() - # for model, items in tracked.items(): - # if model in (models.Domain, models.Alternative, models.Relay): - # if domains & items: - # for fqdn in domains & items: - # print(f'[ERROR] Duplicate domain name used: {fqdn}') - # error = True - # domains.update(items) - - # # delete items not tracked - # if delete_objects: - # for model, items in tracked.items(): - # for item in model.query.all(): - # if not item._dict_pval() in items: - # print(f'Deleted {item!r} {item}') - # db.session.delete(item) - - # # don't commit when running dry - # if dry_run: - # print('Dry run. Not commiting changes.') - # db.session.rollback() - # else: - # db.session.commit() + if delete_objects: + for user in db.session.query(models.User).all(): + if not user.email in tracked_users: + if verbose: + print(f'Deleting user: {user.email}') + db.session.delete(user) + for alias in db.session.query(models.Alias).all(): + if not alias.email in tracked_aliases: + if verbose: + print(f'Deleting alias: {alias.email}') + db.session.delete(alias) + for domain in db.session.query(models.Domain).all(): + if not domain.name in tracked_domains: + if verbose: + print(f'Deleting domain: {domain.name}') + db.session.delete(domain) + db.session.commit() SECTIONS = {'domains', 'relays', 'users', 'aliases'} @@ -304,30 +327,51 @@ SECTIONS = {'domains', 'relays', 'users', 'aliases'} @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext def config_import(verbose=False, dry_run=False, source=None): - """ Import configuration YAML + """ Import configuration from YAML """ + def log(**data): + caller = sys._getframe(1).f_code.co_name # pylint: disable=protected-access + if caller == '_track_import': + print(f'Handling {data["self"].opts.model.__table__} data: {data["data"]!r}') + + def format_errors(store, path=None): + if path is None: + path = [] + for key in sorted(store): + location = path + [str(key)] + value = store[key] + if isinstance(value, dict): + format_errors(value, location) + else: + for message in value: + print(f'[ERROR] {".".join(location)}: {message}') + context = { - 'verbose': verbose, # TODO: use callback function to be verbose? + 'callback': log if verbose else None, 'import': True, } + error = False try: config = MailuSchema(context=context).loads(source) except ValidationError as exc: - print(f'[ERROR] {exc}') - # TODO: show nice errors - from pprint import pprint - pprint(exc.messages) - sys.exit(1) + error = True + format_errors(exc.messages) else: print(config) print(MailuSchema().dumps(config)) - # TODO: does not commit yet. - # TODO: delete other entries? + # TODO: need to delete other entries - # don't commit when running dry - if True: #dry_run: + # TODO: enable commit + error = True + + # don't commit when running dry or validation errors occured + if error: + print('An error occured. Not committing changes.') + db.session.rollback() + sys.exit(2) + elif dry_run: print('Dry run. Not commiting changes.') db.session.rollback() else: @@ -343,10 +387,8 @@ def config_import(verbose=False, dry_run=False, source=None): help='save yaml to file') @click.argument('sections', nargs=-1) @with_appcontext -def config_dump(full=False, secrets=False, dns=False, output=None, sections=None): - """ Dump configuration as YAML to stdout or file - - SECTIONS can be: domains, relays, users, aliases +def config_export(full=False, secrets=False, dns=False, output=None, sections=None): + """ Export configuration as YAML to stdout or file """ if sections: @@ -407,7 +449,7 @@ def alias(localpart, domain_name, destination, wildcard=False): domain=domain, wildcard=wildcard, destination=destination.split(','), - email="%s@%s" % (localpart, domain_name) + email=f'{localpart}@{domain_name}' ) db.session.add(alias) db.session.commit() @@ -438,7 +480,7 @@ def setmanager(domain_name, user_name='manager'): """ Make a user manager of a domain """ domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) + manageruser = models.User.query.get(f'{user_name}@{domain_name}') domain.managers.append(manageruser) db.session.add(domain) db.session.commit() From 8213d044b25860046381c6766220c0855616b88c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 15 Jan 2021 13:53:47 +0100 Subject: [PATCH 040/181] added docstrings, use f-strings, cleanup - idna.encode does not encode upper-case letters, so .lower() has to be called on value not on result - split email-address on '@' only once - converted '*'.format(*) to f-strings - added docstrings - removed from_dict method - code cleanup/style (list concat, exceptions, return&else, line-length) - added TODO comments on possible future changes --- core/admin/mailu/models.py | 376 ++++++++++++------------------------- 1 file changed, 122 insertions(+), 254 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 73d05801..3187f597 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -8,6 +8,7 @@ import json from datetime import date from email.mime import text +from itertools import chain import flask_sqlalchemy import sqlalchemy @@ -30,11 +31,12 @@ class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ + # TODO: String(80) is too small? impl = db.String(80) def process_bind_param(self, value, dialect): """ encode unicode domain name to punycode """ - return idna.encode(value).decode('ascii').lower() + return idna.encode(value.lower()).decode('ascii') def process_result_value(self, value, dialect): """ decode punycode domain name to unicode """ @@ -46,26 +48,21 @@ class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ + # TODO: String(255) is too small? impl = db.String(255) def process_bind_param(self, value, dialect): """ encode unicode domain part of email address to punycode """ - try: - localpart, domain_name = value.split('@') - return '{0}@{1}'.format( - localpart, - idna.encode(domain_name).decode('ascii'), - ).lower() - except ValueError: - pass + localpart, domain_name = value.rsplit('@', 1) + if '@' in localpart: + raise ValueError('email local part must not contain "@"') + domain_name = domain_name.lower() + return f'{localpart}@{idna.encode(domain_name).decode("ascii")}' def process_result_value(self, value, dialect): """ decode punycode domain part of email to unicode """ - localpart, domain_name = value.split('@') - return '{0}@{1}'.format( - localpart, - idna.decode(domain_name), - ) + localpart, domain_name = value.rsplit('@', 1) + return f'{localpart}@{idna.decode(domain_name)}' python_type = str @@ -81,7 +78,7 @@ class CommaSeparatedList(db.TypeDecorator): raise TypeError('Must be a list of strings') for item in value: if ',' in item: - raise ValueError('Item must not contain a comma') + raise ValueError('list item must not contain ","') return ','.join(sorted(value)) def process_result_value(self, value, dialect): @@ -123,173 +120,6 @@ class Base(db.Model): updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) comment = db.Column(db.String(255), nullable=True, default='') - # @classmethod - # def from_dict(cls, data, delete=False): - - # changed = [] - - # pkey = cls._dict_pkey() - - # # handle "primary key" only - # if not isinstance(data, dict): - # data = {pkey: data} - - # # modify input data - # if hasattr(cls, '_dict_input'): - # try: - # cls._dict_input(data) - # except Exception as exc: - # raise ValueError(f'{exc}', cls, None, data) from exc - - # # check for primary key (if not recursed) - # if not getattr(cls, '_dict_recurse', False): - # if not pkey in data: - # raise KeyError(f'primary key {cls.__table__}.{pkey} is missing', cls, pkey, data) - - # # check data keys and values - # for key in list(data.keys()): - - # # check key - # if not hasattr(cls, key) and not key in cls.__mapper__.relationships: - # raise KeyError(f'unknown key {cls.__table__}.{key}', cls, key, data) - - # # check value type - # value = data[key] - # col = cls.__mapper__.columns.get(key) - # if col is not None: - # if not ((value is None and col.nullable) or (isinstance(value, col.type.python_type))): - # raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) - # else: - # rel = cls.__mapper__.relationships.get(key) - # if rel is None: - # itype = getattr(cls, '_dict_types', {}).get(key) - # if itype is not None: - # if itype is False: # ignore value. TODO: emit warning? - # del data[key] - # continue - # elif not isinstance(value, itype): - # raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) - # else: - # raise NotImplementedError(f'type not defined for {cls.__table__}.{key}') - - # # handle relationships - # if key in cls.__mapper__.relationships: - # rel_model = cls.__mapper__.relationships[key].argument - # if not isinstance(rel_model, sqlalchemy.orm.Mapper): - # add = rel_model.from_dict(value, delete) - # assert len(add) == 1 - # rel_item, updated = add[0] - # changed.append((rel_item, updated)) - # data[key] = rel_item - - # # create item if necessary - # created = False - # item = cls.query.get(data[pkey]) if pkey in data else None - # if item is None: - - # # check for mandatory keys - # missing = getattr(cls, '_dict_mandatory', set()) - set(data.keys()) - # if missing: - # raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {cls.__table__} missing', cls, missing, data) - - # # remove mapped relationships from data - # mapped = {} - # for key in list(data.keys()): - # if key in cls.__mapper__.relationships: - # if isinstance(cls.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): - # mapped[key] = data[key] - # del data[key] - - # # create new item - # item = cls(**data) - # created = True - - # # and update mapped relationships (below) - # data = mapped - - # # update item - # updated = [] - # for key, value in data.items(): - - # # skip primary key - # if key == pkey: - # continue - - # if key in cls.__mapper__.relationships: - # # update relationship - # rel_model = cls.__mapper__.relationships[key].argument - # if isinstance(rel_model, sqlalchemy.orm.Mapper): - # rel_model = rel_model.class_ - # # add (and create) referenced items - # cur = getattr(item, key) - # old = sorted(cur, key=id) - # new = [] - # for rel_data in value: - # # get or create related item - # add = rel_model.from_dict(rel_data, delete) - # assert len(add) == 1 - # rel_item, rel_updated = add[0] - # changed.append((rel_item, rel_updated)) - # if rel_item not in cur: - # cur.append(rel_item) - # new.append(rel_item) - - # # delete referenced items missing in yaml - # rel_pkey = rel_model._dict_pkey() - # new_data = list([i.to_dict(True, True, None, True, [rel_pkey]) for i in new]) - # for rel_item in old: - # if rel_item not in new: - # # check if item with same data exists to stabilze import without primary key - # rel_data = rel_item.to_dict(True, True, None, True, [rel_pkey]) - # try: - # same_idx = new_data.index(rel_data) - # except ValueError: - # same = None - # else: - # same = new[same_idx] - - # if same is None: - # # delete items missing in new - # if delete: - # cur.remove(rel_item) - # else: - # new.append(rel_item) - # else: - # # swap found item with same data with newly created item - # new.append(rel_item) - # new_data.append(rel_data) - # new.remove(same) - # del new_data[same_idx] - # for i, (ch_item, _) in enumerate(changed): - # if ch_item is same: - # changed[i] = (rel_item, []) - # db.session.flush() - # db.session.delete(ch_item) - # break - - # # remember changes - # new = sorted(new, key=id) - # if new != old: - # updated.append((key, old, new)) - - # else: - # # update key - # old = getattr(item, key) - # if isinstance(old, list): - # # deduplicate list value - # assert isinstance(value, list) - # value = set(value) - # old = set(old) - # if not delete: - # value = old | value - # if value != old: - # updated.append((key, old, value)) - # setattr(item, key, value) - - # changed.append((item, created if created else updated)) - - # return changed - # Many-to-many association table for domain managers managers = db.Table('manager', Base.metadata, @@ -309,9 +139,7 @@ class Config(Base): # TODO: use sqlalchemy.event.listen() on a store method of object? @sqlalchemy.event.listens_for(db.session, 'after_commit') def store_dkim_key(session): - """ Store DKIM key on commit - """ - + """ Store DKIM key on commit """ for obj in session.identity_map.values(): if isinstance(obj, Domain): if obj._dkim_key_changed: @@ -340,21 +168,27 @@ class Domain(Base): _dkim_key_changed = False def _dkim_file(self): + """ return filename for active DKIM key """ return app.config['DKIM_PATH'].format( - domain=self.name, selector=app.config['DKIM_SELECTOR']) + domain=self.name, + selector=app.config['DKIM_SELECTOR'] + ) @property def dns_mx(self): - hostname = app.config['HOSTNAMES'].split(',')[0] + """ return MX record for domain """ + hostname = app.config['HOSTNAMES'].split(',', 1)[0] return f'{self.name}. 600 IN MX 10 {hostname}.' @property def dns_spf(self): - hostname = app.config['HOSTNAMES'].split(',')[0] + """ return SPF record for domain """ + hostname = app.config['HOSTNAMES'].split(',', 1)[0] return f'{self.name}. 600 IN TXT "v=spf1 mx a:{hostname} ~all"' @property def dns_dkim(self): + """ return DKIM record for domain """ if os.path.exists(self._dkim_file()): selector = app.config['DKIM_SELECTOR'] return ( @@ -364,6 +198,7 @@ class Domain(Base): @property def dns_dmarc(self): + """ return DMARC record for domain """ if os.path.exists(self._dkim_file()): domain = app.config['DOMAIN'] rua = app.config['DMARC_RUA'] @@ -374,6 +209,7 @@ class Domain(Base): @property def dkim_key(self): + """ return private DKIM key """ if self._dkim_key is None: file_path = self._dkim_file() if os.path.exists(file_path): @@ -385,6 +221,7 @@ class Domain(Base): @dkim_key.setter def dkim_key(self, value): + """ set private DKIM key """ old_key = self.dkim_key if value is None: value = b'' @@ -393,36 +230,40 @@ class Domain(Base): @property def dkim_publickey(self): + """ return public part of DKIM key """ dkim_key = self.dkim_key if dkim_key: return dkim.strip_key(dkim_key).decode('utf8') def generate_dkim_key(self): + """ generate and activate new DKIM key """ self.dkim_key = dkim.gen_key() def has_email(self, localpart): - for email in self.users + self.aliases: + """ checks if localpart is configured for domain """ + for email in chain(self.users, self.aliases): if email.localpart == localpart: return True return False def check_mx(self): + """ checks if MX record for domain points to mailu host """ try: - hostnames = app.config['HOSTNAMES'].split(',') + hostnames = set(app.config['HOSTNAMES'].split(',')) return any( - str(rset).split()[-1][:-1] in hostnames + rset.exchange.to_text().rstrip('.') in hostnames for rset in dns.resolver.query(self.name, 'MX') ) - except Exception: + except dns.exception.DNSException: return False def __str__(self): return str(self.name) def __eq__(self, other): - try: - return self.name == other.name - except AttributeError: + if isinstance(other, self.__class__): + return str(self.name) == str(other.name) + else: return NotImplemented def __hash__(self): @@ -432,7 +273,7 @@ class Domain(Base): class Alternative(Base): """ Alternative name for a served domain. - The name "domain alias" was avoided to prevent some confusion. + The name "domain alias" was avoided to prevent some confusion. """ __tablename__ = 'alternative' @@ -454,6 +295,7 @@ class Relay(Base): __tablename__ = 'relay' name = db.Column(IdnaDomain, primary_key=True, nullable=False) + # TODO: String(80) is too small? smtp = db.Column(db.String(80), nullable=True) def __str__(self): @@ -464,10 +306,14 @@ class Email(object): """ Abstraction for an email address (localpart and domain). """ + # TODO: validate max. total length of address (<=254) + + # TODO: String(80) is too large (>64)? localpart = db.Column(db.String(80), nullable=False) @declarative.declared_attr def domain_name(self): + """ the domain part of the email address """ return db.Column(IdnaDomain, db.ForeignKey(Domain.name), nullable=False, default=IdnaDomain) @@ -476,26 +322,18 @@ class Email(object): # especially when the mail server is reading the database. @declarative.declared_attr def email(self): - updater = lambda context: '{0}@{1}'.format( - context.current_parameters['localpart'], - context.current_parameters['domain_name'], - ) + """ the complete email address (localpart@domain) """ + updater = lambda ctx: '{localpart}@{domain_name}'.format(**ctx.current_parameters) return db.Column(IdnaEmail, primary_key=True, nullable=False, - default=updater) + default=updater + ) def sendmail(self, subject, body): - """ Send an email to the address. - """ - from_address = '{0}@{1}'.format( - app.config['POSTMASTER'], - idna.encode(app.config['DOMAIN']).decode('ascii'), - ) + """ send an email to the address """ + from_address = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: - to_address = '{0}@{1}'.format( - self.localpart, - idna.encode(self.domain_name).decode('ascii'), - ) + to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' msg = text.MIMEText(body) msg['Subject'] = subject msg['From'] = from_address @@ -504,7 +342,8 @@ class Email(object): @classmethod def resolve_domain(cls, email): - localpart, domain_name = email.split('@', 1) if '@' in email else (None, email) + """ resolves domain alternative to real domain """ + localpart, domain_name = email.rsplit('@', 1) if '@' in email else (None, email) alternative = Alternative.query.get(domain_name) if alternative: domain_name = alternative.domain_name @@ -512,17 +351,19 @@ class Email(object): @classmethod def resolve_destination(cls, localpart, domain_name, ignore_forward_keep=False): + """ return destination for email address localpart@domain_name """ + localpart_stripped = None stripped_alias = None if os.environ.get('RECIPIENT_DELIMITER') in localpart: localpart_stripped = localpart.rsplit(os.environ.get('RECIPIENT_DELIMITER'), 1)[0] - user = User.query.get('{}@{}'.format(localpart, domain_name)) + user = User.query.get(f'{localpart}@{domain_name}') if not user and localpart_stripped: - user = User.query.get('{}@{}'.format(localpart_stripped, domain_name)) + user = User.query.get(f'{localpart_stripped}@{domain_name}') if user: - email = '{}@{}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' if user.forward_enabled: destination = user.forward_destination @@ -537,11 +378,15 @@ class Email(object): if pure_alias and not pure_alias.wildcard: return pure_alias.destination - elif stripped_alias: + + if stripped_alias: return stripped_alias.destination - elif pure_alias: + + if pure_alias: return pure_alias.destination + return None + def __str__(self): return str(self.email) @@ -586,11 +431,15 @@ class User(Base, Email): is_active = True is_anonymous = False + # TODO: remove unused user.get_id() def get_id(self): + """ return users email address """ return self.email + # TODO: remove unused user.destination @property def destination(self): + """ returns comma separated string of destinations """ if self.forward_enabled: result = list(self.forward_destination) if self.forward_keep: @@ -601,6 +450,7 @@ class User(Base, Email): @property def reply_active(self): + """ returns status of autoreply function """ now = date.today() return ( self.reply_enabled and @@ -608,49 +458,56 @@ class User(Base, Email): self.reply_enddate > now ) - scheme_dict = {'PBKDF2': 'pbkdf2_sha512', - 'BLF-CRYPT': 'bcrypt', - 'SHA512-CRYPT': 'sha512_crypt', - 'SHA256-CRYPT': 'sha256_crypt', - 'MD5-CRYPT': 'md5_crypt', - 'CRYPT': 'des_crypt'} + scheme_dict = { + 'PBKDF2': 'pbkdf2_sha512', + 'BLF-CRYPT': 'bcrypt', + 'SHA512-CRYPT': 'sha512_crypt', + 'SHA256-CRYPT': 'sha256_crypt', + 'MD5-CRYPT': 'md5_crypt', + 'CRYPT': 'des_crypt', + } - def get_password_context(self): + def _get_password_context(self): return passlib.context.CryptContext( schemes=self.scheme_dict.values(), default=self.scheme_dict[app.config['PASSWORD_SCHEME']], ) - def check_password(self, password): - context = self.get_password_context() - reference = re.match('({[^}]+})?(.*)', self.password).group(2) - result = context.verify(password, reference) - if result and context.identify(reference) != context.default_scheme(): - self.set_password(password) + def check_password(self, plain): + """ Check password against stored hash + Update hash when default scheme has changed + """ + context = self._get_password_context() + hashed = re.match('^({[^}]+})?(.*)$', self.password).group(2) + result = context.verify(plain, hashed) + if result and context.identify(hashed) != context.default_scheme(): + self.set_password(plain) db.session.add(self) db.session.commit() return result - def set_password(self, password, hash_scheme=None, raw=False): - """Set password for user with specified encryption scheme - @password: plain text password to encrypt (if raw == True the hash itself) + # TODO: remove kwarg hash_scheme - there is no point in setting a scheme, + # when the next check updates the password to the default scheme. + def set_password(self, new, hash_scheme=None, raw=False): + """ Set password for user with specified encryption scheme + @new: plain text password to encrypt (or, if raw is True: the hash itself) """ + # for the list of hash schemes see https://wiki2.dovecot.org/Authentication/PasswordSchemes if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] - # for the list of hash schemes see https://wiki2.dovecot.org/Authentication/PasswordSchemes - if raw: - self.password = '{'+hash_scheme+'}' + password - else: - self.password = '{'+hash_scheme+'}' + \ - self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme]) + if not raw: + new = self._get_password_context().encrypt(new, self.scheme_dict[hash_scheme]) + self.password = f'{{{hash_scheme}}}{new}' def get_managed_domains(self): + """ return list of domains this user can manage """ if self.global_admin: return Domain.query.all() else: return self.manager_of def get_managed_emails(self, include_aliases=True): + """ returns list of email addresses this user can manage """ emails = [] for domain in self.get_managed_domains(): emails.extend(domain.users) @@ -659,16 +516,18 @@ class User(Base, Email): return emails def send_welcome(self): + """ send welcome email to user """ if app.config['WELCOME']: - self.sendmail(app.config['WELCOME_SUBJECT'], - app.config['WELCOME_BODY']) + self.sendmail(app.config['WELCOME_SUBJECT'], app.config['WELCOME_BODY']) @classmethod def get(cls, email): + """ find user object for email address """ return cls.query.get(email) @classmethod def login(cls, email, password): + """ login user when enabled and password is valid """ user = cls.query.get(email) return user if (user and user.enabled and user.check_password(password)) else None @@ -686,6 +545,8 @@ class Alias(Base, Email): @classmethod def resolve(cls, localpart, domain_name): + """ find aliases matching email address localpart@domain_name """ + alias_preserve_case = cls.query.filter( sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( @@ -709,24 +570,27 @@ class Alias(Base, Email): sqlalchemy.func.lower(cls.localpart) == localpart_lower ), sqlalchemy.and_( cls.wildcard is True, - sqlalchemy.bindparam('l', localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) + sqlalchemy.bindparam('l', localpart_lower).like( + sqlalchemy.func.lower(cls.localpart)) ) ) ) - ).order_by(cls.wildcard, sqlalchemy.func.char_length(sqlalchemy.func.lower(cls.localpart)).desc()).first() + ).order_by(cls.wildcard, sqlalchemy.func.char_length( + sqlalchemy.func.lower(cls.localpart)).desc()).first() if alias_preserve_case and alias_lower_case: - if alias_preserve_case.wildcard: - return alias_lower_case - else: - return alias_preserve_case - elif alias_preserve_case and not alias_lower_case: - return alias_preserve_case - elif alias_lower_case and not alias_preserve_case: - return alias_lower_case - else: - return None + return alias_lower_case if alias_preserve_case.wildcard else alias_preserve_case + if alias_preserve_case and not alias_lower_case: + return alias_preserve_case + + if alias_lower_case and not alias_preserve_case: + return alias_lower_case + + return None + +# TODO: where are Tokens used / validated? +# TODO: what about API tokens? class Token(Base): """ A token is an application password for a given user. """ @@ -739,16 +603,20 @@ class Token(Base): user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) + # TODO: String(80) is too large? ip = db.Column(db.String(255)) def check_password(self, password): + """ verifies password against stored hash """ return passlib.hash.sha256_crypt.verify(password, self.password) + # TODO: use crypt context and default scheme from config? def set_password(self, password): + """ sets password using sha256_crypt(rounds=1000) """ self.password = passlib.hash.sha256_crypt.using(rounds=1000).hash(password) def __str__(self): - return self.comment or self.ip + return str(self.comment or self.ip) class Fetch(Base): From 65b1ad46d97c355ac93879b5b59dc75b77c579b9 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 15 Jan 2021 13:57:20 +0100 Subject: [PATCH 041/181] order yaml data and allow callback on import - in yaml the primary key is now always first - calling a function on import allows import to be more verbose - skip "fetches" when empty --- core/admin/mailu/schemas.py | 88 ++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 16 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index fc08b67c..5dc10e17 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -3,11 +3,12 @@ import re +from collections import OrderedDict from textwrap import wrap import yaml -from marshmallow import pre_load, post_dump, fields, Schema +from marshmallow import pre_load, post_load, post_dump, fields, Schema from marshmallow.exceptions import ValidationError from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts from flask_marshmallow import Marshmallow @@ -25,6 +26,12 @@ ma = Marshmallow() ### yaml render module ### +# allow yaml module to dump OrderedDict +yaml.add_representer( + OrderedDict, + lambda cls, data: cls.represent_mapping('tag:yaml.org,2002:map', data.items()) +) + class RenderYAML: """ Marshmallow YAML Render Module """ @@ -62,6 +69,7 @@ class RenderYAML: 'Dumper': SpacedDumper, 'default_flow_style': False, 'allow_unicode': True, + 'sort_keys': False, } @classmethod def dumps(cls, *args, **kwargs): @@ -195,6 +203,8 @@ class BaseOpts(SQLAlchemyAutoSchemaOpts): def __init__(self, meta, ordered=False): if not hasattr(meta, 'sqla_session'): meta.sqla_session = models.db.session + if not hasattr(meta, 'ordered'): + meta.ordered = True super(BaseOpts, self).__init__(meta, ordered=ordered) class BaseSchema(ma.SQLAlchemyAutoSchema): @@ -206,13 +216,12 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): class Meta: """ Schema config """ - model = None def __init__(self, *args, **kwargs): # context? context = kwargs.get('context', {}) - flags = set([key for key, value in context.items() if value is True]) + flags = {key for key, value in context.items() if value is True} # compile excludes exclude = set(kwargs.get('exclude', [])) @@ -234,7 +243,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # exclude default values if not context.get('full'): - for column in self.Meta.model.__table__.columns: + for column in getattr(self.Meta, 'model').__table__.columns: if column.name not in exclude: self._exclude_by_value.setdefault(column.name, []).append( None if column.default is None else column.default.arg @@ -250,20 +259,48 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # init SQLAlchemyAutoSchema super().__init__(*args, **kwargs) - @post_dump - def _remove_skip_values(self, data, many, **kwargs): # pylint: disable=unused-argument + # init order + if hasattr(self.Meta, 'order'): + # use user-defined order + self._order = list(reversed(getattr(self.Meta, 'order'))) + else: + # default order is: primary_key + other keys alphabetically + self._order = list(sorted(self.fields.keys())) + primary = self.opts.model.__table__.primary_key.columns.values()[0].name + self._order.remove(primary) + self._order.reverse() + self._order.append(primary) + @pre_load + def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument + call = self.context.get('callback') + if call is not None: + call(self=self, data=data, many=many, **kwargs) + return data + + @post_dump + def _hide_and_order(self, data, many, **kwargs): # pylint: disable=unused-argument + + # order output + for key in self._order: + try: + data.move_to_end(key, False) + except KeyError: + pass + + # stop early when not excluding/hiding if not self._exclude_by_value and not self._hide_by_context: return data + # exclude items or hide values full = self.context.get('full') - return { - key: '' if key in self._hide_by_context else value + return type(data)([ + (key, '' if key in self._hide_by_context else value) for key, value in data.items() if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] - } + ]) - # TODO: remove LazyString and change model (IMHO comment should not be nullable) + # TODO: remove LazyStringField and change model (IMHO comment should not be nullable) comment = LazyStringField() @@ -336,13 +373,14 @@ class UserSchema(BaseSchema): exclude_by_value = { 'forward_destination': [[]], 'tokens': [[]], + 'fetches': [[]], 'manager_of': [[]], 'reply_enddate': ['2999-12-31'], 'reply_startdate': ['1900-01-01'], } @pre_load - def _handle_password(self, data, many, **kwargs): # pylint: disable=unused-argument + def _handle_email_and_password(self, data, many, **kwargs): # pylint: disable=unused-argument data = handle_email(data) if 'password' in data: if 'password_hash' in data or 'hash_scheme' in data: @@ -358,16 +396,23 @@ class UserSchema(BaseSchema): elif 'password_hash' in data and 'hash_scheme' in data: if data['hash_scheme'] not in self.Meta.model.scheme_dict: raise ValidationError(f'invalid password scheme {scheme!r}') - data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + data['password'] = f'{{{data["hash_scheme"]}}}{data["password_hash"]}' del data['hash_scheme'] del data['password_hash'] return data + # TODO: verify password (should this be done in model?) + # scheme, hashed = re.match('^(?:{([^}]+)})?(.*)$', self.password).groups() + # if not scheme... + # ctx = passlib.context.CryptContext(schemes=[scheme], default=scheme) + # try: + # ctx.verify('', hashed) + # =>? ValueError: hash could not be identified + tokens = fields.Nested(TokenSchema, many=True) fetches = fields.Nested(FetchSchema, many=True) - class AliasSchema(BaseSchema): """ Marshmallow schema for Alias model """ class Meta: @@ -381,7 +426,7 @@ class AliasSchema(BaseSchema): } @pre_load - def _handle_password(self, data, many, **kwargs): # pylint: disable=unused-argument + def _handle_email(self, data, many, **kwargs): # pylint: disable=unused-argument return handle_email(data) destination = CommaSeparatedListField() @@ -408,9 +453,20 @@ class MailuSchema(Schema): class Meta: """ Schema config """ render_module = RenderYAML + ordered = True + order = ['config', 'domains', 'users', 'aliases', 'relays'] + @post_dump(pass_many=True) + def _order(self, data : OrderedDict, many : bool, **kwargs): # pylint: disable=unused-argument + for key in reversed(self.Meta.order): + try: + data.move_to_end(key, False) + except KeyError: + pass + return data + + config = fields.Nested(ConfigSchema, many=True) domains = fields.Nested(DomainSchema, many=True) - relays = fields.Nested(RelaySchema, many=True) users = fields.Nested(UserSchema, many=True) aliases = fields.Nested(AliasSchema, many=True) - config = fields.Nested(ConfigSchema, many=True) + relays = fields.Nested(RelaySchema, many=True) From 902b398127dc9abb6f3433ee7fccd51ddc0b3873 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 24 Jan 2021 19:07:48 +0100 Subject: [PATCH 042/181] next step for import/export yaml & json --- core/admin/mailu/manage.py | 238 ++++++++++++++++++++++++++++++------ core/admin/mailu/models.py | 176 ++++++++++++++++++-------- core/admin/mailu/schemas.py | 157 +++++++++++++++++------- 3 files changed, 441 insertions(+), 130 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 80a73230..e02d9ad4 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -4,9 +4,15 @@ import sys import os import socket +import json +import logging import uuid +from collections import Counter +from itertools import chain + import click +import sqlalchemy import yaml from flask import current_app as app @@ -14,7 +20,7 @@ from flask.cli import FlaskGroup, with_appcontext from marshmallow.exceptions import ValidationError from . import models -from .schemas import MailuSchema +from .schemas import MailuSchema, get_schema db = models.db @@ -322,60 +328,211 @@ SECTIONS = {'domains', 'relays', 'users', 'aliases'} @mailu.command() -@click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') +@click.option('-v', '--verbose', count=True, help='Increase verbosity') +@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors') +@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext -def config_import(verbose=False, dry_run=False, source=None): - """ Import configuration from YAML +def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=None): + """ Import configuration as YAML or JSON from stdin or file """ - def log(**data): - caller = sys._getframe(1).f_code.co_name # pylint: disable=protected-access - if caller == '_track_import': - print(f'Handling {data["self"].opts.model.__table__} data: {data["data"]!r}') + # verbose + # 0 : show number of changes + # 1 : also show changes + # 2 : also show secrets + # 3 : also show input data + # 4 : also show sql queries + + if quiet: + verbose = -1 + + counter = Counter() + dumper = {} def format_errors(store, path=None): + + res = [] if path is None: path = [] for key in sorted(store): location = path + [str(key)] value = store[key] if isinstance(value, dict): - format_errors(value, location) + res.extend(format_errors(value, location)) else: for message in value: - print(f'[ERROR] {".".join(location)}: {message}') + res.append((".".join(location), message)) - context = { - 'callback': log if verbose else None, + if path: + return res + + fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}' + res = [fmt.format(loc, msg) for loc, msg in res] + num = f'error{["s",""][len(res)==1]}' + res.insert(0, f'[ValidationError] {len(res)} {num} occured during input validation') + + return '\n'.join(res) + + def format_changes(*message): + if counter: + changes = [] + last = None + for (action, what), count in sorted(counter.items()): + if action != last: + if last: + changes.append('/') + changes.append(f'{action}:') + last = action + changes.append(f'{what}({count})') + else: + changes = 'no changes.' + return chain(message, changes) + + def log(action, target, message=None): + if message is None: + message = json.dumps(dumper[target.__class__].dump(target), ensure_ascii=False) + print(f'{action} {target.__table__}: {message}') + + def listen_insert(mapper, connection, target): # pylint: disable=unused-argument + """ callback function to track import """ + counter.update([('Added', target.__table__.name)]) + if verbose >= 1: + log('Added', target) + + def listen_update(mapper, connection, target): # pylint: disable=unused-argument + """ callback function to track import """ + + changed = {} + inspection = sqlalchemy.inspect(target) + for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: + if getattr(inspection.attrs, attr.key).history.has_changes(): + if sqlalchemy.orm.attributes.get_history(target, attr.key)[2]: + before = sqlalchemy.orm.attributes.get_history(target, attr.key)[2].pop() + after = getattr(target, attr.key) + # only remember changed keys + if before != after and (before or after): + if verbose >= 1: + changed[str(attr.key)] = (before, after) + else: + break + + if verbose >= 1: + # use schema with dump_context to hide secrets and sort keys + primary = json.dumps(str(target), ensure_ascii=False) + dumped = get_schema(target)(only=changed.keys(), context=dump_context).dump(target) + for key, value in dumped.items(): + before, after = changed[key] + if value == '': + before = '' if before else before + after = '' if after else after + else: + # TODO: use schema to "convert" before value? + after = value + before = json.dumps(before, ensure_ascii=False) + after = json.dumps(after, ensure_ascii=False) + log('Modified', target, f'{primary} {key}: {before} -> {after}') + + if changed: + counter.update([('Modified', target.__table__.name)]) + + def listen_delete(mapper, connection, target): # pylint: disable=unused-argument + """ callback function to track import """ + counter.update([('Deleted', target.__table__.name)]) + if verbose >= 1: + log('Deleted', target) + + # this listener should not be necessary, when: + # dkim keys should be stored in database and it should be possible to store multiple + # keys per domain. the active key would be also stored on disk on commit. + def listen_dkim(session, flush_context): # pylint: disable=unused-argument + """ callback function to track import """ + for target in session.identity_map.values(): + if not isinstance(target, models.Domain): + continue + primary = json.dumps(str(target), ensure_ascii=False) + before = target._dkim_key_on_disk + after = target._dkim_key + if before != after and (before or after): + if verbose >= 2: + before = before.decode('ascii', 'ignore') + after = after.decode('ascii', 'ignore') + else: + before = '' if before else '' + after = '' if after else '' + before = json.dumps(before, ensure_ascii=False) + after = json.dumps(after, ensure_ascii=False) + log('Modified', target, f'{primary} dkim_key: {before} -> {after}') + counter.update([('Modified', target.__table__.name)]) + + def track_serialize(self, item): + """ callback function to track import """ + log('Handling', self.opts.model, item) + + # configure contexts + dump_context = { + 'secrets': verbose >= 2, + } + load_context = { + 'callback': track_serialize if verbose >= 3 else None, + 'clear': not update, 'import': True, } - error = False + # register listeners + for schema in get_schema(): + model = schema.Meta.model + dumper[model] = schema(context=dump_context) + sqlalchemy.event.listen(model, 'after_insert', listen_insert) + sqlalchemy.event.listen(model, 'after_update', listen_update) + sqlalchemy.event.listen(model, 'after_delete', listen_delete) + + # special listener for dkim_key changes + sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim) + + if verbose >= 4: + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + try: - config = MailuSchema(context=context).loads(source) + with models.db.session.no_autoflush: + config = MailuSchema(only=SECTIONS, context=load_context).loads(source) except ValidationError as exc: - error = True - format_errors(exc.messages) - else: - print(config) + raise click.ClickException(format_errors(exc.messages)) from exc + except Exception as exc: + # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) + raise click.ClickException(f'[{exc.__class__.__name__}] {" ".join(str(exc).split())}') from exc + + # flush session to show/count all changes + if dry_run or verbose >= 1: + db.session.flush() + + # check for duplicate domain names + dup = set() + for fqdn in chain(db.session.query(models.Domain.name), + db.session.query(models.Alternative.name), + db.session.query(models.Relay.name)): + if fqdn in dup: + raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}') + dup.add(fqdn) + + # TODO: implement special update "items" + # -pkey: which - remove item "which" + # -key: null or [] or {} - set key to default + # -pkey: null or [] or {} - remove all existing items in this list + + # don't commit when running dry + if dry_run: + db.session.rollback() + if not quiet: + print(*format_changes('Dry run. Not commiting changes.')) + # TODO: remove debug print(MailuSchema().dumps(config)) - # TODO: need to delete other entries - - # TODO: enable commit - error = True - - # don't commit when running dry or validation errors occured - if error: - print('An error occured. Not committing changes.') - db.session.rollback() - sys.exit(2) - elif dry_run: - print('Dry run. Not commiting changes.') - db.session.rollback() else: db.session.commit() + if not quiet: + print(*format_changes('Commited changes.')) @mailu.command() @@ -385,28 +542,35 @@ def config_import(verbose=False, dry_run=False, source=None): @click.option('-d', '--dns', is_flag=True, help='Include dns records') @click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), help='save yaml to file') +@click.option('-j', '--json', 'as_json', is_flag=True, help='Dump in josn format') @click.argument('sections', nargs=-1) @with_appcontext -def config_export(full=False, secrets=False, dns=False, output=None, sections=None): - """ Export configuration as YAML to stdout or file +def config_export(full=False, secrets=False, dns=False, output=None, as_json=False, sections=None): + """ Export configuration as YAML or JSON to stdout or file """ if sections: for section in sections: if section not in SECTIONS: - print(f'[ERROR] Unknown section: {section!r}') - sys.exit(1) + print(f'[ERROR] Unknown section: {section}') + raise click.exceptions.Exit(1) sections = set(sections) else: sections = SECTIONS - context={ + context = { 'full': full, 'secrets': secrets, 'dns': dns, } - MailuSchema(only=sections, context=context).dumps(models.MailuConfig(), output) + if as_json: + schema = MailuSchema(only=sections, context=context) + schema.opts.render_module = json + print(schema.dumps(models.MailuConfig(), separators=(',',':')), file=output) + + else: + MailuSchema(only=sections, context=context).dumps(models.MailuConfig(), output) @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 3187f597..dac1dc70 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -12,7 +12,8 @@ from itertools import chain import flask_sqlalchemy import sqlalchemy -import passlib +import passlib.context +import passlib.hash import idna import dns @@ -79,11 +80,11 @@ class CommaSeparatedList(db.TypeDecorator): for item in value: if ',' in item: raise ValueError('list item must not contain ","') - return ','.join(sorted(value)) + return ','.join(sorted(set(value))) def process_result_value(self, value, dialect): """ split comma separated string to list """ - return list(filter(bool, value.split(','))) if value else [] + return list(filter(bool, [item.strip() for item in value.split(',')])) if value else [] python_type = list @@ -136,19 +137,11 @@ class Config(Base): value = db.Column(JSONEncoded) -# TODO: use sqlalchemy.event.listen() on a store method of object? -@sqlalchemy.event.listens_for(db.session, 'after_commit') -def store_dkim_key(session): - """ Store DKIM key on commit """ +def _save_dkim_keys(session): + """ store DKIM keys after commit """ for obj in session.identity_map.values(): if isinstance(obj, Domain): - if obj._dkim_key_changed: - file_path = obj._dkim_file() - if obj._dkim_key: - with open(file_path, 'wb') as handle: - handle.write(obj._dkim_key) - elif os.path.exists(file_path): - os.unlink(file_path) + obj.save_dkim_key() class Domain(Base): """ A DNS domain that has mail addresses associated to it. @@ -165,7 +158,7 @@ class Domain(Base): signup_enabled = db.Column(db.Boolean, nullable=False, default=False) _dkim_key = None - _dkim_key_changed = False + _dkim_key_on_disk = None def _dkim_file(self): """ return filename for active DKIM key """ @@ -174,6 +167,17 @@ class Domain(Base): selector=app.config['DKIM_SELECTOR'] ) + def save_dkim_key(self): + """ save changed DKIM key to disk """ + if self._dkim_key != self._dkim_key_on_disk: + file_path = self._dkim_file() + if self._dkim_key: + with open(file_path, 'wb') as handle: + handle.write(self._dkim_key) + elif os.path.exists(file_path): + os.unlink(file_path) + self._dkim_key_on_disk = self._dkim_key + @property def dns_mx(self): """ return MX record for domain """ @@ -189,7 +193,7 @@ class Domain(Base): @property def dns_dkim(self): """ return DKIM record for domain """ - if os.path.exists(self._dkim_file()): + if self.dkim_key: selector = app.config['DKIM_SELECTOR'] return ( f'{selector}._domainkey.{self.name}. 600 IN TXT' @@ -199,7 +203,7 @@ class Domain(Base): @property def dns_dmarc(self): """ return DMARC record for domain """ - if os.path.exists(self._dkim_file()): + if self.dkim_key: domain = app.config['DOMAIN'] rua = app.config['DMARC_RUA'] rua = f' rua=mailto:{rua}@{domain};' if rua else '' @@ -214,19 +218,19 @@ class Domain(Base): file_path = self._dkim_file() if os.path.exists(file_path): with open(file_path, 'rb') as handle: - self._dkim_key = handle.read() + self._dkim_key = self._dkim_key_on_disk = handle.read() else: - self._dkim_key = b'' + self._dkim_key = self._dkim_key_on_disk = b'' return self._dkim_key if self._dkim_key else None @dkim_key.setter def dkim_key(self, value): """ set private DKIM key """ old_key = self.dkim_key - if value is None: - value = b'' - self._dkim_key_changed = value != old_key - self._dkim_key = value + self._dkim_key = value if value is not None else b'' + if self._dkim_key != old_key: + if not sqlalchemy.event.contains(db.session, 'after_commit', _save_dkim_keys): + sqlalchemy.event.listen(db.session, 'after_commit', _save_dkim_keys) @property def dkim_publickey(self): @@ -331,14 +335,14 @@ class Email(object): def sendmail(self, subject, body): """ send an email to the address """ - from_address = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' + f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' msg = text.MIMEText(body) msg['Subject'] = subject - msg['From'] = from_address + msg['From'] = f_addr msg['To'] = to_address - smtp.sendmail(from_address, [to_address], msg.as_string()) + smtp.sendmail(f_addr, [to_address], msg.as_string()) @classmethod def resolve_domain(cls, email): @@ -589,7 +593,6 @@ class Alias(Base, Email): return None -# TODO: where are Tokens used / validated? # TODO: what about API tokens? class Token(Base): """ A token is an application password for a given user. @@ -650,20 +653,22 @@ class MailuConfig: and loading """ - # TODO: add sqlalchemy session updating (.add & .del) class MailuCollection: - """ Provides dict- and list-like access to all instances + """ Provides dict- and list-like access to instances of a sqlalchemy model """ def __init__(self, model : db.Model): - self._model = model + self.model = model + + def __str__(self): + return f'<{self.model.__name__}-Collection>' @cached_property def _items(self): return { inspect(item).identity: item - for item in self._model.query.all() + for item in self.model.query.all() } def __len__(self): @@ -676,8 +681,8 @@ class MailuConfig: return self._items[key] def __setitem__(self, key, item): - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') if key != inspect(item).identity: raise ValueError(f'item identity != key {key!r}') self._items[key] = item @@ -685,23 +690,24 @@ class MailuConfig: def __delitem__(self, key): del self._items[key] - def append(self, item): + def append(self, item, update=False): """ list-like append """ - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') key = inspect(item).identity if key in self._items: - raise ValueError(f'item {key!r} already present in collection') + if not update: + raise ValueError(f'item {key!r} already present in collection') self._items[key] = item - def extend(self, items): + def extend(self, items, update=False): """ list-like extend """ add = {} for item in items: - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') key = inspect(item).identity - if key in self._items: + if not update and key in self._items: raise ValueError(f'item {key!r} already present in collection') add[key] = item self._items.update(add) @@ -721,8 +727,8 @@ class MailuConfig: def remove(self, item): """ list-like remove """ - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') key = inspect(item).identity if not key in self._items: raise ValueError(f'item {key!r} not found in collection') @@ -739,12 +745,11 @@ class MailuConfig: def update(self, items): """ dict-like update """ for key, item in items: - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') if key != inspect(item).identity: raise ValueError(f'item identity != key {key!r}') - if key in self._items: - raise ValueError(f'item {key!r} already present in collection') + self._items.update(items) def setdefault(self, key, item=None): """ dict-like setdefault """ @@ -752,13 +757,86 @@ class MailuConfig: return self._items[key] if item is None: return None - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') if key != inspect(item).identity: raise ValueError(f'item identity != key {key!r}') self._items[key] = item return item + def __init__(self): + + # section-name -> attr + self._sections = { + name: getattr(self, name) + for name in dir(self) + if isinstance(getattr(self, name), self.MailuCollection) + } + + # known models + self._models = tuple(section.model for section in self._sections.values()) + + # model -> attr + self._sections.update({ + section.model: section for section in self._sections.values() + }) + + def _get_model(self, section): + if section is None: + return None + model = self._sections.get(section) + if model is None: + raise ValueError(f'Invalid section: {section!r}') + if isinstance(model, self.MailuCollection): + return model.model + return model + + def _add(self, items, section, update): + + model = self._get_model(section) + if isinstance(items, self._models): + items = [items] + elif not hasattr(items, '__iter__'): + raise ValueError(f'{items!r} is not iterable') + + for item in items: + if model is not None and not isinstance(item, model): + what = item.__class__.__name__.capitalize() + raise ValueError(f'{what} can not be added to section {section!r}') + self._sections[type(item)].append(item, update=update) + + def add(self, items, section=None): + """ add item to config """ + self._add(items, section, update=False) + + def update(self, items, section=None): + """ add or replace item in config """ + self._add(items, section, update=True) + + def remove(self, items, section=None): + """ remove item from config """ + model = self._get_model(section) + if isinstance(items, self._models): + items = [items] + elif not hasattr(items, '__iter__'): + raise ValueError(f'{items!r} is not iterable') + + for item in items: + if isinstance(item, str): + if section is None: + raise ValueError(f'Cannot remove key {item!r} without section') + del self._sections[model][item] + elif model is not None and not isinstance(item, model): + what = item.__class__.__name__.capitalize() + raise ValueError(f'{what} can not be removed from section {section!r}') + self._sections[type(item)].remove(item,) + + def clear(self, models=None): + """ remove complete configuration """ + for model in self._models: + if models is None or model in models: + db.session.query(model).delete() + domains = MailuCollection(Domain) relays = MailuCollection(Relay) users = MailuCollection(User) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 5dc10e17..04512f6d 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -24,6 +24,23 @@ ma = Marshmallow() # - fields which are the primary key => unchangeable when updating +### map model to schema ### + +_model2schema = {} + +def get_schema(model=None): + """ return schema class for model or instance of model """ + if model is None: + return _model2schema.values() + else: + return _model2schema.get(model) or _model2schema.get(model.__class__) + +def mapped(cls): + """ register schema in model2schema map """ + _model2schema[cls.Meta.model] = cls + return cls + + ### yaml render module ### # allow yaml module to dump OrderedDict @@ -79,26 +96,6 @@ class RenderYAML: return yaml.dump(*args, **kwargs) -### functions ### - -def handle_email(data): - """ merge separate localpart and domain to email - """ - - localpart = 'localpart' in data - domain = 'domain' in data - - if 'email' in data: - if localpart or domain: - raise ValidationError('duplicate email and localpart/domain') - elif localpart and domain: - data['email'] = f'{data["localpart"]}@{data["domain"]}' - elif localpart or domain: - raise ValidationError('incomplete localpart/domain') - - return data - - ### field definitions ### class LazyStringField(fields.String): @@ -177,9 +174,7 @@ class DkimKeyField(fields.String): return dkim.gen_key() # remember some keydata for error message - keydata = value - if len(keydata) > 40: - keydata = keydata[:25] + '...' + keydata[-10:] + keydata = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value # wrap value into valid pem layout and check validity value = ( @@ -197,6 +192,26 @@ class DkimKeyField(fields.String): ### base definitions ### +def handle_email(data): + """ merge separate localpart and domain to email + """ + + localpart = 'localpart' in data + domain = 'domain' in data + + if 'email' in data: + if localpart or domain: + raise ValidationError('duplicate email and localpart/domain') + data['localpart'], data['domain_name'] = data['email'].rsplit('@', 1) + elif localpart and domain: + data['domain_name'] = data['domain'] + del data['domain'] + data['email'] = f'{data["localpart"]}@{data["domain_name"]}' + elif localpart or domain: + raise ValidationError('incomplete localpart/domain') + + return data + class BaseOpts(SQLAlchemyAutoSchemaOpts): """ Option class with sqla session """ @@ -238,12 +253,15 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # update excludes kwargs['exclude'] = exclude + # init SQLAlchemyAutoSchema + super().__init__(*args, **kwargs) + # exclude_by_value self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {}) # exclude default values if not context.get('full'): - for column in getattr(self.Meta, 'model').__table__.columns: + for column in getattr(self.opts, 'model').__table__.columns: if column.name not in exclude: self._exclude_by_value.setdefault(column.name, []).append( None if column.default is None else column.default.arg @@ -256,10 +274,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if not flags & set(need): self._hide_by_context |= set(what) - # init SQLAlchemyAutoSchema - super().__init__(*args, **kwargs) - - # init order + # initialize attribute order if hasattr(self.Meta, 'order'): # use user-defined order self._order = list(reversed(getattr(self.Meta, 'order'))) @@ -267,17 +282,35 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # default order is: primary_key + other keys alphabetically self._order = list(sorted(self.fields.keys())) primary = self.opts.model.__table__.primary_key.columns.values()[0].name - self._order.remove(primary) - self._order.reverse() - self._order.append(primary) + if primary in self._order: + self._order.remove(primary) + self._order.reverse() + self._order.append(primary) + + # move pre_load hook "_track_import" to the front + hooks = self._hooks[('pre_load', False)] + if '_track_import' in hooks: + hooks.remove('_track_import') + hooks.insert(0, '_track_import') + # and post_load hook "_fooo" to the end + hooks = self._hooks[('post_load', False)] + if '_add_instance' in hooks: + hooks.remove('_add_instance') + hooks.append('_add_instance') @pre_load def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument - call = self.context.get('callback') - if call is not None: - call(self=self, data=data, many=many, **kwargs) +# TODO: also handle reset, prune and delete in pre_load / post_load hooks! +# print('!!!', repr(data)) + if callback := self.context.get('callback'): + callback(self, data) return data + @post_load + def _add_instance(self, item, many, **kwargs): # pylint: disable=unused-argument + self.opts.sqla_session.add(item) + return item + @post_dump def _hide_and_order(self, data, many, **kwargs): # pylint: disable=unused-argument @@ -306,6 +339,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): ### schema definitions ### +@mapped class DomainSchema(BaseSchema): """ Marshmallow schema for Domain model """ class Meta: @@ -339,6 +373,7 @@ class DomainSchema(BaseSchema): dns_dmarc = fields.String(dump_only=True) +@mapped class TokenSchema(BaseSchema): """ Marshmallow schema for Token model """ class Meta: @@ -347,6 +382,7 @@ class TokenSchema(BaseSchema): load_instance = True +@mapped class FetchSchema(BaseSchema): """ Marshmallow schema for Fetch model """ class Meta: @@ -361,6 +397,7 @@ class FetchSchema(BaseSchema): } +@mapped class UserSchema(BaseSchema): """ Marshmallow schema for User model """ class Meta: @@ -368,7 +405,7 @@ class UserSchema(BaseSchema): model = models.User load_instance = True include_relationships = True - exclude = ['localpart', 'domain', 'quota_bytes_used'] + exclude = ['domain', 'quota_bytes_used'] exclude_by_value = { 'forward_destination': [[]], @@ -395,7 +432,7 @@ class UserSchema(BaseSchema): raise ValidationError(f'invalid hashed password {password!r}') elif 'password_hash' in data and 'hash_scheme' in data: if data['hash_scheme'] not in self.Meta.model.scheme_dict: - raise ValidationError(f'invalid password scheme {scheme!r}') + raise ValidationError(f'invalid password scheme {data["hash_scheme"]!r}') data['password'] = f'{{{data["hash_scheme"]}}}{data["password_hash"]}' del data['hash_scheme'] del data['password_hash'] @@ -409,17 +446,20 @@ class UserSchema(BaseSchema): # ctx.verify('', hashed) # =>? ValueError: hash could not be identified + localpart = fields.Str(load_only=True) + domain_name = fields.Str(load_only=True) tokens = fields.Nested(TokenSchema, many=True) fetches = fields.Nested(FetchSchema, many=True) +@mapped class AliasSchema(BaseSchema): """ Marshmallow schema for Alias model """ class Meta: """ Schema config """ model = models.Alias load_instance = True - exclude = ['localpart'] + exclude = ['domain'] exclude_by_value = { 'destination': [[]], @@ -429,9 +469,12 @@ class AliasSchema(BaseSchema): def _handle_email(self, data, many, **kwargs): # pylint: disable=unused-argument return handle_email(data) + localpart = fields.Str(load_only=True) + domain_name = fields.Str(load_only=True) destination = CommaSeparatedListField() +@mapped class ConfigSchema(BaseSchema): """ Marshmallow schema for Config model """ class Meta: @@ -440,6 +483,7 @@ class ConfigSchema(BaseSchema): load_instance = True +@mapped class RelaySchema(BaseSchema): """ Marshmallow schema for Relay model """ class Meta: @@ -453,18 +497,43 @@ class MailuSchema(Schema): class Meta: """ Schema config """ render_module = RenderYAML + ordered = True order = ['config', 'domains', 'users', 'aliases', 'relays'] - @post_dump(pass_many=True) - def _order(self, data : OrderedDict, many : bool, **kwargs): # pylint: disable=unused-argument - for key in reversed(self.Meta.order): - try: - data.move_to_end(key, False) - except KeyError: - pass + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # order fields + for field_list in self.load_fields, self.dump_fields, self.fields: + for section in reversed(self.Meta.order): + try: + field_list.move_to_end(section, False) + except KeyError: + pass + + @pre_load + def _clear_config(self, data, many, **kwargs): # pylint: disable=unused-argument + """ create config object in context if missing + and clear it if requested + """ + if 'config' not in self.context: + self.context['config'] = models.MailuConfig() + if self.context.get('clear'): + self.context['config'].clear( + models = {field.nested.opts.model for field in self.fields.values()} + ) return data + @post_load + def _make_config(self, data, many, **kwargs): # pylint: disable=unused-argument + """ update and return config object """ + config = self.context['config'] + for section in self.Meta.order: + if section in data: + config.update(data[section], section) + + return config + config = fields.Nested(ConfigSchema, many=True) domains = fields.Nested(DomainSchema, many=True) users = fields.Nested(UserSchema, many=True) From 1c9abf6e481fee92d87be7b8907d8b9fde06479b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 24 Jan 2021 19:27:22 +0100 Subject: [PATCH 043/181] updated requirements for import/export api reqs (flask-restx, ...) are still missing --- core/admin/requirements-prod.txt | 3 +++ core/admin/requirements.txt | 3 +++ 2 files changed, 6 insertions(+) diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index a3c32855..8ad412cf 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -15,6 +15,7 @@ Flask-Bootstrap==3.3.7.1 Flask-DebugToolbar==0.10.1 Flask-Limiter==1.0.1 Flask-Login==0.4.1 +flask-marshmallow==0.14.0 Flask-Migrate==2.4.0 Flask-Script==2.0.6 Flask-SQLAlchemy==2.4.0 @@ -29,6 +30,8 @@ limits==1.3 Mako==1.0.9 MarkupSafe==1.1.1 mysqlclient==1.4.2.post1 +marshmallow==3.10.0 +marshmallow-sqlalchemy==0.24.1 passlib==1.7.1 psycopg2==2.8.2 pycparser==2.19 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index 9739ed3f..59383a07 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -23,3 +23,6 @@ mysqlclient psycopg2 idna srslib +marshmallow +flask-marshmallow +marshmallow-sqlalchemy From 906a051925766058feec60c0b768e274ccd7c862 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 6 Feb 2021 17:23:05 +0100 Subject: [PATCH 044/181] Make rainloop use internal auth --- core/admin/mailu/internal/nginx.py | 19 ++++++++++----- core/admin/mailu/internal/views/auth.py | 12 ++++++++++ core/admin/mailu/models.py | 10 ++++++++ core/admin/mailu/ui/views/base.py | 3 +++ core/nginx/conf/nginx.conf | 20 ++++++++++++++++ webmails/rainloop/Dockerfile | 1 + webmails/rainloop/application.ini | 2 ++ webmails/rainloop/sso.php | 31 +++++++++++++++++++++++++ webmails/rainloop/start.py | 1 + 9 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 webmails/rainloop/sso.php diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 1e0b16c2..a41543d3 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -7,7 +7,6 @@ import ipaddress import socket import tenacity - SUPPORTED_AUTH_METHODS = ["none", "plain"] @@ -49,11 +48,19 @@ def handle_authentication(headers): user = models.User.query.get(user_email) status = False if user: - for token in user.tokens: - if (token.check_password(password) and - (not token.ip or token.ip == ip)): - status = True - if user.check_password(password): + # webmails + if len(password) == 64 and ip == app.config['WEBMAIL_ADDRESS']: + if user.verify_temp_token(password): + status = True + + # All tokens are 32 characters hex lowercase + if len(password) == 32: + for token in user.tokens: + if (token.check_password(password) and + (not token.ip or token.ip == ip)): + status = True + break + if not status and user.check_password(password): status = True if status: if protocol == "imap" and not user.enable_imap: diff --git a/core/admin/mailu/internal/views/auth.py b/core/admin/mailu/internal/views/auth.py index 825dba56..338141d8 100644 --- a/core/admin/mailu/internal/views/auth.py +++ b/core/admin/mailu/internal/views/auth.py @@ -43,6 +43,18 @@ def admin_authentication(): return "" return flask.abort(403) +@internal.route("/auth/user") +def user_authentication(): + """ Fails if the user is not authenticated. + """ + if (not flask_login.current_user.is_anonymous + and flask_login.current_user.enabled): + response = flask.Response() + response.headers["X-User"] = flask_login.current_user.get_id() + response.headers["X-User-Token"] = models.User.get_temp_token(flask_login.current_user.get_id()) + return response + return flask.abort(403) + @internal.route("/auth/basic") def basic_authentication(): diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index bbc00f2d..6230e252 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -12,6 +12,7 @@ import re import time import os import glob +import hmac import smtplib import idna import dns @@ -425,6 +426,15 @@ class User(Base, Email): user = cls.query.get(email) return user if (user and user.enabled and user.check_password(password)) else None + @classmethod + def get_temp_token(cls, email): + user = cls.query.get(email) + return hmac.new(bytearray(app.secret_key,'utf-8'), bytearray("{}|{}".format(datetime.utcnow().strftime("%Y%m%d"), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None + + def verify_temp_token(self, token): + return hmac.compare_digest(b''.fromhex(self.get_temp_token(self.email)), b''.fromhex(token)) + + class Alias(Base, Email): """ An alias is an email address that redirects to some destination. diff --git a/core/admin/mailu/ui/views/base.py b/core/admin/mailu/ui/views/base.py index 7501a883..1aff7db1 100644 --- a/core/admin/mailu/ui/views/base.py +++ b/core/admin/mailu/ui/views/base.py @@ -47,6 +47,9 @@ def announcement(): flask.flash('Your announcement was sent', 'success') return flask.render_template('announcement.html', form=form) +@ui.route('/webmail', methods=['GET']) +def webmail(): + return flask.redirect(app.config['WEB_WEBMAIL']) @ui.route('/client', methods=['GET']) def client(): diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index df598c94..a7a9e134 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -133,7 +133,27 @@ http { {% endif %} include /etc/nginx/proxy.conf; client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; + auth_request /internal/auth/user; proxy_pass http://$webmail; + error_page 403 @webmail_login; + } + location {{ WEB_WEBMAIL }}/sso.php { + {% if WEB_WEBMAIL != '/' %} + rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent; + rewrite ^{{ WEB_WEBMAIL }}/(.*) /$1 break; + {% endif %} + include /etc/nginx/proxy.conf; + client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; + auth_request /internal/auth/user; + auth_request_set $user $upstream_http_x_user; + auth_request_set $token $upstream_http_x_user_token; + proxy_set_header X-Remote-User $user; + proxy_set_header X-Remote-User-Token $token; + proxy_pass http://$webmail; + error_page 403 @webmail_login; + } + location @webmail_login { + return 302 {{ WEB_ADMIN }}/ui/login?next=ui.webmail; } {% endif %} diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile index c67c7496..640d7176 100644 --- a/webmails/rainloop/Dockerfile +++ b/webmails/rainloop/Dockerfile @@ -35,6 +35,7 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists COPY include.php /var/www/html/include.php +COPY sso.php /var/www/html/sso.php COPY php.ini /php.ini COPY application.ini /application.ini diff --git a/webmails/rainloop/application.ini b/webmails/rainloop/application.ini index 5bd9943d..bc953af1 100644 --- a/webmails/rainloop/application.ini +++ b/webmails/rainloop/application.ini @@ -7,6 +7,8 @@ attachment_size_limit = {{ MAX_FILESIZE }} allow_admin_panel = Off [labs] +custom_login_link='sso.php' +custom_logout_link='{{ WEB_ADMIN }}/ui/logout' allow_gravatar = Off [contacts] diff --git a/webmails/rainloop/sso.php b/webmails/rainloop/sso.php new file mode 100644 index 00000000..2415f45c --- /dev/null +++ b/webmails/rainloop/sso.php @@ -0,0 +1,31 @@ + Date: Sat, 6 Feb 2021 18:14:58 +0100 Subject: [PATCH 045/181] Make roundcube use internal auth --- webmails/roundcube/Dockerfile | 1 + webmails/roundcube/config.inc.php | 4 ++- webmails/roundcube/mailu.php | 59 +++++++++++++++++++++++++++++++ webmails/roundcube/start.py | 2 ++ 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 webmails/roundcube/mailu.php diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index 79b911b0..da355cc3 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -46,6 +46,7 @@ RUN apt-get update && apt-get install -y \ COPY php.ini /php.ini COPY config.inc.php /var/www/html/config/ +COPY mailu.php /var/www/html/plugins/mailu/mailu.php COPY start.py /start.py EXPOSE 80/tcp diff --git a/webmails/roundcube/config.inc.php b/webmails/roundcube/config.inc.php index eb40047a..bb1a5e84 100644 --- a/webmails/roundcube/config.inc.php +++ b/webmails/roundcube/config.inc.php @@ -17,7 +17,8 @@ $config['plugins'] = array( 'markasjunk', 'managesieve', 'enigma', - 'carddav' + 'carddav', + 'mailu' ); $front = getenv('FRONT_ADDRESS') ? getenv('FRONT_ADDRESS') : 'front'; @@ -37,6 +38,7 @@ $config['managesieve_usetls'] = false; // Customization settings $config['support_url'] = getenv('WEB_ADMIN') ? '../..' . getenv('WEB_ADMIN') : ''; +$config['sso_logout_url'] = getenv('WEB_ADMIN').'/ui/logout'; $config['product_name'] = 'Mailu Webmail'; // We access the IMAP and SMTP servers locally with internal names, SSL diff --git a/webmails/roundcube/mailu.php b/webmails/roundcube/mailu.php new file mode 100644 index 00000000..bb4d65e9 --- /dev/null +++ b/webmails/roundcube/mailu.php @@ -0,0 +1,59 @@ +add_hook('startup', array($this, 'startup')); + $this->add_hook('authenticate', array($this, 'authenticate')); + $this->add_hook('login_after', array($this, 'login')); + $this->add_hook('login_failed', array($this, 'login_failed')); + $this->add_hook('logout_after', array($this, 'logout')); + } + + function startup($args) + { + if (empty($_SESSION['user_id'])) { + $args['action'] = 'login'; + } + + return $args; + } + + function authenticate($args) + { + if (!in_array('HTTP_X_REMOTE_USER', $_SERVER) || !in_array('HTTP_X_REMOTE_USER_TOKEN', $_SERVER)) { + header('HTTP/1.0 403 Forbidden'); + die(); + } + $args['user'] = $_SERVER['HTTP_X_REMOTE_USER']; + $args['pass'] = $_SERVER['HTTP_X_REMOTE_USER_TOKEN']; + + $args['cookiecheck'] = false; + $args['valid'] = true; + + return $args; + } + + function logout($args) { + // Redirect to global SSO logout path. + $this->load_config(); + + $sso_logout_url = rcmail::get_instance()->config->get('sso_logout_url'); + header("Location: " . $sso_logout_url, true); + exit; + } + + function login($args) + { + header('Location: index.php'); + exit(); + } + function login_failed($args) + { + header('Location: sso.php'); + exit(); + } + +} diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 649f3324..9ce383c8 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -39,6 +39,8 @@ conf.jinja("/php.ini", os.environ, "/usr/local/etc/php/conf.d/roundcube.ini") os.system("mkdir -p /data/gpg /var/www/html/logs") os.system("touch /var/www/html/logs/errors.log") os.system("chown -R www-data:www-data /var/www/html/logs") +os.system("chmod -R a+rX /var/www/html/") +os.system("ln -s /var/www/html/index.php /var/www/html/sso.php") try: print("Initializing database") From 99c7420f923a8b2c44860d25c196fcf2c00280e4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 7 Feb 2021 17:51:19 +0100 Subject: [PATCH 046/181] towncrier --- towncrier/newsfragments/783.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/783.feature diff --git a/towncrier/newsfragments/783.feature b/towncrier/newsfragments/783.feature new file mode 100644 index 00000000..fcafceef --- /dev/null +++ b/towncrier/newsfragments/783.feature @@ -0,0 +1 @@ +Centralize the authentication of webmails behind the admin interface From ef637f51b7f620c88dff403cf1fdb4a74ccbe663 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 7 Feb 2021 17:58:19 +0100 Subject: [PATCH 047/181] derive the SSO keys from a KDF --- core/admin/mailu/__init__.py | 3 +++ core/admin/mailu/models.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 4de3e580..b37b4493 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -3,6 +3,7 @@ import flask_bootstrap from mailu import utils, debug, models, manage, configuration +import hmac def create_app_from_config(config): """ Create a new application based on the given configuration @@ -24,6 +25,8 @@ def create_app_from_config(config): utils.proxy.init_app(app) utils.migrate.init_app(app, models.db) + app.temp_token_key = hmac.new(bytearray(app.secret_key, 'utf-8'), bytearray('WEBMAIL_TEMP_TOKEN_KEY', 'utf-8'), 'sha256').digest() + # Initialize debugging tools if app.config.get("DEBUG"): debug.toolbar.init_app(app) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 6230e252..c3f79b92 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -429,10 +429,10 @@ class User(Base, Email): @classmethod def get_temp_token(cls, email): user = cls.query.get(email) - return hmac.new(bytearray(app.secret_key,'utf-8'), bytearray("{}|{}".format(datetime.utcnow().strftime("%Y%m%d"), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None + return hmac.new(app.temp_token_key, bytearray("{}|{}".format(datetime.utcnow().strftime("%Y%m%d"), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None def verify_temp_token(self, token): - return hmac.compare_digest(b''.fromhex(self.get_temp_token(self.email)), b''.fromhex(token)) + return hmac.compare_digest(self.get_temp_token(self.email), token) From b49554bec1156c69c5f24c5ca62cbb9a818ac8a7 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 7 Feb 2021 18:08:58 +0100 Subject: [PATCH 048/181] merge artifact --- core/admin/mailu/internal/nginx.py | 2 +- core/admin/mailu/ui/views/base.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index a41543d3..b2ee3fba 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -54,7 +54,7 @@ def handle_authentication(headers): status = True # All tokens are 32 characters hex lowercase - if len(password) == 32: + if not status and len(password) == 32: for token in user.tokens: if (token.check_password(password) and (not token.ip or token.ip == ip)): diff --git a/core/admin/mailu/ui/views/base.py b/core/admin/mailu/ui/views/base.py index 1aff7db1..457fd7fb 100644 --- a/core/admin/mailu/ui/views/base.py +++ b/core/admin/mailu/ui/views/base.py @@ -1,6 +1,7 @@ from mailu import models from mailu.ui import ui, forms, access +from flask import current_app as app import flask import flask_login From 2e749abe6118f9a2c414ac239efff099dc3fdb1a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 25 Jan 2021 13:19:21 +0100 Subject: [PATCH 049/181] DNS records for client autoconfiguration (RFC6186) --- core/admin/mailu/ui/templates/domain/details.html | 12 ++++++++++++ towncrier/newsfragments/224.feature | 1 + 2 files changed, 13 insertions(+) create mode 100644 towncrier/newsfragments/224.feature diff --git a/core/admin/mailu/ui/templates/domain/details.html b/core/admin/mailu/ui/templates/domain/details.html index 65c6ec1a..5822ffbc 100644 --- a/core/admin/mailu/ui/templates/domain/details.html +++ b/core/admin/mailu/ui/templates/domain/details.html @@ -50,5 +50,17 @@ 
_dmarc.{{ domain.name }}. 600 IN TXT "v=DMARC1; p=reject;{% if config["DMARC_RUA"] %} rua=mailto:{{ config["DMARC_RUA"] }}@{{ config["DOMAIN"] }};{% endif %}{% if config["DMARC_RUF"] %} ruf=mailto:{{ config["DMARC_RUF"] }}@{{ config["DOMAIN"] }};{% endif %} adkim=s; aspf=s"
{% endif %} + + {% trans %}DNS client auto-configuration (RFC6186) entries{% endtrans %} + + 
_submission._tcp.{{ domain.name }}. 600 IN SRV 1 1 587 {{ config["HOSTNAMES"].split(',')[0] }}.
+ 
_imap._tcp.{{ domain.name }}. 600 IN SRV 100 1 143 {{ config["HOSTNAMES"].split(',')[0] }}.
+ 
_pop3._tcp.{{ domain.name }}. 600 IN SRV 100 1 110 {{ config["HOSTNAMES"].split(',')[0] }}.
+{% if config["TLS_FLAVOR"] != "notls" %} + 
_submissions._tcp.{{ domain.name }}. 600 IN SRV 10 1 465 {{ config["HOSTNAMES"].split(',')[0] }}.
+ 
_imaps._tcp.{{ domain.name }}. 600 IN SRV 10 1 993 {{ config["HOSTNAMES"].split(',')[0] }}.
+ 
_pop3s._tcp.{{ domain.name }}. 600 IN SRV 10 1 995 {{ config["HOSTNAMES"].split(',')[0] }}.
+{% endif %} + {% endcall %} {% endblock %} diff --git a/towncrier/newsfragments/224.feature b/towncrier/newsfragments/224.feature new file mode 100644 index 00000000..9a2f479b --- /dev/null +++ b/towncrier/newsfragments/224.feature @@ -0,0 +1 @@ +Add instructions on how to create DNS records for email client auto-configuration (RFC6186 style) From 80f939cf1ac3fe9863825de1360ee7c5d67926f6 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 8 Feb 2021 10:16:03 +0100 Subject: [PATCH 050/181] Revert to the old behaviour when ADMIN=false --- core/nginx/conf/nginx.conf | 10 +++++++--- webmails/rainloop/application.ini | 4 +++- webmails/roundcube/config.inc.php | 10 ++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index a7a9e134..81f1ac0d 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -133,10 +133,12 @@ http { {% endif %} include /etc/nginx/proxy.conf; client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; - auth_request /internal/auth/user; proxy_pass http://$webmail; + {% if ADMIN == 'true' %} + auth_request /internal/auth/user; error_page 403 @webmail_login; } + location {{ WEB_WEBMAIL }}/sso.php { {% if WEB_WEBMAIL != '/' %} rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent; @@ -152,11 +154,13 @@ http { proxy_pass http://$webmail; error_page 403 @webmail_login; } + location @webmail_login { return 302 {{ WEB_ADMIN }}/ui/login?next=ui.webmail; } - {% endif %} - + {% else %} + } + {% endif %}{% endif %} {% if ADMIN == 'true' %} location {{ WEB_ADMIN }} { return 301 {{ WEB_ADMIN }}/ui; diff --git a/webmails/rainloop/application.ini b/webmails/rainloop/application.ini index bc953af1..0504f174 100644 --- a/webmails/rainloop/application.ini +++ b/webmails/rainloop/application.ini @@ -7,9 +7,11 @@ attachment_size_limit = {{ MAX_FILESIZE }} allow_admin_panel = Off [labs] +allow_gravatar = Off +{% if ADMIN == "true" %} custom_login_link='sso.php' custom_logout_link='{{ WEB_ADMIN }}/ui/logout' -allow_gravatar = Off +{% endif %} [contacts] enable = On diff --git a/webmails/roundcube/config.inc.php b/webmails/roundcube/config.inc.php index bb1a5e84..d8028db3 100644 --- a/webmails/roundcube/config.inc.php +++ b/webmails/roundcube/config.inc.php @@ -17,8 +17,7 @@ $config['plugins'] = array( 'markasjunk', 'managesieve', 'enigma', - 'carddav', - 'mailu' + 'carddav' ); $front = getenv('FRONT_ADDRESS') ? getenv('FRONT_ADDRESS') : 'front'; @@ -37,8 +36,11 @@ $config['managesieve_host'] = $imap; $config['managesieve_usetls'] = false; // Customization settings -$config['support_url'] = getenv('WEB_ADMIN') ? '../..' . getenv('WEB_ADMIN') : ''; -$config['sso_logout_url'] = getenv('WEB_ADMIN').'/ui/logout'; +if (filter_var(getenv('ADMIN'), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE)) { + array_push($config['plugins'], 'mailu'); + $config['support_url'] = getenv('WEB_ADMIN') ? '../..' . getenv('WEB_ADMIN') : ''; + $config['sso_logout_url'] = getenv('WEB_ADMIN').'/ui/logout'; +} $config['product_name'] = 'Mailu Webmail'; // We access the IMAP and SMTP servers locally with internal names, SSL From 0917a6817f845c01a2247d74f111abc2ef54e51a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 8 Feb 2021 10:17:43 +0100 Subject: [PATCH 051/181] Set ADMIN=false to ensure that the tests pass --- tests/compose/rainloop/mailu.env | 2 +- tests/compose/roundcube/mailu.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compose/rainloop/mailu.env b/tests/compose/rainloop/mailu.env index 9c31c8bb..47b0b934 100644 --- a/tests/compose/rainloop/mailu.env +++ b/tests/compose/rainloop/mailu.env @@ -51,7 +51,7 @@ DISABLE_STATISTICS=False ################################### # Expose the admin interface (value: true, false) -ADMIN=true +ADMIN=false # Choose which webmail to run if any (values: roundcube, rainloop, none) WEBMAIL=rainloop diff --git a/tests/compose/roundcube/mailu.env b/tests/compose/roundcube/mailu.env index dc503268..887cebbd 100644 --- a/tests/compose/roundcube/mailu.env +++ b/tests/compose/roundcube/mailu.env @@ -51,7 +51,7 @@ DISABLE_STATISTICS=False ################################### # Expose the admin interface (value: true, false) -ADMIN=true +ADMIN=false # Choose which webmail to run if any (values: roundcube, rainloop, none) WEBMAIL=roundcube From e8f70c12dcdce1064ea063cc5ddcae28a5221613 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 8 Feb 2021 10:22:25 +0100 Subject: [PATCH 052/181] avoid a warning --- webmails/roundcube/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 9ce383c8..45b2aa76 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -40,7 +40,7 @@ os.system("mkdir -p /data/gpg /var/www/html/logs") os.system("touch /var/www/html/logs/errors.log") os.system("chown -R www-data:www-data /var/www/html/logs") os.system("chmod -R a+rX /var/www/html/") -os.system("ln -s /var/www/html/index.php /var/www/html/sso.php") +os.system("ln -sf /var/www/html/index.php /var/www/html/sso.php") try: print("Initializing database") From 6c4fa5432f3f040cc16fca870f0382939ae2817b Mon Sep 17 00:00:00 2001 From: Nils Vogels Date: Thu, 11 Feb 2021 12:03:07 +0100 Subject: [PATCH 053/181] Provide fix in postgresql container for CVE-2021-23240, CVE-2021-3156, CVE-2021-23239 --- optional/postgresql/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/optional/postgresql/Dockerfile b/optional/postgresql/Dockerfile index 95048147..ff25a66f 100644 --- a/optional/postgresql/Dockerfile +++ b/optional/postgresql/Dockerfile @@ -3,6 +3,7 @@ FROM $DISTRO # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip bash py3-multidict \ + && apk add --upgrade sudo \ && pip3 install --upgrade pip # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube From 3b7ecb3a8b4c466ad2f37276107bb49efe978758 Mon Sep 17 00:00:00 2001 From: Nils Vogels Date: Thu, 11 Feb 2021 12:12:06 +0100 Subject: [PATCH 054/181] Add changelog --- towncrier/newsfragments/1760.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 towncrier/newsfragments/1760.bugfix diff --git a/towncrier/newsfragments/1760.bugfix b/towncrier/newsfragments/1760.bugfix new file mode 100644 index 00000000..9d6f38af --- /dev/null +++ b/towncrier/newsfragments/1760.bugfix @@ -0,0 +1,2 @@ +Fix CVE-2021-23240, CVE-2021-3156 and CVE-2021-23239 for postgresql +by force-upgrading sudo. From 68caf501549d08c048a7c232ac72bb8a072dd9e6 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 15 Feb 2021 00:46:59 +0100 Subject: [PATCH 055/181] new import/export using marshmallow --- core/admin/mailu/manage.py | 218 +++++---- core/admin/mailu/models.py | 132 +++--- core/admin/mailu/schemas.py | 608 +++++++++++++++++++++----- docs/cli.rst | 221 ++++++---- tests/compose/core/02_forward_test.sh | 4 +- tests/compose/core/03_alias_test.sh | 4 +- tests/compose/core/04_reply_test.sh | 4 +- towncrier/newsfragments/1604.feature | 2 +- 8 files changed, 851 insertions(+), 342 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index e02d9ad4..a8d1d3cb 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -4,7 +4,6 @@ import sys import os import socket -import json import logging import uuid @@ -20,7 +19,7 @@ from flask.cli import FlaskGroup, with_appcontext from marshmallow.exceptions import ValidationError from . import models -from .schemas import MailuSchema, get_schema +from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, RenderJSON, HIDDEN db = models.db @@ -182,7 +181,7 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() -# TODO: remove this deprecated function +# TODO: remove deprecated config_update function? @mailu.command() @click.option('-v', '--verbose') @click.option('-d', '--delete-objects') @@ -324,17 +323,16 @@ def config_update(verbose=False, delete_objects=False): db.session.commit() -SECTIONS = {'domains', 'relays', 'users', 'aliases'} - - @mailu.command() -@click.option('-v', '--verbose', count=True, help='Increase verbosity') -@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors') -@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config') -@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') +@click.option('-v', '--verbose', count=True, help='Increase verbosity.') +@click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.') +@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') +@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.') +@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.') @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext -def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=None): +def config_import(verbose=0, secrets=False, quiet=False, color=False, update=False, dry_run=False, source=None): """ Import configuration as YAML or JSON from stdin or file """ @@ -344,12 +342,19 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No # 2 : also show secrets # 3 : also show input data # 4 : also show sql queries + # 5 : also show tracebacks if quiet: verbose = -1 + color_cfg = { + 'color': color or sys.stdout.isatty(), + 'lexer': 'python', + 'strip': True, + } + counter = Counter() - dumper = {} + logger = {} def format_errors(store, path=None): @@ -387,19 +392,26 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No last = action changes.append(f'{what}({count})') else: - changes = 'no changes.' + changes = ['No changes.'] return chain(message, changes) def log(action, target, message=None): if message is None: - message = json.dumps(dumper[target.__class__].dump(target), ensure_ascii=False) - print(f'{action} {target.__table__}: {message}') + # TODO: convert nested OrderedDict to dict + # see: flask mailu config-import -nvv yaml/dump4.yaml + try: + message = dict(logger[target.__class__].dump(target)) + except KeyError: + message = target + if not isinstance(message, str): + message = repr(message) + print(f'{action} {target.__table__}: {colorize(message, **color_cfg)}') def listen_insert(mapper, connection, target): # pylint: disable=unused-argument """ callback function to track import """ - counter.update([('Added', target.__table__.name)]) + counter.update([('Created', target.__table__.name)]) if verbose >= 1: - log('Added', target) + log('Created', target) def listen_update(mapper, connection, target): # pylint: disable=unused-argument """ callback function to track import """ @@ -407,32 +419,32 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No changed = {} inspection = sqlalchemy.inspect(target) for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: - if getattr(inspection.attrs, attr.key).history.has_changes(): - if sqlalchemy.orm.attributes.get_history(target, attr.key)[2]: - before = sqlalchemy.orm.attributes.get_history(target, attr.key)[2].pop() - after = getattr(target, attr.key) - # only remember changed keys - if before != after and (before or after): - if verbose >= 1: - changed[str(attr.key)] = (before, after) - else: - break + history = getattr(inspection.attrs, attr.key).history + if history.has_changes() and history.deleted: + before = history.deleted[-1] + after = getattr(target, attr.key) + # TODO: remove special handling of "comment" after modifying model + if attr.key == 'comment' and not before and not after: + pass + # only remember changed keys + elif before != after: + if verbose >= 1: + changed[str(attr.key)] = (before, after) + else: + break if verbose >= 1: # use schema with dump_context to hide secrets and sort keys - primary = json.dumps(str(target), ensure_ascii=False) - dumped = get_schema(target)(only=changed.keys(), context=dump_context).dump(target) + dumped = get_schema(target)(only=changed.keys(), context=diff_context).dump(target) for key, value in dumped.items(): before, after = changed[key] - if value == '': - before = '' if before else before - after = '' if after else after + if value == HIDDEN: + before = HIDDEN if before else before + after = HIDDEN if after else after else: - # TODO: use schema to "convert" before value? + # TODO: need to use schema to "convert" before value? after = value - before = json.dumps(before, ensure_ascii=False) - after = json.dumps(after, ensure_ascii=False) - log('Modified', target, f'{primary} {key}: {before} -> {after}') + log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') if changed: counter.update([('Modified', target.__table__.name)]) @@ -443,47 +455,60 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No if verbose >= 1: log('Deleted', target) - # this listener should not be necessary, when: - # dkim keys should be stored in database and it should be possible to store multiple - # keys per domain. the active key would be also stored on disk on commit. + # TODO: this listener will not be necessary, if dkim keys would be stored in database + _dedupe_dkim = set() def listen_dkim(session, flush_context): # pylint: disable=unused-argument """ callback function to track import """ for target in session.identity_map.values(): - if not isinstance(target, models.Domain): + # look at Domains originally loaded from db + if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: continue - primary = json.dumps(str(target), ensure_ascii=False) before = target._dkim_key_on_disk after = target._dkim_key - if before != after and (before or after): - if verbose >= 2: + if before != after: + if secrets: before = before.decode('ascii', 'ignore') after = after.decode('ascii', 'ignore') else: - before = '' if before else '' - after = '' if after else '' - before = json.dumps(before, ensure_ascii=False) - after = json.dumps(after, ensure_ascii=False) - log('Modified', target, f'{primary} dkim_key: {before} -> {after}') - counter.update([('Modified', target.__table__.name)]) + before = HIDDEN if before else '' + after = HIDDEN if after else '' + # "de-dupe" messages; this event is fired at every flush + if not (target, before, after) in _dedupe_dkim: + _dedupe_dkim.add((target, before, after)) + counter.update([('Modified', target.__table__.name)]) + if verbose >= 1: + log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') - def track_serialize(self, item): + def track_serialize(obj, item): """ callback function to track import """ - log('Handling', self.opts.model, item) + # hide secrets + data = logger[obj.opts.model].hide(item) + if 'hash_password' in data: + data['password'] = HIDDEN + if 'fetches' in data: + for fetch in data['fetches']: + fetch['password'] = HIDDEN + log('Handling', obj.opts.model, data) # configure contexts - dump_context = { - 'secrets': verbose >= 2, + diff_context = { + 'full': True, + 'secrets': secrets, + } + log_context = { + 'secrets': secrets, } load_context = { - 'callback': track_serialize if verbose >= 3 else None, - 'clear': not update, 'import': True, + 'update': update, + 'clear': not update, + 'callback': track_serialize if verbose >= 2 else None, } # register listeners for schema in get_schema(): model = schema.Meta.model - dumper[model] = schema(context=dump_context) + logger[model] = schema(context=log_context) sqlalchemy.event.listen(model, 'after_insert', listen_insert) sqlalchemy.event.listen(model, 'after_update', listen_update) sqlalchemy.event.listen(model, 'after_delete', listen_delete) @@ -491,18 +516,24 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No # special listener for dkim_key changes sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim) - if verbose >= 4: + if verbose >= 3: logging.basicConfig() logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) try: with models.db.session.no_autoflush: - config = MailuSchema(only=SECTIONS, context=load_context).loads(source) + config = MailuSchema(only=MailuSchema.Meta.order, context=load_context).loads(source) except ValidationError as exc: raise click.ClickException(format_errors(exc.messages)) from exc except Exception as exc: - # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) - raise click.ClickException(f'[{exc.__class__.__name__}] {" ".join(str(exc).split())}') from exc + if verbose >= 5: + raise + else: + # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) + raise click.ClickException( + f'[{exc.__class__.__name__}] ' + f'{" ".join(str(exc).split())}' + ) from exc # flush session to show/count all changes if dry_run or verbose >= 1: @@ -510,53 +541,47 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No # check for duplicate domain names dup = set() - for fqdn in chain(db.session.query(models.Domain.name), - db.session.query(models.Alternative.name), - db.session.query(models.Relay.name)): + for fqdn in chain( + db.session.query(models.Domain.name), + db.session.query(models.Alternative.name), + db.session.query(models.Relay.name) + ): if fqdn in dup: raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}') dup.add(fqdn) - # TODO: implement special update "items" - # -pkey: which - remove item "which" - # -key: null or [] or {} - set key to default - # -pkey: null or [] or {} - remove all existing items in this list - # don't commit when running dry if dry_run: - db.session.rollback() if not quiet: print(*format_changes('Dry run. Not commiting changes.')) - # TODO: remove debug - print(MailuSchema().dumps(config)) + db.session.rollback() else: - db.session.commit() if not quiet: - print(*format_changes('Commited changes.')) + print(*format_changes('Committing changes.')) + db.session.commit() @mailu.command() -@click.option('-f', '--full', is_flag=True, help='Include attributes with default value') +@click.option('-f', '--full', is_flag=True, help='Include attributes with default value.') @click.option('-s', '--secrets', is_flag=True, - help='Include secret attributes (dkim-key, passwords)') -@click.option('-d', '--dns', is_flag=True, help='Include dns records') + help='Include secret attributes (dkim-key, passwords).') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') +@click.option('-d', '--dns', is_flag=True, help='Include dns records.') @click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), - help='save yaml to file') -@click.option('-j', '--json', 'as_json', is_flag=True, help='Dump in josn format') -@click.argument('sections', nargs=-1) + help='Save configuration to file.') +@click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.') +@click.argument('only', metavar='[FILTER]...', nargs=-1) @with_appcontext -def config_export(full=False, secrets=False, dns=False, output=None, as_json=False, sections=None): +def config_export(full=False, secrets=False, color=False, dns=False, output=None, as_json=False, only=None): """ Export configuration as YAML or JSON to stdout or file """ - if sections: - for section in sections: - if section not in SECTIONS: - print(f'[ERROR] Unknown section: {section}') - raise click.exceptions.Exit(1) - sections = set(sections) + if only: + for spec in only: + if spec.split('.', 1)[0] not in MailuSchema.Meta.order: + raise click.ClickException(f'[ERROR] Unknown section: {spec}') else: - sections = SECTIONS + only = MailuSchema.Meta.order context = { 'full': full, @@ -564,13 +589,20 @@ def config_export(full=False, secrets=False, dns=False, output=None, as_json=Fal 'dns': dns, } - if as_json: - schema = MailuSchema(only=sections, context=context) - schema.opts.render_module = json - print(schema.dumps(models.MailuConfig(), separators=(',',':')), file=output) + schema = MailuSchema(only=only, context=context) + color_cfg = {'color': color or output.isatty()} - else: - MailuSchema(only=sections, context=context).dumps(models.MailuConfig(), output) + if as_json: + schema.opts.render_module = RenderJSON + color_cfg['lexer'] = 'json' + color_cfg['strip'] = True + + try: + print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output) + except ValueError as exc: + if spec := get_fieldspec(exc): + raise click.ClickException(f'[ERROR] Invalid filter: {spec}') from exc + raise @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index dac1dc70..5799e282 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -19,6 +19,7 @@ import dns from flask import current_app as app from sqlalchemy.ext import declarative +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.inspection import inspect from werkzeug.utils import cached_property @@ -121,6 +122,36 @@ class Base(db.Model): updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) comment = db.Column(db.String(255), nullable=True, default='') + def __str__(self): + pkey = self.__table__.primary_key.columns.values()[0].name + if pkey == 'email': + # ugly hack for email declared attr. _email is not always up2date + return str(f'{self.localpart}@{self.domain_name}') + elif pkey in {'name', 'email'}: + return str(getattr(self, pkey, None)) + else: + return self.__repr__() + return str(getattr(self, self.__table__.primary_key.columns.values()[0].name)) + + def __repr__(self): + return f'<{self.__class__.__name__} {str(self)!r}>' + + def __eq__(self, other): + if isinstance(other, self.__class__): + pkey = self.__table__.primary_key.columns.values()[0].name + this = getattr(self, pkey, None) + other = getattr(other, pkey, None) + return this is not None and other is not None and str(this) == str(other) + else: + return NotImplemented + + def __hash__(self): + primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) + if primary is None: + return NotImplemented + else: + return hash(primary) + # Many-to-many association table for domain managers managers = db.Table('manager', Base.metadata, @@ -261,19 +292,6 @@ class Domain(Base): except dns.exception.DNSException: return False - def __str__(self): - return str(self.name) - - def __eq__(self, other): - if isinstance(other, self.__class__): - return str(self.name) == str(other.name) - else: - return NotImplemented - - def __hash__(self): - return hash(str(self.name)) - - class Alternative(Base): """ Alternative name for a served domain. @@ -287,9 +305,6 @@ class Alternative(Base): domain = db.relationship(Domain, backref=db.backref('alternatives', cascade='all, delete-orphan')) - def __str__(self): - return str(self.name) - class Relay(Base): """ Relayed mail domain. @@ -302,9 +317,6 @@ class Relay(Base): # TODO: String(80) is too small? smtp = db.Column(db.String(80), nullable=True) - def __str__(self): - return str(self.name) - class Email(object): """ Abstraction for an email address (localpart and domain). @@ -312,11 +324,11 @@ class Email(object): # TODO: validate max. total length of address (<=254) - # TODO: String(80) is too large (>64)? + # TODO: String(80) is too large (64)? localpart = db.Column(db.String(80), nullable=False) @declarative.declared_attr - def domain_name(self): + def domain_name(cls): """ the domain part of the email address """ return db.Column(IdnaDomain, db.ForeignKey(Domain.name), nullable=False, default=IdnaDomain) @@ -325,13 +337,33 @@ class Email(object): # It is however very useful for quick lookups without joining tables, # especially when the mail server is reading the database. @declarative.declared_attr - def email(self): + def _email(cls): """ the complete email address (localpart@domain) """ - updater = lambda ctx: '{localpart}@{domain_name}'.format(**ctx.current_parameters) - return db.Column(IdnaEmail, - primary_key=True, nullable=False, - default=updater - ) + + def updater(ctx): + key = f'{cls.__tablename__}_email' + if key in ctx.current_parameters: + return ctx.current_parameters[key] + return '{localpart}@{domain_name}'.format(**ctx.current_parameters) + + return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater) + + # We need to keep email, localpart and domain_name in sync. + # But IMHO using email as primary key was not a good idea in the first place. + @hybrid_property + def email(self): + """ getter for email - gets _email """ + return self._email + + @email.setter + def email(self, value): + """ setter for email - sets _email, localpart and domain_name at once """ + self.localpart, self.domain_name = value.rsplit('@', 1) + self._email = value + + # hack for email declared attr - when _email is not updated yet + def __str__(self): + return str(f'{self.localpart}@{self.domain_name}') def sendmail(self, subject, body): """ send an email to the address """ @@ -391,9 +423,6 @@ class Email(object): return None - def __str__(self): - return str(self.email) - class User(Base, Email): """ A user is an email address that has a password to access a mailbox. @@ -435,12 +464,10 @@ class User(Base, Email): is_active = True is_anonymous = False - # TODO: remove unused user.get_id() def get_id(self): """ return users email address """ return self.email - # TODO: remove unused user.destination @property def destination(self): """ returns comma separated string of destinations """ @@ -471,17 +498,20 @@ class User(Base, Email): 'CRYPT': 'des_crypt', } - def _get_password_context(self): + @classmethod + def get_password_context(cls): + """ Create password context for hashing and verification + """ return passlib.context.CryptContext( - schemes=self.scheme_dict.values(), - default=self.scheme_dict[app.config['PASSWORD_SCHEME']], + schemes=cls.scheme_dict.values(), + default=cls.scheme_dict[app.config['PASSWORD_SCHEME']], ) def check_password(self, plain): """ Check password against stored hash Update hash when default scheme has changed """ - context = self._get_password_context() + context = self.get_password_context() hashed = re.match('^({[^}]+})?(.*)$', self.password).group(2) result = context.verify(plain, hashed) if result and context.identify(hashed) != context.default_scheme(): @@ -490,8 +520,6 @@ class User(Base, Email): db.session.commit() return result - # TODO: remove kwarg hash_scheme - there is no point in setting a scheme, - # when the next check updates the password to the default scheme. def set_password(self, new, hash_scheme=None, raw=False): """ Set password for user with specified encryption scheme @new: plain text password to encrypt (or, if raw is True: the hash itself) @@ -500,7 +528,7 @@ class User(Base, Email): if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] if not raw: - new = self._get_password_context().encrypt(new, self.scheme_dict[hash_scheme]) + new = self.get_password_context().encrypt(new, self.scheme_dict[hash_scheme]) self.password = f'{{{hash_scheme}}}{new}' def get_managed_domains(self): @@ -593,7 +621,7 @@ class Alias(Base, Email): return None -# TODO: what about API tokens? + class Token(Base): """ A token is an application password for a given user. """ @@ -606,20 +634,19 @@ class Token(Base): user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - # TODO: String(80) is too large? + # TODO: String(255) is too large? (43 should be sufficient) ip = db.Column(db.String(255)) def check_password(self, password): """ verifies password against stored hash """ return passlib.hash.sha256_crypt.verify(password, self.password) - # TODO: use crypt context and default scheme from config? def set_password(self, password): """ sets password using sha256_crypt(rounds=1000) """ self.password = passlib.hash.sha256_crypt.using(rounds=1000).hash(password) - def __str__(self): - return str(self.comment or self.ip) + def __repr__(self): + return f'' class Fetch(Base): @@ -644,8 +671,11 @@ class Fetch(Base): last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) - def __str__(self): - return f'{self.protocol}{"s" if self.tls else ""}://{self.username}@{self.host}:{self.port}' + def __repr__(self): + return ( + f'' + ) class MailuConfig: @@ -661,7 +691,7 @@ class MailuConfig: def __init__(self, model : db.Model): self.model = model - def __str__(self): + def __repr__(self): return f'<{self.model.__name__}-Collection>' @cached_property @@ -837,8 +867,8 @@ class MailuConfig: if models is None or model in models: db.session.query(model).delete() - domains = MailuCollection(Domain) - relays = MailuCollection(Relay) - users = MailuCollection(User) - aliases = MailuCollection(Alias) + domain = MailuCollection(Domain) + user = MailuCollection(User) + alias = MailuCollection(Alias) + relay = MailuCollection(Relay) config = MailuCollection(Config) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 04512f6d..54a2e928 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1,27 +1,66 @@ """ Mailu marshmallow fields and schema """ -import re - +from copy import deepcopy from collections import OrderedDict from textwrap import wrap +import re +import json import yaml +import sqlalchemy + from marshmallow import pre_load, post_load, post_dump, fields, Schema +from marshmallow.utils import ensure_text_type from marshmallow.exceptions import ValidationError from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts + from flask_marshmallow import Marshmallow + from OpenSSL import crypto +try: + from pygments import highlight + from pygments.token import Token + from pygments.lexers import get_lexer_by_name + from pygments.lexers.data import YamlLexer + from pygments.formatters import get_formatter_by_name +except ModuleNotFoundError: + COLOR_SUPPORTED = False +else: + COLOR_SUPPORTED = True + from . import models, dkim ma = Marshmallow() -# TODO: how and where to mark keys as "required" while unserializing (on commandline, in api)? -# - fields without default => required -# - fields which are the primary key => unchangeable when updating +# TODO: how and where to mark keys as "required" while unserializing in api? +# - when modifying, nothing is required (only the primary key, but this key is in the uri) +# - the primary key from post data must not differ from the key in the uri +# - when creating all fields without default or auto-increment are required +# TODO: what about deleting list items and prung lists? +# - domain.alternatives, user.forward_destination, user.manager_of, aliases.destination +# TODO: validate everything! + + +### class for hidden values ### + +class _Hidden: + def __bool__(self): + return False + def __copy__(self): + return self + def __deepcopy__(self, _): + return self + def __eq__(self, other): + return str(other) == '' + def __repr__(self): + return '' + __str__ = __repr__ + +HIDDEN = _Hidden() ### map model to schema ### @@ -41,13 +80,90 @@ def mapped(cls): return cls -### yaml render module ### +### helper functions ### + +def get_fieldspec(exc): + """ walk traceback to extract spec of invalid field from marshmallow """ + path = [] + tbck = exc.__traceback__ + while tbck: + if tbck.tb_frame.f_code.co_name == '_serialize': + if 'attr' in tbck.tb_frame.f_locals: + path.append(tbck.tb_frame.f_locals['attr']) + elif tbck.tb_frame.f_code.co_name == '_init_fields': + path = '.'.join(path) + spec = ', '.join([f'{path}.{key}' for key in tbck.tb_frame.f_locals['invalid_fields']]) + return spec + tbck = tbck.tb_next + return None + +def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): + """ add ANSI color to data """ + if color is None: + # autodetect colorize + color = COLOR_SUPPORTED + if not color: + # no color wanted + return data + if not COLOR_SUPPORTED: + # want color, but not supported + raise ValueError('Please install pygments to colorize output') + + scheme = { + Token: ('', ''), + Token.Name.Tag: ('cyan', 'brightcyan'), + Token.Literal.Scalar: ('green', 'green'), + Token.Literal.String: ('green', 'green'), + Token.Keyword.Constant: ('magenta', 'brightmagenta'), + Token.Literal.Number: ('magenta', 'brightmagenta'), + Token.Error: ('red', 'brightred'), + Token.Name: ('red', 'brightred'), + Token.Operator: ('red', 'brightred'), + } + + class MyYamlLexer(YamlLexer): + """ colorize yaml constants and integers """ + def get_tokens(self, text, unfiltered=False): + for typ, value in super().get_tokens(text, unfiltered): + if typ is Token.Literal.Scalar.Plain: + if value in {'true', 'false', 'null'}: + typ = Token.Keyword.Constant + elif value == HIDDEN: + typ = Token.Error + else: + try: + int(value, 10) + except ValueError: + try: + float(value) + except ValueError: + pass + else: + typ = Token.Literal.Number.Float + else: + typ = Token.Literal.Number.Integer + yield typ, value + + res = highlight( + data, + MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer), + get_formatter_by_name(formatter, colorscheme=scheme) + ) + + return res.rstrip('\n') if strip else res + + +### render modules ### # allow yaml module to dump OrderedDict yaml.add_representer( OrderedDict, lambda cls, data: cls.represent_mapping('tag:yaml.org,2002:map', data.items()) ) +yaml.add_representer( + _Hidden, + lambda cls, data: cls.represent_data(str(data)) +) class RenderYAML: """ Marshmallow YAML Render Module @@ -67,19 +183,19 @@ class RenderYAML: return super().increase_indent(flow, False) @staticmethod - def _update_items(dict1, dict2): - """ sets missing keys in dict1 to values of dict2 + def _augment(kwargs, defaults): + """ add default kv's to kwargs if missing """ - for key, value in dict2.items(): - if key not in dict1: - dict1[key] = value + for key, value in defaults.items(): + if key not in kwargs: + kwargs[key] = value _load_defaults = {} @classmethod def loads(cls, *args, **kwargs): """ load yaml data from string """ - cls._update_items(kwargs, cls._load_defaults) + cls._augment(kwargs, cls._load_defaults) return yaml.safe_load(*args, **kwargs) _dump_defaults = { @@ -90,13 +206,52 @@ class RenderYAML: } @classmethod def dumps(cls, *args, **kwargs): - """ dump yaml data to string + """ dump data to yaml string """ - cls._update_items(kwargs, cls._dump_defaults) + cls._augment(kwargs, cls._dump_defaults) return yaml.dump(*args, **kwargs) +class JSONEncoder(json.JSONEncoder): + """ JSONEncoder supporting serialization of HIDDEN """ + def default(self, o): + """ serialize HIDDEN """ + if isinstance(o, _Hidden): + return str(o) + return json.JSONEncoder.default(self, o) -### field definitions ### +class RenderJSON: + """ Marshmallow JSON Render Module + """ + + @staticmethod + def _augment(kwargs, defaults): + """ add default kv's to kwargs if missing + """ + for key, value in defaults.items(): + if key not in kwargs: + kwargs[key] = value + + _load_defaults = {} + @classmethod + def loads(cls, *args, **kwargs): + """ load json data from string + """ + cls._augment(kwargs, cls._load_defaults) + return json.loads(*args, **kwargs) + + _dump_defaults = { + 'separators': (',',':'), + 'cls': JSONEncoder, + } + @classmethod + def dumps(cls, *args, **kwargs): + """ dump data to json string + """ + cls._augment(kwargs, cls._dump_defaults) + return json.dumps(*args, **kwargs) + + +### custom fields ### class LazyStringField(fields.String): """ Field that serializes a "false" value to the empty string @@ -107,9 +262,8 @@ class LazyStringField(fields.String): """ return value if value else '' - class CommaSeparatedListField(fields.Raw): - """ Field that deserializes a string containing comma-separated values to + """ Deserialize a string containing comma-separated values to a list of strings """ @@ -129,10 +283,15 @@ class CommaSeparatedListField(fields.Raw): class DkimKeyField(fields.String): - """ Field that serializes a dkim key to a list of strings (lines) and - deserializes a string or list of strings. + """ Serialize a dkim key to a list of strings (lines) and + Deserialize a string or list of strings to a valid dkim key """ + default_error_messages = { + "invalid": "Not a valid string or list.", + "invalid_utf8": "Not a valid utf-8 string or list.", + } + _clean_re = re.compile( r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)', flags=re.UNICODE @@ -156,11 +315,19 @@ class DkimKeyField(fields.String): # convert list to str if isinstance(value, list): - value = ''.join(value) + try: + value = ''.join([ensure_text_type(item) for item in value]) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc - # only strings are allowed - if not isinstance(value, str): - raise ValidationError(f'invalid type {type(value).__name__!r}') + # only text is allowed + else: + if not isinstance(value, (str, bytes)): + raise self.make_error("invalid") + try: + value = ensure_text_type(value) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc # clean value (remove whitespace and header/footer) value = self._clean_re.sub('', value.strip()) @@ -189,28 +356,53 @@ class DkimKeyField(fields.String): else: return value - -### base definitions ### - -def handle_email(data): - """ merge separate localpart and domain to email +class PasswordField(fields.Str): + """ Serialize a hashed password hash by stripping the obsolete {SCHEME} + Deserialize a plain password or hashed password into a hashed password """ - localpart = 'localpart' in data - domain = 'domain' in data + _hashes = {'PBKDF2', 'BLF-CRYPT', 'SHA512-CRYPT', 'SHA256-CRYPT', 'MD5-CRYPT', 'CRYPT'} - if 'email' in data: - if localpart or domain: - raise ValidationError('duplicate email and localpart/domain') - data['localpart'], data['domain_name'] = data['email'].rsplit('@', 1) - elif localpart and domain: - data['domain_name'] = data['domain'] - del data['domain'] - data['email'] = f'{data["localpart"]}@{data["domain_name"]}' - elif localpart or domain: - raise ValidationError('incomplete localpart/domain') + def _serialize(self, value, attr, obj, **kwargs): + """ strip obsolete {password-hash} when serializing """ + # strip scheme spec if in database - it's obsolete + if value.startswith('{') and (end := value.find('}', 1)) >= 0: + if value[1:end] in self._hashes: + return value[end+1:] + return value - return data + def _deserialize(self, value, attr, data, **kwargs): + """ hashes plain password or checks hashed password + also strips obsolete {password-hash} when deserializing + """ + + # when hashing is requested: use model instance to hash plain password + if data.get('hash_password'): + # hash password using model instance + inst = self.metadata['model']() + inst.set_password(value) + value = inst.password + del inst + + # strip scheme spec when specified - it's obsolete + if value.startswith('{') and (end := value.find('}', 1)) >= 0: + if value[1:end] in self._hashes: + value = value[end+1:] + + # check if algorithm is supported + inst = self.metadata['model'](password=value) + try: + # just check against empty string to see if hash is valid + inst.check_password('') + except ValueError as exc: + # ValueError: hash could not be identified + raise ValidationError(f'invalid password hash {value!r}') from exc + del inst + + return value + + +### base schema ### class BaseOpts(SQLAlchemyAutoSchemaOpts): """ Option class with sqla session @@ -220,6 +412,8 @@ class BaseOpts(SQLAlchemyAutoSchemaOpts): meta.sqla_session = models.db.session if not hasattr(meta, 'ordered'): meta.ordered = True + if not hasattr(meta, 'sibling'): + meta.sibling = False super(BaseOpts, self).__init__(meta, ordered=ordered) class BaseSchema(ma.SQLAlchemyAutoSchema): @@ -231,10 +425,15 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): class Meta: """ Schema config """ + include_by_context = {} + exclude_by_value = {} + hide_by_context = {} + order = [] + sibling = False def __init__(self, *args, **kwargs): - # context? + # get context context = kwargs.get('context', {}) flags = {key for key, value in context.items() if value is True} @@ -261,7 +460,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # exclude default values if not context.get('full'): - for column in getattr(self.opts, 'model').__table__.columns: + for column in self.opts.model.__table__.columns: if column.name not in exclude: self._exclude_by_value.setdefault(column.name, []).append( None if column.default is None else column.default.arg @@ -274,45 +473,239 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if not flags & set(need): self._hide_by_context |= set(what) + # remember primary keys + self._primary = self.opts.model.__table__.primary_key.columns.values()[0].name + # initialize attribute order if hasattr(self.Meta, 'order'): # use user-defined order - self._order = list(reversed(getattr(self.Meta, 'order'))) + self._order = list(reversed(self.Meta.order)) else: # default order is: primary_key + other keys alphabetically self._order = list(sorted(self.fields.keys())) - primary = self.opts.model.__table__.primary_key.columns.values()[0].name - if primary in self._order: - self._order.remove(primary) + if self._primary in self._order: + self._order.remove(self._primary) self._order.reverse() - self._order.append(primary) + self._order.append(self._primary) # move pre_load hook "_track_import" to the front hooks = self._hooks[('pre_load', False)] - if '_track_import' in hooks: - hooks.remove('_track_import') - hooks.insert(0, '_track_import') - # and post_load hook "_fooo" to the end + hooks.remove('_track_import') + hooks.insert(0, '_track_import') + # move pre_load hook "_add_instance" to the end + hooks.remove('_add_required') + hooks.append('_add_required') + + # move post_load hook "_add_instance" to the end hooks = self._hooks[('post_load', False)] - if '_add_instance' in hooks: - hooks.remove('_add_instance') - hooks.append('_add_instance') + hooks.remove('_add_instance') + hooks.append('_add_instance') + + def hide(self, data): + """ helper method to hide input data for logging """ + # always returns a copy of data + return { + key: HIDDEN if key in self._hide_by_context else deepcopy(value) + for key, value in data.items() + } + + def _call_and_store(self, *args, **kwargs): + """ track curent parent field for pruning """ + self.context['parent_field'] = kwargs['field_name'] + return super()._call_and_store(*args, **kwargs) + + # this is only needed to work around the declared attr "email" primary key in model + def get_instance(self, data): + """ lookup item by defined primary key instead of key(s) from model """ + if self.transient: + return None + if keys := getattr(self.Meta, 'primary_keys', None): + filters = {key: data.get(key) for key in keys} + if None not in filters.values(): + return self.session.query(self.opts.model).filter_by(**filters).first() + return super().get_instance(data) + + @pre_load(pass_many=True) + def _patch_input(self, items, many, **kwargs): # pylint: disable=unused-argument + """ - flush sqla session before serializing a section when requested + (make sure all objects that could be referred to later are created) + - when in update mode: patch input data before deserialization + - handle "prune" and "delete" items + - replace values in keys starting with '-' with default + """ + + # flush sqla session + if not self.Meta.sibling: + self.opts.sqla_session.flush() + + # stop early when not updating + if not self.context.get('update'): + return items + + # patch "delete", "prune" and "default" + want_prune = [] + def patch(count, data, prune): + + # don't allow __delete__ coming from input + if '__delete__' in data: + raise ValidationError('Unknown field.', f'{count}.__delete__') + + # handle "prune list" and "delete item" (-pkey: none and -pkey: id) + for key in data: + if key.startswith('-'): + if key[1:] == self._primary: + # delete or prune + if data[key] is None: + # prune + prune.append(True) + return None + # mark item for deletion + return {key[1:]: data[key], '__delete__': True} + + # handle "set to default value" (-key: none) + def set_default(key, value): + if not key.startswith('-'): + return (key, value) + key = key[1:] + if not key in self.opts.model.__table__.columns: + return (key, None) + if value is not None: + raise ValidationError( + 'When resetting to default value must be null.', + f'{count}.{key}' + ) + value = self.opts.model.__table__.columns[key].default + if value is None: + raise ValidationError( + 'Field has no default value.', + f'{count}.{key}' + ) + return (key, value.arg) + + return dict([set_default(key, value) for key, value in data.items()]) + + # convert items to "delete" and filter "prune" item + items = [ + item for item in [ + patch(count, item, want_prune) for count, item in enumerate(items) + ] if item + ] + + # prune: determine if existing items in db need to be added or marked for deletion + add_items = False + del_items = False + if self.Meta.sibling: + # parent prunes automatically + if not want_prune: + # no prune requested => add old items + add_items = True + else: + # parent does not prune automatically + if want_prune: + # prune requested => mark old items for deletion + del_items = True + + if add_items or del_items: + existing = {item[self._primary] for item in items if self._primary in item} + for item in getattr(self.context['parent'], self.context['parent_field']): + key = getattr(item, self._primary) + if key not in existing: + if add_items: + items.append({self._primary: key}) + else: + items.append({self._primary: key, '__delete__': True}) + + return items @pre_load def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument -# TODO: also handle reset, prune and delete in pre_load / post_load hooks! -# print('!!!', repr(data)) + """ call callback function to track import + """ + # callback if callback := self.context.get('callback'): callback(self, data) + return data - @post_load - def _add_instance(self, item, many, **kwargs): # pylint: disable=unused-argument - self.opts.sqla_session.add(item) + @pre_load + def _add_required(self, data, many, **kwargs): # pylint: disable=unused-argument + """ when updating: + allow modification of existing items having required attributes + by loading existing value from db + """ + + if not self.opts.load_instance or not self.context.get('update'): + return data + + # stabilize import of auto-increment primary keys (not required), + # by matching import data to existing items and setting primary key + if not self._primary in data: + for item in getattr(self.context['parent'], self.context['parent_field']): + existing = self.dump(item, many=False) + this = existing.pop(self._primary) + if data == existing: + instance = item + data[self._primary] = this + break + + # try to load instance + instance = self.instance or self.get_instance(data) + if instance is None: + + if '__delete__' in data: + # deletion of non-existent item requested + raise ValidationError( + f'item not found: {data[self._primary]!r}', + field_name=f'?.{self._primary}', + ) + + else: + + if self.context.get('update'): + # remember instance as parent for pruning siblings + if not self.Meta.sibling: + self.context['parent'] = instance + # delete instance when marked + if '__delete__' in data: + self.opts.sqla_session.delete(instance) + + # add attributes required for validation from db + # TODO: this will cause validation errors if value from database does not validate + for attr_name, field_obj in self.load_fields.items(): + if field_obj.required and attr_name not in data: + data[attr_name] = getattr(instance, attr_name) + + return data + + @post_load(pass_original=True) + def _add_instance(self, item, original, many, **kwargs): # pylint: disable=unused-argument + """ add new instances to sqla session """ + + if item in self.opts.sqla_session: + # item was modified + if 'hash_password' in original: + # stabilize import of passwords to be hashed, + # by not re-hashing an unchanged password + if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): + if attr.history.has_changes() and attr.history.deleted: + try: + # reset password hash, if password was not changed + inst = type(item)(password=attr.history.deleted[-1]) + if inst.check_password(original['password']): + item.password = inst.password + except ValueError: + # hash in db is invalid + pass + else: + del inst + else: + # new item + self.opts.sqla_session.add(item) return item @post_dump def _hide_and_order(self, data, many, **kwargs): # pylint: disable=unused-argument + """ hide secrets and order output """ # order output for key in self._order: @@ -325,15 +718,18 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if not self._exclude_by_value and not self._hide_by_context: return data - # exclude items or hide values + # exclude or hide values full = self.context.get('full') return type(data)([ - (key, '' if key in self._hide_by_context else value) + (key, HIDDEN if key in self._hide_by_context else value) for key, value in data.items() if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] ]) - # TODO: remove LazyStringField and change model (IMHO comment should not be nullable) + # this field is used to mark items for deletion + mark_delete = fields.Boolean(data_key='__delete__', load_only=True) + + # TODO: remove LazyStringField (when model was changed - IMHO comment should not be nullable) comment = LazyStringField() @@ -381,6 +777,11 @@ class TokenSchema(BaseSchema): model = models.Token load_instance = True + sibling = True + + password = PasswordField(required=True, metadata={'model': models.User}) + hash_password = fields.Boolean(load_only=True, missing=False) + @mapped class FetchSchema(BaseSchema): @@ -389,6 +790,8 @@ class FetchSchema(BaseSchema): """ Schema config """ model = models.Fetch load_instance = True + + sibling = True include_by_context = { ('full', 'import'): {'last_check', 'error'}, } @@ -405,52 +808,25 @@ class UserSchema(BaseSchema): model = models.User load_instance = True include_relationships = True - exclude = ['domain', 'quota_bytes_used'] + exclude = ['_email', 'domain', 'localpart', 'domain_name', 'quota_bytes_used'] + primary_keys = ['email'] exclude_by_value = { 'forward_destination': [[]], - 'tokens': [[]], - 'fetches': [[]], - 'manager_of': [[]], - 'reply_enddate': ['2999-12-31'], - 'reply_startdate': ['1900-01-01'], + 'tokens': [[]], + 'fetches': [[]], + 'manager_of': [[]], + 'reply_enddate': ['2999-12-31'], + 'reply_startdate': ['1900-01-01'], } - @pre_load - def _handle_email_and_password(self, data, many, **kwargs): # pylint: disable=unused-argument - data = handle_email(data) - if 'password' in data: - if 'password_hash' in data or 'hash_scheme' in data: - raise ValidationError('ambigous key password and password_hash/hash_scheme') - # check (hashed) password - password = data['password'] - if password.startswith('{') and '}' in password: - scheme = password[1:password.index('}')] - if scheme not in self.Meta.model.scheme_dict: - raise ValidationError(f'invalid password scheme {scheme!r}') - else: - raise ValidationError(f'invalid hashed password {password!r}') - elif 'password_hash' in data and 'hash_scheme' in data: - if data['hash_scheme'] not in self.Meta.model.scheme_dict: - raise ValidationError(f'invalid password scheme {data["hash_scheme"]!r}') - data['password'] = f'{{{data["hash_scheme"]}}}{data["password_hash"]}' - del data['hash_scheme'] - del data['password_hash'] - return data - - # TODO: verify password (should this be done in model?) - # scheme, hashed = re.match('^(?:{([^}]+)})?(.*)$', self.password).groups() - # if not scheme... - # ctx = passlib.context.CryptContext(schemes=[scheme], default=scheme) - # try: - # ctx.verify('', hashed) - # =>? ValueError: hash could not be identified - - localpart = fields.Str(load_only=True) - domain_name = fields.Str(load_only=True) + email = fields.String(required=True) tokens = fields.Nested(TokenSchema, many=True) fetches = fields.Nested(FetchSchema, many=True) + password = PasswordField(required=True, metadata={'model': models.User}) + hash_password = fields.Boolean(load_only=True, missing=False) + @mapped class AliasSchema(BaseSchema): @@ -459,18 +835,14 @@ class AliasSchema(BaseSchema): """ Schema config """ model = models.Alias load_instance = True - exclude = ['domain'] + exclude = ['_email', 'domain', 'localpart', 'domain_name'] + primary_keys = ['email'] exclude_by_value = { 'destination': [[]], } - @pre_load - def _handle_email(self, data, many, **kwargs): # pylint: disable=unused-argument - return handle_email(data) - - localpart = fields.Str(load_only=True) - domain_name = fields.Str(load_only=True) + email = fields.String(required=True) destination = CommaSeparatedListField() @@ -499,7 +871,7 @@ class MailuSchema(Schema): render_module = RenderYAML ordered = True - order = ['config', 'domains', 'users', 'aliases', 'relays'] + order = ['domain', 'user', 'alias', 'relay'] # 'config' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -511,6 +883,14 @@ class MailuSchema(Schema): except KeyError: pass + def _call_and_store(self, *args, **kwargs): + """ track current parent and field for pruning """ + self.context.update({ + 'parent': self.context.get('config'), + 'parent_field': kwargs['field_name'], + }) + return super()._call_and_store(*args, **kwargs) + @pre_load def _clear_config(self, data, many, **kwargs): # pylint: disable=unused-argument """ create config object in context if missing @@ -534,8 +914,8 @@ class MailuSchema(Schema): return config - config = fields.Nested(ConfigSchema, many=True) - domains = fields.Nested(DomainSchema, many=True) - users = fields.Nested(UserSchema, many=True) - aliases = fields.Nested(AliasSchema, many=True) - relays = fields.Nested(RelaySchema, many=True) + domain = fields.Nested(DomainSchema, many=True) + user = fields.Nested(UserSchema, many=True) + alias = fields.Nested(AliasSchema, many=True) + relay = fields.Nested(RelaySchema, many=True) +# config = fields.Nested(ConfigSchema, many=True) diff --git a/docs/cli.rst b/docs/cli.rst index 1b2ed14f..497cdfc5 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -10,8 +10,9 @@ Managing users and aliases can be done from CLI using commands: * user * user-import * user-delete -* config-dump * config-update +* config-export +* config-import alias ----- @@ -69,104 +70,160 @@ user-delete docker-compose exec admin flask mailu user-delete foo@example.net -config-dump ------------ - -The purpose of this command is to dump domain-, relay-, alias- and user-configuration to a YAML template. - -.. code-block:: bash - - # docker-compose exec admin flask mailu config-dump --help - - Usage: flask mailu config-dump [OPTIONS] [SECTIONS]... - - dump configuration as YAML-formatted data to stdout - - SECTIONS can be: domains, relays, users, aliases - - Options: - -f, --full Include default attributes - -s, --secrets Include secrets (dkim-key, plain-text / not hashed) - -d, --dns Include dns records - --help Show this message and exit. - -If you want to export secrets (dkim-keys, plain-text / not hashed) you have to add the ``--secrets`` option. -Only non-default attributes are dumped. If you want to dump all attributes use ``--full``. -To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. -Unless you specify some sections all sections are dumped by default. - -.. code-block:: bash - - docker-compose exec admin flask mailu config-dump > mail-config.yml - - docker-compose exec admin flask mailu config-dump --dns domains - config-update ------------- -The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML template. +The sole purpose of this command is for importing users/aliases in bulk and synchronizing DB entries with external YAML template: .. code-block:: bash - # docker-compose exec admin flask mailu config-update --help + cat mail-config.yml | docker-compose exec -T admin flask mailu config-update --delete-objects - Usage: flask mailu config-update [OPTIONS] +where mail-config.yml looks like: - sync configuration with data from YAML-formatted stdin +.. code-block:: bash - Options: - -v, --verbose Increase verbosity - -d, --delete-objects Remove objects not included in yaml - -n, --dry-run Perform a trial run with no changes made - --help Show this message and exit. + users: + - localpart: foo + domain: example.com + password_hash: klkjhumnzxcjkajahsdqweqqwr + hash_scheme: MD5-CRYPT + aliases: + - localpart: alias1 + domain: example.com + destination: "user1@example.com,user2@example.com" + +without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. + +Users +----- + +following are additional parameters that could be defined for users: + +* comment +* quota_bytes +* global_admin +* enable_imap +* enable_pop +* forward_enabled +* forward_destination +* reply_enabled +* reply_subject +* reply_body +* displayed_name +* spam_enabled +* spam_threshold + +Alias +----- + +additional fields: + +* wildcard + +config-export +------------- + +The purpose of this command is to export domain-, relay-, alias- and user-configuration in YAML or JSON format. + +.. code-block:: bash + + # docker-compose exec admin flask mailu config-export --help + + Usage: flask mailu config-export [OPTIONS] [FILTER]... + + Export configuration as YAML or JSON to stdout or file + + Options: + -f, --full Include attributes with default value. + -s, --secrets Include secret attributes (dkim-key, passwords). + -c, --color Force colorized output. + -d, --dns Include dns records. + -o, --output-file FILENAME Save configuration to file. + -j, --json Export configuration in json format. + -?, -h, --help Show this message and exit. + +Only non-default attributes are exported. If you want to export all attributes use ``--full``. +If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option. +To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. +By default all configuration objects are exported (domain, user, alias, relay). You can specify +filters to export only some objects or attributes (try: ``user`` or ``domain.name``). + +.. code-block:: bash + + docker-compose exec admin flask mailu config-export -o mail-config.yml + + docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf + +config-import +------------- + +The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML/JOSN source. + +.. code-block:: bash + + # docker-compose exec admin flask mailu config-import --help + + Usage: flask mailu config-import [OPTIONS] [FILENAME|-] + + Import configuration as YAML or JSON from stdin or file + + Options: + -v, --verbose Increase verbosity. + -s, --secrets Show secret attributes in messages. + -q, --quiet Quiet mode - only show errors. + -c, --color Force colorized output. + -u, --update Update mode - merge input with existing config. + -n, --dry-run Perform a trial run with no changes made. + -?, -h, --help Show this message and exit. The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec instead: .. code-block:: bash - docker exec -i $(docker-compose ps -q admin) flask mailu config-update -nvd < mail-config.yml + docker exec -i $(docker-compose ps -q admin) flask mailu config-import -nv < mail-config.yml - -mail-config.yml looks like this: +mail-config.yml contains the configuration and looks like this: .. code-block:: yaml - - domains: + + domain: - name: example.com alternatives: - alternative.example.com - users: + user: - email: foo@example.com - password_hash: klkjhumnzxcjkajahsdqweqqwr + password_hash: '$2b$12$...' hash_scheme: MD5-CRYPT - aliases: + alias: - email: alias1@example.com - destination: "user1@example.com,user2@example.com" + destination: + - user1@example.com + - user2@example.com - relays: + relay: - name: relay.example.com comment: test smtp: mx.example.com -You can use ``--dry-run`` to test your YAML without comitting any changes to the database. -With ``--verbose`` config-update will show exactly what it changes in the database. -Without ``--delete-object`` option config-update will only add/update changed values but will *not* remove any entries missing in provided YAML input. +config-update shows the number of created/modified/deleted objects after import. +To suppress all messages except error messages use ``--quiet``. +By adding the ``--verbose`` switch (one or more times) the import gets more detailed and shows exactyl what attributes changed. +In all messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to show secrets. +If you want to test what would be done when importing use ``--dry-run``. +By default config-update replaces the whole configuration. You can use ``--update`` to change the existing configuration instead. +When updating you can add new and change existing objects. +To delete an object use ``-key: value`` (To delete the domain example.com ``-name: example.com`` for example). +To reset an attribute to default use ``-key: null`` (To reset enable_imap ``-enable_imap: null`` for example). -This is a complete YAML template with all additional parameters that could be defined: +This is a complete YAML template with all additional parameters that can be defined: .. code-block:: yaml - aliases: - - email: email@example.com - comment: '' - destination: - - address@example.com - wildcard: false - - domains: + domain: - name: example.com alternatives: - alternative.tld @@ -176,13 +233,8 @@ This is a complete YAML template with all additional parameters that could be de max_quota_bytes: 0 max_users: -1 signup_enabled: false - - relays: - - name: relay.example.com - comment: '' - smtp: mx.example.com - - users: + + user: - email: postmaster@example.com comment: '' displayed_name: 'Postmaster' @@ -192,13 +244,16 @@ This is a complete YAML template with all additional parameters that could be de fetches: - id: 1 comment: 'test fetch' - username: fetch-user + error: null host: other.example.com + keep: true + last_check: '2020-12-29T17:09:48.200179' password: 'secret' + hash_password: true port: 993 protocol: imap tls: true - keep: true + username: fetch-user forward_destination: - address@remote.example.com forward_enabled: true @@ -206,12 +261,13 @@ This is a complete YAML template with all additional parameters that could be de global_admin: true manager_of: - example.com - password: '{BLF-CRYPT}$2b$12$...' + password: '$2b$12$...' + hash_password: true quota_bytes: 1000000000 reply_body: '' reply_enabled: false - reply_enddate: 2999-12-31 - reply_startdate: 1900-01-01 + reply_enddate: '2999-12-31' + reply_startdate: '1900-01-01' reply_subject: '' spam_enabled: true spam_threshold: 80 @@ -219,5 +275,16 @@ This is a complete YAML template with all additional parameters that could be de - id: 1 comment: email-client ip: 192.168.1.1 - password: '$5$rounds=1000$...' + password: '$5$rounds=1$...' + aliases: + - email: email@example.com + comment: '' + destination: + - address@example.com + wildcard: false + + relay: + - name: relay.example.com + comment: '' + smtp: mx.example.com diff --git a/tests/compose/core/02_forward_test.sh b/tests/compose/core/02_forward_test.sh index 651e027c..595820cf 100755 --- a/tests/compose/core/02_forward_test.sh +++ b/tests/compose/core/02_forward_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -10,7 +10,7 @@ EOF python3 tests/forward_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" diff --git a/tests/compose/core/03_alias_test.sh b/tests/compose/core/03_alias_test.sh index 2d40903a..dce1918a 100755 --- a/tests/compose/core/03_alias_test.sh +++ b/tests/compose/core/03_alias_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 aliases: - localpart: alltheusers domain: mailu.io @@ -7,6 +7,6 @@ EOF python3 tests/alias_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 aliases: [] EOF diff --git a/tests/compose/core/04_reply_test.sh b/tests/compose/core/04_reply_test.sh index 7615a0f8..83c114f6 100755 --- a/tests/compose/core/04_reply_test.sh +++ b/tests/compose/core/04_reply_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -11,7 +11,7 @@ EOF python3 tests/reply_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" diff --git a/towncrier/newsfragments/1604.feature b/towncrier/newsfragments/1604.feature index 06ee0beb..2b47791a 100644 --- a/towncrier/newsfragments/1604.feature +++ b/towncrier/newsfragments/1604.feature @@ -1 +1 @@ -Added cli command config-dump and enhanced config-update +Add cli commands config-import and config-export From 3937986e76f7079eb4488f12164f83f1966cb0ec Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 15 Feb 2021 10:01:35 +0100 Subject: [PATCH 056/181] Convert OrderedDict to dict for output --- core/admin/mailu/manage.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index a8d1d3cb..37d91f33 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -7,7 +7,7 @@ import socket import logging import uuid -from collections import Counter +from collections import Counter, OrderedDict from itertools import chain import click @@ -396,11 +396,19 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal return chain(message, changes) def log(action, target, message=None): + + def od2d(val): + """ converts OrderedDicts to Dict for logging purposes """ + if isinstance(val, OrderedDict): + return {k: od2d(v) for k, v in val.items()} + elif isinstance(val, list): + return [od2d(v) for v in val] + else: + return val + if message is None: - # TODO: convert nested OrderedDict to dict - # see: flask mailu config-import -nvv yaml/dump4.yaml try: - message = dict(logger[target.__class__].dump(target)) + message = od2d(logger[target.__class__].dump(target)) except KeyError: message = target if not isinstance(message, str): From 8929912dea9c061632a3bde53726e0949ad455b9 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 15 Feb 2021 21:56:58 +0100 Subject: [PATCH 057/181] remove OrderedDict - not necessary in python>=3.7 --- core/admin/mailu/manage.py | 24 ++++++------------ core/admin/mailu/schemas.py | 50 +++++++++++++------------------------ 2 files changed, 25 insertions(+), 49 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 37d91f33..a20c7d6d 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -7,7 +7,7 @@ import socket import logging import uuid -from collections import Counter, OrderedDict +from collections import Counter from itertools import chain import click @@ -397,18 +397,9 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal def log(action, target, message=None): - def od2d(val): - """ converts OrderedDicts to Dict for logging purposes """ - if isinstance(val, OrderedDict): - return {k: od2d(v) for k, v in val.items()} - elif isinstance(val, list): - return [od2d(v) for v in val] - else: - return val - if message is None: try: - message = od2d(logger[target.__class__].dump(target)) + message = logger[target.__class__].dump(target) except KeyError: message = target if not isinstance(message, str): @@ -536,12 +527,11 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal except Exception as exc: if verbose >= 5: raise - else: - # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) - raise click.ClickException( - f'[{exc.__class__.__name__}] ' - f'{" ".join(str(exc).split())}' - ) from exc + # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) + raise click.ClickException( + f'[{exc.__class__.__name__}] ' + f'{" ".join(str(exc).split())}' + ) from exc # flush session to show/count all changes if dry_run or verbose >= 1: diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 54a2e928..8e91b4aa 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -2,7 +2,6 @@ """ from copy import deepcopy -from collections import OrderedDict from textwrap import wrap import re @@ -155,11 +154,7 @@ def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): ### render modules ### -# allow yaml module to dump OrderedDict -yaml.add_representer( - OrderedDict, - lambda cls, data: cls.represent_mapping('tag:yaml.org,2002:map', data.items()) -) +# allow yaml to represent hidden attributes yaml.add_representer( _Hidden, lambda cls, data: cls.represent_data(str(data)) @@ -410,8 +405,6 @@ class BaseOpts(SQLAlchemyAutoSchemaOpts): def __init__(self, meta, ordered=False): if not hasattr(meta, 'sqla_session'): meta.sqla_session = models.db.session - if not hasattr(meta, 'ordered'): - meta.ordered = True if not hasattr(meta, 'sibling'): meta.sibling = False super(BaseOpts, self).__init__(meta, ordered=ordered) @@ -474,19 +467,23 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): self._hide_by_context |= set(what) # remember primary keys - self._primary = self.opts.model.__table__.primary_key.columns.values()[0].name + self._primary = str(self.opts.model.__table__.primary_key.columns.values()[0].name) - # initialize attribute order + # determine attribute order if hasattr(self.Meta, 'order'): # use user-defined order - self._order = list(reversed(self.Meta.order)) + order = self.Meta.order else: # default order is: primary_key + other keys alphabetically - self._order = list(sorted(self.fields.keys())) - if self._primary in self._order: - self._order.remove(self._primary) - self._order.reverse() - self._order.append(self._primary) + order = list(sorted(self.fields.keys())) + if self._primary in order: + order.remove(self._primary) + order.insert(0, self._primary) + + # order dump_fields + for field in order: + if field in self.dump_fields: + self.dump_fields[field] = self.dump_fields.pop(field) # move pre_load hook "_track_import" to the front hooks = self._hooks[('pre_load', False)] @@ -704,16 +701,9 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): return item @post_dump - def _hide_and_order(self, data, many, **kwargs): # pylint: disable=unused-argument + def _hide_values(self, data, many, **kwargs): # pylint: disable=unused-argument """ hide secrets and order output """ - # order output - for key in self._order: - try: - data.move_to_end(key, False) - except KeyError: - pass - # stop early when not excluding/hiding if not self._exclude_by_value and not self._hide_by_context: return data @@ -870,18 +860,14 @@ class MailuSchema(Schema): """ Schema config """ render_module = RenderYAML - ordered = True order = ['domain', 'user', 'alias', 'relay'] # 'config' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # order fields - for field_list in self.load_fields, self.dump_fields, self.fields: - for section in reversed(self.Meta.order): - try: - field_list.move_to_end(section, False) - except KeyError: - pass + # order dump_fields + for field in self.Meta.order: + if field in self.dump_fields: + self.dump_fields[field] = self.dump_fields.pop(field) def _call_and_store(self, *args, **kwargs): """ track current parent and field for pruning """ From 70a1c79f81d4ceecba42e541947b14a2e9980bff Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 15 Feb 2021 22:57:37 +0100 Subject: [PATCH 058/181] handle prune and delete for lists and backrefs --- core/admin/mailu/manage.py | 13 ++++++++--- core/admin/mailu/schemas.py | 43 ++++++++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index a20c7d6d..05eae010 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -478,9 +478,16 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal if verbose >= 1: log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') - def track_serialize(obj, item): + def track_serialize(obj, item, backref=None): """ callback function to track import """ - # hide secrets + # called for backref modification? + if backref is not None: + log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref)) + return + # verbose? + if not verbose >= 2: + return + # hide secrets in data data = logger[obj.opts.model].hide(item) if 'hash_password' in data: data['password'] = HIDDEN @@ -501,7 +508,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal 'import': True, 'update': update, 'clear': not update, - 'callback': track_serialize if verbose >= 2 else None, + 'callback': track_serialize, } # register listeners diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 8e91b4aa..3e15ee26 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -14,6 +14,7 @@ from marshmallow import pre_load, post_load, post_dump, fields, Schema from marshmallow.utils import ensure_text_type from marshmallow.exceptions import ValidationError from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts +from marshmallow_sqlalchemy.fields import RelatedList from flask_marshmallow import Marshmallow @@ -39,8 +40,6 @@ ma = Marshmallow() # - when modifying, nothing is required (only the primary key, but this key is in the uri) # - the primary key from post data must not differ from the key in the uri # - when creating all fields without default or auto-increment are required -# TODO: what about deleting list items and prung lists? -# - domain.alternatives, user.forward_destination, user.manager_of, aliases.destination # TODO: validate everything! @@ -652,7 +651,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if '__delete__' in data: # deletion of non-existent item requested raise ValidationError( - f'item not found: {data[self._primary]!r}', + f'item to delete not found: {data[self._primary]!r}', field_name=f'?.{self._primary}', ) @@ -665,6 +664,44 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # delete instance when marked if '__delete__' in data: self.opts.sqla_session.delete(instance) + # delete item from lists or prune lists + # currently: domain.alternatives, user.forward_destination, + # user.manager_of, aliases.destination + for key, value in data.items(): + if isinstance(value, list): + new_value = set(value) + # handle list pruning + if '-prune-' in value: + value.remove('-prune-') + new_value.remove('-prune-') + else: + for old in getattr(instance, key): + # using str() is okay for now (see above) + new_value.add(str(old)) + # handle item deletion + for item in value: + if item.startswith('-'): + new_value.remove(item) + try: + new_value.remove(item[1:]) + except KeyError as exc: + raise ValidationError( + f'item to delete not found: {item[1:]!r}', + field_name=f'?.{key}', + ) from exc + # deduplicate and sort list + data[key] = sorted(new_value) + # log backref modification not catched by hook + if isinstance(self.fields[key], RelatedList): + if callback := self.context.get('callback'): + callback(self, instance, { + 'key': key, + 'target': str(instance), + 'before': [str(v) for v in getattr(instance, key)], + 'after': data[key], + }) + + # add attributes required for validation from db # TODO: this will cause validation errors if value from database does not validate From 1e2b5f26ab9fed0db8994ca54db7f57ce0792ce8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 16 Feb 2021 13:34:02 +0100 Subject: [PATCH 059/181] don't handle nested lists --- core/admin/mailu/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 3e15ee26..b9b8e393 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -668,7 +668,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # currently: domain.alternatives, user.forward_destination, # user.manager_of, aliases.destination for key, value in data.items(): - if isinstance(value, list): + if not isinstance(self.fields[key], fields.Nested) and isinstance(value, list): new_value = set(value) # handle list pruning if '-prune-' in value: From 10435114ec0206e3558734634d906cffbd49e783 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 16 Feb 2021 15:36:01 +0100 Subject: [PATCH 060/181] updated remarks and docs --- core/admin/mailu/manage.py | 25 ++++++++++++--------- core/admin/mailu/models.py | 12 +++++----- core/admin/mailu/schemas.py | 3 ++- docs/cli.rst | 45 ++++++++++++++++++++++++------------- 4 files changed, 51 insertions(+), 34 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 05eae010..756400ad 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -337,16 +337,19 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal """ # verbose - # 0 : show number of changes - # 1 : also show changes - # 2 : also show secrets - # 3 : also show input data - # 4 : also show sql queries - # 5 : also show tracebacks + # 0 : only show number of changes + # 1 : also show detailed changes + # 2 : also show input data + # 3 : also show sql queries (also needs -s, as sql may contain secrets) + # 4 : also show tracebacks (also needs -s, as tracebacks may contain secrets) if quiet: verbose = -1 + if verbose > 2 and not secrets: + print('[Warning] Verbosity level capped to 2. Specify --secrets to log sql and tracebacks.') + verbose = 2 + color_cfg = { 'color': color or sys.stdout.isatty(), 'lexer': 'python', @@ -376,7 +379,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}' res = [fmt.format(loc, msg) for loc, msg in res] num = f'error{["s",""][len(res)==1]}' - res.insert(0, f'[ValidationError] {len(res)} {num} occured during input validation') + res.insert(0, f'[ValidationError] {len(res)} {num} occurred during input validation') return '\n'.join(res) @@ -484,7 +487,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal if backref is not None: log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref)) return - # verbose? + # show input data? if not verbose >= 2: return # hide secrets in data @@ -532,7 +535,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal except ValidationError as exc: raise click.ClickException(format_errors(exc.messages)) from exc except Exception as exc: - if verbose >= 5: + if verbose >= 3: raise # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) raise click.ClickException( @@ -584,7 +587,7 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None if only: for spec in only: if spec.split('.', 1)[0] not in MailuSchema.Meta.order: - raise click.ClickException(f'[ERROR] Unknown section: {spec}') + raise click.ClickException(f'[ValidationError] Unknown section: {spec}') else: only = MailuSchema.Meta.order @@ -606,7 +609,7 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output) except ValueError as exc: if spec := get_fieldspec(exc): - raise click.ClickException(f'[ERROR] Invalid filter: {spec}') from exc + raise click.ClickException(f'[ValidationError] Invalid filter: {spec}') from exc raise diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 5799e282..4c119984 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -33,7 +33,7 @@ class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: String(80) is too small? + # TODO: use db.String(255)? impl = db.String(80) def process_bind_param(self, value, dialect): @@ -50,7 +50,7 @@ class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: String(255) is too small? + # TODO: use db.String(254)? impl = db.String(255) def process_bind_param(self, value, dialect): @@ -314,7 +314,7 @@ class Relay(Base): __tablename__ = 'relay' name = db.Column(IdnaDomain, primary_key=True, nullable=False) - # TODO: String(80) is too small? + # TODO: use db.String(266)? transport(8):(1)[nexthop(255)](2) smtp = db.Column(db.String(80), nullable=True) @@ -322,9 +322,7 @@ class Email(object): """ Abstraction for an email address (localpart and domain). """ - # TODO: validate max. total length of address (<=254) - - # TODO: String(80) is too large (64)? + # TODO: use db.String(64)? localpart = db.Column(db.String(80), nullable=False) @declarative.declared_attr @@ -634,7 +632,7 @@ class Token(Base): user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - # TODO: String(255) is too large? (43 should be sufficient) + # TODO: use db.String(32)? ip = db.Column(db.String(255)) def check_password(self, password): diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index b9b8e393..7d0393f0 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -36,7 +36,7 @@ from . import models, dkim ma = Marshmallow() -# TODO: how and where to mark keys as "required" while unserializing in api? +# TODO: how and where to mark keys as "required" while deserializing in api? # - when modifying, nothing is required (only the primary key, but this key is in the uri) # - the primary key from post data must not differ from the key in the uri # - when creating all fields without default or auto-increment are required @@ -705,6 +705,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # add attributes required for validation from db # TODO: this will cause validation errors if value from database does not validate + # but there should not be an invalid value in the database for attr_name, field_obj in self.load_fields.items(): if field_obj.required and attr_name not in data: data[attr_name] = getattr(instance, attr_name) diff --git a/docs/cli.rst b/docs/cli.rst index 497cdfc5..6d48c576 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -97,7 +97,7 @@ where mail-config.yml looks like: without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. Users ------ +^^^^^ following are additional parameters that could be defined for users: @@ -116,7 +116,7 @@ following are additional parameters that could be defined for users: * spam_threshold Alias ------ +^^^^^ additional fields: @@ -125,11 +125,11 @@ additional fields: config-export ------------- -The purpose of this command is to export domain-, relay-, alias- and user-configuration in YAML or JSON format. +The purpose of this command is to export the complete configuration in YAML or JSON format. .. code-block:: bash - # docker-compose exec admin flask mailu config-export --help + $ docker-compose exec admin flask mailu config-export --help Usage: flask mailu config-export [OPTIONS] [FILTER]... @@ -152,18 +152,18 @@ filters to export only some objects or attributes (try: ``user`` or ``domain.nam .. code-block:: bash - docker-compose exec admin flask mailu config-export -o mail-config.yml + $ docker-compose exec admin flask mailu config-export -o mail-config.yml - docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf + $ docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf config-import ------------- -The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML/JOSN source. +This command imports configuration data from an external YAML or JSON source. .. code-block:: bash - # docker-compose exec admin flask mailu config-import --help + $ docker-compose exec admin flask mailu config-import --help Usage: flask mailu config-import [OPTIONS] [FILENAME|-] @@ -211,13 +211,28 @@ mail-config.yml contains the configuration and looks like this: config-update shows the number of created/modified/deleted objects after import. To suppress all messages except error messages use ``--quiet``. -By adding the ``--verbose`` switch (one or more times) the import gets more detailed and shows exactyl what attributes changed. -In all messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to show secrets. -If you want to test what would be done when importing use ``--dry-run``. -By default config-update replaces the whole configuration. You can use ``--update`` to change the existing configuration instead. -When updating you can add new and change existing objects. -To delete an object use ``-key: value`` (To delete the domain example.com ``-name: example.com`` for example). -To reset an attribute to default use ``-key: null`` (To reset enable_imap ``-enable_imap: null`` for example). +By adding the ``--verbose`` switch (up to two times) the import gets more detailed and shows exactly what attributes changed. +In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. +If you want to test what would be done when importing without committing any changes, use ``--dry-run``. + +By default config-update replaces the whole configuration. ``--update`` allows to modify the existing configuration instead. +New elements will be added and existing elements will be modified. +It is possible to delete a single element or prune all elements from lists and associative arrays using a special notation: + ++-----------------------------+------------------+--------------------------+ +| Delete what? | notation | example | ++=============================+==================+==========================+ +| specific array object | ``- -key: id`` | ``- -name: example.com`` | ++-----------------------------+------------------+--------------------------+ +| specific list item | ``- -id`` | ``- -user1@example.com`` | ++-----------------------------+------------------+--------------------------+ +| all remaining array objects | ``- -key: null`` | ``- -email: null`` | ++-----------------------------+------------------+--------------------------+ +| all remaining list items | ``- -prune-`` | ``- -prune-`` | ++-----------------------------+------------------+--------------------------+ + +The ``-key: null`` notation can also be used to reset an attribute to its default. +To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``. This is a complete YAML template with all additional parameters that can be defined: From e4c83e162dfa37189b8eac506b07ec6f78a5b5d0 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 16 Feb 2021 17:59:43 +0100 Subject: [PATCH 061/181] fixed colorize auto detection --- core/admin/mailu/manage.py | 6 +++--- core/admin/mailu/schemas.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 756400ad..bef49faa 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -19,7 +19,7 @@ from flask.cli import FlaskGroup, with_appcontext from marshmallow.exceptions import ValidationError from . import models -from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, RenderJSON, HIDDEN +from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, canColorize, RenderJSON, HIDDEN db = models.db @@ -351,7 +351,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal verbose = 2 color_cfg = { - 'color': color or sys.stdout.isatty(), + 'color': color or (canColorize and sys.stdout.isatty()), 'lexer': 'python', 'strip': True, } @@ -598,7 +598,7 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None } schema = MailuSchema(only=only, context=context) - color_cfg = {'color': color or output.isatty()} + color_cfg = {'color': color or (canColorize and output.isatty())} if as_json: schema.opts.render_module = RenderJSON diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 7d0393f0..6a8303c5 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -27,9 +27,9 @@ try: from pygments.lexers.data import YamlLexer from pygments.formatters import get_formatter_by_name except ModuleNotFoundError: - COLOR_SUPPORTED = False + canColorize = False else: - COLOR_SUPPORTED = True + canColorize = True from . import models, dkim @@ -99,11 +99,11 @@ def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): """ add ANSI color to data """ if color is None: # autodetect colorize - color = COLOR_SUPPORTED + color = canColorize if not color: # no color wanted return data - if not COLOR_SUPPORTED: + if not canColorize: # want color, but not supported raise ValueError('Please install pygments to colorize output') From bde7a2b6c4a8a2351b461cee9be413d7683e95dd Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 19 Feb 2021 18:01:02 +0100 Subject: [PATCH 062/181] moved import logging to schema - yaml-import is now logged via schema.Logger - iremoved relative imports - not used in other mailu modules - removed develepment comments - added Mailconfig.check method to check for duplicate domain names - converted .format() to .format_map() where possible - switched to yaml multiline dump for dkim_key - converted dkim_key import from regex to string functions - automatically unhide/unexclude explicitly specified attributes on dump - use field order when loading to stabilize import - fail when using 'hash_password' without 'password' - fixed logging of dkim_key - fixed pruning and deleting of lists - modified error messages - added debug flag and two verbosity levels --- core/admin/mailu/manage.py | 280 ++--------- core/admin/mailu/models.py | 40 +- core/admin/mailu/schemas.py | 922 ++++++++++++++++++++++++------------ docs/cli.rst | 15 +- 4 files changed, 688 insertions(+), 569 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index bef49faa..f9add0f4 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -4,22 +4,16 @@ import sys import os import socket -import logging import uuid -from collections import Counter -from itertools import chain - import click -import sqlalchemy import yaml from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext -from marshmallow.exceptions import ValidationError -from . import models -from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, canColorize, RenderJSON, HIDDEN +from mailu import models +from mailu.schemas import MailuSchema, Logger, RenderJSON db = models.db @@ -326,246 +320,53 @@ def config_update(verbose=False, delete_objects=False): @mailu.command() @click.option('-v', '--verbose', count=True, help='Increase verbosity.') @click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.') +@click.option('-d', '--debug', is_flag=True, help='Enable debug output.') @click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.') @click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.') @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext -def config_import(verbose=0, secrets=False, quiet=False, color=False, update=False, dry_run=False, source=None): +def config_import(verbose=0, secrets=False, debug=False, quiet=False, color=False, + update=False, dry_run=False, source=None): """ Import configuration as YAML or JSON from stdin or file """ - # verbose - # 0 : only show number of changes - # 1 : also show detailed changes - # 2 : also show input data - # 3 : also show sql queries (also needs -s, as sql may contain secrets) - # 4 : also show tracebacks (also needs -s, as tracebacks may contain secrets) + log = Logger(want_color=color or None, can_color=sys.stdout.isatty(), secrets=secrets, debug=debug) + log.lexer = 'python' + log.strip = True + log.verbose = 0 if quiet else verbose + log.quiet = quiet - if quiet: - verbose = -1 - - if verbose > 2 and not secrets: - print('[Warning] Verbosity level capped to 2. Specify --secrets to log sql and tracebacks.') - verbose = 2 - - color_cfg = { - 'color': color or (canColorize and sys.stdout.isatty()), - 'lexer': 'python', - 'strip': True, - } - - counter = Counter() - logger = {} - - def format_errors(store, path=None): - - res = [] - if path is None: - path = [] - for key in sorted(store): - location = path + [str(key)] - value = store[key] - if isinstance(value, dict): - res.extend(format_errors(value, location)) - else: - for message in value: - res.append((".".join(location), message)) - - if path: - return res - - fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}' - res = [fmt.format(loc, msg) for loc, msg in res] - num = f'error{["s",""][len(res)==1]}' - res.insert(0, f'[ValidationError] {len(res)} {num} occurred during input validation') - - return '\n'.join(res) - - def format_changes(*message): - if counter: - changes = [] - last = None - for (action, what), count in sorted(counter.items()): - if action != last: - if last: - changes.append('/') - changes.append(f'{action}:') - last = action - changes.append(f'{what}({count})') - else: - changes = ['No changes.'] - return chain(message, changes) - - def log(action, target, message=None): - - if message is None: - try: - message = logger[target.__class__].dump(target) - except KeyError: - message = target - if not isinstance(message, str): - message = repr(message) - print(f'{action} {target.__table__}: {colorize(message, **color_cfg)}') - - def listen_insert(mapper, connection, target): # pylint: disable=unused-argument - """ callback function to track import """ - counter.update([('Created', target.__table__.name)]) - if verbose >= 1: - log('Created', target) - - def listen_update(mapper, connection, target): # pylint: disable=unused-argument - """ callback function to track import """ - - changed = {} - inspection = sqlalchemy.inspect(target) - for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: - history = getattr(inspection.attrs, attr.key).history - if history.has_changes() and history.deleted: - before = history.deleted[-1] - after = getattr(target, attr.key) - # TODO: remove special handling of "comment" after modifying model - if attr.key == 'comment' and not before and not after: - pass - # only remember changed keys - elif before != after: - if verbose >= 1: - changed[str(attr.key)] = (before, after) - else: - break - - if verbose >= 1: - # use schema with dump_context to hide secrets and sort keys - dumped = get_schema(target)(only=changed.keys(), context=diff_context).dump(target) - for key, value in dumped.items(): - before, after = changed[key] - if value == HIDDEN: - before = HIDDEN if before else before - after = HIDDEN if after else after - else: - # TODO: need to use schema to "convert" before value? - after = value - log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') - - if changed: - counter.update([('Modified', target.__table__.name)]) - - def listen_delete(mapper, connection, target): # pylint: disable=unused-argument - """ callback function to track import """ - counter.update([('Deleted', target.__table__.name)]) - if verbose >= 1: - log('Deleted', target) - - # TODO: this listener will not be necessary, if dkim keys would be stored in database - _dedupe_dkim = set() - def listen_dkim(session, flush_context): # pylint: disable=unused-argument - """ callback function to track import """ - for target in session.identity_map.values(): - # look at Domains originally loaded from db - if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: - continue - before = target._dkim_key_on_disk - after = target._dkim_key - if before != after: - if secrets: - before = before.decode('ascii', 'ignore') - after = after.decode('ascii', 'ignore') - else: - before = HIDDEN if before else '' - after = HIDDEN if after else '' - # "de-dupe" messages; this event is fired at every flush - if not (target, before, after) in _dedupe_dkim: - _dedupe_dkim.add((target, before, after)) - counter.update([('Modified', target.__table__.name)]) - if verbose >= 1: - log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') - - def track_serialize(obj, item, backref=None): - """ callback function to track import """ - # called for backref modification? - if backref is not None: - log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref)) - return - # show input data? - if not verbose >= 2: - return - # hide secrets in data - data = logger[obj.opts.model].hide(item) - if 'hash_password' in data: - data['password'] = HIDDEN - if 'fetches' in data: - for fetch in data['fetches']: - fetch['password'] = HIDDEN - log('Handling', obj.opts.model, data) - - # configure contexts - diff_context = { - 'full': True, - 'secrets': secrets, - } - log_context = { - 'secrets': secrets, - } - load_context = { + context = { 'import': True, 'update': update, 'clear': not update, - 'callback': track_serialize, + 'callback': log.track_serialize, } - # register listeners - for schema in get_schema(): - model = schema.Meta.model - logger[model] = schema(context=log_context) - sqlalchemy.event.listen(model, 'after_insert', listen_insert) - sqlalchemy.event.listen(model, 'after_update', listen_update) - sqlalchemy.event.listen(model, 'after_delete', listen_delete) - - # special listener for dkim_key changes - sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim) - - if verbose >= 3: - logging.basicConfig() - logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + schema = MailuSchema(only=MailuSchema.Meta.order, context=context) try: + # import source with models.db.session.no_autoflush: - config = MailuSchema(only=MailuSchema.Meta.order, context=load_context).loads(source) - except ValidationError as exc: - raise click.ClickException(format_errors(exc.messages)) from exc + config = schema.loads(source) + # flush session to show/count all changes + if not quiet and (dry_run or verbose): + db.session.flush() + # check for duplicate domain names + config.check() except Exception as exc: - if verbose >= 3: - raise - # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) - raise click.ClickException( - f'[{exc.__class__.__name__}] ' - f'{" ".join(str(exc).split())}' - ) from exc - - # flush session to show/count all changes - if dry_run or verbose >= 1: - db.session.flush() - - # check for duplicate domain names - dup = set() - for fqdn in chain( - db.session.query(models.Domain.name), - db.session.query(models.Alternative.name), - db.session.query(models.Relay.name) - ): - if fqdn in dup: - raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}') - dup.add(fqdn) + if msg := log.format_exception(exc): + raise click.ClickException(msg) from exc + raise # don't commit when running dry if dry_run: - if not quiet: - print(*format_changes('Dry run. Not commiting changes.')) + log.changes('Dry run. Not committing changes.') db.session.rollback() else: - if not quiet: - print(*format_changes('Committing changes.')) + log.changes('Committing changes.') db.session.commit() @@ -573,8 +374,8 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal @click.option('-f', '--full', is_flag=True, help='Include attributes with default value.') @click.option('-s', '--secrets', is_flag=True, help='Include secret attributes (dkim-key, passwords).') -@click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-d', '--dns', is_flag=True, help='Include dns records.') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), help='Save configuration to file.') @click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.') @@ -584,32 +385,25 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None """ Export configuration as YAML or JSON to stdout or file """ - if only: - for spec in only: - if spec.split('.', 1)[0] not in MailuSchema.Meta.order: - raise click.ClickException(f'[ValidationError] Unknown section: {spec}') - else: - only = MailuSchema.Meta.order + only = only or MailuSchema.Meta.order context = { 'full': full, 'secrets': secrets, 'dns': dns, } - - schema = MailuSchema(only=only, context=context) - color_cfg = {'color': color or (canColorize and output.isatty())} - - if as_json: - schema.opts.render_module = RenderJSON - color_cfg['lexer'] = 'json' - color_cfg['strip'] = True + log = Logger(want_color=color or None, can_color=output.isatty()) try: - print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output) - except ValueError as exc: - if spec := get_fieldspec(exc): - raise click.ClickException(f'[ValidationError] Invalid filter: {spec}') from exc + schema = MailuSchema(only=only, context=context) + if as_json: + schema.opts.render_module = RenderJSON + log.lexer = 'json' + log.strip = True + print(log.colorize(schema.dumps(models.MailuConfig())), file=output) + except Exception as exc: + if msg := log.format_exception(exc): + raise click.ClickException(msg) from exc raise diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 4c119984..1b3c787a 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -23,7 +23,7 @@ from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.inspection import inspect from werkzeug.utils import cached_property -from . import dkim +from mailu import dkim db = flask_sqlalchemy.SQLAlchemy() @@ -33,7 +33,6 @@ class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: use db.String(255)? impl = db.String(80) def process_bind_param(self, value, dialect): @@ -50,7 +49,6 @@ class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: use db.String(254)? impl = db.String(255) def process_bind_param(self, value, dialect): @@ -127,11 +125,7 @@ class Base(db.Model): if pkey == 'email': # ugly hack for email declared attr. _email is not always up2date return str(f'{self.localpart}@{self.domain_name}') - elif pkey in {'name', 'email'}: - return str(getattr(self, pkey, None)) - else: - return self.__repr__() - return str(getattr(self, self.__table__.primary_key.columns.values()[0].name)) + return str(getattr(self, pkey)) def __repr__(self): return f'<{self.__class__.__name__} {str(self)!r}>' @@ -145,12 +139,15 @@ class Base(db.Model): else: return NotImplemented + # we need hashable instances here for sqlalchemy to update collections + # in collections.bulk_replace, but auto-incrementing don't always have + # a valid primary key, in this case we use the object's id + __hashed = None def __hash__(self): - primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) - if primary is None: - return NotImplemented - else: - return hash(primary) + if self.__hashed is None: + primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) + self.__hashed = id(self) if primary is None else hash(primary) + return self.__hashed # Many-to-many association table for domain managers @@ -314,7 +311,6 @@ class Relay(Base): __tablename__ = 'relay' name = db.Column(IdnaDomain, primary_key=True, nullable=False) - # TODO: use db.String(266)? transport(8):(1)[nexthop(255)](2) smtp = db.Column(db.String(80), nullable=True) @@ -322,7 +318,6 @@ class Email(object): """ Abstraction for an email address (localpart and domain). """ - # TODO: use db.String(64)? localpart = db.Column(db.String(80), nullable=False) @declarative.declared_attr @@ -342,7 +337,7 @@ class Email(object): key = f'{cls.__tablename__}_email' if key in ctx.current_parameters: return ctx.current_parameters[key] - return '{localpart}@{domain_name}'.format(**ctx.current_parameters) + return '{localpart}@{domain_name}'.format_map(ctx.current_parameters) return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater) @@ -632,7 +627,6 @@ class Token(Base): user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - # TODO: use db.String(32)? ip = db.Column(db.String(255)) def check_password(self, password): @@ -865,6 +859,18 @@ class MailuConfig: if models is None or model in models: db.session.query(model).delete() + def check(self): + """ check for duplicate domain names """ + dup = set() + for fqdn in chain( + db.session.query(Domain.name), + db.session.query(Alternative.name), + db.session.query(Relay.name) + ): + if fqdn in dup: + raise ValueError(f'Duplicate domain name: {fqdn}') + dup.add(fqdn) + domain = MailuCollection(Domain) user = MailuCollection(User) alias = MailuCollection(Alias) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 6a8303c5..4c5042ea 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -2,10 +2,10 @@ """ from copy import deepcopy -from textwrap import wrap +from collections import Counter -import re import json +import logging import yaml import sqlalchemy @@ -27,24 +27,309 @@ try: from pygments.lexers.data import YamlLexer from pygments.formatters import get_formatter_by_name except ModuleNotFoundError: - canColorize = False + COLOR_SUPPORTED = False else: - canColorize = True + COLOR_SUPPORTED = True -from . import models, dkim +from mailu import models, dkim ma = Marshmallow() -# TODO: how and where to mark keys as "required" while deserializing in api? -# - when modifying, nothing is required (only the primary key, but this key is in the uri) -# - the primary key from post data must not differ from the key in the uri -# - when creating all fields without default or auto-increment are required -# TODO: validate everything! + +### import logging and schema colorization ### + +_model2schema = {} + +def get_schema(cls=None): + """ return schema class for model """ + if cls is None: + return _model2schema.values() + return _model2schema.get(cls) + +def mapped(cls): + """ register schema in model2schema map """ + _model2schema[cls.Meta.model] = cls + return cls + +class MyYamlLexer(YamlLexer): + """ colorize yaml constants and integers """ + def get_tokens(self, text, unfiltered=False): + for typ, value in super().get_tokens(text, unfiltered): + if typ is Token.Literal.Scalar.Plain: + if value in {'true', 'false', 'null'}: + typ = Token.Keyword.Constant + elif value == HIDDEN: + typ = Token.Error + else: + try: + int(value, 10) + except ValueError: + try: + float(value) + except ValueError: + pass + else: + typ = Token.Literal.Number.Float + else: + typ = Token.Literal.Number.Integer + yield typ, value + +class Logger: + + def __init__(self, want_color=None, can_color=False, debug=False, secrets=False): + + self.lexer = 'yaml' + self.formatter = 'terminal' + self.strip = False + self.verbose = 0 + self.quiet = False + self.secrets = secrets + self.debug = debug + self.print = print + + if want_color and not COLOR_SUPPORTED: + raise ValueError('Please install pygments to colorize output') + + self.color = want_color or (can_color and COLOR_SUPPORTED) + + self._counter = Counter() + self._schemas = {} + + # log contexts + self._diff_context = { + 'full': True, + 'secrets': secrets, + } + log_context = { + 'secrets': secrets, + } + + # register listeners + for schema in get_schema(): + model = schema.Meta.model + self._schemas[model] = schema(context=log_context) + sqlalchemy.event.listen(model, 'after_insert', self._listen_insert) + sqlalchemy.event.listen(model, 'after_update', self._listen_update) + sqlalchemy.event.listen(model, 'after_delete', self._listen_delete) + + # special listener for dkim_key changes + # TODO: _listen_dkim can be removed when dkim keys are stored in database + self._dedupe_dkim = set() + sqlalchemy.event.listen(models.db.session, 'after_flush', self._listen_dkim) + + # register debug logger for sqlalchemy + if self.debug: + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + def _log(self, action, target, message=None): + if message is None: + try: + message = self._schemas[target.__class__].dump(target) + except KeyError: + message = target + if not isinstance(message, str): + message = repr(message) + self.print(f'{action} {target.__table__}: {self.colorize(message)}') + + def _listen_insert(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + self._counter.update([('Created', target.__table__.name)]) + if self.verbose: + self._log('Created', target) + + def _listen_update(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + + changes = {} + inspection = sqlalchemy.inspect(target) + for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: + history = getattr(inspection.attrs, attr.key).history + if history.has_changes() and history.deleted: + before = history.deleted[-1] + after = getattr(target, attr.key) + # TODO: this can be removed when comment is not nullable in model + if attr.key == 'comment' and not before and not after: + pass + # only remember changed keys + elif before != after: + if self.verbose: + changes[str(attr.key)] = (before, after) + else: + break + + if self.verbose: + # use schema to log changed attributes + schema = get_schema(target.__class__) + only = set(changes.keys()) & set(schema().fields.keys()) + if only: + for key, value in schema( + only=only, + context=self._diff_context + ).dump(target).items(): + before, after = changes[key] + if value == HIDDEN: + before = HIDDEN if before else before + after = HIDDEN if after else after + else: + # also hide this + after = value + self._log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') + + if changes: + self._counter.update([('Modified', target.__table__.name)]) + + def _listen_delete(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + self._counter.update([('Deleted', target.__table__.name)]) + if self.verbose: + self._log('Deleted', target) + + # TODO: _listen_dkim can be removed when dkim keys are stored in database + def _listen_dkim(self, session, flush_context): # pylint: disable=unused-argument + """ callback method to track import """ + for target in session.identity_map.values(): + # look at Domains originally loaded from db + if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: + continue + before = target._dkim_key_on_disk + after = target._dkim_key + if before != after: + # "de-dupe" messages; this event is fired at every flush + if not (target, before, after) in self._dedupe_dkim: + self._dedupe_dkim.add((target, before, after)) + self._counter.update([('Modified', target.__table__.name)]) + if self.verbose: + if self.secrets: + before = before.decode('ascii', 'ignore') + after = after.decode('ascii', 'ignore') + else: + before = HIDDEN if before else '' + after = HIDDEN if after else '' + self._log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') + + def track_serialize(self, obj, item, backref=None): + """ callback method to track import """ + # called for backref modification? + if backref is not None: + self._log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format_map(backref)) + return + # show input data? + if self.verbose < 2: + return + # hide secrets in data + if not self.secrets: + item = self._schemas[obj.opts.model].hide(item) + if 'hash_password' in item: + item['password'] = HIDDEN + if 'fetches' in item: + for fetch in item['fetches']: + fetch['password'] = HIDDEN + self._log('Handling', obj.opts.model, item) + + def changes(self, *messages, **kwargs): + """ show changes gathered in counter """ + if self.quiet: + return + if self._counter: + changes = [] + last = None + for (action, what), count in sorted(self._counter.items()): + if action != last: + if last: + changes.append('/') + changes.append(f'{action}:') + last = action + changes.append(f'{what}({count})') + else: + changes = ['No changes.'] + self.print(*messages, *changes, **kwargs) + + def _format_errors(self, store, path=None): + + res = [] + if path is None: + path = [] + for key in sorted(store): + location = path + [str(key)] + value = store[key] + if isinstance(value, dict): + res.extend(self._format_errors(value, location)) + else: + for message in value: + res.append((".".join(location), message)) + + if path: + return res + + maxlen = max([len(loc) for loc, msg in res]) + res = [f' - {loc.ljust(maxlen)} : {msg}' for loc, msg in res] + errors = f'{len(res)} error{["s",""][len(res)==1]}' + res.insert(0, f'[ValidationError] {errors} occurred during input validation') + + return '\n'.join(res) + + def _is_validation_error(self, exc): + """ walk traceback to extract invalid field from marshmallow """ + path = [] + trace = exc.__traceback__ + while trace: + if trace.tb_frame.f_code.co_name == '_serialize': + if 'attr' in trace.tb_frame.f_locals: + path.append(trace.tb_frame.f_locals['attr']) + elif trace.tb_frame.f_code.co_name == '_init_fields': + spec = ', '.join(['.'.join(path + [key]) for key in trace.tb_frame.f_locals['invalid_fields']]) + return f'Invalid filter: {spec}' + trace = trace.tb_next + return None + + def format_exception(self, exc): + """ format ValidationErrors and other exceptions when not debugging """ + if isinstance(exc, ValidationError): + return self._format_errors(exc.messages) + if isinstance(exc, ValueError): + if msg := self._is_validation_error(exc): + return msg + if self.debug: + return None + msg = ' '.join(str(exc).split()) + return f'[{exc.__class__.__name__}] {msg}' + + colorscheme = { + Token: ('', ''), + Token.Name.Tag: ('cyan', 'cyan'), + Token.Literal.Scalar: ('green', 'green'), + Token.Literal.String: ('green', 'green'), + Token.Name.Constant: ('green', 'green'), # multiline strings + Token.Keyword.Constant: ('magenta', 'magenta'), + Token.Literal.Number: ('magenta', 'magenta'), + Token.Error: ('red', 'red'), + Token.Name: ('red', 'red'), + Token.Operator: ('red', 'red'), + } + + def colorize(self, data, lexer=None, formatter=None, color=None, strip=None): + """ add ANSI color to data """ + + if color is False or not self.color: + return data + + lexer = lexer or self.lexer + lexer = MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer) + formatter = get_formatter_by_name(formatter or self.formatter, colorscheme=self.colorscheme) + if strip is None: + strip = self.strip + + res = highlight(data, lexer, formatter) + if strip: + return res.rstrip('\n') + return res -### class for hidden values ### +### marshmallow render modules ### +# hidden attributes class _Hidden: def __bool__(self): return False @@ -58,107 +343,24 @@ class _Hidden: return '' __str__ = __repr__ -HIDDEN = _Hidden() - - -### map model to schema ### - -_model2schema = {} - -def get_schema(model=None): - """ return schema class for model or instance of model """ - if model is None: - return _model2schema.values() - else: - return _model2schema.get(model) or _model2schema.get(model.__class__) - -def mapped(cls): - """ register schema in model2schema map """ - _model2schema[cls.Meta.model] = cls - return cls - - -### helper functions ### - -def get_fieldspec(exc): - """ walk traceback to extract spec of invalid field from marshmallow """ - path = [] - tbck = exc.__traceback__ - while tbck: - if tbck.tb_frame.f_code.co_name == '_serialize': - if 'attr' in tbck.tb_frame.f_locals: - path.append(tbck.tb_frame.f_locals['attr']) - elif tbck.tb_frame.f_code.co_name == '_init_fields': - path = '.'.join(path) - spec = ', '.join([f'{path}.{key}' for key in tbck.tb_frame.f_locals['invalid_fields']]) - return spec - tbck = tbck.tb_next - return None - -def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): - """ add ANSI color to data """ - if color is None: - # autodetect colorize - color = canColorize - if not color: - # no color wanted - return data - if not canColorize: - # want color, but not supported - raise ValueError('Please install pygments to colorize output') - - scheme = { - Token: ('', ''), - Token.Name.Tag: ('cyan', 'brightcyan'), - Token.Literal.Scalar: ('green', 'green'), - Token.Literal.String: ('green', 'green'), - Token.Keyword.Constant: ('magenta', 'brightmagenta'), - Token.Literal.Number: ('magenta', 'brightmagenta'), - Token.Error: ('red', 'brightred'), - Token.Name: ('red', 'brightred'), - Token.Operator: ('red', 'brightred'), - } - - class MyYamlLexer(YamlLexer): - """ colorize yaml constants and integers """ - def get_tokens(self, text, unfiltered=False): - for typ, value in super().get_tokens(text, unfiltered): - if typ is Token.Literal.Scalar.Plain: - if value in {'true', 'false', 'null'}: - typ = Token.Keyword.Constant - elif value == HIDDEN: - typ = Token.Error - else: - try: - int(value, 10) - except ValueError: - try: - float(value) - except ValueError: - pass - else: - typ = Token.Literal.Number.Float - else: - typ = Token.Literal.Number.Integer - yield typ, value - - res = highlight( - data, - MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer), - get_formatter_by_name(formatter, colorscheme=scheme) - ) - - return res.rstrip('\n') if strip else res - - -### render modules ### - -# allow yaml to represent hidden attributes yaml.add_representer( _Hidden, - lambda cls, data: cls.represent_data(str(data)) + lambda dumper, data: dumper.represent_data(str(data)) ) +HIDDEN = _Hidden() + +# multiline attributes +class _Multiline(str): + pass + +yaml.add_representer( + _Multiline, + lambda dumper, data: dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') + +) + +# yaml render module class RenderYAML: """ Marshmallow YAML Render Module """ @@ -178,7 +380,7 @@ class RenderYAML: @staticmethod def _augment(kwargs, defaults): - """ add default kv's to kwargs if missing + """ add defaults to kwargs if missing """ for key, value in defaults.items(): if key not in kwargs: @@ -205,6 +407,7 @@ class RenderYAML: cls._augment(kwargs, cls._dump_defaults) return yaml.dump(*args, **kwargs) +# json encoder class JSONEncoder(json.JSONEncoder): """ JSONEncoder supporting serialization of HIDDEN """ def default(self, o): @@ -213,13 +416,14 @@ class JSONEncoder(json.JSONEncoder): return str(o) return json.JSONEncoder.default(self, o) +# json render module class RenderJSON: """ Marshmallow JSON Render Module """ @staticmethod def _augment(kwargs, defaults): - """ add default kv's to kwargs if missing + """ add defaults to kwargs if missing """ for key, value in defaults.items(): if key not in kwargs: @@ -245,7 +449,7 @@ class RenderJSON: return json.dumps(*args, **kwargs) -### custom fields ### +### schema: custom fields ### class LazyStringField(fields.String): """ Field that serializes a "false" value to the empty string @@ -261,6 +465,11 @@ class CommaSeparatedListField(fields.Raw): a list of strings """ + default_error_messages = { + "invalid": "Not a valid string or list.", + "invalid_utf8": "Not a valid utf-8 string or list.", + } + def _deserialize(self, value, attr, data, **kwargs): """ deserialize comma separated string to list of strings """ @@ -269,16 +478,31 @@ class CommaSeparatedListField(fields.Raw): if not value: return [] - # split string - if isinstance(value, str): - return list([item.strip() for item in value.split(',') if item.strip()]) + # handle list + if isinstance(value, list): + try: + value = [ensure_text_type(item) for item in value] + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + + # handle text else: - return value + if not isinstance(value, (str, bytes)): + raise self.make_error("invalid") + try: + value = ensure_text_type(value) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + else: + value = filter(None, [item.strip() for item in value.split(',')]) + + return list(value) class DkimKeyField(fields.String): - """ Serialize a dkim key to a list of strings (lines) and - Deserialize a string or list of strings to a valid dkim key + """ Serialize a dkim key to a multiline string and + deserialize a dkim key data as string or list of strings + to a valid dkim key """ default_error_messages = { @@ -286,21 +510,26 @@ class DkimKeyField(fields.String): "invalid_utf8": "Not a valid utf-8 string or list.", } - _clean_re = re.compile( - r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)', - flags=re.UNICODE - ) - def _serialize(self, value, attr, obj, **kwargs): - """ serialize dkim key to a list of strings (lines) + """ serialize dkim key as multiline string """ # map empty string and None to None if not value: - return None + return '' - # return list of key lines without header/footer - return value.decode('utf-8').strip().split('\n')[1:-1] + # return multiline string + return _Multiline(value.decode('utf-8')) + + def _wrap_key(self, begin, data, end): + """ generator to wrap key into RFC 7468 format """ + yield begin + pos = 0 + while pos < len(data): + yield data[pos:pos+64] + pos += 64 + yield end + yield '' def _deserialize(self, value, attr, data, **kwargs): """ deserialize a string or list of strings to dkim key data @@ -310,7 +539,7 @@ class DkimKeyField(fields.String): # convert list to str if isinstance(value, list): try: - value = ''.join([ensure_text_type(item) for item in value]) + value = ''.join([ensure_text_type(item) for item in value]).strip() except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc @@ -319,34 +548,53 @@ class DkimKeyField(fields.String): if not isinstance(value, (str, bytes)): raise self.make_error("invalid") try: - value = ensure_text_type(value) + value = ensure_text_type(value).strip() except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc - # clean value (remove whitespace and header/footer) - value = self._clean_re.sub('', value.strip()) + # generate new key? + if value.lower() == '-generate-': + return dkim.gen_key() - # map empty string/list to None + # no key? if not value: return None - # handle special value 'generate' - elif value == 'generate': - return dkim.gen_key() + # remember part of value for ValidationError + bad_key = value - # remember some keydata for error message - keydata = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value + # strip header and footer, clean whitespace and wrap to 64 characters + try: + if value.startswith('-----BEGIN '): + end = value.index('-----', 11) + 5 + header = value[:end] + value = value[end:] + else: + header = '-----BEGIN PRIVATE KEY-----' - # wrap value into valid pem layout and check validity - value = ( - '-----BEGIN PRIVATE KEY-----\n' + - '\n'.join(wrap(value, 64)) + - '\n-----END PRIVATE KEY-----\n' - ).encode('ascii') + if (pos := value.find('-----END ')) >= 0: + end = value.index('-----', pos+9) + 5 + footer = value[pos:end] + value = value[:pos] + else: + footer = '-----END PRIVATE KEY-----' + except ValueError: + raise ValidationError(f'invalid dkim key {bad_key!r}') from exc + + # remove whitespace from key data + value = ''.join(value.split()) + + # remember part of value for ValidationError + bad_key = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value + + # wrap key according to RFC 7468 + value = ('\n'.join(self._wrap_key(header, value, footer))).encode('ascii') + + # check key validity try: crypto.load_privatekey(crypto.FILETYPE_PEM, value) except crypto.Error as exc: - raise ValidationError(f'invalid dkim key {keydata!r}') from exc + raise ValidationError(f'invalid dkim key {bad_key!r}') from exc else: return value @@ -398,6 +646,27 @@ class PasswordField(fields.Str): ### base schema ### +class Storage: + """ Storage class to save information in context + """ + + context = {} + + def _bind(self, key, bind): + if bind is True: + return (self.__class__, key) + if isinstance(bind, str): + return (get_schema(self.recall(bind).__class__), key) + return (bind, key) + + def store(self, key, value, bind=None): + """ store value under key """ + self.context.setdefault('_track', {})[self._bind(key, bind)]= value + + def recall(self, key, bind=None): + """ recall value from key """ + return self.context['_track'][self._bind(key, bind)] + class BaseOpts(SQLAlchemyAutoSchemaOpts): """ Option class with sqla session """ @@ -408,7 +677,7 @@ class BaseOpts(SQLAlchemyAutoSchemaOpts): meta.sibling = False super(BaseOpts, self).__init__(meta, ordered=ordered) -class BaseSchema(ma.SQLAlchemyAutoSchema): +class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): """ Marshmallow base schema with custom exclude logic and option to hide sqla defaults """ @@ -425,6 +694,9 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): def __init__(self, *args, **kwargs): + # prepare only to auto-include explicitly specified attributes + only = set(kwargs.get('only') or []) + # get context context = kwargs.get('context', {}) flags = {key for key, value in context.items() if value is True} @@ -433,13 +705,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): exclude = set(kwargs.get('exclude', [])) # always exclude - exclude.update({'created_at', 'updated_at'}) + exclude.update({'created_at', 'updated_at'} - only) # add include_by_context if context is not None: for need, what in getattr(self.Meta, 'include_by_context', {}).items(): if not flags & set(need): - exclude |= set(what) + exclude |= what - only # update excludes kwargs['exclude'] = exclude @@ -448,12 +720,15 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): super().__init__(*args, **kwargs) # exclude_by_value - self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {}) + self._exclude_by_value = { + key: values for key, values in getattr(self.Meta, 'exclude_by_value', {}).items() + if key not in only + } # exclude default values if not context.get('full'): for column in self.opts.model.__table__.columns: - if column.name not in exclude: + if column.name not in exclude and column.name not in only: self._exclude_by_value.setdefault(column.name, []).append( None if column.default is None else column.default.arg ) @@ -463,7 +738,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if context is not None: for need, what in getattr(self.Meta, 'hide_by_context', {}).items(): if not flags & set(need): - self._hide_by_context |= set(what) + self._hide_by_context |= what - only # remember primary keys self._primary = str(self.opts.model.__table__.primary_key.columns.values()[0].name) @@ -479,20 +754,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): order.remove(self._primary) order.insert(0, self._primary) - # order dump_fields - for field in order: - if field in self.dump_fields: - self.dump_fields[field] = self.dump_fields.pop(field) + # order fieldlists + for fieldlist in (self.fields, self.load_fields, self.dump_fields): + for field in order: + if field in fieldlist: + fieldlist[field] = fieldlist.pop(field) - # move pre_load hook "_track_import" to the front - hooks = self._hooks[('pre_load', False)] - hooks.remove('_track_import') - hooks.insert(0, '_track_import') - # move pre_load hook "_add_instance" to the end - hooks.remove('_add_required') - hooks.append('_add_required') - - # move post_load hook "_add_instance" to the end + # move post_load hook "_add_instance" to the end (after load_instance mixin) hooks = self._hooks[('post_load', False)] hooks.remove('_add_instance') hooks.append('_add_instance') @@ -506,8 +774,8 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): } def _call_and_store(self, *args, **kwargs): - """ track curent parent field for pruning """ - self.context['parent_field'] = kwargs['field_name'] + """ track current parent field for pruning """ + self.store('field', kwargs['field_name'], True) return super()._call_and_store(*args, **kwargs) # this is only needed to work around the declared attr "email" primary key in model @@ -518,11 +786,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if keys := getattr(self.Meta, 'primary_keys', None): filters = {key: data.get(key) for key in keys} if None not in filters.values(): - return self.session.query(self.opts.model).filter_by(**filters).first() - return super().get_instance(data) + res= self.session.query(self.opts.model).filter_by(**filters).first() + return res + res= super().get_instance(data) + return res @pre_load(pass_many=True) - def _patch_input(self, items, many, **kwargs): # pylint: disable=unused-argument + def _patch_many(self, items, many, **kwargs): # pylint: disable=unused-argument """ - flush sqla session before serializing a section when requested (make sure all objects that could be referred to later are created) - when in update mode: patch input data before deserialization @@ -540,12 +810,19 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # patch "delete", "prune" and "default" want_prune = [] - def patch(count, data, prune): + def patch(count, data): # don't allow __delete__ coming from input if '__delete__' in data: raise ValidationError('Unknown field.', f'{count}.__delete__') + # fail when hash_password is specified without password + if 'hash_password' in data and not 'password' in data: + raise ValidationError( + 'Nothing to hash. Field "password" is missing.', + field_name = f'{count}.hash_password', + ) + # handle "prune list" and "delete item" (-pkey: none and -pkey: id) for key in data: if key.startswith('-'): @@ -553,10 +830,10 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # delete or prune if data[key] is None: # prune - prune.append(True) + want_prune.append(True) return None # mark item for deletion - return {key[1:]: data[key], '__delete__': True} + return {key[1:]: data[key], '__delete__': count} # handle "set to default value" (-key: none) def set_default(key, value): @@ -567,7 +844,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): return (key, None) if value is not None: raise ValidationError( - 'When resetting to default value must be null.', + 'Value must be "null" when resetting to default.', f'{count}.{key}' ) value = self.opts.model.__table__.columns[key].default @@ -583,10 +860,128 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # convert items to "delete" and filter "prune" item items = [ item for item in [ - patch(count, item, want_prune) for count, item in enumerate(items) + patch(count, item) for count, item in enumerate(items) ] if item ] + # remember if prune was requested for _prune_items@post_load + self.store('prune', bool(want_prune), True) + + # remember original items to stabilize password-changes in _add_instance@post_load + self.store('original', items, True) + + return items + + @pre_load + def _patch_item(self, data, many, **kwargs): # pylint: disable=unused-argument + """ - call callback function to track import + - stabilize import of items with auto-increment primary key + - delete items + - delete/prune list attributes + - add missing required attributes + """ + + # callback + if callback := self.context.get('callback'): + callback(self, data) + + # stop early when not updating + if not self.opts.load_instance or not self.context.get('update'): + return data + + # stabilize import of auto-increment primary keys (not required), + # by matching import data to existing items and setting primary key + if not self._primary in data: + for item in getattr(self.recall('parent'), self.recall('field', 'parent')): + existing = self.dump(item, many=False) + this = existing.pop(self._primary) + if data == existing: + instance = item + data[self._primary] = this + break + + # try to load instance + instance = self.instance or self.get_instance(data) + if instance is None: + + if '__delete__' in data: + # deletion of non-existent item requested + raise ValidationError( + f'Item to delete not found: {data[self._primary]!r}.', + field_name = f'{data["__delete__"]}.{self._primary}', + ) + + else: + + if self.context.get('update'): + # remember instance as parent for pruning siblings + if not self.Meta.sibling: + self.store('parent', instance) + # delete instance from session when marked + if '__delete__' in data: + self.opts.sqla_session.delete(instance) + # delete item from lists or prune lists + # currently: domain.alternatives, user.forward_destination, + # user.manager_of, aliases.destination + for key, value in data.items(): + if not isinstance(self.fields.get(key), ( + RelatedList, CommaSeparatedListField, fields.Raw) + ) or not isinstance(value, list): + continue + # deduplicate new value + new_value = set(value) + # handle list pruning + if '-prune-' in value: + value.remove('-prune-') + new_value.remove('-prune-') + else: + for old in getattr(instance, key): + # using str() is okay for now (see above) + new_value.add(str(old)) + # handle item deletion + for item in value: + if item.startswith('-'): + new_value.remove(item) + try: + new_value.remove(item[1:]) + except KeyError as exc: + raise ValidationError( + f'Item to delete not found: {item[1:]!r}.', + field_name=f'?.{key}', + ) from exc + # sort list of new values + data[key] = sorted(new_value) + # log backref modification not catched by modify hook + if isinstance(self.fields[key], RelatedList): + if callback := self.context.get('callback'): + before = {str(v) for v in getattr(instance, key)} + after = set(data[key]) + if before != after: + callback(self, instance, { + 'key': key, + 'target': str(instance), + 'before': before, + 'after': after, + }) + + # add attributes required for validation from db + for attr_name, field_obj in self.load_fields.items(): + if field_obj.required and attr_name not in data: + data[attr_name] = getattr(instance, attr_name) + + return data + + @post_load(pass_many=True) + def _prune_items(self, items, many, **kwargs): # pylint: disable=unused-argument + """ handle list pruning """ + + # stop early when not updating + if not self.context.get('update'): + return items + + # get prune flag from _patch_many@pre_load + want_prune = self.recall('prune', True) + # prune: determine if existing items in db need to be added or marked for deletion add_items = False del_items = False @@ -603,144 +998,60 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if add_items or del_items: existing = {item[self._primary] for item in items if self._primary in item} - for item in getattr(self.context['parent'], self.context['parent_field']): + for item in getattr(self.recall('parent'), self.recall('field', 'parent')): key = getattr(item, self._primary) if key not in existing: if add_items: items.append({self._primary: key}) else: - items.append({self._primary: key, '__delete__': True}) + items.append({self._primary: key, '__delete__': '?'}) return items - @pre_load - def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument - """ call callback function to track import - """ - # callback - if callback := self.context.get('callback'): - callback(self, data) - - return data - - @pre_load - def _add_required(self, data, many, **kwargs): # pylint: disable=unused-argument - """ when updating: - allow modification of existing items having required attributes - by loading existing value from db + @post_load + def _add_instance(self, item, many, **kwargs): # pylint: disable=unused-argument + """ - undo password change in existing instances when plain password did not change + - add new instances to sqla session """ - if not self.opts.load_instance or not self.context.get('update'): - return data - - # stabilize import of auto-increment primary keys (not required), - # by matching import data to existing items and setting primary key - if not self._primary in data: - for item in getattr(self.context['parent'], self.context['parent_field']): - existing = self.dump(item, many=False) - this = existing.pop(self._primary) - if data == existing: - instance = item - data[self._primary] = this - break - - # try to load instance - instance = self.instance or self.get_instance(data) - if instance is None: - - if '__delete__' in data: - # deletion of non-existent item requested - raise ValidationError( - f'item to delete not found: {data[self._primary]!r}', - field_name=f'?.{self._primary}', - ) - - else: - - if self.context.get('update'): - # remember instance as parent for pruning siblings - if not self.Meta.sibling: - self.context['parent'] = instance - # delete instance when marked - if '__delete__' in data: - self.opts.sqla_session.delete(instance) - # delete item from lists or prune lists - # currently: domain.alternatives, user.forward_destination, - # user.manager_of, aliases.destination - for key, value in data.items(): - if not isinstance(self.fields[key], fields.Nested) and isinstance(value, list): - new_value = set(value) - # handle list pruning - if '-prune-' in value: - value.remove('-prune-') - new_value.remove('-prune-') - else: - for old in getattr(instance, key): - # using str() is okay for now (see above) - new_value.add(str(old)) - # handle item deletion - for item in value: - if item.startswith('-'): - new_value.remove(item) - try: - new_value.remove(item[1:]) - except KeyError as exc: - raise ValidationError( - f'item to delete not found: {item[1:]!r}', - field_name=f'?.{key}', - ) from exc - # deduplicate and sort list - data[key] = sorted(new_value) - # log backref modification not catched by hook - if isinstance(self.fields[key], RelatedList): - if callback := self.context.get('callback'): - callback(self, instance, { - 'key': key, - 'target': str(instance), - 'before': [str(v) for v in getattr(instance, key)], - 'after': data[key], - }) - - - - # add attributes required for validation from db - # TODO: this will cause validation errors if value from database does not validate - # but there should not be an invalid value in the database - for attr_name, field_obj in self.load_fields.items(): - if field_obj.required and attr_name not in data: - data[attr_name] = getattr(instance, attr_name) - - return data - - @post_load(pass_original=True) - def _add_instance(self, item, original, many, **kwargs): # pylint: disable=unused-argument - """ add new instances to sqla session """ - - if item in self.opts.sqla_session: - # item was modified - if 'hash_password' in original: - # stabilize import of passwords to be hashed, - # by not re-hashing an unchanged password - if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): - if attr.history.has_changes() and attr.history.deleted: - try: - # reset password hash, if password was not changed - inst = type(item)(password=attr.history.deleted[-1]) - if inst.check_password(original['password']): - item.password = inst.password - except ValueError: - # hash in db is invalid - pass - else: - del inst - else: - # new item + if not item in self.opts.sqla_session: self.opts.sqla_session.add(item) + return item + + # stop early if item has no password attribute + if not hasattr(item, 'password'): + return item + + # did we hash a new plaintext password? + original = None + pkey = getattr(item, self._primary) + for data in self.recall('original', True): + if 'hash_password' in data and data.get(self._primary) == pkey: + original = data['password'] + break + if original is None: + # password was hashed by us + return item + + # reset hash if plain password matches hash from db + if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): + if attr.history.has_changes() and attr.history.deleted: + try: + # reset password hash + inst = type(item)(password=attr.history.deleted[-1]) + if inst.check_password(original): + item.password = inst.password + except ValueError: + # hash in db is invalid + pass + else: + del inst + return item @post_dump def _hide_values(self, data, many, **kwargs): # pylint: disable=unused-argument - """ hide secrets and order output """ + """ hide secrets """ # stop early when not excluding/hiding if not self._exclude_by_value and not self._hide_by_context: @@ -757,7 +1068,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # this field is used to mark items for deletion mark_delete = fields.Boolean(data_key='__delete__', load_only=True) - # TODO: remove LazyStringField (when model was changed - IMHO comment should not be nullable) + # TODO: this can be removed when comment is not nullable in model comment = LazyStringField() @@ -892,27 +1203,28 @@ class RelaySchema(BaseSchema): load_instance = True -class MailuSchema(Schema): +@mapped +class MailuSchema(Schema, Storage): """ Marshmallow schema for complete Mailu config """ class Meta: """ Schema config """ + model = models.MailuConfig render_module = RenderYAML order = ['domain', 'user', 'alias', 'relay'] # 'config' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # order dump_fields - for field in self.Meta.order: - if field in self.dump_fields: - self.dump_fields[field] = self.dump_fields.pop(field) + # order fieldlists + for fieldlist in (self.fields, self.load_fields, self.dump_fields): + for field in self.Meta.order: + if field in fieldlist: + fieldlist[field] = fieldlist.pop(field) def _call_and_store(self, *args, **kwargs): """ track current parent and field for pruning """ - self.context.update({ - 'parent': self.context.get('config'), - 'parent_field': kwargs['field_name'], - }) + self.store('field', kwargs['field_name'], True) + self.store('parent', self.context.get('config')) return super()._call_and_store(*args, **kwargs) @pre_load diff --git a/docs/cli.rst b/docs/cli.rst index 6d48c576..891db152 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -138,8 +138,8 @@ The purpose of this command is to export the complete configuration in YAML or J Options: -f, --full Include attributes with default value. -s, --secrets Include secret attributes (dkim-key, passwords). - -c, --color Force colorized output. -d, --dns Include dns records. + -c, --color Force colorized output. -o, --output-file FILENAME Save configuration to file. -j, --json Export configuration in json format. -?, -h, --help Show this message and exit. @@ -147,14 +147,18 @@ The purpose of this command is to export the complete configuration in YAML or J Only non-default attributes are exported. If you want to export all attributes use ``--full``. If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option. To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. + By default all configuration objects are exported (domain, user, alias, relay). You can specify filters to export only some objects or attributes (try: ``user`` or ``domain.name``). +Attributes explicitly specified in filters are automatically exported: there is no need to add ``--secrets`` or ``--full``. .. code-block:: bash - $ docker-compose exec admin flask mailu config-export -o mail-config.yml + $ docker-compose exec admin flask mailu config-export --output mail-config.yml - $ docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf + $ docker-compose exec admin flask mailu config-export domain.dns_mx domain.dns_spf + + $ docker-compose exec admin flask mailu config-export user.spam_threshold config-import ------------- @@ -211,7 +215,7 @@ mail-config.yml contains the configuration and looks like this: config-update shows the number of created/modified/deleted objects after import. To suppress all messages except error messages use ``--quiet``. -By adding the ``--verbose`` switch (up to two times) the import gets more detailed and shows exactly what attributes changed. +By adding the ``--verbose`` switch the import gets more detailed and shows exactly what attributes changed. In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. If you want to test what would be done when importing without committing any changes, use ``--dry-run``. @@ -234,6 +238,9 @@ It is possible to delete a single element or prune all elements from lists and a The ``-key: null`` notation can also be used to reset an attribute to its default. To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``. +A new dkim key can be generated when adding or modifying a domain, by using the special value +``dkim_key: -generate-``. + This is a complete YAML template with all additional parameters that can be defined: .. code-block:: yaml From 0a9f732faa4d1addcc848cda2678976fb244314c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 22 Feb 2021 20:35:23 +0100 Subject: [PATCH 063/181] added docstring to Logger. use generators. --- core/admin/mailu/models.py | 2 +- core/admin/mailu/schemas.py | 48 +++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 1b3c787a..d134086f 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -83,7 +83,7 @@ class CommaSeparatedList(db.TypeDecorator): def process_result_value(self, value, dialect): """ split comma separated string to list """ - return list(filter(bool, [item.strip() for item in value.split(',')])) if value else [] + return list(filter(bool, (item.strip() for item in value.split(',')))) if value else [] python_type = list diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 4c5042ea..68814170 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -76,6 +76,9 @@ class MyYamlLexer(YamlLexer): yield typ, value class Logger: + """ helps with counting and colorizing + imported and exported data + """ def __init__(self, want_color=None, can_color=False, debug=False, secrets=False): @@ -195,25 +198,26 @@ class Logger: continue before = target._dkim_key_on_disk after = target._dkim_key - if before != after: - # "de-dupe" messages; this event is fired at every flush - if not (target, before, after) in self._dedupe_dkim: - self._dedupe_dkim.add((target, before, after)) - self._counter.update([('Modified', target.__table__.name)]) - if self.verbose: - if self.secrets: - before = before.decode('ascii', 'ignore') - after = after.decode('ascii', 'ignore') - else: - before = HIDDEN if before else '' - after = HIDDEN if after else '' - self._log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') + # "de-dupe" messages; this event is fired at every flush + if before == after or (target, before, after) in self._dedupe_dkim: + continue + self._dedupe_dkim.add((target, before, after)) + self._counter.update([('Modified', target.__table__.name)]) + if self.verbose: + if self.secrets: + before = before.decode('ascii', 'ignore') + after = after.decode('ascii', 'ignore') + else: + before = HIDDEN if before else '' + after = HIDDEN if after else '' + self._log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') def track_serialize(self, obj, item, backref=None): """ callback method to track import """ # called for backref modification? if backref is not None: - self._log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format_map(backref)) + self._log( + 'Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format_map(backref)) return # show input data? if self.verbose < 2: @@ -263,7 +267,7 @@ class Logger: if path: return res - maxlen = max([len(loc) for loc, msg in res]) + maxlen = max(len(loc) for loc, msg in res) res = [f' - {loc.ljust(maxlen)} : {msg}' for loc, msg in res] errors = f'{len(res)} error{["s",""][len(res)==1]}' res.insert(0, f'[ValidationError] {errors} occurred during input validation') @@ -279,7 +283,9 @@ class Logger: if 'attr' in trace.tb_frame.f_locals: path.append(trace.tb_frame.f_locals['attr']) elif trace.tb_frame.f_code.co_name == '_init_fields': - spec = ', '.join(['.'.join(path + [key]) for key in trace.tb_frame.f_locals['invalid_fields']]) + spec = ', '.join( + '.'.join(path + [key]) + for key in trace.tb_frame.f_locals['invalid_fields']) return f'Invalid filter: {spec}' trace = trace.tb_next return None @@ -494,7 +500,7 @@ class CommaSeparatedListField(fields.Raw): except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc else: - value = filter(None, [item.strip() for item in value.split(',')]) + value = filter(bool, (item.strip() for item in value.split(','))) return list(value) @@ -539,7 +545,7 @@ class DkimKeyField(fields.String): # convert list to str if isinstance(value, list): try: - value = ''.join([ensure_text_type(item) for item in value]).strip() + value = ''.join(ensure_text_type(item) for item in value).strip() except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc @@ -855,7 +861,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): ) return (key, value.arg) - return dict([set_default(key, value) for key, value in data.items()]) + return dict(set_default(key, value) for key, value in data.items()) # convert items to "delete" and filter "prune" item items = [ @@ -1059,11 +1065,11 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): # exclude or hide values full = self.context.get('full') - return type(data)([ + return type(data)( (key, HIDDEN if key in self._hide_by_context else value) for key, value in data.items() if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] - ]) + ) # this field is used to mark items for deletion mark_delete = fields.Boolean(data_key='__delete__', load_only=True) From b55b53b781350addad83fe93ac68c5c6c7188847 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 26 Feb 2021 20:51:58 +0100 Subject: [PATCH 064/181] optimize generation of transport nexthop --- core/admin/mailu/internal/views/postfix.py | 48 +++++++++++++++++++--- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index a5507830..7f8418cf 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -2,6 +2,7 @@ from mailu import models from mailu.internal import internal import flask +import idna import re import srslib @@ -37,11 +38,48 @@ def postfix_transport(email): return flask.abort(404) localpart, domain_name = models.Email.resolve_domain(email) relay = models.Relay.query.get(domain_name) or flask.abort(404) - ret = "smtp:[{0}]".format(relay.smtp) - if ":" in relay.smtp: - split = relay.smtp.split(':') - ret = "smtp:[{0}]:{1}".format(split[0], split[1]) - return flask.jsonify(ret) + target = relay.smtp.lower() + port = None + if use_mx := target.startswith('mx:'): + target = target[3:] + if target.startswith('['): + if use_mx or ']' not in target: + # invalid target (mx: and []) + flask.abort(400) + host, rest = target[1:].split(']', 1) + if rest.startswith(':'): + port = rest[1:] + elif rest: + # invalid target (rest should be :port) + flask.abort(400) + else: + if ':' in target: + host, port = target.rsplit(':', 1) + else: + host = target + if not host: + host = relay.name.lower() + use_mx = True + if ':' in host: + host = f'ipv6:{host}' + else: + try: + host = idna.encode(host).decode('ascii') + except idna.IDNAError: + # invalid target (fqdn not encodable) + flask.abort(400) + if port is not None: + try: + port = int(port, 10) + if port == 25: + port = None + except ValueError: + # invalid target (port should be numeric) + flask.abort(400) + if not use_mx: + host = f'[{host}]' + port = '' if port is None else f':{port}' + return flask.jsonify(f'smtp:{host}{port}') @internal.route("/postfix/recipient/map/") From e90d5548a651fe933b5977e15c52cd0be954e7dd Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 10 Mar 2021 18:30:28 +0100 Subject: [PATCH 065/181] use RFC3339 for last_check fixed to UTC for now --- core/admin/mailu/schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 68814170..284c1551 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1135,6 +1135,7 @@ class FetchSchema(BaseSchema): """ Schema config """ model = models.Fetch load_instance = True + datetimeformat = '%Y-%m-%dT%H:%M:%S.%fZ' # RFC3339, but fixed to UTC sibling = True include_by_context = { From c17bfae24050e48ae83cc21196346cfe7c6d90bc Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 10 Mar 2021 18:50:25 +0100 Subject: [PATCH 066/181] correct rfc3339 datetime serialization now using correct timezone --- core/admin/mailu/schemas.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 284c1551..277748f7 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -3,6 +3,7 @@ from copy import deepcopy from collections import Counter +from datetime import timezone import json import logging @@ -455,7 +456,20 @@ class RenderJSON: return json.dumps(*args, **kwargs) -### schema: custom fields ### +### marshmallow: custom fields ### + +def _rfc3339(datetime): + """ dump datetime according to rfc3339 """ + if datetime.tzinfo is None: + datetime = datetime.astimezone(timezone.utc) + res = datetime.isoformat() + if res.endswith('+00:00'): + return f'{res[:-6]}Z' + return res + +fields.DateTime.SERIALIZATION_FUNCS['rfc3339'] = _rfc3339 +fields.DateTime.DESERIALIZATION_FUNCS['rfc3339'] = fields.DateTime.DESERIALIZATION_FUNCS['iso'] +fields.DateTime.DEFAULT_FORMAT = 'rfc3339' class LazyStringField(fields.String): """ Field that serializes a "false" value to the empty string @@ -1135,7 +1149,6 @@ class FetchSchema(BaseSchema): """ Schema config """ model = models.Fetch load_instance = True - datetimeformat = '%Y-%m-%dT%H:%M:%S.%fZ' # RFC3339, but fixed to UTC sibling = True include_by_context = { From ce9a9ec572d8220e673b2817f1350ffb53b3215a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 10 Mar 2021 18:50:52 +0100 Subject: [PATCH 067/181] always init Logger first --- core/admin/mailu/manage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index f9c58d26..5708327e 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -375,6 +375,8 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None """ Export configuration as YAML or JSON to stdout or file """ + log = Logger(want_color=color or None, can_color=output.isatty()) + only = only or MailuSchema.Meta.order context = { @@ -382,7 +384,6 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None 'secrets': secrets, 'dns': dns, } - log = Logger(want_color=color or None, can_color=output.isatty()) try: schema = MailuSchema(only=only, context=context) From 9cb6962335d00fbb0505e7d571271232d4f54b97 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 11 Mar 2021 18:12:50 +0100 Subject: [PATCH 068/181] Moved MyYamlLexer into logger now cmdline runs without pygments --- core/admin/mailu/schemas.py | 48 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 277748f7..fdd766b3 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -53,34 +53,34 @@ def mapped(cls): _model2schema[cls.Meta.model] = cls return cls -class MyYamlLexer(YamlLexer): - """ colorize yaml constants and integers """ - def get_tokens(self, text, unfiltered=False): - for typ, value in super().get_tokens(text, unfiltered): - if typ is Token.Literal.Scalar.Plain: - if value in {'true', 'false', 'null'}: - typ = Token.Keyword.Constant - elif value == HIDDEN: - typ = Token.Error - else: - try: - int(value, 10) - except ValueError: - try: - float(value) - except ValueError: - pass - else: - typ = Token.Literal.Number.Float - else: - typ = Token.Literal.Number.Integer - yield typ, value - class Logger: """ helps with counting and colorizing imported and exported data """ + class MyYamlLexer(YamlLexer): + """ colorize yaml constants and integers """ + def get_tokens(self, text, unfiltered=False): + for typ, value in super().get_tokens(text, unfiltered): + if typ is Token.Literal.Scalar.Plain: + if value in {'true', 'false', 'null'}: + typ = Token.Keyword.Constant + elif value == HIDDEN: + typ = Token.Error + else: + try: + int(value, 10) + except ValueError: + try: + float(value) + except ValueError: + pass + else: + typ = Token.Literal.Number.Float + else: + typ = Token.Literal.Number.Integer + yield typ, value + def __init__(self, want_color=None, can_color=False, debug=False, secrets=False): self.lexer = 'yaml' @@ -323,7 +323,7 @@ class Logger: return data lexer = lexer or self.lexer - lexer = MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer) + lexer = Logger.MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer) formatter = get_formatter_by_name(formatter or self.formatter, colorscheme=self.colorscheme) if strip is None: strip = self.strip From 0c38128c4e459aa6fb783c86bea51ae2e19ac9b7 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 11 Mar 2021 18:38:00 +0100 Subject: [PATCH 069/181] Add pygments to requirements --- core/admin/mailu/schemas.py | 20 ++++++-------------- core/admin/requirements-prod.txt | 1 + core/admin/requirements.txt | 1 + 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index fdd766b3..28a5d6f4 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -21,16 +21,11 @@ from flask_marshmallow import Marshmallow from OpenSSL import crypto -try: - from pygments import highlight - from pygments.token import Token - from pygments.lexers import get_lexer_by_name - from pygments.lexers.data import YamlLexer - from pygments.formatters import get_formatter_by_name -except ModuleNotFoundError: - COLOR_SUPPORTED = False -else: - COLOR_SUPPORTED = True +from pygments import highlight +from pygments.token import Token +from pygments.lexers import get_lexer_by_name +from pygments.lexers.data import YamlLexer +from pygments.formatters import get_formatter_by_name from mailu import models, dkim @@ -92,10 +87,7 @@ class Logger: self.debug = debug self.print = print - if want_color and not COLOR_SUPPORTED: - raise ValueError('Please install pygments to colorize output') - - self.color = want_color or (can_color and COLOR_SUPPORTED) + self.color = want_color or can_color self._counter = Counter() self._schemas = {} diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index 7620fd95..5c291e24 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -36,6 +36,7 @@ marshmallow-sqlalchemy==0.24.1 passlib==1.7.4 psycopg2==2.8.2 pycparser==2.19 +Pygments==2.8.1 pyOpenSSL==19.0.0 python-dateutil==2.8.0 python-editor==1.0.4 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index f9d175b3..46500295 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -17,6 +17,7 @@ gunicorn tabulate PyYAML PyOpenSSL +Pygments dnspython bcrypt tenacity From 8bc44455721471a7277196190bb31db9d48d4bce Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 12 Mar 2021 17:56:17 +0100 Subject: [PATCH 070/181] Sync update of localpart, domain_name and email --- core/admin/mailu/models.py | 18 +++++++++++++++--- core/admin/mailu/schemas.py | 4 ++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index b08c5bd2..93efd016 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -355,9 +355,21 @@ class Email(object): self.localpart, self.domain_name = value.rsplit('@', 1) self._email = value - # hack for email declared attr - when _email is not updated yet - def __str__(self): - return str(f'{self.localpart}@{self.domain_name}') + @staticmethod + def _update_localpart(target, value, *_): + if target.domain_name: + target._email = f'{value}@{target.domain_name}' + + @staticmethod + def _update_domain_name(target, value, *_): + if target.localpart: + target._email = f'{target.localpart}@{value}' + + @classmethod + def __declare_last__(cls): + # gets called after mappings are completed + sqlalchemy.event.listen(User.localpart, 'set', cls._update_localpart, propagate=True) + sqlalchemy.event.listen(User.domain_name, 'set', cls._update_domain_name, propagate=True) def sendmail(self, subject, body): """ send an email to the address """ diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 28a5d6f4..2742edf1 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1030,8 +1030,8 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): self.opts.sqla_session.add(item) return item - # stop early if item has no password attribute - if not hasattr(item, 'password'): + # stop early when not updating or item has no password attribute + if not self.context.get('update') or not hasattr(item, 'password'): return item # did we hash a new plaintext password? From 83b1fbb9d67b6752a7d8bb4f8e41e3d35e56ed4d Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 14 Mar 2021 18:09:21 +0100 Subject: [PATCH 071/181] Lazy loading of KVSessionExtension - call cleanup_sessions on first kvstore access this allows to run cmdline actions without redis (and makes it faster) - Allow development using DictStore by setting REDIS_ADDRESS to the empty string in env - don't sign 64bit random session id as suggested by nextgens --- core/admin/mailu/__init__.py | 12 +++--- core/admin/mailu/utils.py | 65 +++++++++++++++++++++++++++++++- core/admin/requirements-prod.txt | 1 + 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index f9ca2466..40cc9cff 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -1,8 +1,8 @@ +""" Mailu admin app +""" + import flask import flask_bootstrap -import redis -from flask_kvsession import KVSessionExtension -from simplekv.memory.redisstore import RedisStore from mailu import utils, debug, models, manage, configuration @@ -20,7 +20,8 @@ def create_app_from_config(config): # Initialize application extensions config.init_app(app) models.db.init_app(app) - KVSessionExtension(RedisStore(redis.StrictRedis().from_url('redis://{0}/3'.format(config['REDIS_ADDRESS']))), app).cleanup_sessions(app) + utils.kvsession.init_kvstore(config) + utils.kvsession.init_app(app) utils.limiter.init_app(app) utils.babel.init_app(app) utils.login.init_app(app) @@ -53,8 +54,7 @@ def create_app_from_config(config): def create_app(): - """ Create a new application based on the config module + """ Create a new application based on the config module """ config = configuration.ConfigManager() return create_app_from_config(config) - diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index ce12a09a..852fe8ad 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -1,11 +1,18 @@ -from mailu import models, limiter +""" Mailu admin app utilities +""" + +from mailu import limiter import flask import flask_login -import flask_script import flask_migrate import flask_babel +import flask_kvsession +import redis +from simplekv.memory import DictStore +from simplekv.memory.redisstore import RedisStore +from itsdangerous.encoding import want_bytes from werkzeug.contrib import fixers @@ -33,6 +40,10 @@ def get_locale(): # Proxy fixer class PrefixMiddleware(object): + """ fix proxy headers """ + def __init__(self): + self.app = None + def __call__(self, environ, start_response): prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '') if prefix: @@ -48,3 +59,53 @@ proxy = PrefixMiddleware() # Data migrate migrate = flask_migrate.Migrate() + + +# session store +class NullSigner(object): + """NullSigner does not sign nor unsign""" + def __init__(self, *args, **kwargs): + pass + def sign(self, value): + """Signs the given string.""" + return want_bytes(value) + def unsign(self, signed_value): + """Unsigns the given string.""" + return want_bytes(signed_value) + +class KVSessionIntf(flask_kvsession.KVSessionInterface): + """ KVSession interface allowing to run int function on first access """ + def __init__(self, app, init_fn=None): + if init_fn: + app.kvsession_init = init_fn + else: + self._first_run(None) + def _first_run(self, app): + if app: + app.kvsession_init() + self.open_session = super().open_session + self.save_session = super().save_session + def open_session(self, app, request): + self._first_run(app) + return super().open_session(app, request) + def save_session(self, app, session, response): + self._first_run(app) + return super().save_session(app, session, response) + +class KVSessionExt(flask_kvsession.KVSessionExtension): + """ Activates Flask-KVSession for an application. """ + def init_kvstore(self, config): + """ Initialize kvstore - fallback to DictStore without REDIS_ADDRESS """ + if addr := config.get('REDIS_ADDRESS'): + self.default_kvstore = RedisStore(redis.StrictRedis().from_url(f'redis://{addr}/3')) + else: + self.default_kvstore = DictStore() + + def init_app(self, app, session_kvstore=None): + """ Initialize application and KVSession. """ + super().init_app(app, session_kvstore) + app.session_interface = KVSessionIntf(app, self.cleanup_sessions) + +kvsession = KVSessionExt() + +flask_kvsession.Signer = NullSigner diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index 54cf9a14..cd084684 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -39,6 +39,7 @@ python-editor==1.0.4 pytz==2019.1 PyYAML==5.1 redis==3.2.1 +simplekv==0.14.1 #alpine3:12 provides six==1.15.0 #six==1.12.0 socrate==0.1.1 From f0f79b23a328064a787d99b3755d43a946a87d9b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 14 Mar 2021 21:38:16 +0100 Subject: [PATCH 072/181] Allow cleanup of sessions by key&value in data This can be used to delete all sessions belonging to a user/login. For no it just iterates over all sessions. This could be enhanced by using a prefix for and deleting by prefix. --- core/admin/mailu/utils.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 852fe8ad..9394c38f 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -1,6 +1,8 @@ """ Mailu admin app utilities """ +from datetime import datetime + from mailu import limiter import flask @@ -22,6 +24,7 @@ login.login_view = "ui.login" @login.unauthorized_handler def handle_needs_login(): + """ redirect unauthorized requests to login page """ return flask.redirect( flask.url_for('ui.login', next=flask.request.endpoint) ) @@ -34,7 +37,8 @@ babel = flask_babel.Babel() @babel.localeselector def get_locale(): - translations = list(map(str, babel.list_translations())) + """ selects locale for translation """ + translations = [str(translation) for translation in babel.list_translations()] return flask.request.accept_languages.best_match(translations) @@ -101,6 +105,32 @@ class KVSessionExt(flask_kvsession.KVSessionExtension): else: self.default_kvstore = DictStore() + def cleanup_sessions(self, app=None, dkey=None, dvalue=None): + """ Remove sessions from the store. """ + if not app: + app = flask.current_app + if dkey is None and dvalue is None: + now = datetime.utcnow() + for key in app.kvsession_store.keys(): + try: + sid = flask_kvsession.SessionID.unserialize(key) + except ValueError: + pass + else: + if sid.has_expired( + app.config['PERMANENT_SESSION_LIFETIME'], + now + ): + app.kvsession_store.delete(key) + elif dkey is not None and dvalue is not None: + for key in app.kvsession_store.keys(): + if app.session_interface.serialization_method.loads( + app.kvsession_store.get(key) + ).get(dkey, None) == dvalue: + app.kvsession_store.delete(key) + else: + raise ValueError('Need dkey and dvalue.') + def init_app(self, app, session_kvstore=None): """ Initialize application and KVSession. """ super().init_app(app, session_kvstore) From 4b71bd56c49dceef38f5cc4267d22936ecaf0bc4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 4 Apr 2021 14:35:31 +0200 Subject: [PATCH 073/181] replace flask_kvsession with mailu's own storage --- core/admin/mailu/__init__.py | 3 +- core/admin/mailu/configuration.py | 5 +- core/admin/mailu/utils.py | 408 +++++++++++++++++++++++++----- core/admin/requirements-prod.txt | 2 - core/admin/requirements.txt | 1 - 5 files changed, 343 insertions(+), 76 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 40cc9cff..690837ea 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -20,8 +20,7 @@ def create_app_from_config(config): # Initialize application extensions config.init_app(app) models.db.init_app(app) - utils.kvsession.init_kvstore(config) - utils.kvsession.init_app(app) + utils.session.init_app(app) utils.limiter.init_app(app) utils.babel.init_app(app) utils.login.init_app(app) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 6f65d17d..3d1b4fb5 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -14,6 +14,7 @@ DEFAULT_CONFIG = { 'DEBUG': False, 'DOMAIN_REGISTRATION': False, 'TEMPLATES_AUTO_RELOAD': True, + 'MEMORY_SESSIONS': False, # Database settings 'DB_FLAVOR': None, 'DB_USER': 'mailu', @@ -55,6 +56,7 @@ DEFAULT_CONFIG = { 'RECAPTCHA_PRIVATE_KEY': '', # Advanced settings 'LOG_LEVEL': 'WARNING', + 'SESSION_KEY_BITS': 128, 'SESSION_LIFETIME': 24, 'SESSION_COOKIE_SECURE': True, 'CREDENTIAL_ROUNDS': 12, @@ -65,7 +67,6 @@ DEFAULT_CONFIG = { 'HOST_SMTP': 'smtp', 'HOST_AUTHSMTP': 'smtp', 'HOST_ADMIN': 'admin', - 'WEBMAIL': 'none', 'HOST_WEBMAIL': 'webmail', 'HOST_WEBDAV': 'webdav:5232', 'HOST_REDIS': 'redis', @@ -136,9 +137,9 @@ class ConfigManager(dict): self.config['RATELIMIT_STORAGE_URL'] = 'redis://{0}/2'.format(self.config['REDIS_ADDRESS']) self.config['QUOTA_STORAGE_URL'] = 'redis://{0}/1'.format(self.config['REDIS_ADDRESS']) + self.config['SESSION_STORAGE_URL'] = 'redis://{0}/3'.format(self.config['REDIS_ADDRESS']) self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True - self.config['SESSION_KEY_BITS'] = 128 self.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=int(self.config['SESSION_LIFETIME'])) # update the app config itself app.config = self diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 9394c38f..1deaa4ae 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -1,7 +1,14 @@ """ Mailu admin app utilities """ -from datetime import datetime +try: + import cPickle as pickle +except ImportError: + import pickle + +import hashlib +import secrets +import time from mailu import limiter @@ -9,12 +16,11 @@ import flask import flask_login import flask_migrate import flask_babel -import flask_kvsession import redis -from simplekv.memory import DictStore -from simplekv.memory.redisstore import RedisStore +from flask.sessions import SessionMixin, SessionInterface from itsdangerous.encoding import want_bytes +from werkzeug.datastructures import CallbackDict from werkzeug.contrib import fixers @@ -65,77 +71,341 @@ proxy = PrefixMiddleware() migrate = flask_migrate.Migrate() -# session store -class NullSigner(object): - """NullSigner does not sign nor unsign""" - def __init__(self, *args, **kwargs): - pass - def sign(self, value): - """Signs the given string.""" - return want_bytes(value) - def unsign(self, signed_value): - """Unsigns the given string.""" - return want_bytes(signed_value) +# session store (inspired by https://github.com/mbr/flask-kvsession) +class RedisStore: + """ Stores session data in a redis db. """ -class KVSessionIntf(flask_kvsession.KVSessionInterface): - """ KVSession interface allowing to run int function on first access """ - def __init__(self, app, init_fn=None): - if init_fn: - app.kvsession_init = init_fn + has_ttl = True + + def __init__(self, redisstore): + self.redis = redisstore + + def get(self, key): + """ load item from store. """ + value = self.redis.get(key) + if value is None: + raise KeyError(key) + return value + + def put(self, key, value, ttl_secs=None): + """ save item to store. """ + if ttl_secs: + self.redis.setex(key, int(ttl_secs), value) else: - self._first_run(None) - def _first_run(self, app): - if app: - app.kvsession_init() - self.open_session = super().open_session - self.save_session = super().save_session - def open_session(self, app, request): - self._first_run(app) - return super().open_session(app, request) - def save_session(self, app, session, response): - self._first_run(app) - return super().save_session(app, session, response) + self.redis.set(key, value) -class KVSessionExt(flask_kvsession.KVSessionExtension): - """ Activates Flask-KVSession for an application. """ - def init_kvstore(self, config): - """ Initialize kvstore - fallback to DictStore without REDIS_ADDRESS """ - if addr := config.get('REDIS_ADDRESS'): - self.default_kvstore = RedisStore(redis.StrictRedis().from_url(f'redis://{addr}/3')) - else: - self.default_kvstore = DictStore() + def delete(self, key): + """ delete item from store. """ + self.redis.delete(key) - def cleanup_sessions(self, app=None, dkey=None, dvalue=None): - """ Remove sessions from the store. """ - if not app: + def list(self, prefix=None): + """ return list of keys starting with prefix """ + if prefix: + prefix += b'*' + return list(self.redis.scan_iter(match=prefix)) + +class DictStore: + """ Stores session data in a python dict. """ + + has_ttl = False + + def __init__(self): + self.dict = {} + + def get(self, key): + """ load item from store. """ + return self.dict[key] + + def put(self, key, value, ttl_secs=None): + """ save item to store. """ + self.dict[key] = value + + def delete(self, key): + """ delete item from store. """ + try: + del self.dict[key] + except KeyError: + pass + + def list(self, prefix=None): + """ return list of keys starting with prefix """ + if prefix is None: + return list(self.dict.keys()) + return [key for key in self.dict if key.startswith(prefix)] + +class MailuSession(CallbackDict, SessionMixin): + """ Custom flask session storage. """ + + # default modified to false + modified = False + + def __init__(self, key=None, app=None): + + self.app = app or flask.current_app + + initial = None + + key = want_bytes(key) + if parsed := self.app.session_config.parse_key(key, self.app): + try: + initial = pickle.loads(app.session_store.get(key)) + except (KeyError, EOFError, pickle.UnpicklingError): + # either the cookie was manipulated or we did not find the + # session in the backend or the pickled data is invalid. + # => start new session + pass + else: + (self._uid, self._sid, self._created) = parsed + self._key = key + + if initial is None: + # start new session + self.new = True + self._uid = None + self._sid = None + self._created = self.app.session_config.gen_created() + self._key = None + + def _on_update(obj): + obj.modified = True + + CallbackDict.__init__(self, initial, _on_update) + + @property + def sid(self): + """ this reflects the session's id. """ + if self._sid is None or self._uid is None or self._created is None: + return None + return b''.join([self._uid, self._sid, self._created]) + + def destroy(self): + """ destroy session for security reasons. """ + + if self._key is not None: + self.app.session_store.delete(self._key) + self._key = None + + self._uid = None + self._sid = None + self._created = None + + self.clear() + + self.modified = False + self.new = False + + def regenerate(self): + """ generate new id for session to avoid `session fixation`. """ + + if self._key is not None: + self.app.session_store.delete(self._key) + self._key = None + + self._sid = None + self._created = self.app.session_config.gen_created() + + self.modified = True + + def save(self): + """ Save session to store. """ + + # don't save if session was destroyed or is not modified + if self._created is None or not self.modified: + return False + + # set uid from dict data + if self._uid is None: + self._uid = self.app.session_config.gen_uid(self.get('user_id', '')) + + # create new session id for new or regenerated sessions + if self._sid is None: + self._sid = self.app.session_config.gen_sid() + + # set created if permanent state changed + if self.permanent: + if self._created: + self._created = b'' + elif not self._created: + self._created = self.app.session_config.gen_created() + + # get new session key + key = self.sid + + # delete old session if key has changed + if key != self._key and self._key is not None: + self.app.session_store.delete(self._key) + + # save session + self.app.session_store.put( + key, + pickle.dumps(dict(self)), + None if self.permanent else self.app.permanent_session_lifetime.total_seconds() + ) + + self._key = key + + self.new = False + self.modified = False + + return True + +class MailuSessionConfig: + """ Stores sessions crypto config """ + + def __init__(self, app=None): + + if app is None: app = flask.current_app - if dkey is None and dvalue is None: - now = datetime.utcnow() - for key in app.kvsession_store.keys(): - try: - sid = flask_kvsession.SessionID.unserialize(key) - except ValueError: - pass - else: - if sid.has_expired( - app.config['PERMANENT_SESSION_LIFETIME'], - now - ): - app.kvsession_store.delete(key) - elif dkey is not None and dvalue is not None: - for key in app.kvsession_store.keys(): - if app.session_interface.serialization_method.loads( - app.kvsession_store.get(key) - ).get(dkey, None) == dvalue: - app.kvsession_store.delete(key) + + bits = app.config.get('SESSION_KEY_BITS', 64) + + if bits < 64: + raise ValueError('Session id entropy must not be less than 64 bits!') + + hash_bytes = bits//8 + (bits%8>0) + time_bytes = 4 # 32 bit timestamp for now + + self._shake_fn = hashlib.shake_256 if bits>128 else hashlib.shake_128 + self._hash_len = hash_bytes + self._hash_b64 = len(self._encode(bytes(hash_bytes))) + self._key_min = 2*self._hash_b64 + self._key_max = self._key_min + len(self._encode(bytes(time_bytes))) + + def gen_sid(self): + """ Generate random session id. """ + return self._encode(secrets.token_bytes(self._hash_len)) + + def gen_uid(self, uid): + """ Generate hashed user id part of session key. """ + return self._encode(self._shake_fn(want_bytes(uid)).digest(self._hash_len)) + + def gen_created(self, now=None): + """ Generate base64 representation of creation time. """ + return self._encode(int(now or time.time()).to_bytes(8, byteorder='big').lstrip(b'\0')) + + def parse_key(self, key, app=None, now=None): + """ Split key into sid, uid and creation time. """ + + if not (isinstance(key, bytes) and self._key_min <= len(key) <= self._key_max): + return None + + uid = key[:self._hash_b64] + sid = key[self._hash_b64:self._key_min] + crt = key[self._key_min:] + + # validate if parts are decodeable + created = self._decode(crt) + if created is None or self._decode(uid) is None or self._decode(sid) is None: + return None + + # validate creation time when requested or store does not support ttl + if now is not None or not app.session_store.has_ttl: + created = int.from_bytes(created, byteorder='big') + if created > 0: + if now is None: + now = int(time.time()) + if created < now < created + app.permanent_session_lifetime.total_seconds(): + return None + + return (uid, sid, crt) + + def _encode(self, value): + return secrets.base64.urlsafe_b64encode(value).rstrip(b'=') + + def _decode(self, value): + try: + return secrets.base64.urlsafe_b64decode(value + b'='*(4-len(value)%4)) + except secrets.binascii.Error: + return None + +class MailuSessionInterface(SessionInterface): + """ Custom flask session interface. """ + + def open_session(self, app, request): + """ Load or create session. """ + return MailuSession(request.cookies.get(app.config['SESSION_COOKIE_NAME'], None), app) + + def save_session(self, app, session, response): + """ Save modified session. """ + + if session.save(): + # session saved. update cookie + response.set_cookie( + key=app.config['SESSION_COOKIE_NAME'], + value=session.sid, + expires=self.get_expiration_time(app, session), + path=self.get_cookie_path(app), + domain=self.get_cookie_domain(app), + secure=app.config['SESSION_COOKIE_SECURE'], + httponly=app.config['SESSION_COOKIE_HTTPONLY'] + ) + +class MailuSessionExtension: + """ Server side session handling """ + + @staticmethod + def cleanup_sessions(app=None): + """ Remove invalid or expired sessions. """ + + app = app or flask.current_app + now = int(time.time()) + + count = 0 + for key in app.session_store.list(): + if not app.session_config.parse_key(key, app, now): + app.session_store.delete(key) + count += 1 + + return count + + @staticmethod + def prune_sessions(uid=None, keep_permanent=False, keep=None, app=None): + """ Remove sessions + uid: remove all sessions (NONE) or sessions belonging to a specific user + keep_permanent: also delete permanent sessions? + keep: keep listed sessions + """ + + keep = keep or set() + app = app or flask.current_app + now = int(time.time()) + + prefix = None if uid is None else app.session_config.gen_uid(uid) + + count = 0 + for key in app.session_store.list(prefix): + if key in keep: + continue + if keep_permanent: + if parsed := app.session_config.parse_key(key, app, now): + if not parsed[2]: + continue + app.session_store.delete(key) + count += 1 + + return count + + def init_app(self, app): + """ Replace session management of application. """ + + if app.config.get('MEMORY_SESSIONS'): + # in-memory session store for use in development + app.session_store = DictStore() + else: - raise ValueError('Need dkey and dvalue.') + # redis-based session store for use in production + app.session_store = RedisStore( + redis.StrictRedis().from_url(app.config['SESSION_STORAGE_URL']) + ) - def init_app(self, app, session_kvstore=None): - """ Initialize application and KVSession. """ - super().init_app(app, session_kvstore) - app.session_interface = KVSessionIntf(app, self.cleanup_sessions) + # clean expired sessions on first use in case lifetime was changed + def cleaner(): + MailuSessionExtension.cleanup_sessions(app) -kvsession = KVSessionExt() + # TODO: hmm. this will clean once per gunicorn worker + app.before_first_request(cleaner) -flask_kvsession.Signer = NullSigner + app.session_config = MailuSessionConfig(app) + app.session_interface = MailuSessionInterface() + +session = MailuSessionExtension() diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index cd084684..f767f431 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -13,7 +13,6 @@ Flask==1.0.2 Flask-Babel==0.12.2 Flask-Bootstrap==3.3.7.1 Flask-DebugToolbar==0.10.1 -Flask-KVSession==0.6.2 Flask-Limiter==1.0.1 Flask-Login==0.4.1 Flask-Migrate==2.4.0 @@ -39,7 +38,6 @@ python-editor==1.0.4 pytz==2019.1 PyYAML==5.1 redis==3.2.1 -simplekv==0.14.1 #alpine3:12 provides six==1.15.0 #six==1.12.0 socrate==0.1.1 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index abb37234..9739ed3f 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -3,7 +3,6 @@ Flask-Login Flask-SQLAlchemy Flask-bootstrap Flask-Babel -Flask-KVSession Flask-migrate Flask-script Flask-wtf From 4b8bbf760b423657a564b478cf72cbbc0125e7c3 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 4 Apr 2021 14:40:49 +0200 Subject: [PATCH 074/181] default to 128 bits --- core/admin/mailu/configuration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 3d1b4fb5..679c6c7e 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -140,6 +140,7 @@ class ConfigManager(dict): self.config['SESSION_STORAGE_URL'] = 'redis://{0}/3'.format(self.config['REDIS_ADDRESS']) self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True + self.config['SESSION_KEY_BITS'] = 128 self.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=int(self.config['SESSION_LIFETIME'])) # update the app config itself app.config = self From 731ce8ede91a7c3e37c8efc0161c0e6b06d330fc Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 4 Apr 2021 18:02:43 +0200 Subject: [PATCH 075/181] fix permanent sessions. hash uid using SECRET_KEY clean session in redis only once when starting --- core/admin/mailu/utils.py | 140 +++++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 61 deletions(-) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 1deaa4ae..30725ff7 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -10,6 +10,8 @@ import hashlib import secrets import time +from multiprocessing import Value + from mailu import limiter import flask @@ -87,10 +89,10 @@ class RedisStore: raise KeyError(key) return value - def put(self, key, value, ttl_secs=None): + def put(self, key, value, ttl=None): """ save item to store. """ - if ttl_secs: - self.redis.setex(key, int(ttl_secs), value) + if ttl: + self.redis.setex(key, int(ttl), value) else: self.redis.set(key, value) @@ -171,6 +173,11 @@ class MailuSession(CallbackDict, SessionMixin): CallbackDict.__init__(self, initial, _on_update) + @property + def saved(self): + """ this reflects if the session was saved. """ + return self._key is not None + @property def sid(self): """ this reflects the session's id. """ @@ -181,9 +188,7 @@ class MailuSession(CallbackDict, SessionMixin): def destroy(self): """ destroy session for security reasons. """ - if self._key is not None: - self.app.session_store.delete(self._key) - self._key = None + self.delete() self._uid = None self._sid = None @@ -191,28 +196,28 @@ class MailuSession(CallbackDict, SessionMixin): self.clear() - self.modified = False + self.modified = True self.new = False def regenerate(self): """ generate new id for session to avoid `session fixation`. """ - if self._key is not None: - self.app.session_store.delete(self._key) - self._key = None + self.delete() self._sid = None self._created = self.app.session_config.gen_created() self.modified = True + def delete(self): + """ Delete stored session. """ + if self.saved: + self.app.session_store.delete(self._key) + self._key = None + def save(self): """ Save session to store. """ - # don't save if session was destroyed or is not modified - if self._created is None or not self.modified: - return False - # set uid from dict data if self._uid is None: self._uid = self.app.session_config.gen_uid(self.get('user_id', '')) @@ -221,25 +226,18 @@ class MailuSession(CallbackDict, SessionMixin): if self._sid is None: self._sid = self.app.session_config.gen_sid() - # set created if permanent state changed - if self.permanent: - if self._created: - self._created = b'' - elif not self._created: - self._created = self.app.session_config.gen_created() - # get new session key key = self.sid # delete old session if key has changed - if key != self._key and self._key is not None: - self.app.session_store.delete(self._key) + if key != self._key: + self.delete() # save session self.app.session_store.put( key, pickle.dumps(dict(self)), - None if self.permanent else self.app.permanent_session_lifetime.total_seconds() + self.app.permanent_session_lifetime.total_seconds() ) self._key = key @@ -247,8 +245,6 @@ class MailuSession(CallbackDict, SessionMixin): self.new = False self.modified = False - return True - class MailuSessionConfig: """ Stores sessions crypto config """ @@ -264,8 +260,9 @@ class MailuSessionConfig: hash_bytes = bits//8 + (bits%8>0) time_bytes = 4 # 32 bit timestamp for now + shaker = hashlib.shake_256 if bits>128 else hashlib.shake_128 - self._shake_fn = hashlib.shake_256 if bits>128 else hashlib.shake_128 + self._shaker = shaker(want_bytes(app.config.get('SECRET_KEY', ''))) self._hash_len = hash_bytes self._hash_b64 = len(self._encode(bytes(hash_bytes))) self._key_min = 2*self._hash_b64 @@ -277,13 +274,15 @@ class MailuSessionConfig: def gen_uid(self, uid): """ Generate hashed user id part of session key. """ - return self._encode(self._shake_fn(want_bytes(uid)).digest(self._hash_len)) + shaker = self._shaker.copy() + shaker.update(want_bytes(uid)) + return self._encode(shaker.digest(self._hash_len)) def gen_created(self, now=None): """ Generate base64 representation of creation time. """ return self._encode(int(now or time.time()).to_bytes(8, byteorder='big').lstrip(b'\0')) - def parse_key(self, key, app=None, now=None): + def parse_key(self, key, app=None, validate=False, now=None): """ Split key into sid, uid and creation time. """ if not (isinstance(key, bytes) and self._key_min <= len(key) <= self._key_max): @@ -299,13 +298,12 @@ class MailuSessionConfig: return None # validate creation time when requested or store does not support ttl - if now is not None or not app.session_store.has_ttl: + if validate or not app.session_store.has_ttl: + if now is None: + now = int(time.time()) created = int.from_bytes(created, byteorder='big') - if created > 0: - if now is None: - now = int(time.time()) - if created < now < created + app.permanent_session_lifetime.total_seconds(): - return None + if not (created < now < created + app.permanent_session_lifetime.total_seconds()): + return None return (uid, sid, crt) @@ -328,17 +326,40 @@ class MailuSessionInterface(SessionInterface): def save_session(self, app, session, response): """ Save modified session. """ - if session.save(): - # session saved. update cookie - response.set_cookie( - key=app.config['SESSION_COOKIE_NAME'], - value=session.sid, - expires=self.get_expiration_time(app, session), - path=self.get_cookie_path(app), - domain=self.get_cookie_domain(app), - secure=app.config['SESSION_COOKIE_SECURE'], - httponly=app.config['SESSION_COOKIE_HTTPONLY'] - ) + # If the session is modified to be empty, remove the cookie. + # If the session is empty, return without setting the cookie. + if not session: + if session.modified: + session.delete() + response.delete_cookie( + app.session_cookie_name, + domain=self.get_cookie_domain(app), + path=self.get_cookie_path(app), + ) + return + + # Add a "Vary: Cookie" header if the session was accessed + if session.accessed: + response.vary.add('Cookie') + + # TODO: set cookie from time to time to prevent expiration in browser + # also update expire in redis + + if not self.should_set_cookie(app, session): + return + + # save session and update cookie + session.save() + response.set_cookie( + app.session_cookie_name, + session.sid, + expires=self.get_expiration_time(app, session), + httponly=self.get_cookie_httponly(app), + domain=self.get_cookie_domain(app), + path=self.get_cookie_path(app), + secure=self.get_cookie_secure(app), + samesite=self.get_cookie_samesite(app) + ) class MailuSessionExtension: """ Server side session handling """ @@ -352,36 +373,29 @@ class MailuSessionExtension: count = 0 for key in app.session_store.list(): - if not app.session_config.parse_key(key, app, now): + if not app.session_config.parse_key(key, app, validate=True, now=now): app.session_store.delete(key) count += 1 return count @staticmethod - def prune_sessions(uid=None, keep_permanent=False, keep=None, app=None): + def prune_sessions(uid=None, keep=None, app=None): """ Remove sessions uid: remove all sessions (NONE) or sessions belonging to a specific user - keep_permanent: also delete permanent sessions? keep: keep listed sessions """ keep = keep or set() app = app or flask.current_app - now = int(time.time()) prefix = None if uid is None else app.session_config.gen_uid(uid) count = 0 for key in app.session_store.list(prefix): - if key in keep: - continue - if keep_permanent: - if parsed := app.session_config.parse_key(key, app, now): - if not parsed[2]: - continue - app.session_store.delete(key) - count += 1 + if key not in keep: + app.session_store.delete(key) + count += 1 return count @@ -398,14 +412,18 @@ class MailuSessionExtension: redis.StrictRedis().from_url(app.config['SESSION_STORAGE_URL']) ) - # clean expired sessions on first use in case lifetime was changed + # clean expired sessions oonce on first use in case lifetime was changed def cleaner(): - MailuSessionExtension.cleanup_sessions(app) + with cleaned.get_lock(): + if not cleaned.value: + cleaned.value = True + flask.current_app.logger.error('cleaning') + MailuSessionExtension.cleanup_sessions(app) - # TODO: hmm. this will clean once per gunicorn worker app.before_first_request(cleaner) app.session_config = MailuSessionConfig(app) app.session_interface = MailuSessionInterface() +cleaned = Value('i', False) session = MailuSessionExtension() From 1f2aee278c1a2dc1ab0d232433f448264a29ba9f Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Thu, 13 May 2021 18:59:18 +0200 Subject: [PATCH 076/181] Reflect override settings for postfix Also added a stumbling stone when changing postfix.cf --- docs/faq.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 9c4f1d75..5b13f191 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -258,9 +258,11 @@ Postfix, Dovecot, Nginx and Rspamd support overriding configuration files. Overr correct syntax. The following file names will be taken as override configuration: - `Postfix`_ : - - ``main.cf`` as ``$ROOT/overrides/postfix/postfix.cf`` - - ``master.cf`` as ``$ROOT/overrides/postfix/postfix.master`` + - ``main.cf`` as ``$ROOT/overrides/postfix.cf`` + - ``master.cf`` as ``$ROOT/overrides/postfix.master`` - All ``$ROOT/overrides/postfix/*.map`` files + - For both ``postfix.cf`` and ``postfix.master``, you need to put one configuration per line, as they are fed line-by-line + to postfix. - `Dovecot`_ - ``dovecot.conf`` in dovecot sub-directory; - `Nginx`_ - All ``*.conf`` files in the ``nginx`` sub-directory; - `Rspamd`_ - All files in the ``rspamd`` sub-directory. From ae9206e968b10de6c456a76f914f8d86dff02f6f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 10 Feb 2021 13:51:07 +0100 Subject: [PATCH 077/181] Implement a simple credential cache --- core/admin/mailu/models.py | 23 +++++++++++++++++++++++ towncrier/newsfragments/1194.feature | 1 + 2 files changed, 24 insertions(+) create mode 100644 towncrier/newsfragments/1194.feature diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a63c33a5..c7787e74 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -305,6 +305,7 @@ class User(Base, Email): """ __tablename__ = "user" _ctx = None + _credential_cache = {} domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) @@ -382,6 +383,17 @@ class User(Base, Email): return User._ctx def check_password(self, password): + cache_result = self._credential_cache.get(self.get_id()) + current_salt = self.password.split('$')[3] if len(self.password.split('$')) == 5 else None + if cache_result and current_salt: + cache_salt, cache_hash = cache_result + if cache_salt == current_salt: + return hash.pbkdf2_sha256.verify(password, cache_hash) + else: + # the cache is local per gunicorn; the password has changed + # so the local cache can be invalidated + del self._credential_cache[self.get_id()] + reference = self.password # strip {scheme} if that's something mailu has added # passlib will identify *crypt based hashes just fine @@ -396,6 +408,17 @@ class User(Base, Email): self.password = new_hash db.session.add(self) db.session.commit() + + if result: + """The credential cache uses a low number of rounds to be fast. +While it's not meant to be persisted to cold-storage, no additional measures +are taken to ensure it isn't (mlock(), encrypted swap, ...) on the basis that +we have little control over GC and string interning anyways. + + An attacker that can dump the process' memory is likely to find credentials +in clear-text regardless of the presence of the cache. + """ + self._credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) return result def set_password(self, password, hash_scheme=None, raw=False): diff --git a/towncrier/newsfragments/1194.feature b/towncrier/newsfragments/1194.feature new file mode 100644 index 00000000..0cd2a9e9 --- /dev/null +++ b/towncrier/newsfragments/1194.feature @@ -0,0 +1 @@ +Add a credential cache to speedup authentication requests. From f52984e4c337e5a101aea82a0654fd731ab94164 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 10 Feb 2021 16:10:10 +0100 Subject: [PATCH 078/181] In fact it could be global --- core/admin/mailu/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index c7787e74..a5ee1f57 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -17,7 +17,7 @@ import dns db = flask_sqlalchemy.SQLAlchemy() - +_credential_cache = {} class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) @@ -383,7 +383,7 @@ class User(Base, Email): return User._ctx def check_password(self, password): - cache_result = self._credential_cache.get(self.get_id()) + cache_result = _credential_cache.get(self.get_id()) current_salt = self.password.split('$')[3] if len(self.password.split('$')) == 5 else None if cache_result and current_salt: cache_salt, cache_hash = cache_result @@ -392,7 +392,7 @@ class User(Base, Email): else: # the cache is local per gunicorn; the password has changed # so the local cache can be invalidated - del self._credential_cache[self.get_id()] + del _credential_cache[self.get_id()] reference = self.password # strip {scheme} if that's something mailu has added @@ -418,7 +418,7 @@ we have little control over GC and string interning anyways. An attacker that can dump the process' memory is likely to find credentials in clear-text regardless of the presence of the cache. """ - self._credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) + _credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) return result def set_password(self, password, hash_scheme=None, raw=False): From 875308d40518dde4000a4f09286059e07f36de71 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 4 Jun 2021 09:51:58 +0200 Subject: [PATCH 079/181] Revert "In fact it could be global" This reverts commit f52984e4c337e5a101aea82a0654fd731ab94164. --- core/admin/mailu/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a5ee1f57..c7787e74 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -17,7 +17,7 @@ import dns db = flask_sqlalchemy.SQLAlchemy() -_credential_cache = {} + class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) @@ -383,7 +383,7 @@ class User(Base, Email): return User._ctx def check_password(self, password): - cache_result = _credential_cache.get(self.get_id()) + cache_result = self._credential_cache.get(self.get_id()) current_salt = self.password.split('$')[3] if len(self.password.split('$')) == 5 else None if cache_result and current_salt: cache_salt, cache_hash = cache_result @@ -392,7 +392,7 @@ class User(Base, Email): else: # the cache is local per gunicorn; the password has changed # so the local cache can be invalidated - del _credential_cache[self.get_id()] + del self._credential_cache[self.get_id()] reference = self.password # strip {scheme} if that's something mailu has added @@ -418,7 +418,7 @@ we have little control over GC and string interning anyways. An attacker that can dump the process' memory is likely to find credentials in clear-text regardless of the presence of the cache. """ - _credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) + self._credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) return result def set_password(self, password, hash_scheme=None, raw=False): From ffa75620799669f7fe23aeaa75e4b90d55c1fe95 Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Mon, 7 Jun 2021 07:57:30 +0200 Subject: [PATCH 080/181] configurations changed place in 1.8 --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 5b13f191..e42bf309 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -258,8 +258,8 @@ Postfix, Dovecot, Nginx and Rspamd support overriding configuration files. Overr correct syntax. The following file names will be taken as override configuration: - `Postfix`_ : - - ``main.cf`` as ``$ROOT/overrides/postfix.cf`` - - ``master.cf`` as ``$ROOT/overrides/postfix.master`` + - ``main.cf`` as ``$ROOT/overrides/postfix/postfix.cf`` + - ``master.cf`` as ``$ROOT/overrides/postfix/postfix.master`` - All ``$ROOT/overrides/postfix/*.map`` files - For both ``postfix.cf`` and ``postfix.master``, you need to put one configuration per line, as they are fed line-by-line to postfix. From 21a362fdaea022404e4f8e55fb1def73f1c24d35 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 8 Jun 2021 07:09:07 +0000 Subject: [PATCH 081/181] Changed config-update to config-import in config-import description. --- docs/cli.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 10669108..957e47e4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -212,13 +212,13 @@ mail-config.yml contains the configuration and looks like this: comment: test smtp: mx.example.com -config-update shows the number of created/modified/deleted objects after import. +config-import shows the number of created/modified/deleted objects after import. To suppress all messages except error messages use ``--quiet``. By adding the ``--verbose`` switch the import gets more detailed and shows exactly what attributes changed. In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. If you want to test what would be done when importing without committing any changes, use ``--dry-run``. -By default config-update replaces the whole configuration. ``--update`` allows to modify the existing configuration instead. +By default config-import replaces the whole configuration. ``--update`` allows to modify the existing configuration instead. New elements will be added and existing elements will be modified. It is possible to delete a single element or prune all elements from lists and associative arrays using a special notation: From 2316ef1162d400fc486ad72df62fbd1fbd46afce Mon Sep 17 00:00:00 2001 From: lub Date: Wed, 16 Jun 2021 14:21:55 +0200 Subject: [PATCH 082/181] update compression algorithms for dovecot 3.3.14 xz is deprecated; lz4 and zstd were not present in our configs before --- core/dovecot/conf/dovecot.conf | 2 +- setup/flavors/compose/mailu.env | 2 +- tests/compose/core/mailu.env | 2 +- tests/compose/fetchmail/mailu.env | 2 +- tests/compose/filters/mailu.env | 2 +- tests/compose/rainloop/mailu.env | 2 +- tests/compose/roundcube/mailu.env | 2 +- tests/compose/webdav/mailu.env | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index ab5cb43a..50657088 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -50,7 +50,7 @@ plugin { fts_autoindex_exclude = \Trash {% endif %} - {% if COMPRESSION in [ 'gz', 'bz2', 'xz', 'lz4' ] %} + {% if COMPRESSION in [ 'gz', 'bz2', 'lz4', 'zstd' ] %} zlib_save = {{ COMPRESSION }} {% endif %} diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 04148b40..0aabf478 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -86,7 +86,7 @@ WELCOME_SUBJECT={{ welcome_subject or 'Welcome to your new email account' }} WELCOME_BODY={{ welcome_body or 'Welcome to your new email account, if you can read this, then it is configured properly!' }} # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION={{ compression }} # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL={{ compression_level }} diff --git a/tests/compose/core/mailu.env b/tests/compose/core/mailu.env index edea6a5c..a78515b8 100644 --- a/tests/compose/core/mailu.env +++ b/tests/compose/core/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/fetchmail/mailu.env b/tests/compose/fetchmail/mailu.env index 4a53ec46..afb57751 100644 --- a/tests/compose/fetchmail/mailu.env +++ b/tests/compose/fetchmail/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/filters/mailu.env b/tests/compose/filters/mailu.env index 1106deb0..4c8c219d 100644 --- a/tests/compose/filters/mailu.env +++ b/tests/compose/filters/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/rainloop/mailu.env b/tests/compose/rainloop/mailu.env index d02b98f2..08b0f8a4 100644 --- a/tests/compose/rainloop/mailu.env +++ b/tests/compose/rainloop/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/roundcube/mailu.env b/tests/compose/roundcube/mailu.env index e1005487..faf1198f 100644 --- a/tests/compose/roundcube/mailu.env +++ b/tests/compose/roundcube/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/webdav/mailu.env b/tests/compose/webdav/mailu.env index 58b9810a..939f453b 100644 --- a/tests/compose/webdav/mailu.env +++ b/tests/compose/webdav/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= From 40ad3ca0325b6920128624665bf0164670f6e31d Mon Sep 17 00:00:00 2001 From: lub Date: Wed, 16 Jun 2021 14:56:53 +0200 Subject: [PATCH 083/181] only load zlib when compression is used --- core/dovecot/conf/dovecot.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 50657088..6b97a086 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -21,7 +21,10 @@ mail_access_groups = mail maildir_stat_dirs = yes mailbox_list_index = yes mail_vsize_bg_after_count = 100 -mail_plugins = $mail_plugins quota quota_clone zlib{{ ' ' }} +mail_plugins = $mail_plugins quota quota_clone{{ ' ' }} + {%- if COMPRESSION -%} + zlib{{ ' ' }} + {%- endif %} {%- if (FULL_TEXT_SEARCH or '').lower() not in ['off', 'false', '0'] -%} fts fts_xapian {%- endif %} From 18f5a2fc11191168c436e862f8f060ba4aeed9c0 Mon Sep 17 00:00:00 2001 From: lub Date: Wed, 16 Jun 2021 15:01:55 +0200 Subject: [PATCH 084/181] update newsfragment #1694 --- towncrier/newsfragments/1694.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/towncrier/newsfragments/1694.feature b/towncrier/newsfragments/1694.feature index 41548707..f7e2013e 100644 --- a/towncrier/newsfragments/1694.feature +++ b/towncrier/newsfragments/1694.feature @@ -1 +1 @@ -Support configuring xz and lz4 compression for dovecot. +Support configuring lz4 and zstd compression for dovecot. From 587901ca51462bf32a4d6df314667a0081a36378 Mon Sep 17 00:00:00 2001 From: lub Date: Wed, 16 Jun 2021 15:03:09 +0200 Subject: [PATCH 085/181] fix comment in compose .env --- docs/compose/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compose/.env b/docs/compose/.env index b4a8b218..27822c37 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -97,7 +97,7 @@ WELCOME_SUBJECT=Welcome to your new email account WELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly! # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= From a1fd44fced2c65a1c379b445fb65f2a5a24efd24 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 16 Jun 2021 16:19:31 +0200 Subject: [PATCH 086/181] added lmtp: prefix and documentation --- core/admin/mailu/internal/views/postfix.py | 39 ++++-- docs/webadministration.rst | 139 +++++++++++---------- 2 files changed, 101 insertions(+), 77 deletions(-) diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index 7f8418cf..c358c37f 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -36,15 +36,23 @@ def postfix_alias_map(alias): def postfix_transport(email): if email == '*' or re.match("(^|.*@)\[.*\]$", email): return flask.abort(404) - localpart, domain_name = models.Email.resolve_domain(email) + _, domain_name = models.Email.resolve_domain(email) relay = models.Relay.query.get(domain_name) or flask.abort(404) target = relay.smtp.lower() port = None - if use_mx := target.startswith('mx:'): + use_lmtp = False + use_mx = False + # strip prefixes mx: and lmtp: + if target.startswith('mx:'): target = target[3:] + use_mx = True + elif target.startswith('lmtp:'): + target = target[5:] + use_lmtp = True + # split host:port or [host]:port if target.startswith('['): if use_mx or ']' not in target: - # invalid target (mx: and []) + # invalid target (mx: and [] or missing ]) flask.abort(400) host, rest = target[1:].split(']', 1) if rest.startswith(':'): @@ -57,29 +65,38 @@ def postfix_transport(email): host, port = target.rsplit(':', 1) else: host = target + # default for empty host part is mx:domain if not host: - host = relay.name.lower() - use_mx = True + if not use_lmtp: + host = relay.name.lower() + use_mx = True + else: + # lmtp: needs a host part + flask.abort(400) + # detect ipv6 address or encode host if ':' in host: host = f'ipv6:{host}' else: try: host = idna.encode(host).decode('ascii') except idna.IDNAError: - # invalid target (fqdn not encodable) + # invalid host (fqdn not encodable) flask.abort(400) + # validate port if port is not None: try: port = int(port, 10) - if port == 25: - port = None except ValueError: - # invalid target (port should be numeric) + # invalid port (should be numeric) flask.abort(400) - if not use_mx: + # create transport + transport = 'lmtp' if use_lmtp else 'smtp' + # use [] when not using MX lookups or host is an ipv6 address + if host.startswith('ipv6:') or (not use_lmtp and not use_mx): host = f'[{host}]' + # create port suffix port = '' if port is None else f':{port}' - return flask.jsonify(f'smtp:{host}{port}') + return flask.jsonify(f'{transport}:{host}{port}') @internal.route("/postfix/recipient/map/") diff --git a/docs/webadministration.rst b/docs/webadministration.rst index 070eb473..86ce41c0 100644 --- a/docs/webadministration.rst +++ b/docs/webadministration.rst @@ -1,7 +1,7 @@ Web administration interface ============================ -The web administration interface is the main website for maintaining your Mailu installation. +The web administration interface is the main website for maintaining your Mailu installation. For brevity the web administration interface will now be mentioned as admin gui. It offers the following configuration options: @@ -30,13 +30,13 @@ It offers the following configuration options: * Configure all email domains served by Mailu, including: * generating dkim and dmarc keys for a domain. - + * view email domain information on how to configure your SPF, DMARC, DKIM and MX dns records for an email domain. - + * Add new email domains. - + * For existing domains, configure users, quotas, aliases, administrators and alternative domain names. - + * access the webmail site. * lookup settings for configuring your email client. @@ -49,7 +49,7 @@ The admin GUI is by default accessed via the URL `https:///admin`, wh To login the admin GUI enter the email address and password of an user. Only global administrator users have access to all configuration settings and the Rspamd webgui. Other users will be presented with settings for only their account, and domains they are managers of. -To create a user who is a global administrator for a new installation, the Mailu.env file can be adapted. +To create a user who is a global administrator for a new installation, the Mailu.env file can be adapted. For more information see the section 'Admin account - automatic creation' in :ref:`the configuration reference `. The following sections are only accessible for global administrators: @@ -69,7 +69,7 @@ The following sections are only accessible for global administrators: Settings -------- -After logging in the web administration interface, the settings page is loaded. +After logging in the web administration interface, the settings page is loaded. On the settings page the settings of the currently logged in user can be changed. Changes are saved and effective immediately after clicking the Save Settings button at the bottom of the page. @@ -77,27 +77,27 @@ Changes are saved and effective immediately after clicking the Save Settings but Display name ```````````` -On the settings page the displayed name can be changed of the logged in user. +On the settings page the displayed name can be changed of the logged in user. This display name is only used within the web administration interface. Antispam ```````` -Under the section `Antispam` the spam filter can be enabled or disabled for the logged in user. By default the spam filter is enabled. +Under the section `Antispam` the spam filter can be enabled or disabled for the logged in user. By default the spam filter is enabled. When the spam filter is disabled, all received email messages will go to the inbox folder of the logged in user. The exception to this rule, are email messages with an extremely high spam score. These email messages are always rejected by Rspamd. When the spam filter is enabled, received email messages will be moved to the logged in user's inbox folder or junk folder depending on the user defined spam filter tolerance. -The user defined spam filter tolerance determines when an email is classified as ham (moved to the inbox folder) or spam (moved to the junk folder). -The default value is 80%. The lower the spam filter tolerance, the more false positives (ham classified as spam). The higher the spam filter tolerance, the more false negatives (spam classified as ham). +The user defined spam filter tolerance determines when an email is classified as ham (moved to the inbox folder) or spam (moved to the junk folder). +The default value is 80%. The lower the spam filter tolerance, the more false positives (ham classified as spam). The higher the spam filter tolerance, the more false negatives (spam classified as ham). For more information see the :ref:`antispam documentation `. Auto-forward ````````````` -Under the section `Auto-forward`, the automatic forwarding of received email messages can be enabled. When enabled, all received email messages are forwarded to the specified email address. +Under the section `Auto-forward`, the automatic forwarding of received email messages can be enabled. When enabled, all received email messages are forwarded to the specified email address. The option "Keep a copy of the emails" can be ticked, to keep a copy of the received email message in the inbox folder. @@ -107,7 +107,7 @@ In the destination textbox, the email addresses can be entered for automatic for Update password --------------- -On the `update password` page, the password of the logged in user can be changed. Changes are effective immediately. +On the `update password` page, the password of the logged in user can be changed. Changes are effective immediately. .. _webadministration_auto-reply: @@ -117,7 +117,7 @@ Auto-reply On the `auto-reply` page, automatic replies can be configured. This is also known as out of office (ooo) or out of facility (oof) replies. -To enable automatic replies tick the checkbox 'Enable automatic reply'. +To enable automatic replies tick the checkbox 'Enable automatic reply'. Under Reply subject the email subject for automatic replies can be configured. When a reply subject is entered, this subject will be used for the automatic reply. @@ -130,12 +130,12 @@ E.g. if the email subject of the received email message is "how are you?", then Fetched accounts ---------------- -This page is only available when the Fetchmail container is part of your Mailu deployment. +This page is only available when the Fetchmail container is part of your Mailu deployment. Fetchmail can be enabled when creating the docker-compose.yml file with the setup utility (https://setup.mailu.io). On the `fetched accounts` page you can configure email accounts from which email messages will be retrieved. -Only unread email messages are retrieved from the specified email account. -By default Fetchmail will retrieve email messages every 10 minutes. This can be changed in the Mailu.env file. +Only unread email messages are retrieved from the specified email account. +By default Fetchmail will retrieve email messages every 10 minutes. This can be changed in the Mailu.env file. For more information on changing the polling interval see :ref:`the configuration reference `. @@ -149,7 +149,7 @@ You can add a fetched account by clicking on the `Add an account` button on the * Enable TLS. Tick this setting if the email server requires TLS/SSL instead of STARTTLS. -* Username. The user name for logging in to the email server. Normally this is the email address or the email address' local-part (the part before @). +* Username. The user name for logging in to the email server. Normally this is the email address or the email address' local-part (the part before @). * Password. The password for logging in to the email server. @@ -166,8 +166,8 @@ The purpose of an authentication token is to create a unique and strong password The application will use this authentication token instead of the logged in user's password for sending/receiving email. This allows safe access to the logged in user's email account. At any moment, the authentication token can be deleted so that the application has no access to the logged in user's email account anymore. -By clicking on the New token button on the top right of the page, a new authentication token can be created. On this page the generated authentication token will only be displayed once. -After saving the application token it is not possible anymore to view the unique password. +By clicking on the New token button on the top right of the page, a new authentication token can be created. On this page the generated authentication token will only be displayed once. +After saving the application token it is not possible anymore to view the unique password. The comment field can be used to enter a description for the authentication token. For example the name of the application the application token is created for. @@ -198,9 +198,9 @@ A global administrator can change `any setting` in the admin GUI. Be careful tha Relayed domains --------------- -On the `relayed domains list` page, destination domains can be added that Mailu will relay email messages for without authentication. -This means that for these destination domains, other email clients or email servers can send email via Mailu unauthenticated via port 25 to this destination domain. -For example if the destination domain example.com is added. Any emails to example.com (john@example.com) will be relayed to example.com. +On the `relayed domains list` page, destination domains can be added that Mailu will relay email messages for without authentication. +This means that for these destination domains, other email clients or email servers can send email via Mailu unauthenticated via port 25 to this destination domain. +For example if the destination domain example.com is added. Any emails to example.com (john@example.com) will be relayed to example.com. Example scenario's are: * relay domain from a backup server. @@ -212,30 +212,37 @@ Example scenario's are: On the new relayed domain page the following options can be entered for a new relayed domain: -* Relayed domain name. The domain name that is relayed. Email messages addressed to this domain (To: John@example.com), will be forwarded to this domain. - No authentication is required. +* Relayed domain name. The domain name that is relayed. Email messages addressed to this domain (To: John@example.com), will be forwarded to this domain. + No authentication is required. -* Remote host (optional). The SMPT server that will be used for relaying the email message. - When this field is blank, the Mailu server will directly send the email message to the relayed domain. - As value can be entered either a hostname or IP address of the SMPT server. - By default port 25 is used. To use a different port append ":port number" to the Remote Host. For example: - 123.45.67.90:2525. +* Remote host (optional). The host that will be used for relaying the email message. + When this field is blank, the Mailu server will directly send the email message to the mail server of the relayed domain. + When a remote host is specified it can be prefixed by ``mx:`` or ``lmtp:`` and followed by a port number: ``:port``). + + ================ ===================================== ========================= + Remote host Description postfix transport:nexthop + ================ ===================================== ========================= + empty use MX of relay domain smtp:domain + :port use MX of relay domain and use port smtp:domain:port + target resolve A/AAAA of target smtp:[target] + target:port resolve A/AAAA of target and use port smtp:[target]:port + mx:target resolve MX of target smtp:target + mx:target:port resolve MX of target and use port smtp:target:port + lmtp:target resolve A/AAAA of target lmtp:target + lmtp:target:port resolve A/AAAA of target and use port lmtp:target:port + ================ ===================================== ========================= + + `target` can also be an IPv4 or IPv6 address (an IPv6 address must be enclosed in []: ``[2001:DB8::]``). * Comment. A text field where a comment can be entered to describe the entry. Changes are effective immediately after clicking the Save button. -NOTE: Due to bug `1588`_ email messages fail to be relayed if no Remote Host is configured. -As a workaround the HOSTNAME or IP Address of the SMPT server of the relayed domain can be entered as Remote Host. -Please note that no MX lookup is performed when entering a hostname as Remote Host. You can use the MX lookup on mxtoolbox.com to find the hostname and IP Address of the SMTP server. - -.. _`1588`: https://github.com/Mailu/Mailu/issues/1588 - Antispam -------- The menu item Antispam opens the Rspamd webgui. For more information how spam filtering works in Mailu see the :ref:`Spam filtering page `. -The spam filtering page also contains a section that describes how to create a local blacklist for blocking email messages from specific domains. +The spam filtering page also contains a section that describes how to create a local blacklist for blocking email messages from specific domains. The Rspamd webgui offers basic functions for setting metric actions, scores, viewing statistics and learning. The following settings are not persisent and are *lost* when the Antispam container is recreated or restarted: @@ -266,31 +273,31 @@ On the `Mail domains` page all the domains served by Mailu are configured. Via t Details ``````` -This page is also accessible for domain managers. On the details page all DNS settings are displayed for configuring your DNS server. It contains information on what to configure as MX record and SPF record. On this page it is also possible to (re-)generate the keys for DKIM and DMARC. The option for generating keys for DKIM and DMARC is only available for global administrators. After generating the keys for DKIM and DMARC, this page will also show the DNS records for configuring the DKIM/DMARC records on the DNS server. +This page is also accessible for domain managers. On the details page all DNS settings are displayed for configuring your DNS server. It contains information on what to configure as MX record and SPF record. On this page it is also possible to (re-)generate the keys for DKIM and DMARC. The option for generating keys for DKIM and DMARC is only available for global administrators. After generating the keys for DKIM and DMARC, this page will also show the DNS records for configuring the DKIM/DMARC records on the DNS server. Edit -```` +```` -This page is only accessible for global administrators. On the edit page, the global settings for the domain can be changed. +This page is only accessible for global administrators. On the edit page, the global settings for the domain can be changed. * Maximum user count. The maximum amount of users that can be created under this domain. Once this limit is reached it is not possible anymore to add users to the domain; and it is also not possible for users to self-register. - + * Maximum alias count. The maximum amount of aliases that can be created for an email account. - + * Maximum user quota. The maximum amount of quota that can be assigned to a user. When creating or editing a user, this sets the limit on the maximum amount of quota that can be assigned to the user. - -* Enable sign-up. When this option is ticked, self-registration is enabled. When the Admin GUI is accessed, in the menu list the option Signup becomes available. - Obviously this menu item is only visible when signed out. On the Signup page a user can create an email account. - If your Admin GUI is available to the public internet, this means your Mailu installation basically becomes a free email provider. + +* Enable sign-up. When this option is ticked, self-registration is enabled. When the Admin GUI is accessed, in the menu list the option Signup becomes available. + Obviously this menu item is only visible when signed out. On the Signup page a user can create an email account. + If your Admin GUI is available to the public internet, this means your Mailu installation basically becomes a free email provider. Use this option with care! - + * Comment. Description for the domain. This description is visible on the parent domains list page. Delete `````` -This page is only accessible for global administrators. This page allows you to delete the domain. The Admin GUI will ask for confirmation if the domain must be really deleted. +This page is only accessible for global administrators. This page allows you to delete the domain. The Admin GUI will ask for confirmation if the domain must be really deleted. Users @@ -326,7 +333,7 @@ For adding a new user the following options can be configured. * Enabled. Tick this checkbox to enable the user account. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail. The email inbox of the user is still retained. This option can be used to temporarily suspend an user account. - + * Quota. The maximum quota for the user's email box. * Allow IMAP access. When ticked, allows email retrieval via the IMAP protocol. @@ -337,7 +344,7 @@ For adding a new user the following options can be configured. Aliases ``````` -This page is also accessible for domain managers. On the aliases page, aliases can be added for email addresses. An alias is a way to disguise another email address. +This page is also accessible for domain managers. On the aliases page, aliases can be added for email addresses. An alias is a way to disguise another email address. Everything sent to an alias email address is actually received in the primary email account's inbox of the destination email address. Aliases can diversify a single email account without having to create multiple email addresses (users). It is also possible to add multiple email addresses to the destination field. All incoming mails will be sent to each users inbox in this case. @@ -348,11 +355,11 @@ The following options are available when adding an alias: * Use SQL LIKE Syntax (e.g. for catch-all aliases). When this option is ticked, you can use SQL LIKE syntax as alias. The SQL LIKE syntax is used to match text values against a pattern using wildcards. There are two wildcards that can be used with SQL LIKE syntax: - + * % - The percent sign represents zero, one, or multiple characters * _ - The underscore represents a single character - - Examples are: + + Examples are: * a% - Finds any values that start with "a" * %a - Finds any values that end with "a" * %or% - Finds any values that have "or" in any position @@ -369,7 +376,7 @@ The following options are available when adding an alias: Managers ```````` -This page is also accessible for domain managers. On the `managers list` page, managers can be added for the domain and can be deleted. +This page is also accessible for domain managers. On the `managers list` page, managers can be added for the domain and can be deleted. Managers have access to configuration settings of the domain. On the `add manager` page you can click on the manager email text box to access a drop down list of users that can be made a manager of the domain. @@ -377,11 +384,11 @@ On the `add manager` page you can click on the manager email text box to access Alternatives ```````````` -This page is only accessible for global administrators. On the alternatives page, alternative domains can be added for the domain. +This page is only accessible for global administrators. On the alternatives page, alternative domains can be added for the domain. An alternative domain acts as a copy of a given domain. -Everything sent to an alternative domain, is actually received in the domain the alternative is created for. -This allows you to receive emails for multiple domains while using a single domain. -For example if the main domain has the email address user@example.com, and the alternative domain is mymail.com, +Everything sent to an alternative domain, is actually received in the domain the alternative is created for. +This allows you to receive emails for multiple domains while using a single domain. +For example if the main domain has the email address user@example.com, and the alternative domain is mymail.com, then email send to user@mymail.com will end up in the email box of user@example.com. New domain @@ -392,16 +399,16 @@ This page is only accessible for global administrators. Via this page a new doma * domain name. The name of the domain. * Maximum user count. The maximum amount of users that can be created under this domain. Once this limit is reached it is not possible anymore to add users to the domain; and it is also not possible for users to self-register. - + * Maximum alias count. The maximum amount of aliases that can be made for an email account. - + * Maximum user quota. The maximum amount of quota that can be assigned to a user. When creating or editing a user, this sets the limit on the maximum amount of quota that can be assigned to the user. - -* Enable sign-up. When this option is ticked, self-registration is enabled. When the Admin GUI is accessed, in the menu list the option Signup becomes available. - Obviously this menu item is only visible when signed out. On the Signup page a user can create an email account. - If your Admin GUI is available to the public internet, this means your Mailu installation basically becomes a free email provider. + +* Enable sign-up. When this option is ticked, self-registration is enabled. When the Admin GUI is accessed, in the menu list the option Signup becomes available. + Obviously this menu item is only visible when signed out. On the Signup page a user can create an email account. + If your Admin GUI is available to the public internet, this means your Mailu installation basically becomes a free email provider. Use this option with care! - + * Comment. Description for the domain. This description is visible on the parent domains list page. @@ -414,7 +421,7 @@ The menu item `Webmail` opens the webmail page. This option is only available if Client setup ------------ -The menu item `Client setup` shows all settings for configuring your email client for connecting to Mailu. +The menu item `Client setup` shows all settings for configuring your email client for connecting to Mailu. Website From 9ef8aaf6989376480b0ecd6c4c6fc2f98a7af408 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 16 Jun 2021 22:06:28 +0200 Subject: [PATCH 087/181] removed double confiog and fixed shaker --- core/admin/mailu/configuration.py | 1 - core/admin/mailu/utils.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 679c6c7e..3d1b4fb5 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -140,7 +140,6 @@ class ConfigManager(dict): self.config['SESSION_STORAGE_URL'] = 'redis://{0}/3'.format(self.config['REDIS_ADDRESS']) self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True - self.config['SESSION_KEY_BITS'] = 128 self.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=int(self.config['SESSION_LIFETIME'])) # update the app config itself app.config = self diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 30725ff7..214a9a2d 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -260,9 +260,8 @@ class MailuSessionConfig: hash_bytes = bits//8 + (bits%8>0) time_bytes = 4 # 32 bit timestamp for now - shaker = hashlib.shake_256 if bits>128 else hashlib.shake_128 - self._shaker = shaker(want_bytes(app.config.get('SECRET_KEY', ''))) + self._shaker = hashlib.shake_128(want_bytes(app.config.get('SECRET_KEY', ''))) self._hash_len = hash_bytes self._hash_b64 = len(self._encode(bytes(hash_bytes))) self._key_min = 2*self._hash_b64 From 3f23e199f6b64bb4960d8368a4c670f16c5f705b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 17 Jun 2021 17:53:15 +0200 Subject: [PATCH 088/181] modified generation of session key and added refresh - the session key is now generated using - a hash of the uid seeded by the apps secret_key (size: SESSION_KEY_BITS) - a random token (size: 128 bits) - the session's creation time (size: 32 bits) - redis server side sessions are now refreshed after 1/2 the session lifetime even if not modified - the cookie is also updated if necessary --- core/admin/mailu/utils.py | 96 +++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 214a9a2d..c7e1f73c 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -6,7 +6,7 @@ try: except ImportError: import pickle -import hashlib +import hmac import secrets import time @@ -218,13 +218,16 @@ class MailuSession(CallbackDict, SessionMixin): def save(self): """ Save session to store. """ + set_cookie = False + # set uid from dict data if self._uid is None: self._uid = self.app.session_config.gen_uid(self.get('user_id', '')) - # create new session id for new or regenerated sessions + # create new session id for new or regenerated sessions and force setting the cookie if self._sid is None: self._sid = self.app.session_config.gen_sid() + set_cookie = True # get new session key key = self.sid @@ -233,6 +236,9 @@ class MailuSession(CallbackDict, SessionMixin): if key != self._key: self.delete() + # remember time to refresh + self['_refresh'] = int(time.time()) + self.app.permanent_session_lifetime.total_seconds()/2 + # save session self.app.session_store.put( key, @@ -245,37 +251,52 @@ class MailuSession(CallbackDict, SessionMixin): self.new = False self.modified = False + return set_cookie + + def needs_refresh(self): + """ Checks if server side session needs to be refreshed. """ + + return int(time.time()) > self.get('_refresh', 0) + class MailuSessionConfig: """ Stores sessions crypto config """ + # default size of session key parts + uid_bits = 64 # default if SESSION_KEY_BITS is not set in config + sid_bits = 128 # for now. must be multiple of 8! + time_bits = 32 # for now. must be multiple of 8! + def __init__(self, app=None): if app is None: app = flask.current_app - bits = app.config.get('SESSION_KEY_BITS', 64) + bits = app.config.get('SESSION_KEY_BITS', self.uid_bits) + if not 64 <= bits <= 256: + raise ValueError('SESSION_KEY_BITS must be between 64 and 256!') - if bits < 64: - raise ValueError('Session id entropy must not be less than 64 bits!') + uid_bytes = bits//8 + (bits%8>0) + sid_bytes = self.sid_bits//8 - hash_bytes = bits//8 + (bits%8>0) - time_bytes = 4 # 32 bit timestamp for now + key = want_bytes(app.secret_key) - self._shaker = hashlib.shake_128(want_bytes(app.config.get('SECRET_KEY', ''))) - self._hash_len = hash_bytes - self._hash_b64 = len(self._encode(bytes(hash_bytes))) - self._key_min = 2*self._hash_b64 - self._key_max = self._key_min + len(self._encode(bytes(time_bytes))) + self._hmac = hmac.new(hmac.digest(key, key, digest='sha256'), digestmod='sha256') + self._uid_len = uid_bytes + self._uid_b64 = len(self._encode(bytes(uid_bytes))) + self._sid_len = sid_bytes + self._sid_b64 = len(self._encode(bytes(sid_bytes))) + self._key_min = self._uid_b64 + self._sid_b64 + self._key_max = self._key_min + len(self._encode(bytes(self.time_bits//8))) def gen_sid(self): """ Generate random session id. """ - return self._encode(secrets.token_bytes(self._hash_len)) + return self._encode(secrets.token_bytes(self._sid_len)) def gen_uid(self, uid): """ Generate hashed user id part of session key. """ - shaker = self._shaker.copy() - shaker.update(want_bytes(uid)) - return self._encode(shaker.digest(self._hash_len)) + _hmac = self._hmac.copy() + _hmac.update(want_bytes(uid)) + return self._encode(_hmac.digest()[:self._uid_len]) def gen_created(self, now=None): """ Generate base64 representation of creation time. """ @@ -287,8 +308,8 @@ class MailuSessionConfig: if not (isinstance(key, bytes) and self._key_min <= len(key) <= self._key_max): return None - uid = key[:self._hash_b64] - sid = key[self._hash_b64:self._key_min] + uid = key[:self._uid_b64] + sid = key[self._uid_b64:self._key_min] crt = key[self._key_min:] # validate if parts are decodeable @@ -301,7 +322,7 @@ class MailuSessionConfig: if now is None: now = int(time.time()) created = int.from_bytes(created, byteorder='big') - if not (created < now < created + app.permanent_session_lifetime.total_seconds()): + if not created < now < created + app.permanent_session_lifetime.total_seconds(): return None return (uid, sid, crt) @@ -341,24 +362,29 @@ class MailuSessionInterface(SessionInterface): if session.accessed: response.vary.add('Cookie') - # TODO: set cookie from time to time to prevent expiration in browser - # also update expire in redis + set_cookie = session.permanent and app.config['SESSION_REFRESH_EACH_REQUEST'] + need_refresh = session.needs_refresh() - if not self.should_set_cookie(app, session): - return + # save modified session or refresh unmodified session + if session.modified or need_refresh: + set_cookie |= session.save() - # save session and update cookie - session.save() - response.set_cookie( - app.session_cookie_name, - session.sid, - expires=self.get_expiration_time(app, session), - httponly=self.get_cookie_httponly(app), - domain=self.get_cookie_domain(app), - path=self.get_cookie_path(app), - secure=self.get_cookie_secure(app), - samesite=self.get_cookie_samesite(app) - ) + # set cookie on refreshed permanent sessions + if need_refresh and session.permanent: + set_cookie = True + + # set or update cookie if necessary + if set_cookie: + response.set_cookie( + app.session_cookie_name, + session.sid, + expires=self.get_expiration_time(app, session), + httponly=self.get_cookie_httponly(app), + domain=self.get_cookie_domain(app), + path=self.get_cookie_path(app), + secure=self.get_cookie_secure(app), + samesite=self.get_cookie_samesite(app) + ) class MailuSessionExtension: """ Server side session handling """ From 49c5c0eba691791c1cc7e4e1b0b4746b38f57074 Mon Sep 17 00:00:00 2001 From: parisni Date: Fri, 18 Jun 2021 23:17:35 +0200 Subject: [PATCH 089/181] Split mailu / roundcube db config There is no reason to share the flavor since at least the dbname shall be different. --- webmails/roundcube/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 649f3324..36502eb6 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -10,7 +10,7 @@ log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576)) -db_flavor=os.environ.get("ROUNDCUBE_DB_FLAVOR",os.environ.get("DB_FLAVOR","sqlite")) +db_flavor=os.environ.get("ROUNDCUBE_DB_FLAVOR","sqlite") if db_flavor=="sqlite": os.environ["DB_DSNW"]="sqlite:////data/roundcube.db" elif db_flavor=="mysql": From 5386e33af303af9e46f18fd7e0c6379f37bd3858 Mon Sep 17 00:00:00 2001 From: parisni Date: Fri, 18 Jun 2021 23:21:24 +0200 Subject: [PATCH 090/181] Reformat python --- webmails/roundcube/start.py | 43 ++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 36502eb6..3e47ce69 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -8,31 +8,29 @@ import subprocess log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) -os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576)) +os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT")) * 0.66 / 1048576)) -db_flavor=os.environ.get("ROUNDCUBE_DB_FLAVOR","sqlite") -if db_flavor=="sqlite": - os.environ["DB_DSNW"]="sqlite:////data/roundcube.db" -elif db_flavor=="mysql": - os.environ["DB_DSNW"]="mysql://%s:%s@%s/%s" % ( - os.environ.get("ROUNDCUBE_DB_USER","roundcube"), +db_flavor = os.environ.get("ROUNDCUBE_DB_FLAVOR", "sqlite") +if db_flavor == "sqlite": + os.environ["DB_DSNW"] = "sqlite:////data/roundcube.db" +elif db_flavor == "mysql": + os.environ["DB_DSNW"] = "mysql://%s:%s@%s/%s" % ( + os.environ.get("ROUNDCUBE_DB_USER", "roundcube"), os.environ.get("ROUNDCUBE_DB_PW"), - os.environ.get("ROUNDCUBE_DB_HOST",os.environ.get("DB_HOST","database")), - os.environ.get("ROUNDCUBE_DB_NAME","roundcube") - ) -elif db_flavor=="postgresql": - os.environ["DB_DSNW"]="pgsql://%s:%s@%s/%s" % ( - os.environ.get("ROUNDCUBE_DB_USER","roundcube"), + os.environ.get("ROUNDCUBE_DB_HOST", os.environ.get("DB_HOST", "database")), + os.environ.get("ROUNDCUBE_DB_NAME", "roundcube") + ) +elif db_flavor == "postgresql": + os.environ["DB_DSNW"] = "pgsql://%s:%s@%s/%s" % ( + os.environ.get("ROUNDCUBE_DB_USER", "roundcube"), os.environ.get("ROUNDCUBE_DB_PW"), - os.environ.get("ROUNDCUBE_DB_HOST",os.environ.get("DB_HOST","database")), - os.environ.get("ROUNDCUBE_DB_NAME","roundcube") - ) + os.environ.get("ROUNDCUBE_DB_HOST", os.environ.get("DB_HOST", "database")), + os.environ.get("ROUNDCUBE_DB_NAME", "roundcube") + ) else: - print("Unknown ROUNDCUBE_DB_FLAVOR: %s",db_flavor) + print("Unknown ROUNDCUBE_DB_FLAVOR: %s", db_flavor) exit(1) - - conf.jinja("/php.ini", os.environ, "/usr/local/etc/php/conf.d/roundcube.ini") # Create dirs, setup permissions @@ -42,7 +40,8 @@ os.system("chown -R www-data:www-data /var/www/html/logs") try: print("Initializing database") - result=subprocess.check_output(["/var/www/html/bin/initdb.sh","--dir","/var/www/html/SQL"],stderr=subprocess.STDOUT) + result = subprocess.check_output(["/var/www/html/bin/initdb.sh", "--dir", "/var/www/html/SQL"], + stderr=subprocess.STDOUT) print(result.decode()) except subprocess.CalledProcessError as e: if "already exists" in e.stdout.decode(): @@ -53,7 +52,7 @@ except subprocess.CalledProcessError as e: try: print("Upgrading database") - subprocess.check_call(["/var/www/html/bin/update.sh","--version=?","-y"],stderr=subprocess.STDOUT) + subprocess.check_call(["/var/www/html/bin/update.sh", "--version=?", "-y"], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: quit(1) @@ -61,7 +60,7 @@ except subprocess.CalledProcessError as e: os.system("chown -R www-data:www-data /data") # Tail roundcube logs -subprocess.Popen(["tail","-f","-n","0","/var/www/html/logs/errors.log"]) +subprocess.Popen(["tail", "-f", "-n", "0", "/var/www/html/logs/errors.log"]) # Run apache os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"]) From 278878d48db01ee581014dd8cc0d102f43fb12c4 Mon Sep 17 00:00:00 2001 From: parisni Date: Fri, 18 Jun 2021 23:36:14 +0200 Subject: [PATCH 091/181] Remove unused deps --- optional/postgresql/start.py | 1 - 1 file changed, 1 deletion(-) diff --git a/optional/postgresql/start.py b/optional/postgresql/start.py index 1f2f2a2b..d318d4d9 100755 --- a/optional/postgresql/start.py +++ b/optional/postgresql/start.py @@ -2,7 +2,6 @@ import anosql import psycopg2 -import jinja2 import glob import os import subprocess From d2803f6f4613df2948c19931793eede3a9c72362 Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 00:38:53 +0200 Subject: [PATCH 092/181] Update setup website --- optional/postgresql/start.py | 1 - setup/flavors/compose/mailu.env | 6 ++++++ setup/static/render.js | 21 +++++++++++++++++++++ setup/templates/steps/database.html | 14 +++++++++++++- 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/optional/postgresql/start.py b/optional/postgresql/start.py index d318d4d9..e34e157e 100755 --- a/optional/postgresql/start.py +++ b/optional/postgresql/start.py @@ -37,7 +37,6 @@ if not os.listdir("/data"): rec.write("restore_command = 'gunzip < /backup/wal_archive/%f > %p'\n") rec.write("standby_mode = off\n") os.system("chown postgres:postgres /data/recovery.conf") - #os.system("sudo -u postgres pg_ctl start -D /data -o '-h \"''\" '") else: # Bootstrap the database os.system("sudo -u postgres initdb -D /data") diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 44452e36..150c70a3 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -175,3 +175,9 @@ DB_HOST={{ db_url }} DB_NAME={{ db_name }} {% endif %} +{% if (postgresql == 'external' or db_flavor == 'mysql') and webmail_type == 'roundcube' %} +ROUNDCUBE_DB_USER={{ roundcube_db_user }} +ROUNDCUBE_DB_PW={{ roundcube_db_pw }} +ROUNDCUBE_DB_HOST={{ roundcube_db_url }} +ROUNDCUBE_DB_NAME={{ roundcube_db_name }} +{% endif %} diff --git a/setup/static/render.js b/setup/static/render.js index a817c4f0..0a0a6675 100644 --- a/setup/static/render.js +++ b/setup/static/render.js @@ -57,6 +57,13 @@ $(document).ready(function() { $("#db_pw").prop('required',true); $("#db_url").prop('required',true); $("#db_name").prop('required',true); + if ($("#webmail").val() == 'roundcube') { + $("#roundcube_external_db").show(); + $("#roundcube_db_user").prop('required',true); + $("#roundcube_db_pw").prop('required',true); + $("#roundcube_db_url").prop('required',true); + $("#roundcube_db_name").prop('required',true); + } } else if (this.value == 'mysql') { $("#postgres_db").hide(); $("#external_db").show(); @@ -64,6 +71,13 @@ $(document).ready(function() { $("#db_pw").prop('required',true); $("#db_url").prop('required',true); $("#db_name").prop('required',true); + if ($("#webmail").val() == 'roundcube') { + $("#roundcube_external_db").show(); + $("#roundcube_db_user").prop('required',true); + $("#roundcube_db_pw").prop('required',true); + $("#roundcube_db_url").prop('required',true); + $("#roundcube_db_name").prop('required',true); + } } }); $("#external_psql").change(function() { @@ -73,6 +87,13 @@ $(document).ready(function() { $("#db_pw").prop('required',true); $("#db_url").prop('required',true); $("#db_name").prop('required',true); + if ($("#webmail").val() == 'roundcube') { + $("#roundcube_external_db").show(); + $("#roundcube_db_user").prop('required',true); + $("#roundcube_db_pw").prop('required',true); + $("#roundcube_db_url").prop('required',true); + $("#roundcube_db_name").prop('required',true); + } } else { $("#external_db").hide(); } diff --git a/setup/templates/steps/database.html b/setup/templates/steps/database.html index ad5411ab..d7184110 100644 --- a/setup/templates/steps/database.html +++ b/setup/templates/steps/database.html @@ -28,7 +28,7 @@
From 84e59c0a6e4d84b1a91c8e84293a7abd12259f6d Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 01:22:23 +0200 Subject: [PATCH 093/181] Add missing roundcube_db_flavor --- setup/flavors/compose/mailu.env | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 150c70a3..4dce34f1 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -176,6 +176,7 @@ DB_NAME={{ db_name }} {% endif %} {% if (postgresql == 'external' or db_flavor == 'mysql') and webmail_type == 'roundcube' %} +ROUNDCUBE_DB_FLAVOR={{ db_flavor }} ROUNDCUBE_DB_USER={{ roundcube_db_user }} ROUNDCUBE_DB_PW={{ roundcube_db_pw }} ROUNDCUBE_DB_HOST={{ roundcube_db_url }} From 14307c83c13877234d3daec11999f26d5bb0efff Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 09:12:46 +0200 Subject: [PATCH 094/181] Document databases variable and deprecation --- docs/configuration.rst | 20 ++++++++++++++++++++ docs/database.rst | 3 ++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index e08675a8..16ea23c3 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -195,4 +195,24 @@ resolved. This can be used to rely on DNS based service discovery with changing When using ``*_ADDRESS``, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to resolve the hostnames. +Database settings +----------------- + +The admin service stores configurations in a database. + +- ``DB_FLAVOR``: the database type for mailu admin service. (``sqlite``, ``postgresql``, ``mysql``) +- ``DB_HOST``: the database host for mailu admin service. (when not ``sqlite``) +- ``DB_PORT``: the database port for mailu admin service. (when not ``sqlite``) +- ``DB_PW``: the database password for mailu admin service. (when not ``sqlite``) +- ``DB_USER``: the database user for mailu admin service. (when not ``sqlite``) +- ``DB_NAME``: the database name for mailu admin service. (when not ``sqlite``) + +The roundcube service stores configurations in a database. + +- ``ROUNDCUBE_DB_FLAVOR``: the database type for roundcube service. (``sqlite``, ``postgresql``, ``mysql``) +- ``ROUNDCUBE_DB_HOST``: the database host for roundcube service. (when not ``sqlite``) +- ``ROUNDCUBE_DB_PORT``: the database port for roundcube service. (when not ``sqlite``) +- ``ROUNDCUBE_DB_PW``: the database password for roundcube service. (when not ``sqlite``) +- ``ROUNDCUBE_DB_USER``: the database user for roundcube service. (when not ``sqlite``) +- ``ROUNDCUBE_DB_NAME``: the database name for roundcube service. (when not ``sqlite``) diff --git a/docs/database.rst b/docs/database.rst index b2526d6f..c13ca0bf 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -8,7 +8,8 @@ This functionality should still be considered experimental! Mailu Postgresql ---------------- -Mailu optionally comes with a pre-configured Postgresql image. +Mailu optionally comes with a pre-configured Postgresql image, wich as of 1.8 is deprecated +will be removed in 1.9 This images has the following features: - Automatic creation of users, db, extensions and password; From a9548e4cbd31a9cd1b04414fc3ad4ca668f05bf6 Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 09:20:23 +0200 Subject: [PATCH 095/181] Remove mailu/roundcube shared host --- webmails/roundcube/start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 3e47ce69..f87e460f 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -17,14 +17,14 @@ elif db_flavor == "mysql": os.environ["DB_DSNW"] = "mysql://%s:%s@%s/%s" % ( os.environ.get("ROUNDCUBE_DB_USER", "roundcube"), os.environ.get("ROUNDCUBE_DB_PW"), - os.environ.get("ROUNDCUBE_DB_HOST", os.environ.get("DB_HOST", "database")), + os.environ.get("ROUNDCUBE_DB_HOST", "database"), os.environ.get("ROUNDCUBE_DB_NAME", "roundcube") ) elif db_flavor == "postgresql": os.environ["DB_DSNW"] = "pgsql://%s:%s@%s/%s" % ( os.environ.get("ROUNDCUBE_DB_USER", "roundcube"), os.environ.get("ROUNDCUBE_DB_PW"), - os.environ.get("ROUNDCUBE_DB_HOST", os.environ.get("DB_HOST", "database")), + os.environ.get("ROUNDCUBE_DB_HOST", "database"), os.environ.get("ROUNDCUBE_DB_NAME", "roundcube") ) else: From f4c76d49c1862894eba341eca318629a2137f274 Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 09:30:32 +0200 Subject: [PATCH 096/181] Add changelog entry --- towncrier/newsfragments/1831.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/1831.bugfix diff --git a/towncrier/newsfragments/1831.bugfix b/towncrier/newsfragments/1831.bugfix new file mode 100644 index 00000000..1094be34 --- /dev/null +++ b/towncrier/newsfragments/1831.bugfix @@ -0,0 +1 @@ +Fix roundcube environment configuration for databases \ No newline at end of file From 58235bcc44e9a7449229ea0d851e35846a5ce176 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 08:25:15 +0000 Subject: [PATCH 097/181] Switch to github actions for CI/CD --- .github/worfklows/CI.yml | 276 +++++++++++++++++++++++ .travis.yml | 56 ----- bors.toml | 3 +- tests/compose/filters/00_create_users.sh | 5 + 4 files changed, 283 insertions(+), 57 deletions(-) create mode 100644 .github/worfklows/CI.yml delete mode 100644 .travis.yml create mode 100755 tests/compose/filters/00_create_users.sh diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml new file mode 100644 index 00000000..3dcaf096 --- /dev/null +++ b/.github/worfklows/CI.yml @@ -0,0 +1,276 @@ + + +name: CI +on: + push: + branches: + - staging + - testing + - '1.5' + - '1.6' + - '1.7' + - '1.8' + - master + # version tags, e.g. 1.7.1 + - '[1-9].[0-9].[0-9]' + # pre-releases, e.g. 1.8-pre1 + - 1.8-pre[0-9] + # test branches, e.g. test-debian + - test-* + +############################################### +# REQUIRED secrets +# DOCKER_UN: ${{ secrets.Docker_Login }} +# Username of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_PW: ${{ secrets.Docker_Password }} +# Password of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_ORG: ${{ secrets.DOCKER_ORG }} +# The docker repository where the images are pushed to. +# +# Add the above secrets to your github repo to determine where the images will be pushed. +################################################ + +jobs: + build: + name: build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - name: check docker-compose version + run: docker-compose -v + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + # In this step, this action saves a list of existing images, + # the cache is created without them in the post run. + # It also restores the cache if it exists. + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: build all docker images + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: docker-compose -f tests/build.yml build + +#NOTE: It appears the filter test depends on the core test. The filter test requires an email user +#that is created by the core test. + core-test: + name: core test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test core suite + run: python tests/compose/test.py core 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + filter-test: + name: filter test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: 'test clamvav' + run: python tests/compose/test.py filters 2 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + fetch-test: + name: fetch test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test fetch + run: python tests/compose/test.py fetchmail 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + rainloop-test: + name: rainloop test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test rainloop + run: python tests/compose/test.py rainloop 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + roundcube-test: + name: roundcube test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test roundcube + run: python tests/compose/test.py roundcube 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + webdav-test: + name: webdav test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test webdav + run: python tests/compose/test.py webdav 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + deploy: + name: deploy step + runs-on: ubuntu-latest + needs: + - build + - core-test + - filter-test + - fetch-test + - rainloop-test + - roundcube-test + - webdav-test + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + - name: build all docker images + run: docker-compose -f tests/build.yml build + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + - name: deploy built docker images + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: bash tests/deploy.sh + + # This job is watched by bors. It only complets if building,testing and deploy worked. + ci-success: + name: CI-Done + #Returns true when none of the **previous** steps have failed or been canceled. + if: ${{ success() }} + needs: + - deploy + runs-on: ubuntu-latest + steps: + - name: CI/CD succeeded. + run: exit 0 + + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f2a85630..00000000 --- a/.travis.yml +++ /dev/null @@ -1,56 +0,0 @@ -branches: - only: - - staging - - testing - - '1.5' - - '1.6' - - '1.7' - - '1.8' - - master - # version tags, e.g. 1.7.1 - - /^1\.[5678]\.\d+$/ - # pre-releases, e.g. 1.8-pre1 - - /^1\.8-pre\d+$/ - # test branches, e.g. test-debian - - /^test-[\w\-\.]+$/ - -sudo: required -services: docker -addons: - apt: - packages: - - docker-ce - -env: - - MAILU_VERSION=${TRAVIS_BRANCH////-} - -language: python -python: - - "3.6" -install: - - pip install -r tests/requirements.txt - - sudo curl -L https://github.com/docker/compose/releases/download/1.23.0-rc3/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose - - sudo chmod +x /usr/local/bin/docker-compose - -before_script: - - docker-compose -v - - echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - - docker-compose -f tests/build.yml build - - sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - -script: -# test.py, test name and timeout between start and tests. - - python tests/compose/test.py core 1 - - python tests/compose/test.py fetchmail 1 - - travis_wait python tests/compose/test.py filters 10 - - python tests/compose/test.py rainloop 1 - - python tests/compose/test.py roundcube 1 - - python tests/compose/test.py webdav 1 - -deploy: - provider: script - script: bash tests/deploy.sh - on: - all_branches: true - condition: -n $DOCKER_UN diff --git a/bors.toml b/bors.toml index 5279fe72..272a6047 100644 --- a/bors.toml +++ b/bors.toml @@ -1,3 +1,4 @@ status = [ - "continuous-integration/travis-ci/push" + "CI-Done" ] + diff --git a/tests/compose/filters/00_create_users.sh b/tests/compose/filters/00_create_users.sh new file mode 100755 index 00000000..3c581685 --- /dev/null +++ b/tests/compose/filters/00_create_users.sh @@ -0,0 +1,5 @@ +echo "Creating user required for next test ..." +# Should not fail and update the password; update mode +docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu admin admin mailu.io 'password' --mode=update || exit 1 +docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu user user mailu.io 'password' || exit 1 +echo "User created successfully" From 54dd4cf224b510f657f0b07696cebf9e8d89d7c2 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 19:16:56 +0000 Subject: [PATCH 098/181] Added new docker repo for test image. Adapted deploy script to use env var for test repo name. Modified travis references to github actions references in docs. Added changelog entry. --- .github/worfklows/CI.yml | 4 +++- docs/contributors/environment.rst | 14 +++++++------- docs/contributors/workflow.rst | 2 +- docs/faq.rst | 2 +- tests/deploy.sh | 2 +- towncrier/newsfragments/1828.misc | 1 + 6 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 towncrier/newsfragments/1828.misc diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml index 3dcaf096..421fec3c 100644 --- a/.github/worfklows/CI.yml +++ b/.github/worfklows/CI.yml @@ -26,7 +26,8 @@ on: # Password of docker login for pushing the images to repo $DOCKER_ORG # DOCKER_ORG: ${{ secrets.DOCKER_ORG }} # The docker repository where the images are pushed to. -# +# DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} +# The docker repository for test images. Only used for the branch TESTING (BORS try). # Add the above secrets to your github repo to determine where the images will be pushed. ################################################ @@ -256,6 +257,7 @@ jobs: DOCKER_UN: ${{ secrets.Docker_Login }} DOCKER_PW: ${{ secrets.Docker_Password }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index 26c04d0b..cef71c6c 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -178,9 +178,9 @@ In the case of a PR from a fellow team member, a single review is enough to initiate merging. In all other cases, two approving reviews are required. There is also a possibility to set the ``review/need2`` to require a second review. -After Travis successfully tests the PR and the required amount of reviews are acquired, +After the Github Action workflow successfully tests the PR and the required amount of reviews are acquired, Mergify will trigger with a ``bors r+`` command. Bors will batch any approved PR's, -merges them with master in a staging branch where Travis builds and tests the result. +merges them with master in a staging branch where the Github Action workflow builds and tests the result. After a successful test, the actual master gets fast-forwarded to that point. System requirements @@ -201,16 +201,16 @@ us on `Matrix`_. Test images ``````````` -All PR's automatically get build by Travis, controlled by `bors-ng`_. +All PR's automatically get build by a Github Action workflow, controlled by `bors-ng`_. Some primitive auto testing is done. The resulting images get uploaded to Docker hub, under the -tag name ``mailutest/:pr-``. +tag name ``mailuci/:pr-``. For example, to test PR #500 against master, reviewers can use: .. code-block:: bash - export DOCKER_ORG="mailutest" + export DOCKER_ORG="mailuci" export MAILU_VERSION="pr-500" docker-compose pull docker-compose up -d @@ -232,8 +232,8 @@ after Bors confirms a successful build. When bors try fails ``````````````````` -Sometimes Travis fails when another PR triggers a ``bors try`` command, -before Travis cloned the git repository. +Sometimes the Github Action workflow fails when another PR triggers a ``bors try`` command, +before the Github Action workflow cloned the git repository. Inspect the build log in the link provided by *bors-ng* to find out the cause. If you see something like the following error on top of the logs, feel free to write a comment with ``bors retry``. diff --git a/docs/contributors/workflow.rst b/docs/contributors/workflow.rst index 16dcef52..31ffd793 100644 --- a/docs/contributors/workflow.rst +++ b/docs/contributors/workflow.rst @@ -41,7 +41,7 @@ PR Workflow ----------- All pull requests have to be against the main ``master`` branch. -The PR gets build by Travis and some primitive auto-testing is done. +The PR gets build by a Github Action workflow and some primitive auto-testing is done. Test images get uploaded to a separate section in Docker hub. Reviewers will check the PR and test the resulting images. See the :ref:`testing` section for more info. diff --git a/docs/faq.rst b/docs/faq.rst index 9c4f1d75..c4cea444 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -61,7 +61,7 @@ have to prevent pushing out something quickly. We currently maintain a strict work flow: #. Someone writes a solution and sends a pull request; -#. We use Travis-CI for some very basic building and testing; +#. We use Github actions for some very basic building and testing; #. The pull request needs to be code-reviewed and tested by at least two members from the contributors team. diff --git a/tests/deploy.sh b/tests/deploy.sh index 21aec444..a836417b 100755 --- a/tests/deploy.sh +++ b/tests/deploy.sh @@ -5,7 +5,7 @@ # Retag in case of `bors try` if [ "$TRAVIS_BRANCH" = "testing" ]; then - export DOCKER_ORG="mailutest" + export DOCKER_ORG=$DOCKER_ORG_TESTS # Commit message is like "Try #99". # This sets the version tag to "pr-99" export MAILU_VERSION="pr-${TRAVIS_COMMIT_MESSAGE//[!0-9]/}" diff --git a/towncrier/newsfragments/1828.misc b/towncrier/newsfragments/1828.misc new file mode 100644 index 00000000..09da59ad --- /dev/null +++ b/towncrier/newsfragments/1828.misc @@ -0,0 +1 @@ +Switched from Travis to Github actions for CI/CD. Improved CI workflow to perform all tests in parallel. From c6a38bbbccafd8e2623645da6c5759c9a23b4382 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Sat, 26 Jun 2021 21:50:55 +0200 Subject: [PATCH 099/181] Update CI.yml --- .github/worfklows/CI.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml index 421fec3c..e0462e1e 100644 --- a/.github/worfklows/CI.yml +++ b/.github/worfklows/CI.yml @@ -1,5 +1,3 @@ - - name: CI on: push: From fb30a62629b6372450b34b6c9a7238d40013ef58 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Sat, 26 Jun 2021 21:52:09 +0200 Subject: [PATCH 100/181] Create CI.yml --- .github/workflows/CI.yml | 276 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 .github/workflows/CI.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 00000000..e0462e1e --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,276 @@ +name: CI +on: + push: + branches: + - staging + - testing + - '1.5' + - '1.6' + - '1.7' + - '1.8' + - master + # version tags, e.g. 1.7.1 + - '[1-9].[0-9].[0-9]' + # pre-releases, e.g. 1.8-pre1 + - 1.8-pre[0-9] + # test branches, e.g. test-debian + - test-* + +############################################### +# REQUIRED secrets +# DOCKER_UN: ${{ secrets.Docker_Login }} +# Username of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_PW: ${{ secrets.Docker_Password }} +# Password of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_ORG: ${{ secrets.DOCKER_ORG }} +# The docker repository where the images are pushed to. +# DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} +# The docker repository for test images. Only used for the branch TESTING (BORS try). +# Add the above secrets to your github repo to determine where the images will be pushed. +################################################ + +jobs: + build: + name: build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - name: check docker-compose version + run: docker-compose -v + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + # In this step, this action saves a list of existing images, + # the cache is created without them in the post run. + # It also restores the cache if it exists. + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: build all docker images + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: docker-compose -f tests/build.yml build + +#NOTE: It appears the filter test depends on the core test. The filter test requires an email user +#that is created by the core test. + core-test: + name: core test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test core suite + run: python tests/compose/test.py core 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + filter-test: + name: filter test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: 'test clamvav' + run: python tests/compose/test.py filters 2 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + fetch-test: + name: fetch test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test fetch + run: python tests/compose/test.py fetchmail 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + rainloop-test: + name: rainloop test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test rainloop + run: python tests/compose/test.py rainloop 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + roundcube-test: + name: roundcube test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test roundcube + run: python tests/compose/test.py roundcube 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + webdav-test: + name: webdav test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test webdav + run: python tests/compose/test.py webdav 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + deploy: + name: deploy step + runs-on: ubuntu-latest + needs: + - build + - core-test + - filter-test + - fetch-test + - rainloop-test + - roundcube-test + - webdav-test + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + - name: build all docker images + run: docker-compose -f tests/build.yml build + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + - name: deploy built docker images + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: bash tests/deploy.sh + + # This job is watched by bors. It only complets if building,testing and deploy worked. + ci-success: + name: CI-Done + #Returns true when none of the **previous** steps have failed or been canceled. + if: ${{ success() }} + needs: + - deploy + runs-on: ubuntu-latest + steps: + - name: CI/CD succeeded. + run: exit 0 + + From 006da4c5e409526ccf5fb3e62ac512d3d5e79593 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Sat, 26 Jun 2021 21:52:51 +0200 Subject: [PATCH 101/181] My mistake. A typo --- .github/worfklows/CI.yml | 276 --------------------------------------- 1 file changed, 276 deletions(-) delete mode 100644 .github/worfklows/CI.yml diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml deleted file mode 100644 index e0462e1e..00000000 --- a/.github/worfklows/CI.yml +++ /dev/null @@ -1,276 +0,0 @@ -name: CI -on: - push: - branches: - - staging - - testing - - '1.5' - - '1.6' - - '1.7' - - '1.8' - - master - # version tags, e.g. 1.7.1 - - '[1-9].[0-9].[0-9]' - # pre-releases, e.g. 1.8-pre1 - - 1.8-pre[0-9] - # test branches, e.g. test-debian - - test-* - -############################################### -# REQUIRED secrets -# DOCKER_UN: ${{ secrets.Docker_Login }} -# Username of docker login for pushing the images to repo $DOCKER_ORG -# DOCKER_PW: ${{ secrets.Docker_Password }} -# Password of docker login for pushing the images to repo $DOCKER_ORG -# DOCKER_ORG: ${{ secrets.DOCKER_ORG }} -# The docker repository where the images are pushed to. -# DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} -# The docker repository for test images. Only used for the branch TESTING (BORS try). -# Add the above secrets to your github repo to determine where the images will be pushed. -################################################ - -jobs: - build: - name: build - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - name: check docker-compose version - run: docker-compose -v - - name: login docker - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - # In this step, this action saves a list of existing images, - # the cache is created without them in the post run. - # It also restores the cache if it exists. - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: build all docker images - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: docker-compose -f tests/build.yml build - -#NOTE: It appears the filter test depends on the core test. The filter test requires an email user -#that is created by the core test. - core-test: - name: core test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test core suite - run: python tests/compose/test.py core 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - filter-test: - name: filter test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: 'test clamvav' - run: python tests/compose/test.py filters 2 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - fetch-test: - name: fetch test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test fetch - run: python tests/compose/test.py fetchmail 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - rainloop-test: - name: rainloop test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test rainloop - run: python tests/compose/test.py rainloop 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - roundcube-test: - name: roundcube test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test roundcube - run: python tests/compose/test.py roundcube 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - webdav-test: - name: webdav test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test webdav - run: python tests/compose/test.py webdav 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - deploy: - name: deploy step - runs-on: ubuntu-latest - needs: - - build - - core-test - - filter-test - - fetch-test - - rainloop-test - - roundcube-test - - webdav-test - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: login docker - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - - name: build all docker images - run: docker-compose -f tests/build.yml build - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - name: deploy built docker images - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: bash tests/deploy.sh - - # This job is watched by bors. It only complets if building,testing and deploy worked. - ci-success: - name: CI-Done - #Returns true when none of the **previous** steps have failed or been canceled. - if: ${{ success() }} - needs: - - deploy - runs-on: ubuntu-latest - steps: - - name: CI/CD succeeded. - run: exit 0 - - From 606c039a6ffbcb63bb28bb1b422d0e622ebc0088 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 21:00:51 +0000 Subject: [PATCH 102/181] Switch back to sequential workflow --- .github/workflows/CI.yml | 126 ++--------------------- tests/compose/filters/00_create_users.sh | 5 - 2 files changed, 6 insertions(+), 125 deletions(-) delete mode 100755 tests/compose/filters/00_create_users.sh diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e0462e1e..8cd9a8d3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -30,8 +30,8 @@ on: ################################################ jobs: - build: - name: build + build-test: + name: build and test runs-on: ubuntu-latest steps: @@ -63,24 +63,6 @@ jobs: DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: docker-compose -f tests/build.yml build -#NOTE: It appears the filter test depends on the core test. The filter test requires an email user -#that is created by the core test. - core-test: - name: core test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - name: copy all certs run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: test core suite @@ -90,74 +72,20 @@ jobs: TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - filter-test: - name: filter test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: 'test clamvav' run: python tests/compose/test.py filters 2 env: MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - fetch-test: - name: fetch test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test fetch run: python tests/compose/test.py fetchmail 1 env: MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - rainloop-test: - name: rainloop test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test rainloop run: python tests/compose/test.py rainloop 1 env: @@ -165,49 +93,13 @@ jobs: TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - roundcube-test: - name: roundcube test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: test roundcube run: python tests/compose/test.py roundcube 1 env: MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - webdav-test: - name: webdav test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test webdav run: python tests/compose/test.py webdav 1 env: @@ -219,13 +111,7 @@ jobs: name: deploy step runs-on: ubuntu-latest needs: - - build - - core-test - - filter-test - - fetch-test - - rainloop-test - - roundcube-test - - webdav-test + - build-test steps: - uses: actions/checkout@v2 - name: Extract branch name diff --git a/tests/compose/filters/00_create_users.sh b/tests/compose/filters/00_create_users.sh deleted file mode 100755 index 3c581685..00000000 --- a/tests/compose/filters/00_create_users.sh +++ /dev/null @@ -1,5 +0,0 @@ -echo "Creating user required for next test ..." -# Should not fail and update the password; update mode -docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu admin admin mailu.io 'password' --mode=update || exit 1 -docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu user user mailu.io 'password' || exit 1 -echo "User created successfully" From 24200ddb670cf500d83695241bb158a4324e94b7 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 21:49:37 +0000 Subject: [PATCH 103/181] Forgot to remove duplicate steps when switching back to sequential workflow --- .github/workflows/CI.yml | 35 +++-------------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8cd9a8d3..71897c52 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -30,7 +30,7 @@ on: ################################################ jobs: - build-test: + build-test-deploy: name: build and test runs-on: ubuntu-latest @@ -106,36 +106,7 @@ jobs: MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - deploy: - name: deploy step - runs-on: ubuntu-latest - needs: - - build-test - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: login docker - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - - name: build all docker images - run: docker-compose -f tests/build.yml build - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + - name: deploy built docker images env: DOCKER_UN: ${{ secrets.Docker_Login }} @@ -153,7 +124,7 @@ jobs: #Returns true when none of the **previous** steps have failed or been canceled. if: ${{ success() }} needs: - - deploy + - build-test-deploy runs-on: ubuntu-latest steps: - name: CI/CD succeeded. From b560d1f36998c4c2f65875cbb88f8fbc319ffbe2 Mon Sep 17 00:00:00 2001 From: Nicolas Paris Date: Sun, 27 Jun 2021 10:38:32 +0200 Subject: [PATCH 104/181] Improve english Co-authored-by: decentral1se <1991377+decentral1se@users.noreply.github.com> --- docs/database.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/database.rst b/docs/database.rst index c13ca0bf..0f8318d5 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -8,8 +8,8 @@ This functionality should still be considered experimental! Mailu Postgresql ---------------- -Mailu optionally comes with a pre-configured Postgresql image, wich as of 1.8 is deprecated -will be removed in 1.9 +Mailu optionally comes with a pre-configured Postgresql image, which as of 1.8, is deprecated +and will be removed in 1.9. This images has the following features: - Automatic creation of users, db, extensions and password; From c0c8c4a55113237274ca4b4e32c9c46e5297998f Mon Sep 17 00:00:00 2001 From: Nicolas Paris Date: Sun, 27 Jun 2021 10:46:28 +0200 Subject: [PATCH 105/181] Fix typo Co-authored-by: decentral1se <1991377+decentral1se@users.noreply.github.com> --- setup/templates/steps/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/templates/steps/database.html b/setup/templates/steps/database.html index d7184110..2727687f 100644 --- a/setup/templates/steps/database.html +++ b/setup/templates/steps/database.html @@ -46,7 +46,7 @@ - + From ab7264df0c9a115a7e9cf63cd4ff55e380c3e7f7 Mon Sep 17 00:00:00 2001 From: Nicolas Paris Date: Sun, 27 Jun 2021 10:46:41 +0200 Subject: [PATCH 106/181] Fix typo Co-authored-by: decentral1se <1991377+decentral1se@users.noreply.github.com> --- setup/templates/steps/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/templates/steps/database.html b/setup/templates/steps/database.html index 2727687f..be32fe0b 100644 --- a/setup/templates/steps/database.html +++ b/setup/templates/steps/database.html @@ -44,7 +44,7 @@ - + From 7386257dedda678419e495c28e382251f7212bfa Mon Sep 17 00:00:00 2001 From: Nicolas Paris Date: Sun, 27 Jun 2021 10:47:28 +0200 Subject: [PATCH 107/181] Fix typo Co-authored-by: decentral1se <1991377+decentral1se@users.noreply.github.com> --- setup/templates/steps/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/templates/steps/database.html b/setup/templates/steps/database.html index be32fe0b..5e942061 100644 --- a/setup/templates/steps/database.html +++ b/setup/templates/steps/database.html @@ -28,7 +28,7 @@
- + diff --git a/setup/templates/steps/config.html b/setup/templates/steps/config.html index 29d8dddd..3769a210 100644 --- a/setup/templates/steps/config.html +++ b/setup/templates/steps/config.html @@ -83,7 +83,7 @@ manage your email domains, users, etc.

- + diff --git a/setup/templates/steps/stack/02_services.html b/setup/templates/steps/stack/02_services.html index 3f5186b0..6fce0ae6 100644 --- a/setup/templates/steps/stack/02_services.html +++ b/setup/templates/steps/stack/02_services.html @@ -55,7 +55,7 @@ the security implications caused by such an increase of attack surface.

Fetchmail allows users to retrieve mail from an external mail-server via IMAP/POP3 and puts it in their inbox. - + From b0fb9d822bed082ee6630baeaf8526ab75934ff4 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 22 Jul 2021 14:44:30 +0000 Subject: [PATCH 155/181] Adapt requirements.txt to use pinned versions. --- setup/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/requirements.txt b/setup/requirements.txt index b6bf2120..f2eb2157 100644 --- a/setup/requirements.txt +++ b/setup/requirements.txt @@ -1,4 +1,4 @@ -flask -flask-bootstrap -redis -gunicorn +Flask==1.0.2 +Flask-Bootstrap==3.3.7.1 +gunicorn==19.9.0 +redis==3.2.1 From 67e00bb1e77fa3350610e855c1396952ea76ea21 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 22 Jul 2021 14:56:30 +0000 Subject: [PATCH 156/181] Add changelog --- towncrier/newsfragments/1880.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/1880.feature diff --git a/towncrier/newsfragments/1880.feature b/towncrier/newsfragments/1880.feature new file mode 100644 index 00000000..212dc906 --- /dev/null +++ b/towncrier/newsfragments/1880.feature @@ -0,0 +1 @@ +Update jquery used in setup. Set pinned versions in requirements.txt for setup. This is a security update. From 9d2629a04e7aaa8c9fc199dac8a6a0631a857fac Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 24 Jul 2021 12:40:38 +0200 Subject: [PATCH 157/181] fix 1884: always lookup a FQDN --- core/postfix/start.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/postfix/start.py b/core/postfix/start.py index 701efec3..7a2b57a1 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -38,7 +38,11 @@ os.environ["ANTISPAM_MILTER_ADDRESS"] = system.get_host_address_from_environment os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525") os.environ["OUTCLEAN"] = os.environ["HOSTNAMES"].split(",")[0] try: - os.environ["OUTCLEAN_ADDRESS"] = system.resolve_hostname(os.environ["OUTCLEAN"]) + _to_lookup = os.environ["OUTCLEAN"] + # Ensure we lookup a FQDN: @see #1884 + if not _to_lookup.endswith('.'): + _to_lookup += '.' + os.environ["OUTCLEAN_ADDRESS"] = system.resolve_hostname(_to_lookup) except: os.environ["OUTCLEAN_ADDRESS"] = "10.10.10.10" From fa915d78624114201074b0bbe919cdb708cca568 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 24 Jul 2021 14:39:40 +0200 Subject: [PATCH 158/181] Fix 1294 ensure podop's socket is owned by postfix --- core/postfix/start.py | 3 ++- towncrier/newsfragments/1294.bugfix | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 towncrier/newsfragments/1294.bugfix diff --git a/core/postfix/start.py b/core/postfix/start.py index 701efec3..125404f6 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -8,12 +8,13 @@ import logging as log import sys from podop import run_server +from pwd import getpwnam from socrate import system, conf log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) def start_podop(): - os.setuid(100) + os.setuid(getpwnam('postfix').pw_uid) url = "http://" + os.environ["ADMIN_ADDRESS"] + "/internal/postfix/" # TODO: Remove verbosity setting from Podop? run_server(0, "postfix", "/tmp/podop.socket", [ diff --git a/towncrier/newsfragments/1294.bugfix b/towncrier/newsfragments/1294.bugfix new file mode 100644 index 00000000..68bb7a8a --- /dev/null +++ b/towncrier/newsfragments/1294.bugfix @@ -0,0 +1 @@ +Ensure that the podop socket is always owned by the postfix user (wasn't the case when build using non-standard base images... typically for arm64) From 8235085848e1e8586a7455bf214bd240f06d549e Mon Sep 17 00:00:00 2001 From: networkException Date: Sat, 24 Jul 2021 15:25:59 +0200 Subject: [PATCH 159/181] Docs: Limit fail2ban matches to front container Previously fail2ban matched against all journal entries. This patch adds a tag to the logdriver and fail2ban filter documentation that limits the matches to entries from the front container --- docs/faq.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 5d975532..f38fdca2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -497,6 +497,8 @@ follow these steps: logging: driver: journald + options: + tag: mailu-front 2. Add the /etc/fail2ban/filter.d/bad-auth.conf @@ -506,6 +508,7 @@ follow these steps: [Definition] failregex = .* client login failed: .+ client:\ ignoreregex = + journalmatch = CONTAINER_TAG=mailu-front 3. Add the /etc/fail2ban/jail.d/bad-auth.conf From 8d9f3214cc5663dc29f7dcf3a03bc373a51d010b Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 24 Jul 2021 15:45:25 +0200 Subject: [PATCH 160/181] Use threads in gunicorn rather than processes This ensures that we share the auth-cache... will enable memory savings and may improve performances when a higher number of cores is available "smarter default" --- core/admin/start.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/start.py b/core/admin/start.py index 2c925e01..0eff3bbe 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -19,7 +19,8 @@ if account is not None and domain is not None and password is not None: os.system("flask mailu admin %s %s '%s' --mode %s" % (account, domain, password, mode)) start_command="".join([ - "gunicorn -w 4 -b :80 ", + "gunicorn --threads ", str(os.cpu_count()), + " -b :80 ", "--access-logfile - " if (log.root.level<=log.INFO) else "", "--error-logfile - ", "--preload ", From ad1b036f2068f5fe70641e549df34cb0b9cc0b21 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Jul 2021 20:21:38 +0200 Subject: [PATCH 161/181] fix Email class --- core/admin/mailu/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index b5ba29c0..fc9fd623 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -371,8 +371,8 @@ class Email(object): @classmethod def __declare_last__(cls): # gets called after mappings are completed - sqlalchemy.event.listen(User.localpart, 'set', cls._update_localpart, propagate=True) - sqlalchemy.event.listen(User.domain_name, 'set', cls._update_domain_name, propagate=True) + sqlalchemy.event.listen(cls.localpart, 'set', cls._update_localpart, propagate=True) + sqlalchemy.event.listen(cls.domain_name, 'set', cls._update_domain_name, propagate=True) def sendmail(self, subject, body): """ send an email to the address """ From 54b46a13c6bc730d5d0faa27d5c540edae9b24d4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 25 Jul 2021 15:51:13 +0200 Subject: [PATCH 162/181] save dkim key after creation --- core/admin/mailu/models.py | 2 +- core/admin/mailu/ui/views/domains.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index b5ba29c0..7a6eb308 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -272,7 +272,7 @@ class Domain(Base): return dkim.strip_key(dkim_key).decode('utf8') def generate_dkim_key(self): - """ generate and activate new DKIM key """ + """ generate new DKIM key """ self.dkim_key = dkim.gen_key() def has_email(self, localpart): diff --git a/core/admin/mailu/ui/views/domains.py b/core/admin/mailu/ui/views/domains.py index 719d3844..f394ce7d 100644 --- a/core/admin/mailu/ui/views/domains.py +++ b/core/admin/mailu/ui/views/domains.py @@ -74,6 +74,8 @@ def domain_details(domain_name): def domain_genkeys(domain_name): domain = models.Domain.query.get(domain_name) or flask.abort(404) domain.generate_dkim_key() + models.db.session.add(domain) + models.db.session.commit() return flask.redirect( flask.url_for(".domain_details", domain_name=domain_name)) From 6856c2c80f1291771a2faed5b72883e62028fcc6 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 30 Jul 2021 22:26:20 +0200 Subject: [PATCH 163/181] treat localpart case insensitive again by lowercasing it where necessary --- core/admin/mailu/models.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index b5ba29c0..4fdf5afb 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -57,10 +57,9 @@ class IdnaEmail(db.TypeDecorator): def process_bind_param(self, value, dialect): """ encode unicode domain part of email address to punycode """ - localpart, domain_name = value.rsplit('@', 1) + localpart, domain_name = value.lower().rsplit('@', 1) if '@' in localpart: raise ValueError('email local part must not contain "@"') - domain_name = domain_name.lower() return f'{localpart}@{idna.encode(domain_name).decode("ascii")}' def process_result_value(self, value, dialect): @@ -277,6 +276,7 @@ class Domain(Base): def has_email(self, localpart): """ checks if localpart is configured for domain """ + localpart = localpart.lower() for email in chain(self.users, self.aliases): if email.localpart == localpart: return True @@ -355,8 +355,8 @@ class Email(object): @email.setter def email(self, value): """ setter for email - sets _email, localpart and domain_name at once """ - self.localpart, self.domain_name = value.rsplit('@', 1) - self._email = value + self._email = value.lower() + self.localpart, self.domain_name = self._email.rsplit('@', 1) @staticmethod def _update_localpart(target, value, *_): @@ -389,8 +389,7 @@ class Email(object): def resolve_domain(cls, email): """ resolves domain alternative to real domain """ localpart, domain_name = email.rsplit('@', 1) if '@' in email else (None, email) - alternative = Alternative.query.get(domain_name) - if alternative: + if alternative := Alternative.query.get(domain_name): domain_name = alternative.domain_name return (localpart, domain_name) @@ -401,12 +400,14 @@ class Email(object): localpart_stripped = None stripped_alias = None - if os.environ.get('RECIPIENT_DELIMITER') in localpart: - localpart_stripped = localpart.rsplit(os.environ.get('RECIPIENT_DELIMITER'), 1)[0] + delim = os.environ.get('RECIPIENT_DELIMITER') + if delim in localpart: + localpart_stripped = localpart.rsplit(delim, 1)[0] user = User.query.get(f'{localpart}@{domain_name}') if not user and localpart_stripped: user = User.query.get(f'{localpart_stripped}@{domain_name}') + if user: email = f'{localpart}@{domain_name}' @@ -416,15 +417,15 @@ class Email(object): destination.append(email) else: destination = [email] + return destination pure_alias = Alias.resolve(localpart, domain_name) - stripped_alias = Alias.resolve(localpart_stripped, domain_name) if pure_alias and not pure_alias.wildcard: return pure_alias.destination - if stripped_alias: + if stripped_alias := Alias.resolve(localpart_stripped, domain_name): return stripped_alias.destination if pure_alias: From f8362d04e4d46c44ab07beffb77cdd041af193c0 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 3 Aug 2021 13:44:56 +0200 Subject: [PATCH 164/181] Switch to openssl to workaround alpine #12763 --- core/admin/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 3153bd9e..a0d3d996 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -24,9 +24,9 @@ RUN mkdir -p /app WORKDIR /app COPY requirements-prod.txt requirements.txt -RUN apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ +RUN apk add --no-cache openssl curl postgresql-libs mariadb-connector-c \ && apk add --no-cache --virtual build-dep \ - libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev \ + openssl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev \ && pip3 install -r requirements.txt \ && apk del --no-cache build-dep From defea3258ddd3faa15563630b7a10625b5ffa4a6 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 3 Aug 2021 13:58:54 +0200 Subject: [PATCH 165/181] update arm builds too --- core/admin/Dockerfile | 2 +- webmails/rainloop/Dockerfile | 2 +- webmails/roundcube/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 0a3b8468..97cf1736 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -2,7 +2,7 @@ ARG DISTRO=alpine:3.14 ARG ARCH="" FROM ${ARCH}node:8 as assets -COPY --from=balenalib/rpi-alpine:3.10 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static +COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static COPY package.json ./ RUN npm install diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile index 9c65f277..9814413d 100644 --- a/webmails/rainloop/Dockerfile +++ b/webmails/rainloop/Dockerfile @@ -3,7 +3,7 @@ ARG QEMU=other # NOTE: only add file if building for arm FROM ${ARCH}php:7.4-apache as build_arm -ONBUILD COPY --from=balenalib/rpi-alpine:3.10 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static +ONBUILD COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static FROM ${ARCH}php:7.4-apache as build_other diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index fae02ce0..4d3e36df 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -2,7 +2,7 @@ ARG ARCH="" ARG QEMU=other FROM ${ARCH}php:7.4-apache as build_arm -ONBUILD COPY --from=balenalib/rpi-alpine:3.10 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static +ONBUILD COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static FROM ${ARCH}php:7.4-apache as build_other From ccb3631622e535c902d22c7ced649cd374d1ac97 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 3 Aug 2021 14:01:44 +0200 Subject: [PATCH 166/181] still need pip3 --- docs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index 11f66b49..29234b19 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -6,7 +6,7 @@ COPY requirements.txt /requirements.txt ARG version=master ENV VERSION=$version -RUN apk add --no-cache nginx curl python3 \ +RUN apk add --no-cache nginx curl python3 py3-pip \ && pip3 install -r /requirements.txt \ && mkdir /run/nginx From 2b63280f59658ca05fd5fb3870126556d6c29004 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 3 Aug 2021 14:16:14 +0200 Subject: [PATCH 167/181] doh --- docs/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index 29234b19..253c8420 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,4 @@ -ARG DISTRO=alpine:3.14 +ARG DISTRO=python:3.8-alpine3.14 FROM $DISTRO COPY requirements.txt /requirements.txt @@ -6,9 +6,8 @@ COPY requirements.txt /requirements.txt ARG version=master ENV VERSION=$version -RUN apk add --no-cache nginx curl python3 py3-pip \ - && pip3 install -r /requirements.txt \ - && mkdir /run/nginx +RUN apk add --no-cache nginx curl \ + && pip3 install -r /requirements.txt COPY ./nginx.conf /etc/nginx/conf.d/default.conf COPY . /docs From 609e0f9f7c5b60e5db498d3b3817b41e65c116c5 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Thu, 5 Aug 2021 00:12:37 +0200 Subject: [PATCH 168/181] Env vars are not shared between jobs --- .github/workflows/CI.yml | 84 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 55595838..19a445b4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -91,6 +91,20 @@ jobs: shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for branch testing + if: ${{ env.BRANCH == 'testing' }} + shell: bash + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for other branches than testing + if: ${{ env.BRANCH != 'testing' }} + shell: bash + env: + MAILU_BRANCH: ${{ env.BRANCH }} + run: | + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -124,6 +138,20 @@ jobs: shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for branch testing + if: ${{ env.BRANCH == 'testing' }} + shell: bash + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for other branches than testing + if: ${{ env.BRANCH != 'testing' }} + shell: bash + env: + MAILU_BRANCH: ${{ env.BRANCH }} + run: | + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -157,6 +185,20 @@ jobs: shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for branch testing + if: ${{ env.BRANCH == 'testing' }} + shell: bash + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for other branches than testing + if: ${{ env.BRANCH != 'testing' }} + shell: bash + env: + MAILU_BRANCH: ${{ env.BRANCH }} + run: | + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -190,6 +232,20 @@ jobs: shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for branch testing + if: ${{ env.BRANCH == 'testing' }} + shell: bash + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for other branches than testing + if: ${{ env.BRANCH != 'testing' }} + shell: bash + env: + MAILU_BRANCH: ${{ env.BRANCH }} + run: | + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -223,6 +279,20 @@ jobs: shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for branch testing + if: ${{ env.BRANCH == 'testing' }} + shell: bash + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for other branches than testing + if: ${{ env.BRANCH != 'testing' }} + shell: bash + env: + MAILU_BRANCH: ${{ env.BRANCH }} + run: | + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -256,6 +326,20 @@ jobs: shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for branch testing + if: ${{ env.BRANCH == 'testing' }} + shell: bash + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION for other branches than testing + if: ${{ env.BRANCH != 'testing' }} + shell: bash + env: + MAILU_BRANCH: ${{ env.BRANCH }} + run: | + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images From 98933f9478aab9f8d04cb0afba5eeac2c4dee9ef Mon Sep 17 00:00:00 2001 From: Erriez Date: Thu, 5 Aug 2021 19:37:06 +0200 Subject: [PATCH 169/181] Optimize docs/Dockerfile - Convert .rst to .html in temporary python:3.8-alpine3.14 build image - Remove all unused packages - Use nginx:1.21-alpine deployment image --- docs/Dockerfile | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index 70c9c3c4..289697da 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,20 +1,28 @@ -ARG DISTRO=alpine:3.8 -FROM $DISTRO - -COPY requirements.txt /requirements.txt +# Convert .rst files to .html in temporary build container +FROM python:3.8-alpine3.14 AS build ARG version=master ENV VERSION=$version -RUN apk add --no-cache nginx curl python3 \ - && pip3 install -r /requirements.txt \ - && mkdir /run/nginx - -COPY ./nginx.conf /etc/nginx/conf.d/default.conf +COPY requirements.txt /requirements.txt COPY . /docs -RUN mkdir -p /build/$VERSION \ - && sphinx-build -W /docs /build/$VERSION +RUN apk add --no-cache --virtual .build-deps \ + gcc musl-dev \ + && pip3 install -r /requirements.txt \ + && mkdir -p /build/$VERSION \ + && sphinx-build -W /docs /build/$VERSION \ + && apk del .build-deps + + +# Build nginx deployment image including generated html +FROM nginx:1.21-alpine + +ARG version=master +ENV VERSION=$version + +COPY ./nginx.conf /etc/nginx/conf.d/default.conf +COPY --from=build /build/$VERSION /build/$VERSION EXPOSE 80/tcp From 588904078edd04ed08db309ceb7dd11e751c37b9 Mon Sep 17 00:00:00 2001 From: Diman0 Date: Fri, 6 Aug 2021 16:27:07 +0200 Subject: [PATCH 170/181] Set default of AUTH_RATELIMIT_SUBNET to False. Increase default AUTH_RATELIMIT value. --- core/admin/mailu/configuration.py | 4 ++-- docs/configuration.rst | 2 +- setup/templates/steps/config.html | 2 +- towncrier/newsfragments/1867.feature | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 towncrier/newsfragments/1867.feature diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 3d1b4fb5..d2d34d88 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -34,8 +34,8 @@ DEFAULT_CONFIG = { 'POSTMASTER': 'postmaster', 'TLS_FLAVOR': 'cert', 'INBOUND_TLS_ENFORCE': False, - 'AUTH_RATELIMIT': '10/minute;1000/hour', - 'AUTH_RATELIMIT_SUBNET': True, + 'AUTH_RATELIMIT': '1000/minute;10000/hour', + 'AUTH_RATELIMIT_SUBNET': False, 'DISABLE_STATISTICS': False, # Mail settings 'DMARC_RUA': None, diff --git a/docs/configuration.rst b/docs/configuration.rst index 16ea23c3..d7ebfc11 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -41,7 +41,7 @@ The ``AUTH_RATELIMIT`` holds a security setting for fighting attackers that try to guess user passwords. The value is the limit of failed authentication attempts that a single IP address can perform against IMAP, POP and SMTP authentication endpoints. -If ``AUTH_RATELIMIT_SUBNET`` is ``True`` (which is the default), the ``AUTH_RATELIMIT`` +If ``AUTH_RATELIMIT_SUBNET`` is ``True`` (default: False), the ``AUTH_RATELIMIT`` rules does also apply to auth requests coming from ``SUBNET``, especially for the webmail. If you disable this, ensure that the rate limit on the webmail is enforced in a different way (e.g. roundcube plug-in), otherwise an attacker can simply bypass the limit using webmail. diff --git a/setup/templates/steps/config.html b/setup/templates/steps/config.html index 3769a210..fee66933 100644 --- a/setup/templates/steps/config.html +++ b/setup/templates/steps/config.html @@ -51,7 +51,7 @@ Or in plain english: if receivers start to classify your mail as spam, this post

/ minute + value="10000" required > / minute

diff --git a/towncrier/newsfragments/1867.feature b/towncrier/newsfragments/1867.feature new file mode 100644 index 00000000..fbd3a7d7 --- /dev/null +++ b/towncrier/newsfragments/1867.feature @@ -0,0 +1 @@ +Changed default value of AUTH_RATELIMIT_SUBNET to false. Increased default value of the rate limit in setup utility (AUTH_RATELIMIT) to a higher value. From 4cfa2dbc2addc1bab34f5402f1b52df2d89ec536 Mon Sep 17 00:00:00 2001 From: Diman0 Date: Fri, 6 Aug 2021 16:44:18 +0200 Subject: [PATCH 171/181] Increase width of rate limiting text box. --- setup/templates/steps/config.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/templates/steps/config.html b/setup/templates/steps/config.html index fee66933..72b83915 100644 --- a/setup/templates/steps/config.html +++ b/setup/templates/steps/config.html @@ -50,7 +50,7 @@ Or in plain english: if receivers start to classify your mail as spam, this post
-

/ minute

From a7d99bdedd3decc7dc02609fc070337451302eb2 Mon Sep 17 00:00:00 2001 From: Diman0 Date: Fri, 6 Aug 2021 22:35:37 +0200 Subject: [PATCH 172/181] Update CHANGELOG.md and process towncrier newsfragments. --- CHANGELOG.md | 32 +++++++++++++++++++++++++--- towncrier/newsfragments/1660.bugfix | 1 - towncrier/newsfragments/1686.bugfix | 1 - towncrier/newsfragments/1720.bugfix | 2 -- towncrier/newsfragments/1783.misc | 1 - towncrier/newsfragments/1837.bugfix | 1 - towncrier/newsfragments/1841.feature | 1 - towncrier/newsfragments/1845.feature | 1 - towncrier/newsfragments/1857.doc | 1 - towncrier/newsfragments/1861.bugfix | 1 - towncrier/newsfragments/1867.feature | 1 - towncrier/newsfragments/1874.bugfix | 1 - towncrier/newsfragments/1880.feature | 1 - towncrier/newsfragments/191.bugfix | 1 - 14 files changed, 29 insertions(+), 17 deletions(-) delete mode 100644 towncrier/newsfragments/1660.bugfix delete mode 100644 towncrier/newsfragments/1686.bugfix delete mode 100644 towncrier/newsfragments/1720.bugfix delete mode 100644 towncrier/newsfragments/1783.misc delete mode 100644 towncrier/newsfragments/1837.bugfix delete mode 100644 towncrier/newsfragments/1841.feature delete mode 100644 towncrier/newsfragments/1845.feature delete mode 100644 towncrier/newsfragments/1857.doc delete mode 100644 towncrier/newsfragments/1861.bugfix delete mode 100644 towncrier/newsfragments/1867.feature delete mode 100644 towncrier/newsfragments/1874.bugfix delete mode 100644 towncrier/newsfragments/1880.feature delete mode 100644 towncrier/newsfragments/191.bugfix diff --git a/CHANGELOG.md b/CHANGELOG.md index 579f3e82..09b9f68f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,18 +4,44 @@ Changelog Upgrade should run fine as long as you generate a new compose or stack configuration and upgrade your mailu.env. -Please note that the current 1.8 is what we call a "soft release": It’s there for everyone to see and use, but to limit possible user-impact of this very big release, it’s not yet the default in the setup-utility for new users. When upgrading, please treat it with some care, and be sure to always have backups! - There are some changes to the configuration overrides. Override files are now mounted read-only into the containers. The Dovecot and Postfix overrides are moved in their own sub-directory. If there are local override files, they will need to be moved from overrides/ to overrides/dovecot and overrides/postfix/. See https://mailu.io/1.8/faq.html#how-can-i-override-settings for all the mappings. +<<<<<<< HEAD Please note that the shipped image for PostgreSQL database is deprecated. We advise to switch to an external database server. +======= +One major change for the docker compose file is that the antispam needs a fixed hostname [#1837](https://github.com/Mailu/Mailu/issues/1837). +This is handled when you regenerate the docker-compose file. A fixed hostname is required to retain rspamd history. + +Please not that the shipped image for PostgreSQL database is deprecated. +We advise to switch to an external PostgreSQL database server. +>>>>>>> afaacf5a... Update CHANGELOG.md and process towncrier newsfragments. -v1.8.0 - 2020-09-28 +1.8.0 - 2021-08-06 +-------------------- + +- Features: Update version of roundcube webmail and carddav plugin. This is a security update. ([#1841](https://github.com/Mailu/Mailu/issues/1841)) +- Features: Update version of rainloop webmail to 1.16.0. This is a security update. ([#1845](https://github.com/Mailu/Mailu/issues/1845)) +- Features: Changed default value of AUTH_RATELIMIT_SUBNET to false. Increased default value of the rate limit in setup utility (AUTH_RATELIMIT) to a higher value. ([#1867](https://github.com/Mailu/Mailu/issues/1867)) +- Features: Update jquery used in setup. Set pinned versions in requirements.txt for setup. This is a security update. ([#1880](https://github.com/Mailu/Mailu/issues/1880)) +- Bugfixes: Replace PUBLIC_HOSTNAME and PUBLIC_IP in "Received" headers to ensure that no undue spam points are attributed ([#191](https://github.com/Mailu/Mailu/issues/191)) +- Bugfixes: Don't replace nested headers (typically in attached emails) ([#1660](https://github.com/Mailu/Mailu/issues/1660)) +- Bugfixes: Fix letsencrypt access to certbot for the mail-letsencrypt flavour ([#1686](https://github.com/Mailu/Mailu/issues/1686)) +- Bugfixes: Fix CVE-2020-25275 and CVE-2020-24386 by using alpine 3.13 for + dovecot which contains a fixed dovecot version. ([#1720](https://github.com/Mailu/Mailu/issues/1720)) +- Bugfixes: Antispam service now uses a static hostname. Rspamd history is only retained when the service has a fixed hostname. ([#1837](https://github.com/Mailu/Mailu/issues/1837)) +- Bugfixes: Fix a bug preventing colons from being used in passwords when using radicale/webdav. ([#1861](https://github.com/Mailu/Mailu/issues/1861)) +- Bugfixes: Remove dot in blueprint name to prevent critical flask startup error in setup. ([#1874](https://github.com/Mailu/Mailu/issues/1874)) +- Bugfixes: fix punycode encoding of domain names ([#1891](https://github.com/Mailu/Mailu/issues/1891)) +- Improved Documentation: Update fail2ban documentation to use systemd backend instead of filepath for journald ([#1857](https://github.com/Mailu/Mailu/issues/1857)) +- Misc: ([#1783](https://github.com/Mailu/Mailu/issues/1783)) + + +v1.8.0rc - 2020-09-28 -------------------- - Features: Add support for backward-forwarding using SRS ([#328](https://github.com/Mailu/Mailu/issues/328)) diff --git a/towncrier/newsfragments/1660.bugfix b/towncrier/newsfragments/1660.bugfix deleted file mode 100644 index a90fb099..00000000 --- a/towncrier/newsfragments/1660.bugfix +++ /dev/null @@ -1 +0,0 @@ -Don't replace nested headers (typically in attached emails) diff --git a/towncrier/newsfragments/1686.bugfix b/towncrier/newsfragments/1686.bugfix deleted file mode 100644 index 932d7d7c..00000000 --- a/towncrier/newsfragments/1686.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix letsencrypt access to certbot for the mail-letsencrypt flavour diff --git a/towncrier/newsfragments/1720.bugfix b/towncrier/newsfragments/1720.bugfix deleted file mode 100644 index 0bf2b8e6..00000000 --- a/towncrier/newsfragments/1720.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fix CVE-2020-25275 and CVE-2020-24386 by using alpine 3.13 for -dovecot which contains a fixed dovecot version. diff --git a/towncrier/newsfragments/1783.misc b/towncrier/newsfragments/1783.misc deleted file mode 100644 index 2ee4c97f..00000000 --- a/towncrier/newsfragments/1783.misc +++ /dev/null @@ -1 +0,0 @@ -Switch from client side sessions (cookies) to server-side sessions (Redis). This simplies the security model a lot and allows for an easier recovery should a cookie ever land in the hands of an attacker. diff --git a/towncrier/newsfragments/1837.bugfix b/towncrier/newsfragments/1837.bugfix deleted file mode 100644 index dcabcc6b..00000000 --- a/towncrier/newsfragments/1837.bugfix +++ /dev/null @@ -1 +0,0 @@ -Antispam service now uses a static hostname. Rspamd history is only retained when the service has a fixed hostname. diff --git a/towncrier/newsfragments/1841.feature b/towncrier/newsfragments/1841.feature deleted file mode 100644 index c91f805f..00000000 --- a/towncrier/newsfragments/1841.feature +++ /dev/null @@ -1 +0,0 @@ -Update version of roundcube webmail and carddav plugin. This is a security update. \ No newline at end of file diff --git a/towncrier/newsfragments/1845.feature b/towncrier/newsfragments/1845.feature deleted file mode 100644 index afde9313..00000000 --- a/towncrier/newsfragments/1845.feature +++ /dev/null @@ -1 +0,0 @@ -Update version of rainloop webmail to 1.16.0. This is a security update. diff --git a/towncrier/newsfragments/1857.doc b/towncrier/newsfragments/1857.doc deleted file mode 100644 index 06cb91ab..00000000 --- a/towncrier/newsfragments/1857.doc +++ /dev/null @@ -1 +0,0 @@ -Update fail2ban documentation to use systemd backend instead of filepath for journald \ No newline at end of file diff --git a/towncrier/newsfragments/1861.bugfix b/towncrier/newsfragments/1861.bugfix deleted file mode 100644 index 1e28d1b6..00000000 --- a/towncrier/newsfragments/1861.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug preventing colons from being used in passwords when using radicale/webdav. diff --git a/towncrier/newsfragments/1867.feature b/towncrier/newsfragments/1867.feature deleted file mode 100644 index fbd3a7d7..00000000 --- a/towncrier/newsfragments/1867.feature +++ /dev/null @@ -1 +0,0 @@ -Changed default value of AUTH_RATELIMIT_SUBNET to false. Increased default value of the rate limit in setup utility (AUTH_RATELIMIT) to a higher value. diff --git a/towncrier/newsfragments/1874.bugfix b/towncrier/newsfragments/1874.bugfix deleted file mode 100644 index a301835e..00000000 --- a/towncrier/newsfragments/1874.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove dot in blueprint name to prevent critical flask startup error in setup. diff --git a/towncrier/newsfragments/1880.feature b/towncrier/newsfragments/1880.feature deleted file mode 100644 index 212dc906..00000000 --- a/towncrier/newsfragments/1880.feature +++ /dev/null @@ -1 +0,0 @@ -Update jquery used in setup. Set pinned versions in requirements.txt for setup. This is a security update. diff --git a/towncrier/newsfragments/191.bugfix b/towncrier/newsfragments/191.bugfix deleted file mode 100644 index 185d3074..00000000 --- a/towncrier/newsfragments/191.bugfix +++ /dev/null @@ -1 +0,0 @@ -Replace PUBLIC_HOSTNAME and PUBLIC_IP in "Received" headers to ensure that no undue spam points are attributed From 4b89143362d9ef8cfb985c030047ce74642f7952 Mon Sep 17 00:00:00 2001 From: Diman0 Date: Fri, 6 Aug 2021 23:00:27 +0200 Subject: [PATCH 173/181] Update documentation config and release notes page. --- docs/conf.py | 2 +- docs/releases.rst | 65 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 8f174b64..db7008b3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,7 +36,7 @@ html_context = { 'github_user': 'mailu', 'github_repo': 'mailu', 'github_version': version, - 'stable_version': '1.7', + 'stable_version': '1.8', 'versions': [ ('1.5', '/1.5/'), ('1.6', '/1.6/'), diff --git a/docs/releases.rst b/docs/releases.rst index 7a15d1fa..7473b033 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -1,8 +1,69 @@ Release notes ============= -Mailu 1.8 - 2020-10-02 ----------------------- +Mailu 1.8 - 2021-08-7 +--------------------- + +The full 1.8 release is finally ready. There have been some changes in the contributors team. Many people from the contributors team have stepped back due to changed priorities in their life. +We are very grateful for all their contributions and hope we will see them back again in the future. +This is the main reason why it took so long for 1.8 to be fully released. + +Fortunately more people have decided to join the project. Some very nice contributions have been made which will become part of the next 1.9 release. +We hope that future Mailu releases will be released more quickly now we have more active contributors again. + +For a list of all changes refer to `CHANGELOG.md` in the root folder of the Mailu github project. Please read the 'Override location changes' section further on this page. It contains important information for the people who use the overrides folder. + +New Functionality & Improvements +```````````````````````````````` + +Here’s a short summary of new features: + +- Roundcube and Rainloop have been updated. +- All dependencies have been updated to the latest security update. +- Fail2ban documentation has been improved. +- Switch from client side (cookie) sessions to server side sessions. +- Full-text-search is back after having been disabled for a while due to nasty bugs. It can still be disabled via the mailu.env file. +- Tons of documentation improvements, especially geared towards new users. +- (Experimental) support for different architectures, such as ARM. +- Improvements around webmails, such as CardDAV, GPG and a new skin for an updated roundcube, and support for MySQL for it. Updated Rainloop, too. +- Improvements around relaying, such as AUTH LOGIN and non-standard port support. +- Update to alpine:3.14 as baseimage for most containers. +- Setup warns users about compose-IPv6 deployments which have caused open relays in the past. +- Improved handling of upper-vs-lowercase aliases and user-addresses. +- Improved rate-limiting system. +- Support for SRS. +- Japanese localisation is now available. + + +Upgrading +````````` + +Upgrade should run fine as long as you generate a new compose or stack +configuration and upgrade your mailu.env. + +Please not that the shipped image for PostgreSQL database is deprecated. +The shipped image for PostgreSQL is not maintained anymore from release 1.8. +We recommend switching to an external PostgreSQL image as soon as possible. + +Override location changes +^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you have regenerated the Docker compose and environment files, there are some changes to the configuration overrides. +Override files are now mounted read-only into the containers. The Dovecot and Postfix overrides are moved in their own sub-directory. If there are local override files, they will need to be moved from ``overrides/`` to ``overrides/dovecot`` and ``overrides/postfix/``. + +Update your DNS SPF Records +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It has become known that the SPF DNS records generated by the admin interface are not completely standard compliant anymore. Please check the DNS records for your domains and compare them to what the new admin-interface instructs you to use. In most cases, this should be a simple copy-paste operation for you …. + +Fixed hostname for antispam service +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For history to be retained in Rspamd, the antispam container requires a static hostname. When you re-generate your docker-compose.yml file (or helm-chart), this will be covered. + + +Mailu 1.8rc - 2020-10-02 +------------------------ Release 1.8 has come a long way again. Due to corona the project slowed down to a crawl. Fortunately new contributors have joined the team what enabled us to still release Mailu 1.8 this year. From 6581f8f087d62b841e9939ea3cf3814ea9e9a518 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Fri, 6 Aug 2021 23:17:41 +0200 Subject: [PATCH 174/181] Resolve merge conflict --- CHANGELOG.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09b9f68f..3ad0061b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,18 +9,14 @@ The Dovecot and Postfix overrides are moved in their own sub-directory. If there are local override files, they will need to be moved from overrides/ to overrides/dovecot and overrides/postfix/. See https://mailu.io/1.8/faq.html#how-can-i-override-settings for all the mappings. -<<<<<<< HEAD -Please note that the shipped image for PostgreSQL database is deprecated. -We advise to switch to an external database server. -======= -One major change for the docker compose file is that the antispam needs a fixed hostname [#1837](https://github.com/Mailu/Mailu/issues/1837). -This is handled when you regenerate the docker-compose file. A fixed hostname is required to retain rspamd history. +One major change for the docker compose file is that the antispam container needs a fixed hostname [#1837](https://github.com/Mailu/Mailu/issues/1837). +This is handled when you regenerate the docker-compose file. A fixed hostname is required to retain rspamd history. +This is also handled in the helm-chart repo. Please not that the shipped image for PostgreSQL database is deprecated. We advise to switch to an external PostgreSQL database server. ->>>>>>> afaacf5a... Update CHANGELOG.md and process towncrier newsfragments. - + 1.8.0 - 2021-08-06 -------------------- From f0997ed0fd5b3be5e8a6964bfd3197ced1aa5ef9 Mon Sep 17 00:00:00 2001 From: Diman0 Date: Sat, 7 Aug 2021 09:12:43 +0200 Subject: [PATCH 175/181] Improved changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad0061b..0a128163 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ We advise to switch to an external PostgreSQL database server. - Bugfixes: Replace PUBLIC_HOSTNAME and PUBLIC_IP in "Received" headers to ensure that no undue spam points are attributed ([#191](https://github.com/Mailu/Mailu/issues/191)) - Bugfixes: Don't replace nested headers (typically in attached emails) ([#1660](https://github.com/Mailu/Mailu/issues/1660)) - Bugfixes: Fix letsencrypt access to certbot for the mail-letsencrypt flavour ([#1686](https://github.com/Mailu/Mailu/issues/1686)) -- Bugfixes: Fix CVE-2020-25275 and CVE-2020-24386 by using alpine 3.13 for +- Bugfixes: Fix CVE-2020-25275 and CVE-2020-24386 by upgrading alpine for dovecot which contains a fixed dovecot version. ([#1720](https://github.com/Mailu/Mailu/issues/1720)) - Bugfixes: Antispam service now uses a static hostname. Rspamd history is only retained when the service has a fixed hostname. ([#1837](https://github.com/Mailu/Mailu/issues/1837)) - Bugfixes: Fix a bug preventing colons from being used in passwords when using radicale/webdav. ([#1861](https://github.com/Mailu/Mailu/issues/1861)) From 21e7a338e75437afd59ae4185c87f59d18f8f4e9 Mon Sep 17 00:00:00 2001 From: Diman0 Date: Sat, 7 Aug 2021 09:14:09 +0200 Subject: [PATCH 176/181] Fixed typing error. --- CHANGELOG.md | 2 +- docs/releases.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a128163..82f04acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ One major change for the docker compose file is that the antispam container need This is handled when you regenerate the docker-compose file. A fixed hostname is required to retain rspamd history. This is also handled in the helm-chart repo. -Please not that the shipped image for PostgreSQL database is deprecated. +Please note that the shipped image for PostgreSQL database is deprecated. We advise to switch to an external PostgreSQL database server. diff --git a/docs/releases.rst b/docs/releases.rst index 7473b033..3ae25f48 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -41,7 +41,7 @@ Upgrading Upgrade should run fine as long as you generate a new compose or stack configuration and upgrade your mailu.env. -Please not that the shipped image for PostgreSQL database is deprecated. +Please note that the shipped image for PostgreSQL database is deprecated. The shipped image for PostgreSQL is not maintained anymore from release 1.8. We recommend switching to an external PostgreSQL image as soon as possible. From 14a18715111e4d21fd9cac0c7e18a293771f3fd0 Mon Sep 17 00:00:00 2001 From: Diman0 Date: Sat, 7 Aug 2021 09:25:40 +0200 Subject: [PATCH 177/181] enhanced security changelog entry and added recommendation to recreate secret_key --- CHANGELOG.md | 11 ++++++++++- docs/releases.rst | 14 +++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82f04acc..da945c72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,15 @@ One major change for the docker compose file is that the antispam container need This is handled when you regenerate the docker-compose file. A fixed hostname is required to retain rspamd history. This is also handled in the helm-chart repo. +Improvements have been made to protect again session-fixation attacks. +To be fully protected, it is required to change your SECRET_KEY in Mailu.env after upgrading. +A new SECRET_KEY is generated when you recreate your docker-compose.yml & mailu.env file via setup.mailu.io. + +The SECRET_KEY is an uppercase alphanumeric string of length 16. You can manually create such a string via +```cat /dev/urandom | tr -dc 'A-Z0-9' | fold -w ${1:-16} | head -n 1``` + +After changing mailu.env, it is required to recreate all containers for the changes to be propagated. + Please note that the shipped image for PostgreSQL database is deprecated. We advise to switch to an external PostgreSQL database server. @@ -34,7 +43,7 @@ We advise to switch to an external PostgreSQL database server. - Bugfixes: Remove dot in blueprint name to prevent critical flask startup error in setup. ([#1874](https://github.com/Mailu/Mailu/issues/1874)) - Bugfixes: fix punycode encoding of domain names ([#1891](https://github.com/Mailu/Mailu/issues/1891)) - Improved Documentation: Update fail2ban documentation to use systemd backend instead of filepath for journald ([#1857](https://github.com/Mailu/Mailu/issues/1857)) -- Misc: ([#1783](https://github.com/Mailu/Mailu/issues/1783)) +- Misc: Switch from client side (cookie) sessions to server side sessions and protect against session-fixation attacks. We recommend that you change your SECRET_KEY after upgrading. ([#1783](https://github.com/Mailu/Mailu/issues/1783)) v1.8.0rc - 2020-09-28 diff --git a/docs/releases.rst b/docs/releases.rst index 3ae25f48..6c672538 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -21,7 +21,7 @@ Here’s a short summary of new features: - Roundcube and Rainloop have been updated. - All dependencies have been updated to the latest security update. - Fail2ban documentation has been improved. -- Switch from client side (cookie) sessions to server side sessions. +- Switch from client side (cookie) sessions to server side sessions and protect against session-fixation attacks. We recommend that you change your SECRET_KEY after upgrading. - Full-text-search is back after having been disabled for a while due to nasty bugs. It can still be disabled via the mailu.env file. - Tons of documentation improvements, especially geared towards new users. - (Experimental) support for different architectures, such as ARM. @@ -51,6 +51,18 @@ Override location changes If you have regenerated the Docker compose and environment files, there are some changes to the configuration overrides. Override files are now mounted read-only into the containers. The Dovecot and Postfix overrides are moved in their own sub-directory. If there are local override files, they will need to be moved from ``overrides/`` to ``overrides/dovecot`` and ``overrides/postfix/``. +Recreate SECRET_KEY after upgrading +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Improvements have been made to protect again session-fixation attacks. +To be fully protected, it is required to change your SECRET_KEY in Mailu.env after upgrading. +A new SECRET_KEY is generated when you recreate your docker-compose.yml & mailu.env file via setup.mailu.io. + +The SECRET_KEY is an uppercase alphanumeric string of length 16. You can manually create such a string via +```cat /dev/urandom | tr -dc 'A-Z0-9' | fold -w ${1:-16} | head -n 1``` + +After changing mailu.env, it is required to recreate all containers for the changes to be propagated. + Update your DNS SPF Records ^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 3157fc3623424f7af991f2de86981c2aeced4cca Mon Sep 17 00:00:00 2001 From: Diman0 Date: Sat, 7 Aug 2021 09:27:47 +0200 Subject: [PATCH 178/181] Give docker containers in each test one more minute for starting. --- .github/workflows/CI.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 19a445b4..e2a535dd 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -121,7 +121,7 @@ jobs: - name: Copy all certs run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: Test core suite - run: python tests/compose/test.py core 1 + run: python tests/compose/test.py core 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} @@ -168,7 +168,7 @@ jobs: - name: Copy all certs run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: Test fetch - run: python tests/compose/test.py fetchmail 1 + run: python tests/compose/test.py fetchmail 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} @@ -215,7 +215,7 @@ jobs: - name: Copy all certs run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: Test clamvav - run: python tests/compose/test.py filters 2 + run: python tests/compose/test.py filters 3 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} @@ -262,7 +262,7 @@ jobs: - name: Copy all certs run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: Test rainloop - run: python tests/compose/test.py rainloop 1 + run: python tests/compose/test.py rainloop 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} @@ -309,7 +309,7 @@ jobs: - name: Copy all certs run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: Test roundcube - run: python tests/compose/test.py roundcube 1 + run: python tests/compose/test.py roundcube 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} @@ -356,7 +356,7 @@ jobs: - name: Copy all certs run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: Test webdav - run: python tests/compose/test.py webdav 1 + run: python tests/compose/test.py webdav 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} From 6b3c208fc9e4166b45c7ef21eacf75a86239bf24 Mon Sep 17 00:00:00 2001 From: Erriez Date: Sun, 8 Aug 2021 14:50:20 +0200 Subject: [PATCH 179/181] Update Alpine version from 3.10 to 3.14 --- tests/build_arm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build_arm.sh b/tests/build_arm.sh index 04836ddb..32dba421 100755 --- a/tests/build_arm.sh +++ b/tests/build_arm.sh @@ -1,6 +1,6 @@ #!/bin/bash -x -ALPINE_VER="3.10" +ALPINE_VER="3.14" DISTRO="balenalib/rpi-alpine:$ALPINE_VER" # Used for webmails QEMU="arm" From bcdc137677efef597ff8729ca40626924ee042f9 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 8 Aug 2021 19:18:33 +0200 Subject: [PATCH 180/181] Alpine has removed support for btree and hash --- core/postfix/conf/main.cf | 4 ++-- core/postfix/conf/sasl_passwd | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 8f35f609..9cd4010e 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -32,7 +32,7 @@ mydestination = relayhost = {{ RELAYHOST }} {% if RELAYUSER %} smtp_sasl_auth_enable = yes -smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd +smtp_sasl_password_maps = lmdb:/etc/postfix/sasl_passwd smtp_sasl_security_options = noanonymous {% endif %} @@ -58,7 +58,7 @@ tls_ssl_options = NO_COMPRESSION smtp_tls_security_level = {{ OUTBOUND_TLS_LEVEL|default('may') }} smtp_tls_mandatory_protocols = !SSLv2, !SSLv3 smtp_tls_protocols =!SSLv2,!SSLv3 -smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache +smtp_tls_session_cache_database = lmdb:${data_directory}/smtp_scache ############### # Virtual diff --git a/core/postfix/conf/sasl_passwd b/core/postfix/conf/sasl_passwd index e19d0657..1e32322a 100644 --- a/core/postfix/conf/sasl_passwd +++ b/core/postfix/conf/sasl_passwd @@ -1 +1,2 @@ -{{ RELAYHOST }} {{ RELAYUSER }}:{{ RELAYPASSWORD }} \ No newline at end of file +{{ RELAYHOST }} {{ RELAYUSER }}:{{ RELAYPASSWORD }} + From 1029cad04807371ac11133a43d95307005c003dc Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 8 Aug 2021 19:21:55 +0200 Subject: [PATCH 181/181] towncrier --- towncrier/newsfragments/1917.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/1917.bugfix diff --git a/towncrier/newsfragments/1917.bugfix b/towncrier/newsfragments/1917.bugfix new file mode 100644 index 00000000..68187d61 --- /dev/null +++ b/towncrier/newsfragments/1917.bugfix @@ -0,0 +1 @@ +Alpine has removed support for btree and hash in postfix... please use lmdb instead