merged changes from api without api

core/admin/mailu/manage.py

@@ -1,42 +1,52 @@
-from mailu import models
+""" Mailu command line interface
+"""
 
-from flask import current_app as app
+import sys
-from flask import cli as flask_cli
-
-import flask
 import os
 import socket
+import logging
 import uuid
+
+from collections import Counter
+from itertools import chain
+
 import click
+import sqlalchemy
 import yaml
-import sys
+
+from flask import current_app as app
+from flask.cli import FlaskGroup, with_appcontext
+from marshmallow.exceptions import ValidationError
+
+from . import models
+from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, RenderJSON, HIDDEN
 
 
 db = models.db
 
 
-@click.group()
+@click.group(cls=FlaskGroup, context_settings={'help_option_names': ['-?', '-h', '--help']})
-def mailu(cls=flask_cli.FlaskGroup):
+def mailu():
     """ Mailu command line
     """
 
 
 @mailu.command()
-@flask_cli.with_appcontext
+@with_appcontext
 def advertise():
     """ Advertise this server against statistic services.
     """
-    if os.path.isfile(app.config["INSTANCE_ID_PATH"]):
+    if os.path.isfile(app.config['INSTANCE_ID_PATH']):
-        with open(app.config["INSTANCE_ID_PATH"], "r") as handle:
+        with open(app.config['INSTANCE_ID_PATH'], 'r') as handle:
             instance_id = handle.read()
     else:
         instance_id = str(uuid.uuid4())
-        with open(app.config["INSTANCE_ID_PATH"], "w") as handle:
+        with open(app.config['INSTANCE_ID_PATH'], 'w') as handle:
             handle.write(instance_id)
-    if not app.config["DISABLE_STATISTICS"]:
+    if not app.config['DISABLE_STATISTICS']:
         try:
-            socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id))
+            socket.gethostbyname(app.config['STATS_ENDPOINT'].format(instance_id))
-        except:
+        except OSError:
             pass
 
 
@@ -45,7 +55,7 @@ def advertise():
 @click.argument('domain_name')
 @click.argument('password')
 @click.option('-m', '--mode')
-@flask_cli.with_appcontext
+@with_appcontext
 def admin(localpart, domain_name, password, mode='create'):
     """ Create an admin user
         'mode' can be:
@@ -60,7 +70,7 @@ def admin(localpart, domain_name, password, mode='create'):
 
     user = None
     if mode == 'ifmissing' or mode == 'update':
-        email = '{}@{}'.format(localpart, domain_name)
+        email = f'{localpart}@{domain_name}'
         user = models.User.query.get(email)
 
         if user and mode == 'ifmissing':
@@ -89,7 +99,7 @@ def admin(localpart, domain_name, password, mode='create'):
 @click.argument('domain_name')
 @click.argument('password')
 @click.argument('hash_scheme', required=False)
-@flask_cli.with_appcontext
+@with_appcontext
 def user(localpart, domain_name, password, hash_scheme=None):
     """ Create a user
     """
@@ -114,18 +124,18 @@ def user(localpart, domain_name, password, hash_scheme=None):
 @click.argument('domain_name')
 @click.argument('password')
 @click.argument('hash_scheme', required=False)
-@flask_cli.with_appcontext
+@with_appcontext
 def password(localpart, domain_name, password, hash_scheme=None):
     """ Change the password of an user
     """
-    email = '{0}@{1}'.format(localpart, domain_name)
+    email = f'{localpart}@{domain_name}'
-    user   = models.User.query.get(email)
+    user = models.User.query.get(email)
     if hash_scheme is None:
         hash_scheme = app.config['PASSWORD_SCHEME']
     if user:
         user.set_password(password, hash_scheme=hash_scheme)
     else:
-        print("User " + email + " not found.")
+        print(f'User {email} not found.')
     db.session.commit()
 
 
@@ -134,7 +144,7 @@ def password(localpart, domain_name, password, hash_scheme=None):
 @click.option('-u', '--max-users')
 @click.option('-a', '--max-aliases')
 @click.option('-q', '--max-quota-bytes')
-@flask_cli.with_appcontext
+@with_appcontext
 def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0):
     """ Create a domain
     """
@@ -151,9 +161,9 @@ def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0):
 @click.argument('domain_name')
 @click.argument('password_hash')
 @click.argument('hash_scheme')
-@flask_cli.with_appcontext
+@with_appcontext
 def user_import(localpart, domain_name, password_hash, hash_scheme = None):
-    """ Import a user along with password hash.
+    """ Import a user along with password hash
     """
     if hash_scheme is None:
         hash_scheme = app.config['PASSWORD_SCHEME']
@@ -171,179 +181,441 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None):
     db.session.commit()
 
 
-yaml_sections = [
+# TODO: remove deprecated config_update function?
-    ('domains', models.Domain),
+@mailu.command()
-    ('relays', models.Relay),
+@click.option('-v', '--verbose')
-    ('users', models.User),
+@click.option('-d', '--delete-objects')
-    ('aliases', models.Alias),
+@with_appcontext
-#    ('config', models.Config),
+def config_update(verbose=False, delete_objects=False):
-]
+    """ Sync configuration with data from YAML (deprecated)
+    """
+    new_config = yaml.safe_load(sys.stdin)
+    # print new_config
+    domains = new_config.get('domains', [])
+    tracked_domains = set()
+    for domain_config in domains:
+        if verbose:
+            print(str(domain_config))
+        domain_name = domain_config['name']
+        max_users = domain_config.get('max_users', -1)
+        max_aliases = domain_config.get('max_aliases', -1)
+        max_quota_bytes = domain_config.get('max_quota_bytes', 0)
+        tracked_domains.add(domain_name)
+        domain = models.Domain.query.get(domain_name)
+        if not domain:
+            domain = models.Domain(name=domain_name,
+                                   max_users=max_users,
+                                   max_aliases=max_aliases,
+                                   max_quota_bytes=max_quota_bytes)
+            db.session.add(domain)
+            print(f'Added {domain_config}')
+        else:
+            domain.max_users = max_users
+            domain.max_aliases = max_aliases
+            domain.max_quota_bytes = max_quota_bytes
+            db.session.add(domain)
+            print(f'Updated {domain_config}')
+
+    users = new_config.get('users', [])
+    tracked_users = set()
+    user_optional_params = ('comment', 'quota_bytes', 'global_admin',
+                            'enable_imap', 'enable_pop', 'forward_enabled',
+                            'forward_destination', 'reply_enabled',
+                            'reply_subject', 'reply_body', 'displayed_name',
+                            'spam_enabled', 'email', 'spam_threshold')
+    for user_config in users:
+        if verbose:
+            print(str(user_config))
+        localpart = user_config['localpart']
+        domain_name = user_config['domain']
+        password_hash = user_config.get('password_hash', None)
+        hash_scheme = user_config.get('hash_scheme', None)
+        domain = models.Domain.query.get(domain_name)
+        email = f'{localpart}@{domain_name}'
+        optional_params = {}
+        for k in user_optional_params:
+            if k in user_config:
+                optional_params[k] = user_config[k]
+        if not domain:
+            domain = models.Domain(name=domain_name)
+            db.session.add(domain)
+        user = models.User.query.get(email)
+        tracked_users.add(email)
+        tracked_domains.add(domain_name)
+        if not user:
+            user = models.User(
+                localpart=localpart,
+                domain=domain,
+                **optional_params
+            )
+        else:
+            for k in optional_params:
+                setattr(user, k, optional_params[k])
+        user.set_password(password_hash, hash_scheme=hash_scheme, raw=True)
+        db.session.add(user)
+
+    aliases = new_config.get('aliases', [])
+    tracked_aliases = set()
+    for alias_config in aliases:
+        if verbose:
+            print(str(alias_config))
+        localpart = alias_config['localpart']
+        domain_name = alias_config['domain']
+        if isinstance(alias_config['destination'], str):
+            destination = alias_config['destination'].split(',')
+        else:
+            destination = alias_config['destination']
+        wildcard = alias_config.get('wildcard', False)
+        domain = models.Domain.query.get(domain_name)
+        email = f'{localpart}@{domain_name}'
+        if not domain:
+            domain = models.Domain(name=domain_name)
+            db.session.add(domain)
+        alias = models.Alias.query.get(email)
+        tracked_aliases.add(email)
+        tracked_domains.add(domain_name)
+        if not alias:
+            alias = models.Alias(
+                localpart=localpart,
+                domain=domain,
+                wildcard=wildcard,
+                destination=destination,
+                email=email
+            )
+        else:
+            alias.destination = destination
+            alias.wildcard = wildcard
+        db.session.add(alias)
+
+    db.session.commit()
+
+    managers = new_config.get('managers', [])
+    # tracked_managers=set()
+    for manager_config in managers:
+        if verbose:
+            print(str(manager_config))
+        domain_name = manager_config['domain']
+        user_name = manager_config['user']
+        domain = models.Domain.query.get(domain_name)
+        manageruser = models.User.query.get(f'{user_name}@{domain_name}')
+        if manageruser not in domain.managers:
+            domain.managers.append(manageruser)
+        db.session.add(domain)
+
+    db.session.commit()
+
+    if delete_objects:
+        for user in db.session.query(models.User).all():
+            if not user.email in tracked_users:
+                if verbose:
+                    print(f'Deleting user: {user.email}')
+                db.session.delete(user)
+        for alias in db.session.query(models.Alias).all():
+            if not alias.email in tracked_aliases:
+                if verbose:
+                    print(f'Deleting alias: {alias.email}')
+                db.session.delete(alias)
+        for domain in db.session.query(models.Domain).all():
+            if not domain.name in tracked_domains:
+                if verbose:
+                    print(f'Deleting domain: {domain.name}')
+                db.session.delete(domain)
+    db.session.commit()
+
 
 @mailu.command()
-@click.option('-v', '--verbose', is_flag=True, help='Increase verbosity')
+@click.option('-v', '--verbose', count=True, help='Increase verbosity.')
-@click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml')
+@click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.')
-@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made')
+@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.')
-@flask_cli.with_appcontext
+@click.option('-c', '--color', is_flag=True, help='Force colorized output.')
-def config_update(verbose=False, delete_objects=False, dry_run=False, file=None):
+@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.')
-    """sync configuration with data from YAML-formatted stdin"""
+@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.')
+@click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin)
+@with_appcontext
+def config_import(verbose=0, secrets=False, quiet=False, color=False, update=False, dry_run=False, source=None):
+    """ Import configuration as YAML or JSON from stdin or file
+    """
 
-    out = (lambda *args: print('(DRY RUN)', *args)) if dry_run else print
+    # verbose
+    # 0 : only show number of changes
+    # 1 : also show detailed changes
+    # 2 : also show input data
+    # 3 : also show sql queries (also needs -s, as sql may contain secrets)
+    # 4 : also show tracebacks (also needs -s, as tracebacks may contain secrets)
+
+    if quiet:
+        verbose = -1
+
+    if verbose > 2 and not secrets:
+        print('[Warning] Verbosity level capped to 2. Specify --secrets to log sql and tracebacks.')
+        verbose = 2
+
+    color_cfg = {
+        'color': color or sys.stdout.isatty(),
+        'lexer': 'python',
+        'strip': True,
+    }
+
+    counter = Counter()
+    logger = {}
+
+    def format_errors(store, path=None):
+
+        res = []
+        if path is None:
+            path = []
+        for key in sorted(store):
+            location = path + [str(key)]
+            value = store[key]
+            if isinstance(value, dict):
+                res.extend(format_errors(value, location))
+            else:
+                for message in value:
+                    res.append((".".join(location), message))
+
+        if path:
+            return res
+
+        fmt = f'     - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}'
+        res = [fmt.format(loc, msg) for loc, msg in res]
+        num = f'error{["s",""][len(res)==1]}'
+        res.insert(0, f'[ValidationError] {len(res)} {num} occurred during input validation')
+
+        return '\n'.join(res)
+
+    def format_changes(*message):
+        if counter:
+            changes = []
+            last = None
+            for (action, what), count in sorted(counter.items()):
+                if action != last:
+                    if last:
+                        changes.append('/')
+                    changes.append(f'{action}:')
+                    last = action
+                changes.append(f'{what}({count})')
+        else:
+            changes = ['No changes.']
+        return chain(message, changes)
+
+    def log(action, target, message=None):
+
+        if message is None:
+            try:
+                message = logger[target.__class__].dump(target)
+            except KeyError:
+                message = target
+        if not isinstance(message, str):
+            message = repr(message)
+        print(f'{action} {target.__table__}: {colorize(message, **color_cfg)}')
+
+    def listen_insert(mapper, connection, target): # pylint: disable=unused-argument
+        """ callback function to track import """
+        counter.update([('Created', target.__table__.name)])
+        if verbose >= 1:
+            log('Created', target)
+
+    def listen_update(mapper, connection, target): # pylint: disable=unused-argument
+        """ callback function to track import """
+
+        changed = {}
+        inspection = sqlalchemy.inspect(target)
+        for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs:
+            history = getattr(inspection.attrs, attr.key).history
+            if history.has_changes() and history.deleted:
+                before = history.deleted[-1]
+                after = getattr(target, attr.key)
+                # TODO: remove special handling of "comment" after modifying model
+                if attr.key == 'comment' and not before and not after:
+                    pass
+                # only remember changed keys
+                elif before != after:
+                    if verbose >= 1:
+                        changed[str(attr.key)] = (before, after)
+                    else:
+                        break
+
+        if verbose >= 1:
+            # use schema with dump_context to hide secrets and sort keys
+            dumped = get_schema(target)(only=changed.keys(), context=diff_context).dump(target)
+            for key, value in dumped.items():
+                before, after = changed[key]
+                if value == HIDDEN:
+                    before = HIDDEN if before else before
+                    after = HIDDEN if after else after
+                else:
+                    # TODO: need to use schema to "convert" before value?
+                    after = value
+                log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}')
+
+        if changed:
+            counter.update([('Modified', target.__table__.name)])
+
+    def listen_delete(mapper, connection, target): # pylint: disable=unused-argument
+        """ callback function to track import """
+        counter.update([('Deleted', target.__table__.name)])
+        if verbose >= 1:
+            log('Deleted', target)
+
+    # TODO: this listener will not be necessary, if dkim keys would be stored in database
+    _dedupe_dkim = set()
+    def listen_dkim(session, flush_context): # pylint: disable=unused-argument
+        """ callback function to track import """
+        for target in session.identity_map.values():
+            # look at Domains originally loaded from db
+            if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path:
+                continue
+            before = target._dkim_key_on_disk
+            after = target._dkim_key
+            if before != after:
+                if secrets:
+                    before = before.decode('ascii', 'ignore')
+                    after = after.decode('ascii', 'ignore')
+                else:
+                    before = HIDDEN if before else ''
+                    after = HIDDEN if after else ''
+                # "de-dupe" messages; this event is fired at every flush
+                if not (target, before, after) in _dedupe_dkim:
+                    _dedupe_dkim.add((target, before, after))
+                    counter.update([('Modified', target.__table__.name)])
+                    if verbose >= 1:
+                        log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}')
+
+    def track_serialize(obj, item, backref=None):
+        """ callback function to track import """
+        # called for backref modification?
+        if backref is not None:
+            log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref))
+            return
+        # show input data?
+        if not verbose >= 2:
+            return
+        # hide secrets in data
+        data = logger[obj.opts.model].hide(item)
+        if 'hash_password' in data:
+            data['password'] = HIDDEN
+        if 'fetches' in data:
+            for fetch in data['fetches']:
+                fetch['password'] = HIDDEN
+        log('Handling', obj.opts.model, data)
+
+    # configure contexts
+    diff_context = {
+        'full': True,
+        'secrets': secrets,
+    }
+    log_context = {
+        'secrets': secrets,
+    }
+    load_context = {
+        'import': True,
+        'update': update,
+        'clear': not update,
+        'callback': track_serialize,
+    }
+
+    # register listeners
+    for schema in get_schema():
+        model = schema.Meta.model
+        logger[model] = schema(context=log_context)
+        sqlalchemy.event.listen(model, 'after_insert', listen_insert)
+        sqlalchemy.event.listen(model, 'after_update', listen_update)
+        sqlalchemy.event.listen(model, 'after_delete', listen_delete)
+
+    # special listener for dkim_key changes
+    sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim)
+
+    if verbose >= 3:
+        logging.basicConfig()
+        logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
 
     try:
-        new_config = yaml.safe_load(sys.stdin)
+        with models.db.session.no_autoflush:
-    except (yaml.scanner.ScannerError, yaml.parser.ParserError) as reason:
+            config = MailuSchema(only=MailuSchema.Meta.order, context=load_context).loads(source)
-        out(f'[ERROR] Invalid yaml: {reason}')
+    except ValidationError as exc:
-        sys.exit(1)
+        raise click.ClickException(format_errors(exc.messages)) from exc
-    else:
+    except Exception as exc:
-        if type(new_config) is str:
+        if verbose >= 3:
-            out(f'[ERROR] Invalid yaml: {new_config!r}')
+            raise
-            sys.exit(1)
+        # (yaml.scanner.ScannerError, UnicodeDecodeError, ...)
-        elif new_config is None or not len(new_config):
+        raise click.ClickException(
-            out('[ERROR] Empty yaml: Please pipe yaml into stdin')
+            f'[{exc.__class__.__name__}] '
-            sys.exit(1)
+            f'{" ".join(str(exc).split())}'
+        ) from exc
 
-    error = False
+    # flush session to show/count all changes
-    tracked = {}
+    if dry_run or verbose >= 1:
-    for section, model in yaml_sections:
+        db.session.flush()
 
-        items = new_config.get(section)
+    # check for duplicate domain names
-        if items is None:
+    dup = set()
-            if delete_objects:
+    for fqdn in chain(
-                out(f'[ERROR] Invalid yaml: Section "{section}" is missing')
+        db.session.query(models.Domain.name),
-                error = True
+        db.session.query(models.Alternative.name),
-                break
+        db.session.query(models.Relay.name)
-            else:
+    ):
-                continue
+        if fqdn in dup:
-
+            raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}')
-        del new_config[section]
+        dup.add(fqdn)
-
-        if type(items) is not list:
-            out(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}')
-            error = True
-            break
-        elif not items:
-            continue
-
-        # create items
-        for data in items:
-
-            if verbose:
-                out(f'Handling {model.__table__} data: {data!r}')
-
-            try:
-                changed = model.from_dict(data, delete_objects)
-            except Exception as reason:
-                out(f'[ERROR] {reason.args[0]} in data: {data}')
-                error = True
-                break
-
-            for item, created in changed:
-
-                if created is True:
-                    # flush newly created item
-                    db.session.add(item)
-                    db.session.flush()
-                    if verbose:
-                        out(f'Added {item!r}: {item.to_dict()}')
-                    else:
-                        out(f'Added {item!r}')
-
-                elif len(created):
-                    # modified instance
-                    if verbose:
-                        for key, old, new in created:
-                            out(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}')
-                    else:
-                        out(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}')
-
-                # track primary key of all items
-                tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()]))
-
-        if error:
-            break
-
-    # on error: stop early
-    if error:
-        out('An error occured. Not committing changes.')
-        db.session.rollback()
-        sys.exit(1)
-
-    # are there sections left in new_config?
-    if new_config:
-        out(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}')
-        error = True
-
-    # test for conflicting domains
-    domains = set()
-    for model, items in tracked.items():
-        if model in (models.Domain, models.Alternative, models.Relay):
-            if domains & items:
-                for domain in domains & items:
-                    out(f'[ERROR] Duplicate domain name used: {domain}')
-                error = True
-            domains.update(items)
-
-    # delete items not tracked
-    if delete_objects:
-        for model, items in tracked.items():
-            for item in model.query.all():
-                if not item._dict_pval() in items:
-                    out(f'Deleted {item!r} {item}')
-                    db.session.delete(item)
 
     # don't commit when running dry
     if dry_run:
+        if not quiet:
+            print(*format_changes('Dry run. Not commiting changes.'))
         db.session.rollback()
     else:
+        if not quiet:
+            print(*format_changes('Committing changes.'))
         db.session.commit()
 
 
 @mailu.command()
-@click.option('-f', '--full', is_flag=True, help='Include default attributes')
+@click.option('-f', '--full', is_flag=True, help='Include attributes with default value.')
-@click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)')
+@click.option('-s', '--secrets', is_flag=True,
-@click.option('-d', '--dns', is_flag=True, help='Include dns records')
+              help='Include secret attributes (dkim-key, passwords).')
-@click.argument('sections', nargs=-1)
+@click.option('-c', '--color', is_flag=True, help='Force colorized output.')
-@flask_cli.with_appcontext
+@click.option('-d', '--dns', is_flag=True, help='Include dns records.')
-def config_dump(full=False, secrets=False, dns=False, sections=None):
+@click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'),
-    """dump configuration as YAML-formatted data to stdout
+              help='Save configuration to file.')
-
+@click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.')
-    SECTIONS can be: domains, relays, users, aliases
+@click.argument('only', metavar='[FILTER]...', nargs=-1)
+@with_appcontext
+def config_export(full=False, secrets=False, color=False, dns=False, output=None, as_json=False, only=None):
+    """ Export configuration as YAML or JSON to stdout or file
     """
 
-    class spacedDumper(yaml.Dumper):
+    if only:
+        for spec in only:
+            if spec.split('.', 1)[0] not in MailuSchema.Meta.order:
+                raise click.ClickException(f'[ValidationError] Unknown section: {spec}')
+    else:
+        only = MailuSchema.Meta.order
 
-        def write_line_break(self, data=None):
+    context = {
-            super().write_line_break(data)
+        'full': full,
-            if len(self.indents) == 1:
+        'secrets': secrets,
-                super().write_line_break()
+        'dns': dns,
+    }
 
-        def increase_indent(self, flow=False, indentless=False):
+    schema = MailuSchema(only=only, context=context)
-            return super().increase_indent(flow, False)
+    color_cfg = {'color': color or output.isatty()}
 
-    if sections:
+    if as_json:
-        check = dict(yaml_sections)
+        schema.opts.render_module = RenderJSON
-        for section in sections:
+        color_cfg['lexer'] = 'json'
-            if section not in check:
+        color_cfg['strip'] = True
-                print(f'[ERROR] Invalid section: {section}')
-                return 1
 
-    extra = []
+    try:
-    if dns:
+        print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output)
-        extra.append('dns')
+    except ValueError as exc:
-
+        if spec := get_fieldspec(exc):
-    config = {}
+            raise click.ClickException(f'[ValidationError] Invalid filter: {spec}') from exc
-    for section, model in yaml_sections:
+        raise
-        if not sections or section in sections:
-            dump = [item.to_dict(full, secrets, extra) for item in model.query.all()]
-            if len(dump):
-                config[section] = dump
-
-    yaml.dump(config, sys.stdout, Dumper=spacedDumper, default_flow_style=False, allow_unicode=True)
 
 
 @mailu.command()
 @click.argument('email')
-@flask_cli.with_appcontext
+@with_appcontext
 def user_delete(email):
     """delete user"""
     user = models.User.query.get(email)
@@ -354,7 +626,7 @@ def user_delete(email):
 
 @mailu.command()
 @click.argument('email')
-@flask_cli.with_appcontext
+@with_appcontext
 def alias_delete(email):
     """delete alias"""
     alias = models.Alias.query.get(email)
@@ -368,7 +640,7 @@ def alias_delete(email):
 @click.argument('domain_name')
 @click.argument('destination')
 @click.option('-w', '--wildcard', is_flag=True)
-@flask_cli.with_appcontext
+@with_appcontext
 def alias(localpart, domain_name, destination, wildcard=False):
     """ Create an alias
     """
@@ -381,7 +653,7 @@ def alias(localpart, domain_name, destination, wildcard=False):
         domain=domain,
         wildcard=wildcard,
         destination=destination.split(','),
-        email="%s@%s" % (localpart, domain_name)
+        email=f'{localpart}@{domain_name}'
     )
     db.session.add(alias)
     db.session.commit()
@@ -392,7 +664,7 @@ def alias(localpart, domain_name, destination, wildcard=False):
 @click.argument('max_users')
 @click.argument('max_aliases')
 @click.argument('max_quota_bytes')
-@flask_cli.with_appcontext
+@with_appcontext
 def setlimits(domain_name, max_users, max_aliases, max_quota_bytes):
     """ Set domain limits
     """
@@ -407,12 +679,12 @@ def setlimits(domain_name, max_users, max_aliases, max_quota_bytes):
 @mailu.command()
 @click.argument('domain_name')
 @click.argument('user_name')
-@flask_cli.with_appcontext
+@with_appcontext
 def setmanager(domain_name, user_name='manager'):
     """ Make a user manager of a domain
     """
     domain = models.Domain.query.get(domain_name)
-    manageruser = models.User.query.get(user_name + '@' + domain_name)
+    manageruser = models.User.query.get(f'{user_name}@{domain_name}')
     domain.managers.append(manageruser)
     db.session.add(domain)
     db.session.commit()

core/admin/mailu/schemas.py

@@ -0,0 +1,945 @@
+""" Mailu marshmallow fields and schema
+"""
+
+from copy import deepcopy
+from textwrap import wrap
+
+import re
+import json
+import yaml
+
+import sqlalchemy
+
+from marshmallow import pre_load, post_load, post_dump, fields, Schema
+from marshmallow.utils import ensure_text_type
+from marshmallow.exceptions import ValidationError
+from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts
+from marshmallow_sqlalchemy.fields import RelatedList
+
+from flask_marshmallow import Marshmallow
+
+from OpenSSL import crypto
+
+try:
+    from pygments import highlight
+    from pygments.token import Token
+    from pygments.lexers import get_lexer_by_name
+    from pygments.lexers.data import YamlLexer
+    from pygments.formatters import get_formatter_by_name
+except ModuleNotFoundError:
+    COLOR_SUPPORTED = False
+else:
+    COLOR_SUPPORTED = True
+
+from . import models, dkim
+
+
+ma = Marshmallow()
+
+# TODO: how and where to mark keys as "required" while deserializing in api?
+# - when modifying, nothing is required (only the primary key, but this key is in the uri)
+#   - the primary key from post data must not differ from the key in the uri
+# - when creating all fields without default or auto-increment are required
+# TODO: validate everything!
+
+
+### class for hidden values ###
+
+class _Hidden:
+    def __bool__(self):
+        return False
+    def __copy__(self):
+        return self
+    def __deepcopy__(self, _):
+        return self
+    def __eq__(self, other):
+        return str(other) == '<hidden>'
+    def __repr__(self):
+        return '<hidden>'
+    __str__ = __repr__
+
+HIDDEN = _Hidden()
+
+
+### map model to schema ###
+
+_model2schema = {}
+
+def get_schema(model=None):
+    """ return schema class for model or instance of model """
+    if model is None:
+        return _model2schema.values()
+    else:
+        return _model2schema.get(model) or _model2schema.get(model.__class__)
+
+def mapped(cls):
+    """ register schema in model2schema map """
+    _model2schema[cls.Meta.model] = cls
+    return cls
+
+
+### helper functions ###
+
+def get_fieldspec(exc):
+    """ walk traceback to extract spec of invalid field from marshmallow """
+    path = []
+    tbck = exc.__traceback__
+    while tbck:
+        if tbck.tb_frame.f_code.co_name == '_serialize':
+            if 'attr' in tbck.tb_frame.f_locals:
+                path.append(tbck.tb_frame.f_locals['attr'])
+        elif tbck.tb_frame.f_code.co_name == '_init_fields':
+            path = '.'.join(path)
+            spec = ', '.join([f'{path}.{key}' for key in tbck.tb_frame.f_locals['invalid_fields']])
+            return spec
+        tbck = tbck.tb_next
+    return None
+
+def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False):
+    """ add ANSI color to data """
+    if color is None:
+        # autodetect colorize
+        color = COLOR_SUPPORTED
+    if not color:
+        # no color wanted
+        return data
+    if not COLOR_SUPPORTED:
+        # want color, but not supported
+        raise ValueError('Please install pygments to colorize output')
+
+    scheme = {
+        Token:                  ('',        ''),
+        Token.Name.Tag:         ('cyan',    'brightcyan'),
+        Token.Literal.Scalar:   ('green',   'green'),
+        Token.Literal.String:   ('green',   'green'),
+        Token.Keyword.Constant: ('magenta', 'brightmagenta'),
+        Token.Literal.Number:   ('magenta', 'brightmagenta'),
+        Token.Error:            ('red',     'brightred'),
+        Token.Name:             ('red',     'brightred'),
+        Token.Operator:         ('red',     'brightred'),
+    }
+
+    class MyYamlLexer(YamlLexer):
+        """ colorize yaml constants and integers """
+        def get_tokens(self, text, unfiltered=False):
+            for typ, value in super().get_tokens(text, unfiltered):
+                if typ is Token.Literal.Scalar.Plain:
+                    if value in {'true', 'false', 'null'}:
+                        typ = Token.Keyword.Constant
+                    elif value == HIDDEN:
+                        typ = Token.Error
+                    else:
+                        try:
+                            int(value, 10)
+                        except ValueError:
+                            try:
+                                float(value)
+                            except ValueError:
+                                pass
+                            else:
+                                typ = Token.Literal.Number.Float
+                        else:
+                            typ = Token.Literal.Number.Integer
+                yield typ, value
+
+    res = highlight(
+        data,
+        MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer),
+        get_formatter_by_name(formatter, colorscheme=scheme)
+    )
+
+    return res.rstrip('\n') if strip else res
+
+
+### render modules ###
+
+# allow yaml to represent hidden attributes
+yaml.add_representer(
+    _Hidden,
+    lambda cls, data: cls.represent_data(str(data))
+)
+
+class RenderYAML:
+    """ Marshmallow YAML Render Module
+    """
+
+    class SpacedDumper(yaml.Dumper):
+        """ YAML Dumper to add a newline between main sections
+            and double the indent used
+        """
+
+        def write_line_break(self, data=None):
+            super().write_line_break(data)
+            if len(self.indents) == 1:
+                super().write_line_break()
+
+        def increase_indent(self, flow=False, indentless=False):
+            return super().increase_indent(flow, False)
+
+    @staticmethod
+    def _augment(kwargs, defaults):
+        """ add default kv's to kwargs if missing
+        """
+        for key, value in defaults.items():
+            if key not in kwargs:
+                kwargs[key] = value
+
+    _load_defaults = {}
+    @classmethod
+    def loads(cls, *args, **kwargs):
+        """ load yaml data from string
+        """
+        cls._augment(kwargs, cls._load_defaults)
+        return yaml.safe_load(*args, **kwargs)
+
+    _dump_defaults = {
+        'Dumper': SpacedDumper,
+        'default_flow_style': False,
+        'allow_unicode': True,
+        'sort_keys': False,
+    }
+    @classmethod
+    def dumps(cls, *args, **kwargs):
+        """ dump data to yaml string
+        """
+        cls._augment(kwargs, cls._dump_defaults)
+        return yaml.dump(*args, **kwargs)
+
+class JSONEncoder(json.JSONEncoder):
+    """ JSONEncoder supporting serialization of HIDDEN """
+    def default(self, o):
+        """ serialize HIDDEN """
+        if isinstance(o, _Hidden):
+            return str(o)
+        return json.JSONEncoder.default(self, o)
+
+class RenderJSON:
+    """ Marshmallow JSON Render Module
+    """
+
+    @staticmethod
+    def _augment(kwargs, defaults):
+        """ add default kv's to kwargs if missing
+        """
+        for key, value in defaults.items():
+            if key not in kwargs:
+                kwargs[key] = value
+
+    _load_defaults = {}
+    @classmethod
+    def loads(cls, *args, **kwargs):
+        """ load json data from string
+        """
+        cls._augment(kwargs, cls._load_defaults)
+        return json.loads(*args, **kwargs)
+
+    _dump_defaults = {
+        'separators': (',',':'),
+        'cls': JSONEncoder,
+    }
+    @classmethod
+    def dumps(cls, *args, **kwargs):
+        """ dump data to json string
+        """
+        cls._augment(kwargs, cls._dump_defaults)
+        return json.dumps(*args, **kwargs)
+
+
+### custom fields ###
+
+class LazyStringField(fields.String):
+    """ Field that serializes a "false" value to the empty string
+    """
+
+    def _serialize(self, value, attr, obj, **kwargs):
+        """ serialize None to the empty string
+        """
+        return value if value else ''
+
+class CommaSeparatedListField(fields.Raw):
+    """ Deserialize a string containing comma-separated values to
+        a list of strings
+    """
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        """ deserialize comma separated string to list of strings
+        """
+
+        # empty
+        if not value:
+            return []
+
+        # split string
+        if isinstance(value, str):
+            return list([item.strip() for item in value.split(',') if item.strip()])
+        else:
+            return value
+
+
+class DkimKeyField(fields.String):
+    """ Serialize a dkim key to a list of strings (lines) and
+        Deserialize a string or list of strings to a valid dkim key
+    """
+
+    default_error_messages = {
+        "invalid": "Not a valid string or list.",
+        "invalid_utf8": "Not a valid utf-8 string or list.",
+    }
+
+    _clean_re = re.compile(
+        r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)',
+        flags=re.UNICODE
+    )
+
+    def _serialize(self, value, attr, obj, **kwargs):
+        """ serialize dkim key to a list of strings (lines)
+        """
+
+        # map empty string and None to None
+        if not value:
+            return None
+
+        # return list of key lines without header/footer
+        return value.decode('utf-8').strip().split('\n')[1:-1]
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        """ deserialize a string or list of strings to dkim key data
+            with verification
+        """
+
+        # convert list to str
+        if isinstance(value, list):
+            try:
+                value = ''.join([ensure_text_type(item) for item in value])
+            except UnicodeDecodeError as exc:
+                raise self.make_error("invalid_utf8") from exc
+
+        # only text is allowed
+        else:
+            if not isinstance(value, (str, bytes)):
+                raise self.make_error("invalid")
+            try:
+                value = ensure_text_type(value)
+            except UnicodeDecodeError as exc:
+                raise self.make_error("invalid_utf8") from exc
+
+        # clean value (remove whitespace and header/footer)
+        value = self._clean_re.sub('', value.strip())
+
+        # map empty string/list to None
+        if not value:
+            return None
+
+        # handle special value 'generate'
+        elif value == 'generate':
+            return dkim.gen_key()
+
+        # remember some keydata for error message
+        keydata = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value
+
+        # wrap value into valid pem layout and check validity
+        value = (
+            '-----BEGIN PRIVATE KEY-----\n' +
+            '\n'.join(wrap(value, 64)) +
+            '\n-----END PRIVATE KEY-----\n'
+        ).encode('ascii')
+        try:
+            crypto.load_privatekey(crypto.FILETYPE_PEM, value)
+        except crypto.Error as exc:
+            raise ValidationError(f'invalid dkim key {keydata!r}') from exc
+        else:
+            return value
+
+class PasswordField(fields.Str):
+    """ Serialize a hashed password hash by stripping the obsolete {SCHEME}
+        Deserialize a plain password or hashed password into a hashed password
+    """
+
+    _hashes = {'PBKDF2', 'BLF-CRYPT', 'SHA512-CRYPT', 'SHA256-CRYPT', 'MD5-CRYPT', 'CRYPT'}
+
+    def _serialize(self, value, attr, obj, **kwargs):
+        """ strip obsolete {password-hash} when serializing """
+        # strip scheme spec if in database - it's obsolete
+        if value.startswith('{') and (end := value.find('}', 1)) >= 0:
+            if value[1:end] in self._hashes:
+                return value[end+1:]
+        return value
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        """ hashes plain password or checks hashed password
+            also strips obsolete {password-hash} when deserializing
+        """
+
+        # when hashing is requested: use model instance to hash plain password
+        if data.get('hash_password'):
+            # hash password using model instance
+            inst = self.metadata['model']()
+            inst.set_password(value)
+            value = inst.password
+            del inst
+
+        # strip scheme spec when specified - it's obsolete
+        if value.startswith('{') and (end := value.find('}', 1)) >= 0:
+            if value[1:end] in self._hashes:
+                value = value[end+1:]
+
+        # check if algorithm is supported
+        inst = self.metadata['model'](password=value)
+        try:
+            # just check against empty string to see if hash is valid
+            inst.check_password('')
+        except ValueError as exc:
+            # ValueError: hash could not be identified
+            raise ValidationError(f'invalid password hash {value!r}') from exc
+        del inst
+
+        return value
+
+
+### base schema ###
+
+class BaseOpts(SQLAlchemyAutoSchemaOpts):
+    """ Option class with sqla session
+    """
+    def __init__(self, meta, ordered=False):
+        if not hasattr(meta, 'sqla_session'):
+            meta.sqla_session = models.db.session
+        if not hasattr(meta, 'sibling'):
+            meta.sibling = False
+        super(BaseOpts, self).__init__(meta, ordered=ordered)
+
+class BaseSchema(ma.SQLAlchemyAutoSchema):
+    """ Marshmallow base schema with custom exclude logic
+        and option to hide sqla defaults
+    """
+
+    OPTIONS_CLASS = BaseOpts
+
+    class Meta:
+        """ Schema config """
+        include_by_context = {}
+        exclude_by_value = {}
+        hide_by_context = {}
+        order = []
+        sibling = False
+
+    def __init__(self, *args, **kwargs):
+
+        # get context
+        context = kwargs.get('context', {})
+        flags = {key for key, value in context.items() if value is True}
+
+        # compile excludes
+        exclude = set(kwargs.get('exclude', []))
+
+        # always exclude
+        exclude.update({'created_at', 'updated_at'})
+
+        # add include_by_context
+        if context is not None:
+            for need, what in getattr(self.Meta, 'include_by_context', {}).items():
+                if not flags & set(need):
+                    exclude |= set(what)
+
+        # update excludes
+        kwargs['exclude'] = exclude
+
+        # init SQLAlchemyAutoSchema
+        super().__init__(*args, **kwargs)
+
+        # exclude_by_value
+        self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {})
+
+        # exclude default values
+        if not context.get('full'):
+            for column in self.opts.model.__table__.columns:
+                if column.name not in exclude:
+                    self._exclude_by_value.setdefault(column.name, []).append(
+                        None if column.default is None else column.default.arg
+                    )
+
+        # hide by context
+        self._hide_by_context = set()
+        if context is not None:
+            for need, what in getattr(self.Meta, 'hide_by_context', {}).items():
+                if not flags & set(need):
+                    self._hide_by_context |= set(what)
+
+        # remember primary keys
+        self._primary = str(self.opts.model.__table__.primary_key.columns.values()[0].name)
+
+        # determine attribute order
+        if hasattr(self.Meta, 'order'):
+            # use user-defined order
+            order = self.Meta.order
+        else:
+            # default order is: primary_key + other keys alphabetically
+            order = list(sorted(self.fields.keys()))
+            if self._primary in order:
+                order.remove(self._primary)
+                order.insert(0, self._primary)
+
+        # order dump_fields
+        for field in order:
+            if field in self.dump_fields:
+                self.dump_fields[field] = self.dump_fields.pop(field)
+
+        # move pre_load hook "_track_import" to the front
+        hooks = self._hooks[('pre_load', False)]
+        hooks.remove('_track_import')
+        hooks.insert(0, '_track_import')
+        # move pre_load hook "_add_instance" to the end
+        hooks.remove('_add_required')
+        hooks.append('_add_required')
+
+        # move post_load hook "_add_instance" to the end
+        hooks = self._hooks[('post_load', False)]
+        hooks.remove('_add_instance')
+        hooks.append('_add_instance')
+
+    def hide(self, data):
+        """ helper method to hide input data for logging """
+        # always returns a copy of data
+        return {
+            key: HIDDEN if key in self._hide_by_context else deepcopy(value)
+            for key, value in data.items()
+        }
+
+    def _call_and_store(self, *args, **kwargs):
+        """ track curent parent field for pruning """
+        self.context['parent_field'] = kwargs['field_name']
+        return super()._call_and_store(*args, **kwargs)
+
+    # this is only needed to work around the declared attr "email" primary key in model
+    def get_instance(self, data):
+        """ lookup item by defined primary key instead of key(s) from model """
+        if self.transient:
+            return None
+        if keys := getattr(self.Meta, 'primary_keys', None):
+            filters = {key: data.get(key) for key in keys}
+            if None not in filters.values():
+                return self.session.query(self.opts.model).filter_by(**filters).first()
+        return super().get_instance(data)
+
+    @pre_load(pass_many=True)
+    def _patch_input(self, items, many, **kwargs): # pylint: disable=unused-argument
+        """ - flush sqla session before serializing a section when requested
+              (make sure all objects that could be referred to later are created)
+            - when in update mode: patch input data before deserialization
+              - handle "prune" and "delete" items
+              - replace values in keys starting with '-' with default
+        """
+
+        # flush sqla session
+        if not self.Meta.sibling:
+            self.opts.sqla_session.flush()
+
+        # stop early when not updating
+        if not self.context.get('update'):
+            return items
+
+        # patch "delete", "prune" and "default"
+        want_prune = []
+        def patch(count, data, prune):
+
+            # don't allow __delete__ coming from input
+            if '__delete__' in data:
+                raise ValidationError('Unknown field.', f'{count}.__delete__')
+
+            # handle "prune list" and "delete item" (-pkey: none and -pkey: id)
+            for key in data:
+                if key.startswith('-'):
+                    if key[1:] == self._primary:
+                        # delete or prune
+                        if data[key] is None:
+                            # prune
+                            prune.append(True)
+                            return None
+                        # mark item for deletion
+                        return {key[1:]: data[key], '__delete__': True}
+
+            # handle "set to default value" (-key: none)
+            def set_default(key, value):
+                if not key.startswith('-'):
+                    return (key, value)
+                key = key[1:]
+                if not key in self.opts.model.__table__.columns:
+                    return (key, None)
+                if value is not None:
+                    raise ValidationError(
+                        'When resetting to default value must be null.',
+                        f'{count}.{key}'
+                    )
+                value = self.opts.model.__table__.columns[key].default
+                if value is None:
+                    raise ValidationError(
+                        'Field has no default value.',
+                        f'{count}.{key}'
+                    )
+                return (key, value.arg)
+
+            return dict([set_default(key, value) for key, value in data.items()])
+
+        # convert items to "delete" and filter "prune" item
+        items = [
+            item for item in [
+                patch(count, item, want_prune) for count, item in enumerate(items)
+            ] if item
+        ]
+
+        # prune: determine if existing items in db need to be added or marked for deletion
+        add_items = False
+        del_items = False
+        if self.Meta.sibling:
+            # parent prunes automatically
+            if not want_prune:
+                # no prune requested => add old items
+                add_items = True
+        else:
+            # parent does not prune automatically
+            if want_prune:
+                # prune requested => mark old items for deletion
+                del_items = True
+
+        if add_items or del_items:
+            existing = {item[self._primary] for item in items if self._primary in item}
+            for item in getattr(self.context['parent'], self.context['parent_field']):
+                key = getattr(item, self._primary)
+                if key not in existing:
+                    if add_items:
+                        items.append({self._primary: key})
+                    else:
+                        items.append({self._primary: key, '__delete__': True})
+
+        return items
+
+    @pre_load
+    def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument
+        """ call callback function to track import
+        """
+        # callback
+        if callback := self.context.get('callback'):
+            callback(self, data)
+
+        return data
+
+    @pre_load
+    def _add_required(self, data, many, **kwargs): # pylint: disable=unused-argument
+        """ when updating:
+            allow modification of existing items having required attributes
+            by loading existing value from db
+        """
+
+        if not self.opts.load_instance or not self.context.get('update'):
+            return data
+
+        # stabilize import of auto-increment primary keys (not required),
+        # by matching import data to existing items and setting primary key
+        if not self._primary in data:
+            for item in getattr(self.context['parent'], self.context['parent_field']):
+                existing = self.dump(item, many=False)
+                this = existing.pop(self._primary)
+                if data == existing:
+                    instance = item
+                    data[self._primary] = this
+                    break
+
+        # try to load instance
+        instance = self.instance or self.get_instance(data)
+        if instance is None:
+
+            if '__delete__' in data:
+                # deletion of non-existent item requested
+                raise ValidationError(
+                    f'item to delete not found: {data[self._primary]!r}',
+                    field_name=f'?.{self._primary}',
+                )
+
+        else:
+
+            if self.context.get('update'):
+                # remember instance as parent for pruning siblings
+                if not self.Meta.sibling:
+                    self.context['parent'] = instance
+                # delete instance when marked
+                if '__delete__' in data:
+                    self.opts.sqla_session.delete(instance)
+                # delete item from lists or prune lists
+                # currently: domain.alternatives, user.forward_destination,
+                # user.manager_of, aliases.destination
+                for key, value in data.items():
+                    if not isinstance(self.fields[key], fields.Nested) and isinstance(value, list):
+                        new_value = set(value)
+                        # handle list pruning
+                        if '-prune-' in value:
+                            value.remove('-prune-')
+                            new_value.remove('-prune-')
+                        else:
+                            for old in getattr(instance, key):
+                                # using str() is okay for now (see above)
+                                new_value.add(str(old))
+                        # handle item deletion
+                        for item in value:
+                            if item.startswith('-'):
+                                new_value.remove(item)
+                                try:
+                                    new_value.remove(item[1:])
+                                except KeyError as exc:
+                                    raise ValidationError(
+                                        f'item to delete not found: {item[1:]!r}',
+                                        field_name=f'?.{key}',
+                                    ) from exc
+                        # deduplicate and sort list
+                        data[key] = sorted(new_value)
+                        # log backref modification not catched by hook
+                        if isinstance(self.fields[key], RelatedList):
+                            if callback := self.context.get('callback'):
+                                callback(self, instance, {
+                                    'key': key,
+                                    'target': str(instance),
+                                    'before': [str(v) for v in getattr(instance, key)],
+                                    'after': data[key],
+                                })
+
+
+
+            # add attributes required for validation from db
+            # TODO: this will cause validation errors if value from database does not validate
+            # but there should not be an invalid value in the database
+            for attr_name, field_obj in self.load_fields.items():
+                if field_obj.required and attr_name not in data:
+                    data[attr_name] = getattr(instance, attr_name)
+
+        return data
+
+    @post_load(pass_original=True)
+    def _add_instance(self, item, original, many, **kwargs): # pylint: disable=unused-argument
+        """ add new instances to sqla session """
+
+        if item in self.opts.sqla_session:
+            # item was modified
+            if 'hash_password' in original:
+                # stabilize import of passwords to be hashed,
+                # by not re-hashing an unchanged password
+                if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None):
+                    if attr.history.has_changes() and attr.history.deleted:
+                        try:
+                            # reset password hash, if password was not changed
+                            inst = type(item)(password=attr.history.deleted[-1])
+                            if inst.check_password(original['password']):
+                                item.password = inst.password
+                        except ValueError:
+                            # hash in db is invalid
+                            pass
+                        else:
+                            del inst
+        else:
+            # new item
+            self.opts.sqla_session.add(item)
+        return item
+
+    @post_dump
+    def _hide_values(self, data, many, **kwargs): # pylint: disable=unused-argument
+        """ hide secrets and order output """
+
+        # stop early when not excluding/hiding
+        if not self._exclude_by_value and not self._hide_by_context:
+            return data
+
+        # exclude or hide values
+        full = self.context.get('full')
+        return type(data)([
+            (key, HIDDEN if key in self._hide_by_context else value)
+            for key, value in data.items()
+            if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key]
+        ])
+
+    # this field is used to mark items for deletion
+    mark_delete = fields.Boolean(data_key='__delete__', load_only=True)
+
+    # TODO: remove LazyStringField (when model was changed - IMHO comment should not be nullable)
+    comment = LazyStringField()
+
+
+### schema definitions ###
+
+@mapped
+class DomainSchema(BaseSchema):
+    """ Marshmallow schema for Domain model """
+    class Meta:
+        """ Schema config """
+        model = models.Domain
+        load_instance = True
+        include_relationships = True
+        exclude = ['users', 'managers', 'aliases']
+
+        include_by_context = {
+            ('dns',): {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'},
+        }
+        hide_by_context = {
+            ('secrets',): {'dkim_key'},
+        }
+        exclude_by_value = {
+            'alternatives': [[]],
+            'dkim_key': [None],
+            'dkim_publickey': [None],
+            'dns_mx': [None],
+            'dns_spf': [None],
+            'dns_dkim': [None],
+            'dns_dmarc': [None],
+        }
+
+    dkim_key = DkimKeyField(allow_none=True)
+    dkim_publickey = fields.String(dump_only=True)
+    dns_mx = fields.String(dump_only=True)
+    dns_spf = fields.String(dump_only=True)
+    dns_dkim = fields.String(dump_only=True)
+    dns_dmarc = fields.String(dump_only=True)
+
+
+@mapped
+class TokenSchema(BaseSchema):
+    """ Marshmallow schema for Token model """
+    class Meta:
+        """ Schema config """
+        model = models.Token
+        load_instance = True
+
+        sibling = True
+
+    password = PasswordField(required=True, metadata={'model': models.User})
+    hash_password = fields.Boolean(load_only=True, missing=False)
+
+
+@mapped
+class FetchSchema(BaseSchema):
+    """ Marshmallow schema for Fetch model """
+    class Meta:
+        """ Schema config """
+        model = models.Fetch
+        load_instance = True
+
+        sibling = True
+        include_by_context = {
+            ('full', 'import'): {'last_check', 'error'},
+        }
+        hide_by_context = {
+            ('secrets',): {'password'},
+        }
+
+
+@mapped
+class UserSchema(BaseSchema):
+    """ Marshmallow schema for User model """
+    class Meta:
+        """ Schema config """
+        model = models.User
+        load_instance = True
+        include_relationships = True
+        exclude = ['_email', 'domain', 'localpart', 'domain_name', 'quota_bytes_used']
+
+        primary_keys = ['email']
+        exclude_by_value = {
+            'forward_destination': [[]],
+            'tokens':              [[]],
+            'fetches':             [[]],
+            'manager_of':          [[]],
+            'reply_enddate':       ['2999-12-31'],
+            'reply_startdate':     ['1900-01-01'],
+        }
+
+    email = fields.String(required=True)
+    tokens = fields.Nested(TokenSchema, many=True)
+    fetches = fields.Nested(FetchSchema, many=True)
+
+    password = PasswordField(required=True, metadata={'model': models.User})
+    hash_password = fields.Boolean(load_only=True, missing=False)
+
+
+@mapped
+class AliasSchema(BaseSchema):
+    """ Marshmallow schema for Alias model """
+    class Meta:
+        """ Schema config """
+        model = models.Alias
+        load_instance = True
+        exclude = ['_email', 'domain', 'localpart', 'domain_name']
+
+        primary_keys = ['email']
+        exclude_by_value = {
+            'destination': [[]],
+        }
+
+    email = fields.String(required=True)
+    destination = CommaSeparatedListField()
+
+
+@mapped
+class ConfigSchema(BaseSchema):
+    """ Marshmallow schema for Config model """
+    class Meta:
+        """ Schema config """
+        model = models.Config
+        load_instance = True
+
+
+@mapped
+class RelaySchema(BaseSchema):
+    """ Marshmallow schema for Relay model """
+    class Meta:
+        """ Schema config """
+        model = models.Relay
+        load_instance = True
+
+
+class MailuSchema(Schema):
+    """ Marshmallow schema for complete Mailu config """
+    class Meta:
+        """ Schema config """
+        render_module = RenderYAML
+
+        order = ['domain', 'user', 'alias', 'relay'] # 'config'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # order dump_fields
+        for field in self.Meta.order:
+            if field in self.dump_fields:
+                self.dump_fields[field] = self.dump_fields.pop(field)
+
+    def _call_and_store(self, *args, **kwargs):
+        """ track current parent and field for pruning """
+        self.context.update({
+            'parent': self.context.get('config'),
+            'parent_field': kwargs['field_name'],
+        })
+        return super()._call_and_store(*args, **kwargs)
+
+    @pre_load
+    def _clear_config(self, data, many, **kwargs): # pylint: disable=unused-argument
+        """ create config object in context if missing
+            and clear it if requested
+        """
+        if 'config' not in self.context:
+            self.context['config'] = models.MailuConfig()
+        if self.context.get('clear'):
+            self.context['config'].clear(
+                models = {field.nested.opts.model for field in self.fields.values()}
+            )
+        return data
+
+    @post_load
+    def _make_config(self, data, many, **kwargs): # pylint: disable=unused-argument
+        """ update and return config object """
+        config = self.context['config']
+        for section in self.Meta.order:
+            if section in data:
+                config.update(data[section], section)
+
+        return config
+
+    domain = fields.Nested(DomainSchema, many=True)
+    user = fields.Nested(UserSchema, many=True)
+    alias = fields.Nested(AliasSchema, many=True)
+    relay = fields.Nested(RelaySchema, many=True)
+#    config = fields.Nested(ConfigSchema, many=True)

core/admin/requirements-prod.txt

@@ -15,6 +15,7 @@ Flask-Bootstrap==3.3.7.1
 Flask-DebugToolbar==0.10.1
 Flask-Limiter==1.0.1
 Flask-Login==0.4.1
+flask-marshmallow==0.14.0
 Flask-Migrate==2.4.0
 Flask-Script==2.0.6
 Flask-SQLAlchemy==2.4.0
@@ -29,6 +30,8 @@ limits==1.3
 Mako==1.0.9
 MarkupSafe==1.1.1
 mysqlclient==1.4.2.post1
+marshmallow==3.10.0
+marshmallow-sqlalchemy==0.24.1
 passlib==1.7.1
 psycopg2==2.8.2
 pycparser==2.19

core/admin/requirements.txt

@@ -23,3 +23,6 @@ mysqlclient
 psycopg2
 idna
 srslib
+marshmallow
+flask-marshmallow
+marshmallow-sqlalchemy

docs/cli.rst

@@ -10,8 +10,9 @@ Managing users and aliases can be done from CLI using commands:
 * user
 * user-import
 * user-delete
-* config-dump
 * config-update
+* config-export
+* config-import
 
 alias
 -----
@@ -69,104 +70,175 @@ user-delete
 
   docker-compose exec admin flask mailu user-delete foo@example.net
 
-config-dump
------------
-
-The purpose of this command is to dump domain-, relay-, alias- and user-configuration to a YAML template.
-
-.. code-block:: bash
-
-  # docker-compose exec admin flask mailu config-dump --help
-
-  Usage: flask mailu config-dump [OPTIONS] [SECTIONS]...
-
-    dump configuration as YAML-formatted data to stdout
-
-    SECTIONS can be: domains, relays, users, aliases
-
-  Options:
-    -f, --full     Include default attributes
-    -s, --secrets  Include secrets (dkim-key, plain-text / not hashed)
-    -d, --dns      Include dns records
-    --help         Show this message and exit.
-
-If you want to export secrets (dkim-keys, plain-text / not hashed) you have to add the ``--secrets`` option.
-Only non-default attributes are dumped. If you want to dump all attributes use ``--full``.
-To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option.
-Unless you specify some sections all sections are dumped by default.
-
-.. code-block:: bash
-
-  docker-compose exec admin flask mailu config-dump > mail-config.yml
-
-  docker-compose exec admin flask mailu config-dump --dns domains
-
 config-update
 -------------
 
-The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML template.
+The sole purpose of this command is for importing users/aliases in bulk and synchronizing DB entries with external YAML template:
 
 .. code-block:: bash
 
-  # docker-compose exec admin flask mailu config-update --help
+  cat mail-config.yml | docker-compose exec -T admin flask mailu config-update --delete-objects
 
-  Usage: flask mailu config-update [OPTIONS]
+where mail-config.yml looks like:
 
-    sync configuration with data from YAML-formatted stdin
+.. code-block:: bash
 
-  Options:
+  users:
-    -v, --verbose         Increase verbosity
+    - localpart: foo
-    -d, --delete-objects  Remove objects not included in yaml
+      domain: example.com
-    -n, --dry-run         Perform a trial run with no changes made
+      password_hash: klkjhumnzxcjkajahsdqweqqwr
-    --help                Show this message and exit.
+      hash_scheme: MD5-CRYPT
 
+  aliases:
+    - localpart: alias1
+      domain: example.com
+      destination: "user1@example.com,user2@example.com"
+
+without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input.
+
+Users
+^^^^^
+
+following are additional parameters that could be defined for users:
+
+* comment
+* quota_bytes
+* global_admin
+* enable_imap
+* enable_pop
+* forward_enabled
+* forward_destination
+* reply_enabled
+* reply_subject
+* reply_body
+* displayed_name
+* spam_enabled
+* spam_threshold
+
+Alias
+^^^^^
+
+additional fields:
+
+* wildcard
+
+config-export
+-------------
+
+The purpose of this command is to export the complete configuration in YAML or JSON format.
+
+.. code-block:: bash
+
+  $ docker-compose exec admin flask mailu config-export --help
+
+ Usage: flask mailu config-export [OPTIONS] [FILTER]...
+
+   Export configuration as YAML or JSON to stdout or file
+
+ Options:
+   -f, --full                  Include attributes with default value.
+   -s, --secrets               Include secret attributes (dkim-key, passwords).
+   -c, --color                 Force colorized output.
+   -d, --dns                   Include dns records.
+   -o, --output-file FILENAME  Save configuration to file.
+   -j, --json                  Export configuration in json format.
+   -?, -h, --help              Show this message and exit.
+
+Only non-default attributes are exported. If you want to export all attributes use ``--full``.
+If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option.
+To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option.
+By default all configuration objects are exported (domain, user, alias, relay). You can specify
+filters to export only some objects or attributes (try: ``user`` or ``domain.name``).
+
+.. code-block:: bash
+
+  $ docker-compose exec admin flask mailu config-export -o mail-config.yml
+
+  $ docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf
+
+config-import
+-------------
+
+This command imports configuration data from an external YAML or JSON source.
+
+.. code-block:: bash
+
+  $ docker-compose exec admin flask mailu config-import --help
+
+ Usage: flask mailu config-import [OPTIONS] [FILENAME|-]
+
+   Import configuration as YAML or JSON from stdin or file
+
+ Options:
+   -v, --verbose   Increase verbosity.
+   -s, --secrets   Show secret attributes in messages.
+   -q, --quiet     Quiet mode - only show errors.
+   -c, --color     Force colorized output.
+   -u, --update    Update mode - merge input with existing config.
+   -n, --dry-run   Perform a trial run with no changes made.
+   -?, -h, --help  Show this message and exit.
 
 The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec instead:
 
 .. code-block:: bash
 
-  docker exec -i $(docker-compose ps -q admin) flask mailu config-update -nvd < mail-config.yml
+  docker exec -i $(docker-compose ps -q admin) flask mailu config-import -nv < mail-config.yml
 
-
+mail-config.yml contains the configuration and looks like this:
-mail-config.yml looks like this:
 
 .. code-block:: yaml
 
-  domains:
+  domain:
     - name: example.com
       alternatives:
         - alternative.example.com
 
-  users:
+  user:
     - email: foo@example.com
-      password_hash: klkjhumnzxcjkajahsdqweqqwr
+      password_hash: '$2b$12$...'
       hash_scheme: MD5-CRYPT
 
-  aliases:
+  alias:
     - email: alias1@example.com
-      destination: "user1@example.com,user2@example.com"
+      destination:
+        - user1@example.com
+        - user2@example.com
 
-  relays:
+  relay:
     - name: relay.example.com
       comment: test
       smtp: mx.example.com
 
-You can use ``--dry-run`` to test your YAML without comitting any changes to the database.
+config-update shows the number of created/modified/deleted objects after import.
-With ``--verbose`` config-update will show exactly what it changes in the database.
+To suppress all messages except error messages use ``--quiet``.
-Without ``--delete-object`` option config-update will only add/update changed values but will *not* remove any entries missing in provided YAML input.
+By adding the ``--verbose`` switch (up to two times) the import gets more detailed and shows exactly what attributes changed.
+In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets.
+If you want to test what would be done when importing without committing any changes, use ``--dry-run``.
 
-This is a complete YAML template with all additional parameters that could be defined:
+By default config-update replaces the whole configuration. ``--update`` allows to modify the existing configuration instead.
+New elements will be added and existing elements will be modified.
+It is possible to delete a single element or prune all elements from lists and associative arrays using a special notation:
+
++-----------------------------+------------------+--------------------------+
+| Delete what?                | notation         | example                  |
++=============================+==================+==========================+
+| specific array object       | ``- -key: id``   | ``- -name: example.com`` |
++-----------------------------+------------------+--------------------------+
+| specific list item          | ``- -id``        | ``- -user1@example.com`` |
++-----------------------------+------------------+--------------------------+
+| all remaining array objects | ``- -key: null`` | ``- -email: null``       |
++-----------------------------+------------------+--------------------------+
+| all remaining list items    | ``- -prune-``    | ``- -prune-``            |
++-----------------------------+------------------+--------------------------+
+
+The ``-key: null`` notation can also be used to reset an attribute to its default.
+To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``.
+
+This is a complete YAML template with all additional parameters that can be defined:
 
 .. code-block:: yaml
 
-  aliases:
+  domain:
-    - email: email@example.com
-      comment: ''
-      destination:
-        - address@example.com
-      wildcard: false
-  
-  domains:
     - name: example.com
       alternatives:
         - alternative.tld
@@ -177,12 +249,7 @@ This is a complete YAML template with all additional parameters that could be de
       max_users: -1
       signup_enabled: false
 
-  relays:
+  user:
-    - name: relay.example.com
-      comment: ''
-      smtp: mx.example.com
-  
-  users:
     - email: postmaster@example.com
       comment: ''
       displayed_name: 'Postmaster'
@@ -192,13 +259,16 @@ This is a complete YAML template with all additional parameters that could be de
       fetches:
         - id: 1
           comment: 'test fetch'
-          username: fetch-user
+          error: null
           host: other.example.com
+          keep: true
+          last_check: '2020-12-29T17:09:48.200179'
           password: 'secret'
+          hash_password: true
           port: 993
           protocol: imap
           tls: true
-          keep: true
+          username: fetch-user
       forward_destination:
         - address@remote.example.com
       forward_enabled: true
@@ -206,12 +276,13 @@ This is a complete YAML template with all additional parameters that could be de
       global_admin: true
       manager_of:
         - example.com
-      password: '{BLF-CRYPT}$2b$12$...'
+      password: '$2b$12$...'
+      hash_password: true
       quota_bytes: 1000000000
       reply_body: ''
       reply_enabled: false
-      reply_enddate: 2999-12-31
+      reply_enddate: '2999-12-31'
-      reply_startdate: 1900-01-01
+      reply_startdate: '1900-01-01'
       reply_subject: ''
       spam_enabled: true
       spam_threshold: 80
@@ -219,5 +290,16 @@ This is a complete YAML template with all additional parameters that could be de
         - id: 1
           comment: email-client
           ip: 192.168.1.1
-          password: '$5$rounds=1000$...'
+          password: '$5$rounds=1$...'
 
+  aliases:
+    - email: email@example.com
+      comment: ''
+      destination:
+        - address@example.com
+      wildcard: false
+
+  relay:
+    - name: relay.example.com
+      comment: ''
+      smtp: mx.example.com

tests/compose/core/02_forward_test.sh

@@ -1,4 +1,4 @@
-cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose
+cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1
 users:
   - localpart: forwardinguser
     password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/"
@@ -10,7 +10,7 @@ EOF
 
 python3 tests/forward_test.py
 
-cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose
+cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1
 users:
   - localpart: forwardinguser
     password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/"

tests/compose/core/03_alias_test.sh

@@ -1,4 +1,4 @@
-cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose
+cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1
 aliases:
   - localpart: alltheusers
     domain: mailu.io
@@ -7,6 +7,6 @@ EOF
 
 python3 tests/alias_test.py
 
-cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose
+cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1
 aliases: []
 EOF

tests/compose/core/04_reply_test.sh

@@ -1,4 +1,4 @@
-cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose
+cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1
 users:
   - localpart: replyuser
     password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/"
@@ -11,7 +11,7 @@ EOF
 
 python3 tests/reply_test.py
 
-cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose
+cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1
 users:
   - localpart: replyuser
     password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/"

towncrier/newsfragments/1604.feature

@@ -1 +1 @@
-Added cli command config-dump and enhanced config-update
+Add cli commands config-import and config-export