self-hosted/Mailu
1
0
Fork 0
mirror of https://github.com/Mailu/Mailu.git synced 2025-03-11 14:49:19 +02:00
Code Issues Releases Activity

moved import logging to schema

Browse Source 
- yaml-import is now logged via schema.Logger
- iremoved relative imports - not used in other mailu modules
- removed develepment comments
- added Mailconfig.check method to check for duplicate domain names
- converted .format() to .format_map() where possible
- switched to yaml multiline dump for dkim_key
- converted dkim_key import from regex to string functions
- automatically unhide/unexclude explicitly specified attributes on dump
- use field order when loading to stabilize import
- fail when using 'hash_password' without 'password'
- fixed logging of dkim_key
- fixed pruning and deleting of lists
- modified error messages
- added debug flag and two verbosity levels
This commit is contained in:
Alexander Graf 2021-02-19 18:01:02 +01:00
parent e4c83e162d
commit bde7a2b6c4
4 changed files with 688 additions and 569 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

280
core/admin/mailu/manage.py
View File

@ -4,22 +4,16 @@
import sys
import os
import socket
import logging
import uuid
from collections import Counter
from itertools import chain
import click
import sqlalchemy
import yaml
from flask import current_app as app
from flask.cli import FlaskGroup, with_appcontext
from marshmallow.exceptions import ValidationError
from . import models
from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, canColorize, RenderJSON, HIDDEN
from mailu import models
from mailu.schemas import MailuSchema, Logger, RenderJSON
db = models.db
@ -326,246 +320,53 @@ def config_update(verbose=False, delete_objects=False):
@mailu.command()
@click.option('-v', '--verbose', count=True, help='Increase verbosity.')
@click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.')
@click.option('-d', '--debug', is_flag=True, help='Enable debug output.')
@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.')
@click.option('-c', '--color', is_flag=True, help='Force colorized output.')
@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.')
@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.')
@click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin)
@with_appcontext
def config_import(verbose=0, secrets=False, quiet=False, color=False, update=False, dry_run=False, source=None):
def config_import(verbose=0, secrets=False, debug=False, quiet=False, color=False,
update=False, dry_run=False, source=None):
""" Import configuration as YAML or JSON from stdin or file
"""
# verbose
# 0 : only show number of changes
# 1 : also show detailed changes
# 2 : also show input data
# 3 : also show sql queries (also needs -s, as sql may contain secrets)
# 4 : also show tracebacks (also needs -s, as tracebacks may contain secrets)
log = Logger(want_color=color or None, can_color=sys.stdout.isatty(), secrets=secrets, debug=debug)
log.lexer = 'python'
log.strip = True
log.verbose = 0 if quiet else verbose
log.quiet = quiet
if quiet:
verbose = -1
if verbose > 2 and not secrets:
print('[Warning] Verbosity level capped to 2. Specify --secrets to log sql and tracebacks.')
verbose = 2
color_cfg = {
'color': color or (canColorize and sys.stdout.isatty()),
'lexer': 'python',
'strip': True,
}
counter = Counter()
logger = {}
def format_errors(store, path=None):
res = []
if path is None:
path = []
for key in sorted(store):
location = path + [str(key)]
value = store[key]
if isinstance(value, dict):
res.extend(format_errors(value, location))
else:
for message in value:
res.append((".".join(location), message))
if path:
return res
fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}'
res = [fmt.format(loc, msg) for loc, msg in res]
num = f'error{["s",""][len(res)==1]}'
res.insert(0, f'[ValidationError] {len(res)} {num} occurred during input validation')
return '\n'.join(res)
def format_changes(*message):
if counter:
changes = []
last = None
for (action, what), count in sorted(counter.items()):
if action != last:
if last:
changes.append('/')
changes.append(f'{action}:')
last = action
changes.append(f'{what}({count})')
else:
changes = ['No changes.']
return chain(message, changes)
def log(action, target, message=None):
if message is None:
try:
message = logger[target.__class__].dump(target)
except KeyError:
message = target
if not isinstance(message, str):
message = repr(message)
print(f'{action} {target.__table__}: {colorize(message, **color_cfg)}')
def listen_insert(mapper, connection, target): # pylint: disable=unused-argument
""" callback function to track import """
counter.update([('Created', target.__table__.name)])
if verbose >= 1:
log('Created', target)
def listen_update(mapper, connection, target): # pylint: disable=unused-argument
""" callback function to track import """
changed = {}
inspection = sqlalchemy.inspect(target)
for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs:
history = getattr(inspection.attrs, attr.key).history
if history.has_changes() and history.deleted:
before = history.deleted[-1]
after = getattr(target, attr.key)
# TODO: remove special handling of "comment" after modifying model
if attr.key == 'comment' and not before and not after:
pass
# only remember changed keys
elif before != after:
if verbose >= 1:
changed[str(attr.key)] = (before, after)
else:
break
if verbose >= 1:
# use schema with dump_context to hide secrets and sort keys
dumped = get_schema(target)(only=changed.keys(), context=diff_context).dump(target)
for key, value in dumped.items():
before, after = changed[key]
if value == HIDDEN:
before = HIDDEN if before else before
after = HIDDEN if after else after
else:
# TODO: need to use schema to "convert" before value?
after = value
log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}')
if changed:
counter.update([('Modified', target.__table__.name)])
def listen_delete(mapper, connection, target): # pylint: disable=unused-argument
""" callback function to track import """
counter.update([('Deleted', target.__table__.name)])
if verbose >= 1:
log('Deleted', target)
# TODO: this listener will not be necessary, if dkim keys would be stored in database
_dedupe_dkim = set()
def listen_dkim(session, flush_context): # pylint: disable=unused-argument
""" callback function to track import """
for target in session.identity_map.values():
# look at Domains originally loaded from db
if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path:
continue
before = target._dkim_key_on_disk
after = target._dkim_key
if before != after:
if secrets:
before = before.decode('ascii', 'ignore')
after = after.decode('ascii', 'ignore')
else:
before = HIDDEN if before else ''
after = HIDDEN if after else ''
# "de-dupe" messages; this event is fired at every flush
if not (target, before, after) in _dedupe_dkim:
_dedupe_dkim.add((target, before, after))
counter.update([('Modified', target.__table__.name)])
if verbose >= 1:
log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}')
def track_serialize(obj, item, backref=None):
""" callback function to track import """
# called for backref modification?
if backref is not None:
log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref))
return
# show input data?
if not verbose >= 2:
return
# hide secrets in data
data = logger[obj.opts.model].hide(item)
if 'hash_password' in data:
data['password'] = HIDDEN
if 'fetches' in data:
for fetch in data['fetches']:
fetch['password'] = HIDDEN
log('Handling', obj.opts.model, data)
# configure contexts
diff_context = {
'full': True,
'secrets': secrets,
}
log_context = {
'secrets': secrets,
}
load_context = {
context = {
'import': True,
'update': update,
'clear': not update,
'callback': track_serialize,
'callback': log.track_serialize,
}
# register listeners
for schema in get_schema():
model = schema.Meta.model
logger[model] = schema(context=log_context)
sqlalchemy.event.listen(model, 'after_insert', listen_insert)
sqlalchemy.event.listen(model, 'after_update', listen_update)
sqlalchemy.event.listen(model, 'after_delete', listen_delete)
# special listener for dkim_key changes
sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim)
if verbose >= 3:
logging.basicConfig()
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
schema = MailuSchema(only=MailuSchema.Meta.order, context=context)
try:
# import source
with models.db.session.no_autoflush:
config = MailuSchema(only=MailuSchema.Meta.order, context=load_context).loads(source)
except ValidationError as exc:
raise click.ClickException(format_errors(exc.messages)) from exc
config = schema.loads(source)
# flush session to show/count all changes
if not quiet and (dry_run or verbose):
db.session.flush()
# check for duplicate domain names
config.check()
except Exception as exc:
if verbose >= 3:
raise
# (yaml.scanner.ScannerError, UnicodeDecodeError, ...)
raise click.ClickException(
f'[{exc.__class__.__name__}] '
f'{" ".join(str(exc).split())}'
) from exc
# flush session to show/count all changes
if dry_run or verbose >= 1:
db.session.flush()
# check for duplicate domain names
dup = set()
for fqdn in chain(
db.session.query(models.Domain.name),
db.session.query(models.Alternative.name),
db.session.query(models.Relay.name)
):
if fqdn in dup:
raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}')
dup.add(fqdn)
if msg := log.format_exception(exc):
raise click.ClickException(msg) from exc
raise
# don't commit when running dry
if dry_run:
if not quiet:
print(*format_changes('Dry run. Not commiting changes.'))
log.changes('Dry run. Not committing changes.')
db.session.rollback()
else:
if not quiet:
print(*format_changes('Committing changes.'))
log.changes('Committing changes.')
db.session.commit()
@ -573,8 +374,8 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal
@click.option('-f', '--full', is_flag=True, help='Include attributes with default value.')
@click.option('-s', '--secrets', is_flag=True,
help='Include secret attributes (dkim-key, passwords).')
@click.option('-c', '--color', is_flag=True, help='Force colorized output.')
@click.option('-d', '--dns', is_flag=True, help='Include dns records.')
@click.option('-c', '--color', is_flag=True, help='Force colorized output.')
@click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'),
help='Save configuration to file.')
@click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.')
@ -584,32 +385,25 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None
""" Export configuration as YAML or JSON to stdout or file
"""
if only:
for spec in only:
if spec.split('.', 1)[0] not in MailuSchema.Meta.order:
raise click.ClickException(f'[ValidationError] Unknown section: {spec}')
else:
only = MailuSchema.Meta.order
only = only or MailuSchema.Meta.order
context = {
'full': full,
'secrets': secrets,
'dns': dns,
}
schema = MailuSchema(only=only, context=context)
color_cfg = {'color': color or (canColorize and output.isatty())}
if as_json:
schema.opts.render_module = RenderJSON
color_cfg['lexer'] = 'json'
color_cfg['strip'] = True
log = Logger(want_color=color or None, can_color=output.isatty())
try:
print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output)
except ValueError as exc:
if spec := get_fieldspec(exc):
raise click.ClickException(f'[ValidationError] Invalid filter: {spec}') from exc
schema = MailuSchema(only=only, context=context)
if as_json:
schema.opts.render_module = RenderJSON
log.lexer = 'json'
log.strip = True
print(log.colorize(schema.dumps(models.MailuConfig())), file=output)
except Exception as exc:
if msg := log.format_exception(exc):
raise click.ClickException(msg) from exc
raise

40
core/admin/mailu/models.py
View File

@ -23,7 +23,7 @@ from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.inspection import inspect
from werkzeug.utils import cached_property
from . import dkim
from mailu import dkim
db = flask_sqlalchemy.SQLAlchemy()
@ -33,7 +33,6 @@ class IdnaDomain(db.TypeDecorator):
""" Stores a Unicode string in it's IDNA representation (ASCII only)
"""
# TODO: use db.String(255)?
impl = db.String(80)
def process_bind_param(self, value, dialect):
@ -50,7 +49,6 @@ class IdnaEmail(db.TypeDecorator):
""" Stores a Unicode string in it's IDNA representation (ASCII only)
"""
# TODO: use db.String(254)?
impl = db.String(255)
def process_bind_param(self, value, dialect):
@ -127,11 +125,7 @@ class Base(db.Model):
if pkey == 'email':
# ugly hack for email declared attr. _email is not always up2date
return str(f'{self.localpart}@{self.domain_name}')
elif pkey in {'name', 'email'}:
return str(getattr(self, pkey, None))
else:
return self.__repr__()
return str(getattr(self, self.__table__.primary_key.columns.values()[0].name))
return str(getattr(self, pkey))
def __repr__(self):
return f'<{self.__class__.__name__} {str(self)!r}>'
@ -145,12 +139,15 @@ class Base(db.Model):
else:
return NotImplemented
# we need hashable instances here for sqlalchemy to update collections
# in collections.bulk_replace, but auto-incrementing don't always have
# a valid primary key, in this case we use the object's id
__hashed = None
def __hash__(self):
primary = getattr(self, self.__table__.primary_key.columns.values()[0].name)
if primary is None:
return NotImplemented
else:
return hash(primary)
if self.__hashed is None:
primary = getattr(self, self.__table__.primary_key.columns.values()[0].name)
self.__hashed = id(self) if primary is None else hash(primary)
return self.__hashed
# Many-to-many association table for domain managers
@ -314,7 +311,6 @@ class Relay(Base):
__tablename__ = 'relay'
name = db.Column(IdnaDomain, primary_key=True, nullable=False)
# TODO: use db.String(266)? transport(8):(1)[nexthop(255)](2)
smtp = db.Column(db.String(80), nullable=True)
@ -322,7 +318,6 @@ class Email(object):
""" Abstraction for an email address (localpart and domain).
"""
# TODO: use db.String(64)?
localpart = db.Column(db.String(80), nullable=False)
@declarative.declared_attr
@ -342,7 +337,7 @@ class Email(object):
key = f'{cls.__tablename__}_email'
if key in ctx.current_parameters:
return ctx.current_parameters[key]
return '{localpart}@{domain_name}'.format(**ctx.current_parameters)
return '{localpart}@{domain_name}'.format_map(ctx.current_parameters)
return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater)
@ -632,7 +627,6 @@ class Token(Base):
user = db.relationship(User,
backref=db.backref('tokens', cascade='all, delete-orphan'))
password = db.Column(db.String(255), nullable=False)
# TODO: use db.String(32)?
ip = db.Column(db.String(255))
def check_password(self, password):
@ -865,6 +859,18 @@ class MailuConfig:
if models is None or model in models:
db.session.query(model).delete()
def check(self):
""" check for duplicate domain names """
dup = set()
for fqdn in chain(
db.session.query(Domain.name),
db.session.query(Alternative.name),
db.session.query(Relay.name)
):
if fqdn in dup:
raise ValueError(f'Duplicate domain name: {fqdn}')
dup.add(fqdn)
domain = MailuCollection(Domain)
user = MailuCollection(User)
alias = MailuCollection(Alias)

922
core/admin/mailu/schemas.py
View File

File diff suppressed because it is too large Load Diff

15
docs/cli.rst
View File

@ -138,8 +138,8 @@ The purpose of this command is to export the complete configuration in YAML or J
Options:
-f, --full Include attributes with default value.
-s, --secrets Include secret attributes (dkim-key, passwords).
-c, --color Force colorized output.
-d, --dns Include dns records.
-c, --color Force colorized output.
-o, --output-file FILENAME Save configuration to file.
-j, --json Export configuration in json format.
-?, -h, --help Show this message and exit.
@ -147,14 +147,18 @@ The purpose of this command is to export the complete configuration in YAML or J
Only non-default attributes are exported. If you want to export all attributes use ``--full``.
If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option.
To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option.
By default all configuration objects are exported (domain, user, alias, relay). You can specify
filters to export only some objects or attributes (try: ``user`` or ``domain.name``).
Attributes explicitly specified in filters are automatically exported: there is no need to add ``--secrets`` or ``--full``.
.. code-block:: bash
$ docker-compose exec admin flask mailu config-export -o mail-config.yml
$ docker-compose exec admin flask mailu config-export --output mail-config.yml
$ docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf
$ docker-compose exec admin flask mailu config-export domain.dns_mx domain.dns_spf
$ docker-compose exec admin flask mailu config-export user.spam_threshold
config-import
-------------
@ -211,7 +215,7 @@ mail-config.yml contains the configuration and looks like this:
config-update shows the number of created/modified/deleted objects after import.
To suppress all messages except error messages use ``--quiet``.
By adding the ``--verbose`` switch (up to two times) the import gets more detailed and shows exactly what attributes changed.
By adding the ``--verbose`` switch the import gets more detailed and shows exactly what attributes changed.
In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets.
If you want to test what would be done when importing without committing any changes, use ``--dry-run``.
@ -234,6 +238,9 @@ It is possible to delete a single element or prune all elements from lists and a
The ``-key: null`` notation can also be used to reset an attribute to its default.
To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``.
A new dkim key can be generated when adding or modifying a domain, by using the special value
``dkim_key: -generate-``.
This is a complete YAML template with all additional parameters that can be defined:
.. code-block:: yaml