From 7413f9b7b42d8797f5de6203260761f2561859f0 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 13 Jan 2021 00:05:43 +0100 Subject: [PATCH] config_dump now using marshmallow --- core/admin/mailu/manage.py | 46 +--- core/admin/mailu/models.py | 4 +- core/admin/mailu/schemas.py | 498 ++++++++++++++++++++++++------------ 3 files changed, 346 insertions(+), 202 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 569c161c..a67abdf7 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,5 +1,5 @@ from mailu import models -from .schemas import schemas +from .schemas import MailuConfig, MailuSchema from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext @@ -310,41 +310,17 @@ def config_dump(full=False, secrets=False, dns=False, sections=None): SECTIONS can be: domains, relays, users, aliases """ - class spacedDumper(yaml.Dumper): + try: + config = MailuConfig(sections) + except ValueError as reason: + print(f'[ERROR] {reason}') + return 1 - def write_line_break(self, data=None): - super().write_line_break(data) - if len(self.indents) == 1: - super().write_line_break() - - def increase_indent(self, flow=False, indentless=False): - return super().increase_indent(flow, False) - - if sections: - for section in sections: - if section not in schemas: - print(f'[ERROR] Invalid section: {section}') - return 1 - else: - sections = sorted(schemas.keys()) - -# TODO: create World Schema and dump only this with Word.dumps ? - - for section in sections: - schema = schemas[section](many=True) - schema.context.update({ - 'full': full, - 'secrets': secrets, - 'dns': dns, - }) - yaml.dump( - {section: schema.dump(schema.Meta.model.query.all())}, - sys.stdout, - Dumper=spacedDumper, - default_flow_style=False, - allow_unicode=True - ) - sys.stdout.write('\n') + MailuSchema(context={ + 'full': full, + 'secrets': secrets, + 'dns': dns, + }).dumps(config, sys.stdout) @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 625d1fd6..8b77b011 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -110,7 +110,7 @@ class Base(db.Model): created_at = db.Column(db.Date, nullable=False, default=date.today) updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) - comment = db.Column(db.String(255), nullable=True) + comment = db.Column(db.String(255), nullable=True, default='') @classmethod def _dict_pkey(cls): @@ -171,7 +171,7 @@ class Base(db.Model): if self.__mapper__.relationships[key].query_class is not None: if hasattr(items, 'all'): items = items.all() - if full or len(items): + if full or items: if key in secret: res[key] = '' else: diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index c2a3d03d..bd9d228a 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1,63 +1,252 @@ -import marshmallow -import sqlalchemy -import flask_marshmallow +""" +Mailu marshmallow schema +""" -from . import models +from textwrap import wrap + +import re +import yaml + +from marshmallow import post_dump, fields, Schema +from flask_marshmallow import Marshmallow +from OpenSSL import crypto + +from . import models, dkim -ma = flask_marshmallow.Marshmallow() +ma = Marshmallow() +# TODO: +# how to mark keys as "required" while unserializing (in certain use cases/API)? +# - fields withoud default => required +# - fields which are the primary key => unchangeable when updating + + +### yaml render module ### + +class RenderYAML: + """ Marshmallow YAML Render Module + """ + + class SpacedDumper(yaml.Dumper): + """ YAML Dumper to add a newline between main sections + and double the indent used + """ + + def write_line_break(self, data=None): + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + @staticmethod + def _update_dict(dict1, dict2): + """ sets missing keys in dict1 to values of dict2 + """ + for key, value in dict2.items(): + if key not in dict1: + dict1[key] = value + + _load_defaults = {} + @classmethod + def loads(cls, *args, **kwargs): + """ load yaml data from string + """ + cls._update_dict(kwargs, cls._load_defaults) + return yaml.load(*args, **kwargs) + + _dump_defaults = { + 'Dumper': SpacedDumper, + 'default_flow_style': False, + 'allow_unicode': True, + } + @classmethod + def dumps(cls, *args, **kwargs): + """ dump yaml data to string + """ + cls._update_dict(kwargs, cls._dump_defaults) + return yaml.dump(*args, **kwargs) + + +### field definitions ### + +class LazyString(fields.String): + """ Field that serializes a "false" value to the empty string + """ + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize None to the empty string + """ + return value if value else '' + + +class CommaSeparatedList(fields.Raw): + """ Field that deserializes a string containing comma-separated values to + a list of strings + """ + # TODO: implement this + + +class DkimKey(fields.String): + """ Field that serializes a dkim key to a list of strings (lines) and + deserializes a string or list of strings. + """ + + _clean_re = re.compile( + r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)', + flags=re.UNICODE + ) + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize dkim key to a list of strings (lines) + """ + + # map empty string and None to None + if not value: + return None + + # return list of key lines without header/footer + return value.decode('utf-8').strip().split('\n')[1:-1] + + def _deserialize(self, value, attr, data, **kwargs): + """ deserialize a string or list of strings to dkim key data + with verification + """ + + # convert list to str + if isinstance(value, list): + value = ''.join(value) + + # only strings are allowed + if not isinstance(value, str): + raise TypeError(f'invalid type: {type(value).__name__!r}') + + # clean value (remove whitespace and header/footer) + value = self._clean_re.sub('', value.strip()) + + # map empty string/list to None + if not value: + return None + + # handle special value 'generate' + elif value == 'generate': + return dkim.gen_key() + + # wrap value into valid pem layout and check validity + value = ( + '-----BEGIN PRIVATE KEY-----\n' + + '\n'.join(wrap(value, 64)) + + '\n-----END PRIVATE KEY-----\n' + ).encode('ascii') + try: + crypto.load_privatekey(crypto.FILETYPE_PEM, value) + except crypto.Error as exc: + raise ValueError('invalid dkim key') from exc + else: + return value + + +### schema definitions ### -import collections class BaseSchema(ma.SQLAlchemyAutoSchema): + """ Marshmallow base schema with custom exclude logic + and option to hide sqla defaults + """ class Meta: - base_hide_always = {'created_at', 'updated_at'} - base_hide_secrets = set() - base_hide_by_value = { -# 'comment': {'', None} - } + """ Schema config """ + model = None - @marshmallow.post_dump - def remove_skip_values(self, data, many, **kwargs): -# print(repr(data), self.context) + def __init__(self, *args, **kwargs): - # always hide - hide_by_key = self.Meta.base_hide_always | set(getattr(self.Meta, 'hide_always', ())) + # get and remove config from kwargs + context = kwargs.get('context', {}) - # hide secrets - if not self.context.get('secrets'): - hide_by_key |= self.Meta.base_hide_secrets - hide_by_key |= set(getattr(self.Meta, 'hide_secrets', ())) + # compile excludes + exclude = set(kwargs.get('exclude', [])) - # hide by value - hide_by_value = self.Meta.base_hide_by_value | getattr(self.Meta, 'hide_by_value', {}) + # always exclude + exclude.update({'created_at', 'updated_at'}) - # hide defaults - if not self.context.get('full'): + # add include_by_context + if context is not None: + for ctx, what in getattr(self.Meta, 'include_by_context', {}).items(): + if not context.get(ctx): + exclude |= set(what) + + # update excludes + kwargs['exclude'] = exclude + + # exclude_by_value + self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {}) + + # exclude default values + if not context.get('full'): for column in self.Meta.model.__table__.columns: -# print(column.name, column.default.arg if isinstance(column.default, sqlalchemy.sql.schema.ColumnDefault) else column.default) -# alias.destiantion has default [] - is this okay. how to check it? - if column.name not in hide_by_key: - hide_by_value.setdefault(column.name, set()).add(None if column.default is None else column.default.arg) + if column.name not in exclude: + self._exclude_by_value.setdefault(column.name, []).append( + None if column.default is None else column.default.arg + ) + # hide by context + self._hide_by_context = set() + if context is not None: + for ctx, what in getattr(self.Meta, 'hide_by_context', {}).items(): + if not context.get(ctx): + self._hide_by_context |= set(what) + + # init SQLAlchemyAutoSchema + super().__init__(*args, **kwargs) + + @post_dump + def _remove_skip_values(self, data, many, **kwargs): # pylint: disable=unused-argument + + if not self._exclude_by_value and not self._hide_by_context: + return data + + full = self.context.get('full') return { - key: value for key, value in data.items() - if - not isinstance(value, collections.Hashable) - or( - key not in hide_by_key - and - (key not in hide_by_value or value not in hide_by_value[key])) + key: '' if key in self._hide_by_context else value + for key, value in data.items() + if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] } + # TODO: remove LazyString and fix model definition (comment should not be nullable) + comment = LazyString() + class DomainSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Domain model """ + class Meta: + """ Schema config """ model = models.Domain + include_relationships = True + #include_fk = True + exclude = ['users', 'managers', 'aliases'] + + include_by_context = { + 'dns': {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}, + } + hide_by_context = { + 'secrets': {'dkim_key'}, + } + exclude_by_value = { + 'alternatives': [[]], + 'dkim_key': [None], + 'dkim_publickey': [None], + 'dns_mx': [None], + 'dns_spf': [None], + 'dns_dkim': [None], + 'dns_dmarc': [None], + } + + dkim_key = DkimKey() + dkim_publickey = fields.String(dump_only=True) + dns_mx = fields.String(dump_only=True) + dns_spf = fields.String(dump_only=True) + dns_dkim = fields.String(dump_only=True) + dns_dmarc = fields.String(dump_only=True) - # _dict_hide = {'users', 'managers', 'aliases'} - # _dict_show = {'dkim_key'} - # _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} - # _dict_secret = {'dkim_key'} # _dict_types = { # 'dkim_key': (bytes, type(None)), # 'dkim_publickey': False, @@ -66,50 +255,62 @@ class DomainSchema(BaseSchema): # 'dns_dkim': False, # 'dns_dmarc': False, # } - # _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} - # @staticmethod - # def _dict_input(data): - # if 'dkim_key' in data: - # key = data['dkim_key'] - # if key is not None: - # if type(key) is list: - # key = ''.join(key) - # if type(key) is str: - # key = ''.join(key.strip().split()) # removes all whitespace - # if key == 'generate': - # data['dkim_key'] = dkim.gen_key() - # elif key: - # m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) - # if m is not None: - # key = key[m.end():] - # m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) - # if m is not None: - # key = key[:m.start()] - # key = '\n'.join(wrap(key, 64)) - # key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') - # try: - # dkim.strip_key(key) - # except: - # raise ValueError('invalid dkim key') - # else: - # data['dkim_key'] = key - # else: - # data['dkim_key'] = None - # name = db.Column(IdnaDomain, primary_key=True, nullable=False) - # managers = db.relationship('User', secondary=managers, - # backref=db.backref('manager_of'), lazy='dynamic') - # max_users = db.Column(db.Integer, nullable=False, default=-1) - # max_aliases = db.Column(db.Integer, nullable=False, default=-1) - # max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) - # signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + +class TokenSchema(BaseSchema): + """ Marshmallow schema for Token model """ + class Meta: + """ Schema config """ + model = models.Token + + # _dict_recurse = True + # _dict_hide = {'user', 'user_email'} + # _dict_mandatory = {'password'} + + # id = db.Column(db.Integer(), primary_key=True) + # user_email = db.Column(db.String(255), db.ForeignKey(User.email), + # nullable=False) + # user = db.relationship(User, + # backref=db.backref('tokens', cascade='all, delete-orphan')) + # password = db.Column(db.String(255), nullable=False) + # ip = db.Column(db.String(255)) + + +class FetchSchema(BaseSchema): + """ Marshmallow schema for Fetch model """ + class Meta: + """ Schema config """ + model = models.Fetch + include_by_context = { + 'full': {'last_check', 'error'}, + } + hide_by_context = { + 'secrets': {'password'}, + } + +# TODO: What about mandatory keys? + # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} class UserSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for User model """ + class Meta: + """ Schema config """ model = models.User + include_relationships = True + exclude = ['localpart', 'domain', 'quota_bytes_used'] - # _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} + exclude_by_value = { + 'forward_destination': [[]], + 'tokens': [[]], + 'reply_enddate': ['2999-12-31'], + 'reply_startdate': ['1900-01-01'], + } + + tokens = fields.Nested(TokenSchema, many=True) + fetches = fields.Nested(FetchSchema, many=True) + +# TODO: deserialize password/password_hash! What about mandatory keys? # _dict_mandatory = {'localpart', 'domain', 'password'} # @classmethod # def _dict_input(cls, data): @@ -133,44 +334,19 @@ class UserSchema(BaseSchema): # del data['hash_scheme'] # del data['password_hash'] - # domain = db.relationship(Domain, - # backref=db.backref('users', cascade='all, delete-orphan')) - # password = db.Column(db.String(255), nullable=False) - # quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) - # quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) - # global_admin = db.Column(db.Boolean(), nullable=False, default=False) - # enabled = db.Column(db.Boolean(), nullable=False, default=True) - - # # Features - # enable_imap = db.Column(db.Boolean(), nullable=False, default=True) - # enable_pop = db.Column(db.Boolean(), nullable=False, default=True) - - # # Filters - # forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) - # forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) - # forward_keep = db.Column(db.Boolean(), nullable=False, default=True) - # reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) - # reply_subject = db.Column(db.String(255), nullable=True, default=None) - # reply_body = db.Column(db.Text(), nullable=True, default=None) - # reply_startdate = db.Column(db.Date, nullable=False, - # default=date(1900, 1, 1)) - # reply_enddate = db.Column(db.Date, nullable=False, - # default=date(2999, 12, 31)) - - # # Settings - # displayed_name = db.Column(db.String(160), nullable=False, default='') - # spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) - # spam_threshold = db.Column(db.Integer(), nullable=False, default=80) class AliasSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Alias model """ + class Meta: + """ Schema config """ model = models.Alias - hide_always = {'localpart'} - hide_secrets = {'wildcard'} - hide_by_value = { - 'destination': set([]) # always hide empty lists?! + exclude = ['localpart'] + + exclude_by_value = { + 'destination': [[]], } +# TODO: deserialize destination! # @staticmethod # def _dict_input(data): # Email._dict_input(data) @@ -180,65 +356,57 @@ class AliasSchema(BaseSchema): # data['destination'] = list([adr.strip() for adr in dst.split(',')]) -class TokenSchema(BaseSchema): - class Meta(BaseSchema.Meta): - model = models.Token - - # _dict_recurse = True - # _dict_hide = {'user', 'user_email'} - # _dict_mandatory = {'password'} - - # id = db.Column(db.Integer(), primary_key=True) - # user_email = db.Column(db.String(255), db.ForeignKey(User.email), - # nullable=False) - # user = db.relationship(User, - # backref=db.backref('tokens', cascade='all, delete-orphan')) - # password = db.Column(db.String(255), nullable=False) - # ip = db.Column(db.String(255)) - - -class FetchSchema(BaseSchema): - class Meta(BaseSchema.Meta): - model = models.Fetch - - # _dict_recurse = True - # _dict_hide = {'user_email', 'user', 'last_check', 'error'} - # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} - # _dict_secret = {'password'} - - # id = db.Column(db.Integer(), primary_key=True) - # user_email = db.Column(db.String(255), db.ForeignKey(User.email), - # nullable=False) - # user = db.relationship(User, - # backref=db.backref('fetches', cascade='all, delete-orphan')) - # protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) - # host = db.Column(db.String(255), nullable=False) - # port = db.Column(db.Integer(), nullable=False) - # tls = db.Column(db.Boolean(), nullable=False, default=False) - # username = db.Column(db.String(255), nullable=False) - # password = db.Column(db.String(255), nullable=False) - # keep = db.Column(db.Boolean(), nullable=False, default=False) - # last_check = db.Column(db.DateTime, nullable=True) - # error = db.Column(db.String(1023), nullable=True) - - class ConfigSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Config model """ + class Meta: + """ Schema config """ model = models.Config -# TODO: how to mark keys as "required" while unserializing (in certain use cases/API) - name = ma.auto_field(required=True) - value = ma.auto_field(required=True) class RelaySchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Relay model """ + class Meta: + """ Schema config """ model = models.Relay -schemas = { - 'domains': DomainSchema, - 'relays': RelaySchema, - 'users': UserSchema, - 'aliases': AliasSchema, -# 'config': ConfigSchema, -} +class MailuSchema(Schema): + """ Marshmallow schema for Mailu config """ + class Meta: + """ Schema config """ + render_module = RenderYAML + domains = fields.Nested(DomainSchema, many=True) + relays = fields.Nested(RelaySchema, many=True) + users = fields.Nested(UserSchema, many=True) + aliases = fields.Nested(AliasSchema, many=True) + config = fields.Nested(ConfigSchema, many=True) + + +### config class ### + +class MailuConfig: + """ Class which joins whole Mailu config for dumping + """ + + _models = { + 'domains': models.Domain, + 'relays': models.Relay, + 'users': models.User, + 'aliases': models.Alias, +# 'config': models.Config, + } + + def __init__(self, sections): + if sections: + for section in sections: + if section not in self._models: + raise ValueError(f'Unknown section: {section!r}') + self._sections = set(sections) + else: + self._sections = set(self._models.keys()) + + def __getattr__(self, section): + if section in self._sections: + return self._models[section].query.all() + else: + raise AttributeError