Merge #2518

2518: Add dev runner for admin container r=mergify[bot] a=ghostwheel42

## What type of PR?

development feature

## What does this PR do?

This adds a shell script (run_dev.sh) to run a live development environment in a container.


Co-authored-by: Alexander Graf <ghostwheel42@users.noreply.github.com>

core/admin/.gitignore

@@ -2,3 +2,4 @@
 lib64
 .vscode
 tags
+dev

core/admin/assets/Dockerfile

@@ -4,13 +4,19 @@ FROM node:16-alpine3.16
 
 WORKDIR /work
 
-COPY content/ ./
+COPY package.json ./
+COPY webpack.config.js ./
 
 RUN set -euxo pipefail \
- && npm config set update-notifier false \
- && npm install --no-audit --no-fund \
- && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \
- && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \
+  ; npm config set update-notifier false \
+  ; npm install --no-audit --no-fund \
+  ; sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \
+  ; mkdir assets \
+  ; for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \
       cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \
-    done \
- && node_modules/.bin/webpack-cli --color
+    done
+
+COPY assets/ ./assets/
+
+RUN set -euxo pipefail \
+  ; node_modules/.bin/webpack-cli --color

core/admin/mailu/__init__.py

@@ -44,8 +44,10 @@ def create_app_from_config(config):
     # Initialize debugging tools
     if app.config.get("DEBUG"):
         debug.toolbar.init_app(app)
-        # TODO: add a specific configuration variable for profiling
-        # debug.profiler.init_app(app)
+    if app.config.get("DEBUG_PROFILER"):
+        debug.profiler.init_app(app)
+    if assets := app.config.get('DEBUG_ASSETS'):
+        app.static_folder = assets
 
     # Inject the default variables in the Jinja parser
     # TODO: move this to blueprints when needed

core/admin/mailu/configuration.py

@@ -11,8 +11,9 @@ DEFAULT_CONFIG = {
     'BABEL_DEFAULT_TIMEZONE': 'UTC',
     'BOOTSTRAP_SERVE_LOCAL': True,
     'RATELIMIT_STORAGE_URL': '',
-    'QUOTA_STORAGE_URL': '',
     'DEBUG': False,
+    'DEBUG_PROFILER': False,
+    'DEBUG_ASSETS': '',
     'DOMAIN_REGISTRATION': False,
     'TEMPLATES_AUTO_RELOAD': True,
     'MEMORY_SESSIONS': False,
@@ -149,8 +150,9 @@ class ConfigManager:
             template = self.DB_TEMPLATES[self.config['DB_FLAVOR']]
             self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config)
 
+        if not self.config.get('RATELIMIT_STORAGE_URL'):
             self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2'
-        self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1'
+
         self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3'
         self.config['SESSION_COOKIE_SAMESITE'] = 'Strict'
         self.config['SESSION_COOKIE_HTTPONLY'] = True
@@ -159,9 +161,9 @@ class ConfigManager:
         self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME'])
         self.config['AUTH_RATELIMIT_IP_V4_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V4_MASK'])
         self.config['AUTH_RATELIMIT_IP_V6_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V6_MASK'])
-        hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')]
         self.config['AUTH_RATELIMIT_EXEMPTION'] = set(ipaddress.ip_network(cidr, False) for cidr in (cidr.strip() for cidr in self.config['AUTH_RATELIMIT_EXEMPTION'].split(',')) if cidr)
         self.config['MESSAGE_RATELIMIT_EXEMPTION'] = set([s for s in self.config['MESSAGE_RATELIMIT_EXEMPTION'].lower().replace(' ', '').split(',') if s])
+        hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')]
         self.config['HOSTNAMES'] = ','.join(hostnames)
         self.config['HOSTNAME'] = hostnames[0]
         self.config['DEFAULT_SPAM_THRESHOLD'] = int(self.config['DEFAULT_SPAM_THRESHOLD'])

core/admin/run_dev.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+### CONFIG
+
+DEV_NAME="${DEV_NAME:-mailu-dev}"
+DEV_DB="${DEV_DB:-}"
+DEV_PROFILER="${DEV_PROFILER:-false}"
+DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}"
+[[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}"
+DEV_ADMIN="${DEV_ADMIN:-admin@example.com}"
+DEV_PASSWORD="${DEV_PASSWORD:-letmein}"
+
+### MAIN
+
+[[ -n "${DEV_DB}" ]] && {
+    [[ -f "${DEV_DB}" ]] || {
+        echo "Sorry, can't find DEV_DB: '${DEV_DB}'"
+        exit 1
+    }
+    DEV_DB="$(realpath "${DEV_DB}")"
+}
+
+docker="$(command -v podman || command -v docker || echo false)"
+[[ "${docker}" == "false" ]] && {
+    echo "Sorry, you'll need podman or docker to run this."
+    exit 1
+}
+
+tmp="$(mktemp -d)"
+[[ -n "${tmp}" && -d "${tmp}" ]] || {
+    echo "Sorry, can't create temporary folder."
+    exit 1
+}
+trap "rm -rf '${tmp}'" INT TERM EXIT
+
+admin="$(realpath "$(pwd)/${0%/*}")"
+base="${admin}/../base"
+assets="${admin}/assets"
+
+cd "${tmp}"
+
+# base
+cp "${base}"/requirements-* .
+cp -r "${base}"/libs .
+sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Dockerfile
+
+# assets
+cp "${assets}/package.json" .
+cp -r "${assets}/assets/" .
+awk '/new compress/{f=1}!f{print}/}),/{f=0}' <"${assets}/webpack.config.js" >webpack.config.js
+sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile
+
+# admin
+sed -E '/^#/d;/^(COPY|EXPOSE|HEALTHCHECK|VOLUME|CMD) /d; s:^(.* )[^ ]*pybabel[^\\]*(.*):\1true \2:' "${admin}/Dockerfile" >>Dockerfile
+
+# development
+cat >>Dockerfile <<EOF
+COPY --from=assets /work/static/ ./static/
+
+RUN set -euxo pipefail \
+  ; mkdir /data \
+  ; ln -s /app/audit.py / \
+  ; ln -s /app/start.py /
+
+ENV \
+    FLASK_ENV="development" \
+    MEMORY_SESSIONS="true" \
+    RATELIMIT_STORAGE_URL="memory://" \
+    SESSION_COOKIE_SECURE="false" \
+    \
+    DEBUG="true" \
+    DEBUG_PROFILER="${DEV_PROFILER}" \
+    DEBUG_ASSETS="/app/static" \
+    DEBUG_TB_ENABLED="true" \
+    \
+    IMAP_ADDRESS="127.0.0.1" \
+    POP3_ADDRESS="127.0.0.1" \
+    AUTHSMTP_ADDRESS="127.0.0.1" \
+    SMTP_ADDRESS="127.0.0.1" \
+    REDIS_ADDRESS="127.0.0.1" \
+    WEBMAIL_ADDRESS="127.0.0.1"
+
+CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null && flask run --host=0.0.0.0 --port=8080"]
+EOF
+
+# build
+chmod -R u+rwX,go+rX .
+"${docker}" build --tag "${DEV_NAME}:latest" .
+
+# gather volumes to map into container
+volumes=()
+
+[[ -n "${DEV_DB}" ]] && volumes+=( --volume "${DEV_DB}:/data/main.db" )
+
+for vol in audit.py start.py mailu/ migrations/; do
+    volumes+=( --volume "${admin}/${vol}:/app/${vol}" )
+done
+
+for file in "${assets}/assets"/*; do
+    [[ ! -f "${file}" || "${file}" == */vendor.js ]] && continue
+    volumes+=( --volume "${file}:/app/static/${file/*\//}" )
+done
+
+# show configuration
+cat <<EOF
+
+=============================================================================
+The "${DEV_NAME}" container was built using this configuration:
+
+DEV_NAME="${DEV_NAME}"
+DEV_DB="${DEV_DB}"
+DEV_PROFILER="${DEV_PROFILER}"
+DEV_LISTEN="${DEV_LISTEN}"
+DEV_ADMIN="${DEV_ADMIN}"
+DEV_PASSWORD="${DEV_PASSWORD}"
+=============================================================================
+
+=============================================================================
+You can start the container later using this commandline:
+
+${docker/*\/} run --rm -it --name "${DEV_NAME}" --publish ${DEV_LISTEN}:8080$(printf " %q" "${volumes[@]}") "${DEV_NAME}"
+=============================================================================
+
+=============================================================================
+The Mailu UI can be found here: http://${DEV_LISTEN}/sso/login
+EOF
+[[ -z "${DEV_DB}" ]] && echo "You can log in with user ${DEV_ADMIN} and password ${DEV_PASSWORD}"
+cat <<EOF
+=============================================================================
+
+Starting mailu dev environment...
+EOF
+
+# run
+"${docker}" run --rm -it --name "${DEV_NAME}" --publish "${DEV_LISTEN}:8080" "${volumes[@]}" "${DEV_NAME}"
+

docs/contributors/environment.rst

@@ -313,6 +313,48 @@ If git opens a editor for a commit message just save and exit as-is. If you have
 see above and do the complete procedure from ``git fetch`` onward again.
 
 
+Web administration development
+------------------------------
+
+The administration web interface requires a proper dev environment that can easily
+be setup using the ``run_dev.sh`` shell script. You need ``docker`` or ``podman``
+to run it. It will create a local webserver listening at port 8080:
+
+.. code-block:: bash
+
+  cd core/admin
+  ./run_dev.sh
+  pip install -r requirements.txt
+  [...]
+  =============================================================================
+  The "mailu-dev" container was built using this configuration:
+
+  DEV_NAME="mailu-dev"
+  DEV_DB=""
+  DEV_PROFILER="false"
+  DEV_LISTEN="127.0.0.1:8080"
+  DEV_ADMIN="admin@example.com"
+  DEV_PASSWORD="letmein"
+  =============================================================================
+  [...]
+  =============================================================================
+  The Mailu UI can be found here: http://127.0.0.1:8080/sso/login
+  You can log in with user admin@example.com and password letmein
+  =============================================================================
+
+The container will use an empty database and a default user/password unless you
+specify a database file to use by setting ``$DEV_DB``.
+
+.. code-block:: bash
+
+  DEV_DB="/path/to/dev.db" ./run_dev.sh
+
+Any change to the files will automatically restart the Web server and reload the files.
+
+When using the development environment, a debugging toolbar is displayed on the right
+side of the screen, where you can access query details, internal variables, etc.
+
+
 Documentation
 -------------