Merge #3032

3032: Update all python dependencies r=mergify[bot] a=Diman0

## What type of PR?

enhancement

## What does this PR do?
Updates all dependencies. Unfortunately multiple things were deprecated such as
* `@babel.localeselector` decorator
* app.app.session_cookie_name
* app.before_first_request

Unfortunately flask_debugtoolbar is not compatible with the updated version of flask.
Flask-SQLAlchemy cannot be updated to 3.0.5 because this introduces an error/bug.

### Related issue(s)

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [n/a] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.


Co-authored-by: Dimitri Huisman <diman@huisman.xyz>

.gitignore

@@ -5,6 +5,7 @@ pip-selfcheck.json
 /core/admin/lib*
 /core/admin/bin
 /core/admin/include
+/core/base/.venv
 /docs/lib*
 /docs/bin
 /docs/include

core/admin/mailu/__init__.py

@@ -40,7 +40,7 @@ def create_app_from_config(config):
     models.db.init_app(app)
     utils.session.init_app(app)
     utils.limiter.init_app(app)
-    utils.babel.init_app(app)
+    utils.babel.init_app(app, locale_selector=utils.get_locale)
     utils.login.init_app(app)
     utils.login.user_loader(models.User.get)
     utils.proxy.init_app(app)
@@ -52,13 +52,14 @@ def create_app_from_config(config):
     app.truncated_pw_key = hmac.new(bytearray(app.secret_key, 'utf-8'), bytearray('TRUNCATED_PW_KEY', 'utf-8'), 'sha256').digest()
 
     # Initialize list of translations
-    app.config.translations = {
-        str(locale): locale
-        for locale in sorted(
-            utils.babel.list_translations(),
-            key=lambda l: l.get_language_name().title()
-        )
-    }
+    with app.app_context():
+        app.config.translations = {
+            str(locale): locale
+            for locale in sorted(
+                utils.babel.list_translations(),
+                key=lambda l: l.get_language_name().title()
+            )
+        }
 
     # Initialize debugging tools
     if app.config.get("DEBUG"):

core/admin/mailu/debug.py

@@ -1,10 +1,11 @@
-import flask_debugtoolbar
+#Note: Currently flask_debugtoolbar is not compatible with flask.
+#import flask_debugtoolbar
 
 from werkzeug.middleware.profiler import ProfilerMiddleware
 
 
 # Debugging toolbar
-toolbar = flask_debugtoolbar.DebugToolbarExtension()
+#toolbar = flask_debugtoolbar.DebugToolbarExtension()
 
 
 # Profiler

core/admin/mailu/utils.py

@@ -100,7 +100,6 @@ def is_ip_in_subnet(ip, subnets=[]):
 # Application translation
 babel = flask_babel.Babel()
 
-@babel.localeselector
 def get_locale():
     """ selects locale for translation """
     if not app.config['SESSION_COOKIE_NAME'] in flask.request.cookies:
@@ -310,7 +309,7 @@ class MailuSessionConfig:
     # default size of session key parts
     uid_bits = 64 # default if SESSION_KEY_BITS is not set in config
     sid_bits = 128 # for now. must be multiple of 8!
-    time_bits = 32 # for now. must be multiple of 8!  
+    time_bits = 32 # for now. must be multiple of 8!
 
     def __init__(self, app=None):
 
@@ -400,7 +399,7 @@ class MailuSessionInterface(SessionInterface):
             if session.modified:
                 session.delete()
                 response.delete_cookie(
-                    app.session_cookie_name,
+                    app.config['SESSION_COOKIE_NAME'],
                     domain=self.get_cookie_domain(app),
                     path=self.get_cookie_path(app),
                 )
@@ -413,7 +412,7 @@ class MailuSessionInterface(SessionInterface):
         # save session and update cookie if necessary
         if session.save():
             response.set_cookie(
-                app.session_cookie_name,
+               app.config['SESSION_COOKIE_NAME'],
                 session.sid,
                 expires=datetime.now()+timedelta(seconds=app.config['PERMANENT_SESSION_LIFETIME']),
                 httponly=self.get_cookie_httponly(app),
@@ -473,29 +472,30 @@ class MailuSessionExtension:
     def init_app(self, app):
         """ Replace session management of application. """
 
+        redis_session = False
+
         if app.config.get('MEMORY_SESSIONS'):
             # in-memory session store for use in development
             app.session_store = DictStore()
 
         else:
             # redis-based session store for use in production
+            redis_session = True
             app.session_store = RedisStore(
                 redis.StrictRedis().from_url(app.config['SESSION_STORAGE_URL'])
             )
 
+        app.session_config = MailuSessionConfig(app)
+        app.session_interface = MailuSessionInterface()
+        if redis_session:
             # clean expired sessions once on first use in case lifetime was changed
-            def cleaner():
+            with app.app_context():
                 with cleaned.get_lock():
                     if not cleaned.value:
                         cleaned.value = True
                         app.logger.info('cleaning session store')
                         MailuSessionExtension.cleanup_sessions(app)
 
-            app.before_first_request(cleaner)
-
-        app.session_config = MailuSessionConfig(app)
-        app.session_interface = MailuSessionInterface()
-
 cleaned = Value('i', False)
 session = MailuSessionExtension()
 

core/base/requirements-build.txt

@@ -1,3 +1,3 @@
-pip==22.3.1
-setuptools==65.6.3
-wheel==0.38.4
+pip==23.3.1
+setuptools==68.2.2
+wheel==0.41.3

core/base/requirements-prod.txt

@@ -1,86 +1,93 @@
-aiodns==3.0.0
-aiohttp==3.8.5
-aiosignal==1.2.0
-alembic==1.8.1
-async-timeout==4.0.2
-attrs==22.1.0
-Babel==2.11.0
+aiodns==3.1.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+alembic==1.12.1
+aniso8601==9.0.1
+async-timeout==4.0.3
+attrs==23.1.0
+Babel==2.13.1
 bcrypt==4.0.1
-blinker==1.5
+blinker==1.6.3
 certifi==2023.7.22
-cffi==1.15.1
-charset-normalizer==2.1.1
-click==8.1.3
+cffi==1.16.0
+charset-normalizer==3.3.1
+click==8.1.7
 colorclass==2.2.2
-cryptography==41.0.3
-decorator==5.1.1
+cryptography==41.0.5
 defusedxml==0.7.1
-Deprecated==1.2.13
-dnspython==2.2.1
-dominate==2.7.0
+Deprecated==1.2.14
+dnspython==2.4.2
+dominate==2.8.0
 easygui==0.98.3
-email-validator==1.3.0
-Flask==2.2.5
-Flask-Babel==2.0.0
+email-validator==2.1.0.post1
+Flask==2.3.3
+flask-babel==4.0.0
 Flask-Bootstrap==3.3.7.1
-Flask-DebugToolbar==0.13.1
-Flask-Login==0.6.2
-flask-marshmallow==0.14.0
-Flask-Migrate==3.1.0
-Flask-RESTX==1.0.5
+#Flask-DebugToolbar is not compatible with Flask 2.3.3+
+#Flask-DebugToolbar==0.13.1
+Flask-Login==0.6.3
+flask-marshmallow==0.15.0
+Flask-Migrate==4.0.5
+flask-restx==1.1.0
 Flask-SQLAlchemy==2.5.1
-Flask-WTF==1.0.1
-frozenlist==1.3.1
-greenlet==2.0.0
-gunicorn==20.1.0
+# >2.5.1 bug with parsing models.py. Could otherwise be 3.0.5
+Flask-WTF==1.2.1
+frozenlist==1.4.0
+greenlet==3.0.1
+gunicorn==21.2.0
 idna==3.4
+importlib-resources==6.1.0
 infinity==1.5
 intervals==0.9.2
 itsdangerous==2.1.2
 Jinja2==3.1.2
-limits==2.7.1
-Mako==1.2.3
-MarkupSafe==2.1.1
-marshmallow==3.18.0
-marshmallow-sqlalchemy==0.28.1
+jsonschema==4.19.2
+jsonschema-specifications==2023.7.1
+limits==3.6.0
+Mako==1.2.4
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+marshmallow-sqlalchemy==0.29
 msoffcrypto-tool==5.1.1
-multidict==6.0.2
-mysql-connector-python==8.0.32
+multidict==6.0.4
+mysql-connector-python==8.2.0
 olefile==0.46
 oletools==0.60.1
-packaging==21.3
+packaging==23.2
 passlib==1.7.4
 pcodedmp==1.2.6
 podop @ file:///app/libs/podop
-postfix-mta-sts-resolver==1.1.4
-protobuf==3.20.2
-psycopg2-binary==2.9.5
-pycares==4.2.2
+postfix-mta-sts-resolver==1.4.0
+protobuf==4.21.12
+psycopg2-binary==2.9.9
+pycares==4.4.0
 pycparser==2.21
-Pygments==2.15.0
+Pygments==2.16.1
 pyparsing==2.4.7
 python-dateutil==2.8.2
 python-magic==0.4.27
-pytz==2022.6
+pytz==2023.3.post1
 PyYAML==6.0.1
 Radicale==3.1.8
-redis==4.4.4
+redis==5.0.1
+referencing==0.30.2
 requests==2.31.0
+rpds-py==0.10.6
 six==1.16.0
 socrate @ file:///app/libs/socrate
-SQLAlchemy==1.4.42
+SQLAlchemy==1.4.50
 srslib==0.1.4
 tabulate==0.9.0
-tenacity==8.1.0
-typing_extensions==4.4.0
-urllib3==1.26.12
-validators==0.20.0
+tenacity==8.2.3
+typing_extensions==4.8.0
+urllib3==2.0.7
+validators==0.22.0
 visitor==0.1.3
 vobject==0.9.6.1
-watchdog==2.1.9
-Werkzeug==2.2.3
-wrapt==1.14.1
-WTForms==3.0.1
+watchdog==3.0.0
+Werkzeug===2.3.7
+wrapt==1.15.0
+WTForms==3.1.0
 WTForms-Components==0.10.5
 xmltodict==0.13.0
-yarl==1.8.1
+yarl==1.9.2

towncrier/newsfragments/3032.misc

@@ -0,0 +1 @@
+Update all python dependencies in preparation of next Mailu release.