self-hosted/Mailu
1
0
Fork 0
mirror of https://github.com/Mailu/Mailu.git synced 2025-07-15 01:24:34 +02:00
Code Issues Releases Activity

Merge remote-tracking branch 'upstream/master' into upgrade-alpine

Browse Source
This commit is contained in:
Florent Daigniere
2021-08-03 13:46:47 +02:00
parent f8362d04e4 6ea4e3217a
commit d44608ed04
28 changed files with 115 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
.github/workflows/CI.yml vendored
View File

@ -13,7 +13,7 @@ on:
- '[1-9].[0-9].[0-9]'
# pre-releases, e.g. 1.8-pre1
- 1.8-pre[0-9]
# test branches, e.g. test-debian
# test branches, e.g. test-debian
- test-*
###############################################
@ -39,6 +39,21 @@ jobs:
shell: bash
run: |
echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
#For branch TESTING, we set the image tag to PR-xxxx
- name: Derive MAILU_VERSION for branch testing
if: ${{ env.BRANCH == 'testing' }}
shell: bash
env:
COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
run: |
echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV
- name: Derive MAILU_VERSION for other branches than testing
if: ${{ env.BRANCH != 'testing' }}
shell: bash
env:
MAILU_BRANCH: ${{ env.BRANCH }}
run: |
echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV
- name: Create folder for storing images
run: |
sudo mkdir -p /images
@ -58,7 +73,7 @@ jobs:
run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin
- name: Build all docker images
env:
MAILU_VERSION: ${{ env.BRANCH }}
MAILU_VERSION: ${{ env.MAILU_VERSION }}
TRAVIS_BRANCH: ${{ env.BRANCH }}
DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
run: docker-compose -f tests/build.yml build
@ -94,7 +109,7 @@ jobs:
- name: Test core suite
run: python tests/compose/test.py core 1
env:
MAILU_VERSION: ${{ env.BRANCH }}
MAILU_VERSION: ${{ env.MAILU_VERSION }}
TRAVIS_BRANCH: ${{ env.BRANCH }}
DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
@ -127,7 +142,7 @@ jobs:
- name: Test fetch
run: python tests/compose/test.py fetchmail 1
env:
MAILU_VERSION: ${{ env.BRANCH }}
MAILU_VERSION: ${{ env.MAILU_VERSION }}
TRAVIS_BRANCH: ${{ env.BRANCH }}
DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
@ -160,7 +175,7 @@ jobs:
- name: Test clamvav
run: python tests/compose/test.py filters 2
env:
MAILU_VERSION: ${{ env.BRANCH }}
MAILU_VERSION: ${{ env.MAILU_VERSION }}
TRAVIS_BRANCH: ${{ env.BRANCH }}
DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
@ -193,7 +208,7 @@ jobs:
- name: Test rainloop
run: python tests/compose/test.py rainloop 1
env:
MAILU_VERSION: ${{ env.BRANCH }}
MAILU_VERSION: ${{ env.MAILU_VERSION }}
TRAVIS_BRANCH: ${{ env.BRANCH }}
DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
@ -226,7 +241,7 @@ jobs:
- name: Test roundcube
run: python tests/compose/test.py roundcube 1
env:
MAILU_VERSION: ${{ env.BRANCH }}
MAILU_VERSION: ${{ env.MAILU_VERSION }}
TRAVIS_BRANCH: ${{ env.BRANCH }}
DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
@ -259,7 +274,7 @@ jobs:
- name: Test webdav
run: python tests/compose/test.py webdav 1
env:
MAILU_VERSION: ${{ env.BRANCH }}
MAILU_VERSION: ${{ env.MAILU_VERSION }}
TRAVIS_BRANCH: ${{ env.BRANCH }}
DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
@ -280,6 +295,21 @@ jobs:
shell: bash
run: |
echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
#For branch TESTING, we set the image tag to PR-xxxx
- name: Derive MAILU_VERSION for branch testing
if: ${{ env.BRANCH == 'testing' }}
shell: bash
env:
COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
run: |
echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV
- name: Derive MAILU_VERSION for other branches than testing
if: ${{ env.BRANCH != 'testing' }}
shell: bash
env:
MAILU_BRANCH: ${{ env.BRANCH }}
run: |
echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV
- name: Create folder for storing images
run: |
sudo mkdir -p /images
@ -300,9 +330,8 @@ jobs:
DOCKER_PW: ${{ secrets.Docker_Password }}
DOCKER_ORG: ${{ secrets.DOCKER_ORG }}
DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }}
MAILU_VERSION: ${{ env.BRANCH }}
MAILU_VERSION: ${{ env.MAILU_VERSION }}
TRAVIS_BRANCH: ${{ env.BRANCH }}
TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
run: bash tests/deploy.sh
# This job is watched by bors. It only complets if building,testing and deploy worked.

2
PULL_REQUEST_TEMPLATE.md
View File

@ -13,4 +13,4 @@ Before we can consider review and merge, please make sure the following list is
If an entry in not applicable, you can check it or remove it from the list.
- [ ] In case of feature or enhancement: documentation updated accordingly
- [ ] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/guide.html#changelog) entry file.
- [ ] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.

2
core/admin/Dockerfile
View File

@ -26,7 +26,7 @@ WORKDIR /app
COPY requirements-prod.txt requirements.txt
RUN apk add --no-cache openssl curl postgresql-libs mariadb-connector-c \
&& apk add --no-cache --virtual build-dep \
openssl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev \
openssl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \
&& pip3 install -r requirements.txt \
&& apk del --no-cache build-dep

2
core/admin/mailu/internal/views/auth.py
View File

@ -63,7 +63,7 @@ def basic_authentication():
authorization = flask.request.headers.get("Authorization")
if authorization and authorization.startswith("Basic "):
encoded = authorization.replace("Basic ", "")
user_email, password = base64.b64decode(encoded).split(b":")
user_email, password = base64.b64decode(encoded).split(b":", 1)
user = models.User.query.get(user_email.decode("utf8"))
if nginx.check_credentials(user, password.decode('utf-8'), flask.request.remote_addr, "web"):
response = flask.Response()

27
core/admin/mailu/models.py
View File

@ -57,10 +57,9 @@ class IdnaEmail(db.TypeDecorator):
def process_bind_param(self, value, dialect):
""" encode unicode domain part of email address to punycode """
localpart, domain_name = value.rsplit('@', 1)
localpart, domain_name = value.lower().rsplit('@', 1)
if '@' in localpart:
raise ValueError('email local part must not contain "@"')
domain_name = domain_name.lower()
return f'{localpart}@{idna.encode(domain_name).decode("ascii")}'
def process_result_value(self, value, dialect):
@ -272,11 +271,12 @@ class Domain(Base):
return dkim.strip_key(dkim_key).decode('utf8')
def generate_dkim_key(self):
""" generate and activate new DKIM key """
""" generate new DKIM key """
self.dkim_key = dkim.gen_key()
def has_email(self, localpart):
""" checks if localpart is configured for domain """
localpart = localpart.lower()
for email in chain(self.users, self.aliases):
if email.localpart == localpart:
return True
@ -355,8 +355,8 @@ class Email(object):
@email.setter
def email(self, value):
""" setter for email - sets _email, localpart and domain_name at once """
self.localpart, self.domain_name = value.rsplit('@', 1)
self._email = value
self._email = value.lower()
self.localpart, self.domain_name = self._email.rsplit('@', 1)
@staticmethod
def _update_localpart(target, value, *_):
@ -371,8 +371,8 @@ class Email(object):
@classmethod
def __declare_last__(cls):
# gets called after mappings are completed
sqlalchemy.event.listen(User.localpart, 'set', cls._update_localpart, propagate=True)
sqlalchemy.event.listen(User.domain_name, 'set', cls._update_domain_name, propagate=True)
sqlalchemy.event.listen(cls.localpart, 'set', cls._update_localpart, propagate=True)
sqlalchemy.event.listen(cls.domain_name, 'set', cls._update_domain_name, propagate=True)
def sendmail(self, subject, body):
""" send an email to the address """
@ -389,8 +389,7 @@ class Email(object):
def resolve_domain(cls, email):
""" resolves domain alternative to real domain """
localpart, domain_name = email.rsplit('@', 1) if '@' in email else (None, email)
alternative = Alternative.query.get(domain_name)
if alternative:
if alternative := Alternative.query.get(domain_name):
domain_name = alternative.domain_name
return (localpart, domain_name)
@ -401,12 +400,14 @@ class Email(object):
localpart_stripped = None
stripped_alias = None
if os.environ.get('RECIPIENT_DELIMITER') in localpart:
localpart_stripped = localpart.rsplit(os.environ.get('RECIPIENT_DELIMITER'), 1)[0]
delim = os.environ.get('RECIPIENT_DELIMITER')
if delim in localpart:
localpart_stripped = localpart.rsplit(delim, 1)[0]
user = User.query.get(f'{localpart}@{domain_name}')
if not user and localpart_stripped:
user = User.query.get(f'{localpart_stripped}@{domain_name}')
if user:
email = f'{localpart}@{domain_name}'
@ -416,15 +417,15 @@ class Email(object):
destination.append(email)
else:
destination = [email]
return destination
pure_alias = Alias.resolve(localpart, domain_name)
stripped_alias = Alias.resolve(localpart_stripped, domain_name)
if pure_alias and not pure_alias.wildcard:
return pure_alias.destination
if stripped_alias:
if stripped_alias := Alias.resolve(localpart_stripped, domain_name):
return stripped_alias.destination
if pure_alias:

2
core/admin/mailu/ui/templates/user/reply.html
View File

@ -14,7 +14,7 @@
{{ form.hidden_tag() }}
{{ macros.form_field(form.reply_enabled,
onchange="if(this.checked){$('#reply_subject,#reply_body,#reply_enddate,#reply_startdate').removeAttr('readonly')}
else{$('#reply_subject,#reply_body,#reply_enddate').attr('readonly', '')}") }}
else{$('#reply_subject,#reply_body,#reply_enddate,#reply_startdate').attr('readonly', '')}") }}
{{ macros.form_field(form.reply_subject,
**{("rw" if user.reply_enabled else "readonly"): ""}) }}
{{ macros.form_field(form.reply_body, rows=10,

2
core/admin/mailu/ui/views/domains.py
View File

@ -74,6 +74,8 @@ def domain_details(domain_name):
def domain_genkeys(domain_name):
domain = models.Domain.query.get(domain_name) or flask.abort(404)
domain.generate_dkim_key()
models.db.session.add(domain)
models.db.session.commit()
return flask.redirect(
flask.url_for(".domain_details", domain_name=domain_name))

36
core/admin/package.json
View File

@ -2,34 +2,30 @@
"name": "mailu",
"version": "1.0.0",
"description": "Mailu admin assets",
"main": "assest/index.js",
"main": "assets/index.js",
"directories": {
"lib": "lib"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"@babel/core": "^7.4.4",
"@babel/preset-env": "^7.4.4",
"admin-lte": "^2.4.10",
"babel-loader": "^8.0.5",
"bootstrap": "^3.4.1",
"@babel/core": "^7.14.6",
"admin-lte": "^2.4.18",
"babel-loader": "^8.0.6",
"css-loader": "^2.1.1",
"expose-loader": "^0.7.5",
"file-loader": "^3.0.1",
"font-awesome": "^4.7.0",
"font-awesome-loader": "^1.0.2",
"jQuery": "^1.7.4",
"less": "^3.9.0",
"jquery": "^3.6.0",
"less": "^3.13.1",
"less-loader": "^5.0.0",
"mini-css-extract-plugin": "^0.6.0",
"node-sass": "^4.12.0",
"popper.js": "^1.15.0",
"sass-loader": "^7.1.0",
"select2": "^4.0.7-rc.0",
"style-loader": "^0.23.1",
"url-loader": "^1.1.2",
"webpack": "^4.30.0",
"webpack-cli": "^3.3.2"
"mini-css-extract-plugin": "^1.2.1",
"node-sass": "^4.13.1",
"sass-loader": "^7.3.1",
"select2": "^4.0.13",
"url-loader": "^2.3.0",
"webpack": "^4.33.0",
"webpack-cli": "^3.3.12"
}
}

8
core/admin/requirements-prod.txt
View File

@ -5,7 +5,7 @@ bcrypt==3.1.6
blinker==1.4
cffi==1.12.3
Click==7.0
cryptography==3.2
cryptography==3.4.7
decorator==4.4.0
dnspython==1.16.0
dominate==2.3.5
@ -25,7 +25,7 @@ idna==2.8
infinity==1.4
intervals==0.8.1
itsdangerous==1.1.0
Jinja2==2.10.1
Jinja2==2.11.3
limits==1.3
Mako==1.0.9
MarkupSafe==1.1.1
@ -36,11 +36,11 @@ passlib==1.7.4
psycopg2==2.8.2
pycparser==2.19
Pygments==2.8.1
pyOpenSSL==19.0.0
pyOpenSSL==20.0.1
python-dateutil==2.8.0
python-editor==1.0.4
pytz==2019.1
PyYAML==5.1
PyYAML==5.4.1
redis==3.2.1
#alpine3:12 provides six==1.15.0
#six==1.12.0

3
core/admin/start.py
View File

@ -19,7 +19,8 @@ if account is not None and domain is not None and password is not None:
os.system("flask mailu admin %s %s '%s' --mode %s" % (account, domain, password, mode))
start_command="".join([
"gunicorn -w 4 -b :80 ",
"gunicorn --threads ", str(os.cpu_count()),
" -b :80 ",
"--access-logfile - " if (log.root.level<=log.INFO) else "",
"--error-logfile - ",
"--preload ",

2
core/postfix/conf/outclean_header_filter.cf
View File

@ -4,7 +4,7 @@
# Remove the first line of the Received: header. Note that we cannot fully remove the Received: header
# because OpenDKIM requires that a header be present when signing outbound mail. The first line is
# where the user's home IP address would be.
/^\s*Received:[^\n]*(.*)/ REPLACE Received: from authenticated-user (PRIMARY_HOSTNAME [PUBLIC_IP])$1
/^\s*Received:[^\n]*(.*)/ REPLACE Received: from authenticated-user ({{OUTCLEAN}} [{{OUTCLEAN_ADDRESS}}])$1
# Remove other typically private information.
/^\s*User-Agent:/ IGNORE

12
core/postfix/start.py
View File

@ -8,12 +8,13 @@ import logging as log
import sys
from podop import run_server
from pwd import getpwnam
from socrate import system, conf
log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
def start_podop():
os.setuid(100)
os.setuid(getpwnam('postfix').pw_uid)
url = "http://" + os.environ["ADMIN_ADDRESS"] + "/internal/postfix/"
# TODO: Remove verbosity setting from Podop?
run_server(0, "postfix", "/tmp/podop.socket", [
@ -36,6 +37,15 @@ os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT",
os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
os.environ["ANTISPAM_MILTER_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_MILTER", "antispam:11332")
os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
os.environ["OUTCLEAN"] = os.environ["HOSTNAMES"].split(",")[0]
try:
_to_lookup = os.environ["OUTCLEAN"]
# Ensure we lookup a FQDN: @see #1884
if not _to_lookup.endswith('.'):
_to_lookup += '.'
os.environ["OUTCLEAN_ADDRESS"] = system.resolve_hostname(_to_lookup)
except:
os.environ["OUTCLEAN_ADDRESS"] = "10.10.10.10"
for postfix_file in glob.glob("/conf/*.cf"):
conf.jinja(postfix_file, os.environ, os.path.join("/etc/postfix", os.path.basename(postfix_file)))

5
docs/faq.rst
View File

@ -497,6 +497,8 @@ follow these steps:
logging:
driver: journald
options:
tag: mailu-front
2. Add the /etc/fail2ban/filter.d/bad-auth.conf
@ -506,6 +508,7 @@ follow these steps:
[Definition]
failregex = .* client login failed: .+ client:\ <HOST>
ignoreregex =
journalmatch = CONTAINER_TAG=mailu-front
3. Add the /etc/fail2ban/jail.d/bad-auth.conf
@ -513,8 +516,8 @@ follow these steps:
[bad-auth]
enabled = true
backend = systemd
filter = bad-auth
logpath = /var/log/messages
bantime = 604800
findtime = 300
maxretry = 10

2
docs/kubernetes/mailu/front.yaml
View File

@ -1,4 +1,4 @@
apiVersion: apps/v1beta2
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: mailu-front

8
setup/requirements.txt
View File

@ -1,4 +1,4 @@
flask
flask-bootstrap
redis
gunicorn
Flask==1.0.2
Flask-Bootstrap==3.3.7.1
gunicorn==19.9.0
redis==3.2.1

4
setup/server.py
View File

@ -54,11 +54,11 @@ def build_app(path):
@app.context_processor
def app_context():
return dict(
versions=os.getenv("VERSIONS","master").split(','),
versions=os.getenv("VERSIONS","master").split(','),
stable_version = os.getenv("stable_version", "master")
)
prefix_bp = flask.Blueprint(version, __name__)
prefix_bp = flask.Blueprint(version.replace(".", "_"), __name__)
prefix_bp.jinja_loader = jinja2.ChoiceLoader([
jinja2.FileSystemLoader(os.path.join(path, "templates")),
jinja2.FileSystemLoader(os.path.join(path, "flavors"))

2
setup/templates/steps/compose/02_services.html
View File

@ -59,7 +59,7 @@ the security implications caused by such an increase of attack surface.<p>
<i>Fetchmail allows users to retrieve mail from an external mail-server via IMAP/POP3 and puts it in their inbox.</i>
</div>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
<script type="text/javascript" src="{{ url_for('static', filename='render.js') }}"></script>

2
setup/templates/steps/config.html
View File

@ -83,7 +83,7 @@ manage your email domains, users, etc.</p>
<input class="form-control" type="text" name="admin_path" id="admin_path" style="display: none">
</div>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
<script type="text/javascript" src="{{ url_for('static', filename='render.js') }}"></script>

2
setup/templates/steps/stack/02_services.html
View File

@ -55,7 +55,7 @@ the security implications caused by such an increase of attack surface.<p>
<i>Fetchmail allows users to retrieve mail from an external mail-server via IMAP/POP3 and puts it in their inbox.</i>
</div>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
<script type="text/javascript" src="{{ url_for('static', filename='render.js') }}"></script>

9
tests/deploy.sh
View File

@ -3,14 +3,5 @@
# Skip deploy for staging branch
[ "$TRAVIS_BRANCH" = "staging" ] && exit 0
# Retag in case of `bors try`
if [ "$TRAVIS_BRANCH" = "testing" ]; then
export DOCKER_ORG=$DOCKER_ORG_TESTS
# Commit message is like "Try #99".
# This sets the version tag to "pr-99"
export MAILU_VERSION="pr-${TRAVIS_COMMIT_MESSAGE//[!0-9]/}"
docker-compose -f tests/build.yml build
fi
docker login -u $DOCKER_UN -p $DOCKER_PW
docker-compose -f tests/build.yml push

1
towncrier/newsfragments/1294.bugfix Normal file
View File

@ -0,0 +1 @@
Ensure that the podop socket is always owned by the postfix user (wasn't the case when build using non-standard base images... typically for arm64)

1
towncrier/newsfragments/1845.feature Normal file
View File

@ -0,0 +1 @@
Update version of rainloop webmail to 1.16.0. This is a security update.

1
towncrier/newsfragments/1857.doc Normal file
View File

@ -0,0 +1 @@
Update fail2ban documentation to use systemd backend instead of filepath for journald

1
towncrier/newsfragments/1861.bugfix Normal file
View File

@ -0,0 +1 @@
Fix a bug preventing colons from being used in passwords when using radicale/webdav.

1
towncrier/newsfragments/1874.bugfix Normal file
View File

@ -0,0 +1 @@
Remove dot in blueprint name to prevent critical flask startup error in setup.

1
towncrier/newsfragments/1880.feature Normal file
View File

@ -0,0 +1 @@
Update jquery used in setup. Set pinned versions in requirements.txt for setup. This is a security update.

1
towncrier/newsfragments/191.bugfix Normal file
View File

@ -0,0 +1 @@
Replace PUBLIC_HOSTNAME and PUBLIC_IP in "Received" headers to ensure that no undue spam points are attributed

2
webmails/rainloop/Dockerfile
View File

@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y \
# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
RUN pip3 install socrate
ENV RAINLOOP_URL https://github.com/RainLoop/rainloop-webmail/releases/download/v1.14.0/rainloop-community-1.14.0.zip
ENV RAINLOOP_URL https://github.com/RainLoop/rainloop-webmail/releases/download/v1.16.0/rainloop-community-1.16.0.zip
RUN apt-get update && apt-get install -y \
unzip python3-jinja2 \