1
0
mirror of https://github.com/mailcow/mailcow-dockerized.git synced 2024-12-02 09:02:42 +02:00

Merge pull request #5689 from mailcow/staging

2024-01c
This commit is contained in:
Niklas Meyer 2024-02-02 15:52:33 +01:00 committed by GitHub
commit 1e09df20b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 379 additions and 85 deletions

View File

@ -22,7 +22,7 @@ jobs:
bash helper-scripts/update_postscreen_whitelist.sh
- name: Create Pull Request
uses: peter-evans/create-pull-request@v5
uses: peter-evans/create-pull-request@v6
with:
token: ${{ secrets.mailcow_action_Update_postscreen_access_cidr_pat }}
commit-message: update postscreen_access.cidr

1
.gitignore vendored
View File

@ -13,6 +13,7 @@ data/conf/dovecot/acl_anyone
data/conf/dovecot/dovecot-master.passwd
data/conf/dovecot/dovecot-master.userdb
data/conf/dovecot/extra.conf
data/conf/dovecot/mail_replica.conf
data/conf/dovecot/global_sieve_*
data/conf/dovecot/last_login
data/conf/dovecot/lua

View File

@ -335,6 +335,15 @@ sys.exit()
EOF
fi
# Set mail_replica for HA setups
if [[ -n ${MAILCOW_REPLICA_IP} && -n ${DOVEADM_REPLICA_PORT} ]]; then
cat <<EOF > /etc/dovecot/mail_replica.conf
# Autogenerated by mailcow
mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT}
EOF
fi
# 401 is user dovecot
if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem

View File

@ -21,28 +21,6 @@ from modules.IPTables import IPTables
from modules.NFTables import NFTables
# connect to redis
while True:
try:
redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
if "".__eq__(redis_slaveof_ip):
r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
else:
r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
r.ping()
except Exception as ex:
print('%s - trying again in 3 seconds' % (ex))
time.sleep(3)
else:
break
pubsub = r.pubsub()
# rename fail2ban to netfilter
if r.exists('F2B_LOG'):
r.rename('F2B_LOG', 'NETFILTER_LOG')
# globals
WHITELIST = []
BLACKLIST= []
@ -50,18 +28,10 @@ bans = {}
quit_now = False
exit_code = 0
lock = Lock()
# init Logger
logger = Logger(r)
# init backend
backend = sys.argv[1]
if backend == "nftables":
logger.logInfo('Using NFTables backend')
tables = NFTables("MAILCOW", logger)
else:
logger.logInfo('Using IPTables backend')
tables = IPTables("MAILCOW", logger)
chain_name = "MAILCOW"
r = None
pubsub = None
clear_before_quit = False
def refreshF2boptions():
@ -250,17 +220,21 @@ def clear():
with lock:
tables.clearIPv4Table()
tables.clearIPv6Table()
r.delete('F2B_ACTIVE_BANS')
r.delete('F2B_PERM_BANS')
pubsub.unsubscribe()
try:
if r is not None:
r.delete('F2B_ACTIVE_BANS')
r.delete('F2B_PERM_BANS')
except Exception as ex:
logger.logWarn('Error clearing redis keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
def watch():
logger.logInfo('Watching Redis channel F2B_CHANNEL')
pubsub.subscribe('F2B_CHANNEL')
global pubsub
global quit_now
global exit_code
logger.logInfo('Watching Redis channel F2B_CHANNEL')
pubsub.subscribe('F2B_CHANNEL')
while not quit_now:
try:
for item in pubsub.listen():
@ -280,6 +254,7 @@ def watch():
ban(addr)
except Exception as ex:
logger.logWarn('Error reading log line from pubsub: %s' % ex)
pubsub = None
quit_now = True
exit_code = 2
@ -403,21 +378,76 @@ def blacklistUpdate():
permBan(net=net, unban=True)
time.sleep(60.0 - ((time.time() - start_time) % 60.0))
def quit(signum, frame):
global quit_now
quit_now = True
def sigterm_quit(signum, frame):
global clear_before_quit
clear_before_quit = True
sys.exit(exit_code)
def berfore_quit():
if clear_before_quit:
clear()
if pubsub is not None:
pubsub.unsubscribe()
if __name__ == '__main__':
refreshF2boptions()
atexit.register(berfore_quit)
signal.signal(signal.SIGTERM, sigterm_quit)
# init Logger
logger = Logger(None)
# init backend
backend = sys.argv[1]
if backend == "nftables":
logger.logInfo('Using NFTables backend')
tables = NFTables(chain_name, logger)
else:
logger.logInfo('Using IPTables backend')
tables = IPTables(chain_name, logger)
# In case a previous session was killed without cleanup
clear()
# Reinit MAILCOW chain
# Is called before threads start, no locking
logger.logInfo("Initializing mailcow netfilter chain")
tables.initChainIPv4()
tables.initChainIPv6()
if os.getenv("DISABLE_NETFILTER_ISOLATION_RULE").lower() in ("y", "yes"):
logger.logInfo(f"Skipping {chain_name} isolation")
else:
logger.logInfo(f"Setting {chain_name} isolation")
tables.create_mailcow_isolation_rule("br-mailcow", [3306, 6379, 8983, 12345], os.getenv("MAILCOW_REPLICA_IP"))
# connect to redis
while True:
try:
redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
if "".__eq__(redis_slaveof_ip):
r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
else:
r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
r.ping()
pubsub = r.pubsub()
except Exception as ex:
print('%s - trying again in 3 seconds' % (ex))
time.sleep(3)
else:
break
Logger.r = r
# rename fail2ban to netfilter
if r.exists('F2B_LOG'):
r.rename('F2B_LOG', 'NETFILTER_LOG')
# clear bans in redis
r.delete('F2B_ACTIVE_BANS')
r.delete('F2B_PERM_BANS')
refreshF2boptions()
watch_thread = Thread(target=watch)
watch_thread.daemon = True
watch_thread.start()
@ -460,9 +490,6 @@ if __name__ == '__main__':
whitelistupdate_thread.daemon = True
whitelistupdate_thread.start()
signal.signal(signal.SIGTERM, quit)
atexit.register(clear)
while not quit_now:
time.sleep(0.5)

View File

@ -1,5 +1,6 @@
import iptc
import time
import os
class IPTables:
def __init__(self, chain_name, logger):
@ -211,3 +212,41 @@ class IPTables:
target = rule.create_target("SNAT")
target.to_source = snat_target
return rule
def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
try:
chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
# insert mailcow isolation rule
rule = iptc.Rule()
rule.in_interface = f'! {_interface}'
rule.out_interface = _interface
rule.protocol = 'tcp'
rule.create_target("DROP")
match = rule.create_match("multiport")
match.dports = ','.join(map(str, _dports))
if rule in chain.rules:
chain.delete_rule(rule)
chain.insert_rule(rule, position=0)
# insert mailcow isolation exception rule
if _allow != "":
rule = iptc.Rule()
rule.src = _allow
rule.in_interface = f'! {_interface}'
rule.out_interface = _interface
rule.protocol = 'tcp'
rule.create_target("ACCEPT")
match = rule.create_match("multiport")
match.dports = ','.join(map(str, _dports))
if rule in chain.rules:
chain.delete_rule(rule)
chain.insert_rule(rule, position=0)
return True
except Exception as e:
self.logger.logCrit(f"Error adding {self.chain_name} isolation: {e}")
return False

View File

@ -10,7 +10,8 @@ class Logger:
tolog['time'] = int(round(time.time()))
tolog['priority'] = priority
tolog['message'] = message
self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
if self.r:
self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
print(message)
def logWarn(self, message):

View File

@ -1,5 +1,6 @@
import nftables
import ipaddress
import os
class NFTables:
def __init__(self, chain_name, logger):
@ -266,6 +267,17 @@ class NFTables:
return self.nft_exec_dict(delete_command)
def delete_filter_rule(self, _family:str, _chain: str, _handle:str):
delete_command = self.get_base_dict()
_rule_opts = {'family': _family,
'table': 'filter',
'chain': _chain,
'handle': _handle }
_delete = {'delete': {'rule': _rule_opts} }
delete_command["nftables"].append(_delete)
return self.nft_exec_dict(delete_command)
def snat_rule(self, _family: str, snat_target: str, source_address: str):
chain_name = self.nft_chain_names[_family]['nat']['postrouting']
@ -381,7 +393,7 @@ class NFTables:
break
return chain_handle
def get_rules_handle(self, _family: str, _table: str, chain_name: str):
def get_rules_handle(self, _family: str, _table: str, chain_name: str, _comment_filter = "mailcow"):
rule_handle = []
# Command: 'nft list chain {family} {table} {chain_name}'
_chain_opts = {'family': _family, 'table': _table, 'name': chain_name}
@ -397,7 +409,7 @@ class NFTables:
rule = _object["rule"]
if rule["family"] == _family and rule["table"] == _table and rule["chain"] == chain_name:
if rule.get("comment") and rule["comment"] == "mailcow":
if rule.get("comment") and rule["comment"] == _comment_filter:
rule_handle.append(rule["handle"])
return rule_handle
@ -493,3 +505,152 @@ class NFTables:
position+=1
return position if rule_found else False
def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
family = "ip"
table = "filter"
comment_filter_drop = "mailcow isolation"
comment_filter_allow = "mailcow isolation allow"
json_command = self.get_base_dict()
# Delete old mailcow isolation rules
handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_drop)
for handle in handles:
self.delete_filter_rule(family, self.chain_name, handle)
handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_allow)
for handle in handles:
self.delete_filter_rule(family, self.chain_name, handle)
# insert mailcow isolation rule
_match_dict_drop = [
{
"match": {
"op": "!=",
"left": {
"meta": {
"key": "iifname"
}
},
"right": _interface
}
},
{
"match": {
"op": "==",
"left": {
"meta": {
"key": "oifname"
}
},
"right": _interface
}
},
{
"match": {
"op": "==",
"left": {
"payload": {
"protocol": "tcp",
"field": "dport"
}
},
"right": {
"set": _dports
}
}
},
{
"counter": {
"packets": 0,
"bytes": 0
}
},
{
"drop": None
}
]
rule_drop = { "insert": { "rule": {
"family": family,
"table": table,
"chain": self.chain_name,
"comment": comment_filter_drop,
"expr": _match_dict_drop
}}}
json_command["nftables"].append(rule_drop)
# insert mailcow isolation allow rule
if _allow != "":
_match_dict_allow = [
{
"match": {
"op": "==",
"left": {
"payload": {
"protocol": "ip",
"field": "saddr"
}
},
"right": _allow
}
},
{
"match": {
"op": "!=",
"left": {
"meta": {
"key": "iifname"
}
},
"right": _interface
}
},
{
"match": {
"op": "==",
"left": {
"meta": {
"key": "oifname"
}
},
"right": _interface
}
},
{
"match": {
"op": "==",
"left": {
"payload": {
"protocol": "tcp",
"field": "dport"
}
},
"right": {
"set": _dports
}
}
},
{
"counter": {
"packets": 0,
"bytes": 0
}
},
{
"accept": None
}
]
rule_allow = { "insert": { "rule": {
"family": family,
"table": table,
"chain": self.chain_name,
"comment": comment_filter_allow,
"expr": _match_dict_allow
}}}
json_command["nftables"].append(rule_allow)
success = self.nft_exec_dict(json_command)
if success == False:
self.logger.logCrit(f"Error adding {self.chain_name} isolation")
return False
return True

View File

@ -1,7 +1,8 @@
FROM debian:bullseye-slim
FROM debian:bookworm-slim
LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
ARG DEBIAN_FRONTEND=noninteractive
ARG DEBIAN_VERSION=bookworm
ARG SOGO_DEBIAN_REPOSITORY=http://www.axis.cz/linux/debian
# renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
ARG GOSU_VERSION=1.17
@ -21,7 +22,7 @@ RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \
syslog-ng-core \
syslog-ng-mod-redis \
dirmngr \
netcat \
netcat-traditional \
psmisc \
wget \
patch \
@ -32,7 +33,7 @@ RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \
&& mkdir /usr/share/doc/sogo \
&& touch /usr/share/doc/sogo/empty.sh \
&& apt-key adv --keyserver keys.openpgp.org --recv-key 74FFC6D72B925A34B5D356BDF8A27B36A6E2EAE9 \
&& echo "deb [trusted=yes] ${SOGO_DEBIAN_REPOSITORY} bullseye sogo-v5" > /etc/apt/sources.list.d/sogo.list \
&& echo "deb [trusted=yes] ${SOGO_DEBIAN_REPOSITORY} ${DEBIAN_VERSION} sogo-v5" > /etc/apt/sources.list.d/sogo.list \
&& apt-get update && apt-get install -y --no-install-recommends \
sogo \
sogo-activesync \

View File

@ -247,6 +247,9 @@ plugin {
mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
mail_log_fields = uid box msgid size
mail_log_cached_only = yes
# Try set mail_replica
!include_try /etc/dovecot/mail_replica.conf
}
service quota-warning {
executable = script /usr/local/bin/quota_notify.py

View File

@ -1,9 +1,10 @@
# Whitelist generated by Postwhite v3.4 on Mon Jan 1 00:15:22 UTC 2024
# Whitelist generated by Postwhite v3.4 on Thu Feb 1 00:13:50 UTC 2024
# https://github.com/stevejenkins/postwhite/
# 2052 total rules
# 2089 total rules
2a00:1450:4000::/36 permit
2a01:111:f400::/48 permit
2a01:111:f403:8000::/50 permit
2a01:111:f403:8000::/51 permit
2a01:111:f403::/49 permit
2a01:111:f403:c000::/51 permit
2a01:111:f403:f000::/52 permit
@ -116,7 +117,6 @@
40.92.0.0/16 permit
40.107.0.0/16 permit
40.112.65.63 permit
40.117.80.0/24 permit
43.228.184.0/22 permit
44.206.138.57 permit
44.209.42.157 permit
@ -206,7 +206,6 @@
52.95.49.88/29 permit
52.96.91.34 permit
52.96.111.82 permit
52.96.172.98 permit
52.96.214.50 permit
52.96.222.194 permit
52.96.222.226 permit
@ -405,7 +404,6 @@
66.196.81.228/30 permit
66.196.81.232/31 permit
66.196.81.234 permit
66.211.168.230/31 permit
66.211.170.88/29 permit
66.211.184.0/23 permit
66.218.74.64/30 permit
@ -594,9 +592,10 @@
74.112.67.243 permit
74.125.0.0/16 permit
74.202.227.40 permit
74.208.4.192/26 permit
74.208.5.64/26 permit
74.208.122.0/26 permit
74.208.4.200 permit
74.208.4.201 permit
74.208.4.220 permit
74.208.4.221 permit
74.209.250.0/24 permit
75.2.70.75 permit
76.223.128.0/19 permit
@ -624,12 +623,20 @@
77.238.189.148/30 permit
81.7.169.128/25 permit
81.223.46.0/27 permit
82.165.159.0/24 permit
82.165.159.0/26 permit
82.165.229.31 permit
82.165.229.130 permit
82.165.230.21 permit
82.165.230.22 permit
82.165.159.2 permit
82.165.159.3 permit
82.165.159.4 permit
82.165.159.12 permit
82.165.159.13 permit
82.165.159.14 permit
82.165.159.34 permit
82.165.159.35 permit
82.165.159.40 permit
82.165.159.41 permit
82.165.159.42 permit
82.165.159.45 permit
82.165.159.130 permit
82.165.159.131 permit
84.116.6.0/23 permit
84.116.36.0/24 permit
84.116.50.0/23 permit
@ -1430,6 +1437,7 @@
135.84.216.0/22 permit
136.143.160.0/24 permit
136.143.161.0/24 permit
136.143.178.49 permit
136.143.182.0/23 permit
136.143.184.0/24 permit
136.143.188.0/24 permit
@ -1952,12 +1960,41 @@
212.82.111.228/31 permit
212.82.111.230 permit
212.123.28.40 permit
212.227.15.0/24 permit
212.227.15.0/25 permit
212.227.17.0/27 permit
212.227.126.128/25 permit
212.227.15.3 permit
212.227.15.4 permit
212.227.15.5 permit
212.227.15.6 permit
212.227.15.14 permit
212.227.15.15 permit
212.227.15.18 permit
212.227.15.19 permit
212.227.15.25 permit
212.227.15.26 permit
212.227.15.29 permit
212.227.15.44 permit
212.227.15.45 permit
212.227.15.46 permit
212.227.15.47 permit
212.227.15.50 permit
212.227.15.52 permit
212.227.15.53 permit
212.227.15.54 permit
212.227.15.55 permit
212.227.17.11 permit
212.227.17.12 permit
212.227.17.18 permit
212.227.17.19 permit
212.227.17.20 permit
212.227.17.21 permit
212.227.17.22 permit
212.227.17.26 permit
212.227.17.28 permit
212.227.17.29 permit
212.227.126.224 permit
212.227.126.225 permit
212.227.126.226 permit
212.227.126.227 permit
213.46.255.0/24 permit
213.165.64.0/23 permit
213.199.128.139 permit
213.199.128.145 permit
213.199.138.181 permit
@ -2021,10 +2058,10 @@
216.203.30.55 permit
216.203.33.178/31 permit
216.205.24.0/24 permit
216.221.160.0/19 permit
216.239.32.0/19 permit
217.72.192.64/26 permit
217.72.192.248/29 permit
217.72.207.0/27 permit
217.72.192.77 permit
217.72.192.78 permit
217.77.141.52 permit
217.77.141.59 permit
217.175.194.0/24 permit

View File

@ -21,6 +21,7 @@ services:
image: mariadb:10.5
depends_on:
- unbound-mailcow
- netfilter-mailcow
stop_grace_period: 45s
volumes:
- mysql-vol-1:/var/lib/mysql/
@ -46,6 +47,8 @@ services:
volumes:
- redis-vol-1:/data/
restart: always
depends_on:
- netfilter-mailcow
ports:
- "${REDIS_PORT:-127.0.0.1:7654}:6379"
environment:
@ -172,7 +175,7 @@ services:
- phpfpm
sogo-mailcow:
image: mailcow/sogo:1.121
image: mailcow/sogo:1.122
environment:
- DBNAME=${DBNAME}
- DBUSER=${DBUSER}
@ -219,9 +222,10 @@ services:
- sogo
dovecot-mailcow:
image: mailcow/dovecot:1.27
image: mailcow/dovecot:1.28
depends_on:
- mysql-mailcow
- netfilter-mailcow
dns:
- ${IPV4_NETWORK:-172.22.1}.254
cap_add:
@ -242,6 +246,8 @@ services:
environment:
- DOVECOT_MASTER_USER=${DOVECOT_MASTER_USER:-}
- DOVECOT_MASTER_PASS=${DOVECOT_MASTER_PASS:-}
- MAILCOW_REPLICA_IP=${MAILCOW_REPLICA_IP:-}
- DOVEADM_REPLICA_PORT=${DOVEADM_REPLICA_PORT:-}
- LOG_LINES=${LOG_LINES:-9999}
- DBNAME=${DBNAME}
- DBUSER=${DBUSER}
@ -435,14 +441,8 @@ services:
- acme
netfilter-mailcow:
image: mailcow/netfilter:1.55
image: mailcow/netfilter:1.56
stop_grace_period: 30s
depends_on:
- dovecot-mailcow
- postfix-mailcow
- sogo-mailcow
- php-fpm-mailcow
- redis-mailcow
restart: always
privileged: true
environment:
@ -453,6 +453,8 @@ services:
- SNAT6_TO_SOURCE=${SNAT6_TO_SOURCE:-n}
- REDIS_SLAVEOF_IP=${REDIS_SLAVEOF_IP:-}
- REDIS_SLAVEOF_PORT=${REDIS_SLAVEOF_PORT:-}
- MAILCOW_REPLICA_IP=${MAILCOW_REPLICA_IP:-}
- DISABLE_NETFILTER_ISOLATION_RULE=${DISABLE_NETFILTER_ISOLATION_RULE:-n}
network_mode: "host"
volumes:
- /lib/modules:/lib/modules:ro
@ -553,6 +555,8 @@ services:
solr-mailcow:
image: mailcow/solr:1.8.2
restart: always
depends_on:
- netfilter-mailcow
volumes:
- solr-vol-1:/opt/solr/server/solr/dovecot-fts/data
ports:

View File

@ -494,6 +494,9 @@ WEBAUTHN_ONLY_TRUSTED_VENDORS=n
# Otherwise it will work normally.
SPAMHAUS_DQS_KEY=
# Prevent netfilter from setting an iptables/nftables rule to isolate the mailcow docker network - y/n
# CAUTION: Disabling this may expose container ports to other neighbors on the same subnet, even if the ports are bound to localhost
DISABLE_NETFILTER_ISOLATION_RULE=n
EOF
mkdir -p data/assets/ssl

View File

@ -481,6 +481,7 @@ CONFIG_ARRAY=(
"WEBAUTHN_ONLY_TRUSTED_VENDORS"
"SPAMHAUS_DQS_KEY"
"SKIP_UNBOUND_HEALTHCHECK"
"DISABLE_NETFILTER_ISOLATION_RULE"
)
detect_bad_asn
@ -754,6 +755,13 @@ for option in ${CONFIG_ARRAY[@]}; do
echo '# Skip Unbound (DNS Resolver) Healthchecks (NOT Recommended!) - y/n' >> mailcow.conf
echo 'SKIP_UNBOUND_HEALTHCHECK=n' >> mailcow.conf
fi
elif [[ ${option} == "DISABLE_NETFILTER_ISOLATION_RULE" ]]; then
if ! grep -q ${option} mailcow.conf; then
echo "Adding new option \"${option}\" to mailcow.conf"
echo '# Prevent netfilter from setting an iptables/nftables rule to isolate the mailcow docker network - y/n' >> mailcow.conf
echo '# CAUTION: Disabling this may expose container ports to other neighbors on the same subnet, even if the ports are bound to localhost' >> mailcow.conf
echo 'DISABLE_NETFILTER_ISOLATION_RULE=n' >> mailcow.conf
fi
elif ! grep -q ${option} mailcow.conf; then
echo "Adding new option \"${option}\" to mailcow.conf"
echo "${option}=n" >> mailcow.conf