Merge pull request #5696 from mailcow/fix/netfilter

[Netfilter] add mailcow isolation rule to MAILCOW chain

.gitignore

@@ -13,6 +13,7 @@ data/conf/dovecot/acl_anyone
 data/conf/dovecot/dovecot-master.passwd
 data/conf/dovecot/dovecot-master.userdb
 data/conf/dovecot/extra.conf
+data/conf/dovecot/mail_replica.conf
 data/conf/dovecot/global_sieve_*
 data/conf/dovecot/last_login
 data/conf/dovecot/lua

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -335,6 +335,15 @@ sys.exit()
 EOF
 fi
 
+# Set mail_replica for HA setups
+if [[ -n ${MAILCOW_REPLICA_IP} && -n ${DOVEADM_REPLICA_PORT} ]]; then
+  cat <<EOF > /etc/dovecot/mail_replica.conf
+# Autogenerated by mailcow
+mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT}
+EOF
+fi
+
+
 # 401 is user dovecot
 if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
 	openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem

data/Dockerfiles/netfilter/main.py

@@ -21,28 +21,6 @@ from modules.IPTables import IPTables
 from modules.NFTables import NFTables
 
 
-# connect to redis
-while True:
-  try:
-    redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
-    redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
-    if "".__eq__(redis_slaveof_ip):
-      r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
-    else:
-      r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
-    r.ping()
-  except Exception as ex:
-    print('%s - trying again in 3 seconds'  % (ex))
-    time.sleep(3)
-  else:
-    break
-pubsub = r.pubsub()
-
-# rename fail2ban to netfilter
-if r.exists('F2B_LOG'):
-  r.rename('F2B_LOG', 'NETFILTER_LOG')
-
-
 # globals
 WHITELIST = []
 BLACKLIST= []
@@ -50,18 +28,10 @@ bans = {}
 quit_now = False
 exit_code = 0
 lock = Lock()
-
+chain_name = "MAILCOW"
-
+r = None
-# init Logger
+pubsub = None
-logger = Logger(r)
+clear_before_quit = False
-# init backend
-backend = sys.argv[1]
-if backend == "nftables":
-  logger.logInfo('Using NFTables backend')
-  tables = NFTables("MAILCOW", logger)
-else:
-  logger.logInfo('Using IPTables backend')
-  tables = IPTables("MAILCOW", logger)
 
 
 def refreshF2boptions():
@@ -250,17 +220,21 @@ def clear():
   with lock:
     tables.clearIPv4Table()
     tables.clearIPv6Table()
-    r.delete('F2B_ACTIVE_BANS')
+    try:
-    r.delete('F2B_PERM_BANS')
+      if r is not None:
-    pubsub.unsubscribe()
+        r.delete('F2B_ACTIVE_BANS')
+        r.delete('F2B_PERM_BANS')
+    except Exception as ex:
+      logger.logWarn('Error clearing redis keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
 
 def watch():
-  logger.logInfo('Watching Redis channel F2B_CHANNEL')
+  global pubsub
-  pubsub.subscribe('F2B_CHANNEL')
-
   global quit_now
   global exit_code
 
+  logger.logInfo('Watching Redis channel F2B_CHANNEL')
+  pubsub.subscribe('F2B_CHANNEL')
+
   while not quit_now:
     try:
       for item in pubsub.listen():
@@ -280,6 +254,7 @@ def watch():
               ban(addr)
     except Exception as ex:
       logger.logWarn('Error reading log line from pubsub: %s' % ex)
+      pubsub = None
       quit_now = True
       exit_code = 2
 
@@ -403,21 +378,76 @@ def blacklistUpdate():
           permBan(net=net, unban=True)
     time.sleep(60.0 - ((time.time() - start_time) % 60.0))
 
-def quit(signum, frame):
+def sigterm_quit(signum, frame):
-  global quit_now
+  global clear_before_quit
-  quit_now = True
+  clear_before_quit = True
+  sys.exit(exit_code)
+
+def berfore_quit():
+  if clear_before_quit:
+    clear()
+  if pubsub is not None:
+    pubsub.unsubscribe()
 
 
 if __name__ == '__main__':
-  refreshF2boptions()
+  atexit.register(berfore_quit)
+  signal.signal(signal.SIGTERM, sigterm_quit)
+
+  # init Logger
+  logger = Logger(None)
+
+  # init backend
+  backend = sys.argv[1]
+  if backend == "nftables":
+    logger.logInfo('Using NFTables backend')
+    tables = NFTables(chain_name, logger)
+  else:
+    logger.logInfo('Using IPTables backend')
+    tables = IPTables(chain_name, logger)
+
   # In case a previous session was killed without cleanup
   clear()
+
   # Reinit MAILCOW chain
   # Is called before threads start, no locking
   logger.logInfo("Initializing mailcow netfilter chain")
   tables.initChainIPv4()
   tables.initChainIPv6()
 
+  if os.getenv("DISABLE_NETFILTER_ISOLATION_RULE").lower() in ("y", "yes"):
+    logger.logInfo(f"Skipping {chain_name} isolation")
+  else:
+    logger.logInfo(f"Setting {chain_name} isolation")
+    tables.create_mailcow_isolation_rule("br-mailcow", [3306, 6379, 8983, 12345], os.getenv("MAILCOW_REPLICA_IP"))
+
+  # connect to redis
+  while True:
+    try:
+      redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
+      redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
+      if "".__eq__(redis_slaveof_ip):
+        r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
+      else:
+        r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
+      r.ping()
+      pubsub = r.pubsub()
+    except Exception as ex:
+      print('%s - trying again in 3 seconds'  % (ex))
+      time.sleep(3)
+    else:
+      break
+  Logger.r = r
+
+  # rename fail2ban to netfilter
+  if r.exists('F2B_LOG'):
+    r.rename('F2B_LOG', 'NETFILTER_LOG')
+  # clear bans in redis
+  r.delete('F2B_ACTIVE_BANS')
+  r.delete('F2B_PERM_BANS')
+  
+  refreshF2boptions()
+
   watch_thread = Thread(target=watch)
   watch_thread.daemon = True
   watch_thread.start()
@@ -460,9 +490,6 @@ if __name__ == '__main__':
   whitelistupdate_thread.daemon = True
   whitelistupdate_thread.start()
 
-  signal.signal(signal.SIGTERM, quit)
-  atexit.register(clear)
-
   while not quit_now:
     time.sleep(0.5)
 

data/Dockerfiles/netfilter/modules/IPTables.py

@@ -1,5 +1,6 @@
 import iptc
 import time
+import os
 
 class IPTables:
   def __init__(self, chain_name, logger):
@@ -211,3 +212,41 @@ class IPTables:
     target = rule.create_target("SNAT")
     target.to_source = snat_target
     return rule
+
+  def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
+    try:
+      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+
+      # insert mailcow isolation rule
+      rule = iptc.Rule()
+      rule.in_interface = f'! {_interface}'
+      rule.out_interface = _interface
+      rule.protocol = 'tcp'
+      rule.create_target("DROP")
+      match = rule.create_match("multiport")
+      match.dports = ','.join(map(str, _dports))
+
+      if rule in chain.rules:
+        chain.delete_rule(rule)
+      chain.insert_rule(rule, position=0)
+
+      # insert mailcow isolation exception rule
+      if _allow != "":
+        rule = iptc.Rule()
+        rule.src = _allow
+        rule.in_interface = f'! {_interface}'
+        rule.out_interface = _interface
+        rule.protocol = 'tcp'
+        rule.create_target("ACCEPT")
+        match = rule.create_match("multiport")
+        match.dports = ','.join(map(str, _dports))
+
+        if rule in chain.rules:
+          chain.delete_rule(rule)
+        chain.insert_rule(rule, position=0)
+
+
+      return True
+    except Exception as e:
+      self.logger.logCrit(f"Error adding {self.chain_name} isolation: {e}")
+      return False

data/Dockerfiles/netfilter/modules/Logger.py

@@ -10,7 +10,8 @@ class Logger:
     tolog['time'] = int(round(time.time()))
     tolog['priority'] = priority
     tolog['message'] = message
-    self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
+    if self.r:
+      self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
     print(message)
 
   def logWarn(self, message):

data/Dockerfiles/netfilter/modules/NFTables.py

@@ -1,5 +1,6 @@
 import nftables
 import ipaddress
+import os
 
 class NFTables:
   def __init__(self, chain_name, logger):
@@ -266,6 +267,17 @@ class NFTables:
 
     return self.nft_exec_dict(delete_command)
 
+  def delete_filter_rule(self, _family:str, _chain: str, _handle:str):
+    delete_command = self.get_base_dict()
+    _rule_opts = {'family': _family,
+                  'table': 'filter',
+                  'chain': _chain,
+                  'handle': _handle  }
+    _delete = {'delete': {'rule': _rule_opts} }
+    delete_command["nftables"].append(_delete)
+
+    return self.nft_exec_dict(delete_command)
+
   def snat_rule(self, _family: str, snat_target: str, source_address: str):
     chain_name = self.nft_chain_names[_family]['nat']['postrouting']
 
@@ -381,7 +393,7 @@ class NFTables:
           break
     return chain_handle
 
-  def get_rules_handle(self, _family: str, _table: str, chain_name: str):
+  def get_rules_handle(self, _family: str, _table: str, chain_name: str, _comment_filter = "mailcow"):
     rule_handle = []
     # Command: 'nft list chain {family} {table} {chain_name}'
     _chain_opts = {'family': _family, 'table': _table, 'name': chain_name}
@@ -397,7 +409,7 @@ class NFTables:
 
         rule = _object["rule"]
         if rule["family"] == _family and rule["table"] == _table and rule["chain"] == chain_name:
-          if rule.get("comment") and rule["comment"] == "mailcow":
+          if rule.get("comment") and rule["comment"] == _comment_filter:
             rule_handle.append(rule["handle"])
     return rule_handle
 
@@ -493,3 +505,152 @@ class NFTables:
         position+=1
 
     return position if rule_found else False
+
+  def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
+    family = "ip"
+    table = "filter"
+    comment_filter_drop = "mailcow isolation"
+    comment_filter_allow = "mailcow isolation allow"
+    json_command = self.get_base_dict()
+
+    # Delete old mailcow isolation rules
+    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_drop)
+    for handle in handles:
+      self.delete_filter_rule(family, self.chain_name, handle)
+    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_allow)
+    for handle in handles:
+      self.delete_filter_rule(family, self.chain_name, handle)
+
+    # insert mailcow isolation rule
+    _match_dict_drop = [
+      {
+        "match": {
+          "op": "!=",
+          "left": {
+            "meta": {
+              "key": "iifname"
+            }
+          },
+          "right": _interface
+        }
+      },
+      {
+        "match": {
+          "op": "==",
+          "left": {
+            "meta": {
+              "key": "oifname"
+            }
+          },
+          "right": _interface
+        }
+      },
+      {
+        "match": {
+          "op": "==",
+          "left": {
+            "payload": {
+              "protocol": "tcp",
+              "field": "dport"
+            }
+          },
+          "right": {
+            "set": _dports
+          }
+        }
+      },
+      {
+        "counter": {
+          "packets": 0,
+          "bytes": 0
+        }
+      },
+      {
+        "drop": None
+      }
+    ]
+    rule_drop = { "insert": { "rule": {
+      "family": family,
+      "table": table,
+      "chain": self.chain_name,
+      "comment": comment_filter_drop,
+      "expr": _match_dict_drop
+    }}}
+    json_command["nftables"].append(rule_drop)
+
+    # insert mailcow isolation allow rule
+    if _allow != "":
+      _match_dict_allow = [
+        {
+          "match": {
+            "op": "==",
+            "left": {
+              "payload": {
+                "protocol": "ip",
+                "field": "saddr"
+              }
+            },
+            "right": _allow
+          }
+        },
+        {
+          "match": {
+            "op": "!=",
+            "left": {
+              "meta": {
+                "key": "iifname"
+              }
+            },
+            "right": _interface
+          }
+        },
+        {
+          "match": {
+            "op": "==",
+            "left": {
+              "meta": {
+                "key": "oifname"
+              }
+            },
+            "right": _interface
+          }
+        },
+        {
+          "match": {
+            "op": "==",
+            "left": {
+              "payload": {
+                "protocol": "tcp",
+                "field": "dport"
+              }
+            },
+            "right": {
+              "set": _dports
+            }
+          }
+        },
+        {
+          "counter": {
+            "packets": 0,
+            "bytes": 0
+          }
+        },
+        {
+          "accept": None
+        }
+      ]
+      rule_allow = { "insert": { "rule": {
+        "family": family,
+        "table": table,
+        "chain": self.chain_name,
+        "comment": comment_filter_allow,
+        "expr": _match_dict_allow
+      }}}
+      json_command["nftables"].append(rule_allow)
+
+    success = self.nft_exec_dict(json_command)
+    if success == False:
+      self.logger.logCrit(f"Error adding {self.chain_name} isolation")
+      return False
+
+    return True

data/conf/dovecot/dovecot.conf

@@ -247,6 +247,9 @@ plugin {
   mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
   mail_log_fields = uid box msgid size
   mail_log_cached_only = yes
+
+  # Try set mail_replica
+  !include_try /etc/dovecot/mail_replica.conf
 }
 service quota-warning {
   executable = script /usr/local/bin/quota_notify.py

docker-compose.yml

@@ -21,6 +21,7 @@ services:
       image: mariadb:10.5
       depends_on:
         - unbound-mailcow
+        - netfilter-mailcow
       stop_grace_period: 45s
       volumes:
         - mysql-vol-1:/var/lib/mysql/
@@ -46,6 +47,8 @@ services:
       volumes:
         - redis-vol-1:/data/
       restart: always
+      depends_on:
+        - netfilter-mailcow
       ports:
         - "${REDIS_PORT:-127.0.0.1:7654}:6379"
       environment:
@@ -219,9 +222,10 @@ services:
             - sogo
 
     dovecot-mailcow:
-      image: mailcow/dovecot:1.27
+      image: mailcow/dovecot:1.28
       depends_on:
         - mysql-mailcow
+        - netfilter-mailcow
       dns:
         - ${IPV4_NETWORK:-172.22.1}.254
       cap_add:
@@ -242,6 +246,8 @@ services:
       environment:
         - DOVECOT_MASTER_USER=${DOVECOT_MASTER_USER:-}
         - DOVECOT_MASTER_PASS=${DOVECOT_MASTER_PASS:-}
+        - MAILCOW_REPLICA_IP=${MAILCOW_REPLICA_IP:-}
+        - DOVEADM_REPLICA_PORT=${DOVEADM_REPLICA_PORT:-}
         - LOG_LINES=${LOG_LINES:-9999}
         - DBNAME=${DBNAME}
         - DBUSER=${DBUSER}
@@ -435,14 +441,8 @@ services:
             - acme
 
     netfilter-mailcow:
-      image: mailcow/netfilter:1.55
+      image: mailcow/netfilter:1.56
       stop_grace_period: 30s
-      depends_on:
-        - dovecot-mailcow
-        - postfix-mailcow
-        - sogo-mailcow
-        - php-fpm-mailcow
-        - redis-mailcow
       restart: always
       privileged: true
       environment:
@@ -453,6 +453,8 @@ services:
         - SNAT6_TO_SOURCE=${SNAT6_TO_SOURCE:-n}
         - REDIS_SLAVEOF_IP=${REDIS_SLAVEOF_IP:-}
         - REDIS_SLAVEOF_PORT=${REDIS_SLAVEOF_PORT:-}
+        - MAILCOW_REPLICA_IP=${MAILCOW_REPLICA_IP:-}
+        - DISABLE_NETFILTER_ISOLATION_RULE=${DISABLE_NETFILTER_ISOLATION_RULE:-n}
       network_mode: "host"
       volumes:
         - /lib/modules:/lib/modules:ro
@@ -553,6 +555,8 @@ services:
     solr-mailcow:
       image: mailcow/solr:1.8.2
       restart: always
+      depends_on:
+        - netfilter-mailcow
       volumes:
         - solr-vol-1:/opt/solr/server/solr/dovecot-fts/data
       ports:

generate_config.sh

@@ -494,6 +494,9 @@ WEBAUTHN_ONLY_TRUSTED_VENDORS=n
 # Otherwise it will work normally.
 SPAMHAUS_DQS_KEY=
 
+# Prevent netfilter from setting an iptables/nftables rule to isolate the mailcow docker network - y/n
+# CAUTION: Disabling this may expose container ports to other neighbors on the same subnet, even if the ports are bound to localhost
+DISABLE_NETFILTER_ISOLATION_RULE=n
 EOF
 
 mkdir -p data/assets/ssl

update.sh

@@ -481,6 +481,7 @@ CONFIG_ARRAY=(
   "WEBAUTHN_ONLY_TRUSTED_VENDORS"
   "SPAMHAUS_DQS_KEY"
   "SKIP_UNBOUND_HEALTHCHECK"
+  "DISABLE_NETFILTER_ISOLATION_RULE"
 )
 
 detect_bad_asn
@@ -754,6 +755,13 @@ for option in ${CONFIG_ARRAY[@]}; do
       echo '# Skip Unbound (DNS Resolver) Healthchecks (NOT Recommended!) - y/n' >> mailcow.conf
       echo 'SKIP_UNBOUND_HEALTHCHECK=n' >> mailcow.conf
     fi
+  elif [[ ${option} == "DISABLE_NETFILTER_ISOLATION_RULE" ]]; then
+    if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
+      echo '# Prevent netfilter from setting an iptables/nftables rule to isolate the mailcow docker network - y/n' >> mailcow.conf
+      echo '# CAUTION: Disabling this may expose container ports to other neighbors on the same subnet, even if the ports are bound to localhost' >> mailcow.conf
+      echo 'DISABLE_NETFILTER_ISOLATION_RULE=n' >> mailcow.conf
+    fi 
   elif ! grep -q ${option} mailcow.conf; then
     echo "Adding new option \"${option}\" to mailcow.conf"
     echo "${option}=n" >> mailcow.conf