Merge pull request #5313 from mailcow/feat/f2b-banlist

[Web] add f2b_banlist endpoint
This commit is contained in:
Patrick Schult 2023-12-11 12:36:06 +01:00 committed by GitHub
commit c2e5dfd933
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 617 additions and 466 deletions

View File

@ -1,463 +1,469 @@
#!/usr/bin/env python3
import re
import os
import sys
import time
import atexit
import signal
import ipaddress
from collections import Counter
from random import randint
from threading import Thread
from threading import Lock
import redis
import json
import dns.resolver
import dns.exception
from modules.Logger import Logger
from modules.IPTables import IPTables
from modules.NFTables import NFTables
# connect to redis
while True:
try:
redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
if "".__eq__(redis_slaveof_ip):
r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
else:
r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
r.ping()
except Exception as ex:
print('%s - trying again in 3 seconds' % (ex))
time.sleep(3)
else:
break
pubsub = r.pubsub()
# rename fail2ban to netfilter
if r.exists('F2B_LOG'):
r.rename('F2B_LOG', 'NETFILTER_LOG')
# globals
WHITELIST = []
BLACKLIST= []
bans = {}
quit_now = False
exit_code = 0
lock = Lock()
# init Logger
logger = Logger(r)
# init backend
backend = sys.argv[1]
if backend == "nftables":
logger.logInfo('Using NFTables backend')
tables = NFTables("MAILCOW", logger)
else:
logger.logInfo('Using IPTables backend')
tables = IPTables("MAILCOW", logger)
def refreshF2boptions():
global f2boptions
global quit_now
global exit_code
f2boptions = {}
if not r.get('F2B_OPTIONS'):
f2boptions['ban_time'] = r.get('F2B_BAN_TIME')
f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME')
f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT')
f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS')
f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW')
f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4')
f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6')
else:
try:
f2boptions = json.loads(r.get('F2B_OPTIONS'))
except ValueError:
logger.logCrit('Error loading F2B options: F2B_OPTIONS is not json')
quit_now = True
exit_code = 2
verifyF2boptions(f2boptions)
r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
def verifyF2boptions(f2boptions):
verifyF2boption(f2boptions,'ban_time', 1800)
verifyF2boption(f2boptions,'max_ban_time', 10000)
verifyF2boption(f2boptions,'ban_time_increment', True)
verifyF2boption(f2boptions,'max_attempts', 10)
verifyF2boption(f2boptions,'retry_window', 600)
verifyF2boption(f2boptions,'netban_ipv4', 32)
verifyF2boption(f2boptions,'netban_ipv6', 128)
def verifyF2boption(f2boptions, f2boption, f2bdefault):
f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault
def refreshF2bregex():
global f2bregex
global quit_now
global exit_code
if not r.get('F2B_REGEX'):
f2bregex = {}
f2bregex[1] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
f2bregex[2] = 'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
f2bregex[3] = 'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
f2bregex[4] = 'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
f2bregex[5] = 'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
f2bregex[6] = '-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),'
f2bregex[7] = '-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
f2bregex[8] = '-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
f2bregex[9] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
f2bregex[10] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
else:
try:
f2bregex = {}
f2bregex = json.loads(r.get('F2B_REGEX'))
except ValueError:
logger.logCrit('Error loading F2B options: F2B_REGEX is not json')
quit_now = True
exit_code = 2
def get_ip(address):
ip = ipaddress.ip_address(address)
if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped:
ip = ip.ipv4_mapped
if ip.is_private or ip.is_loopback:
return False
return ip
def ban(address):
global lock
refreshF2boptions()
BAN_TIME = int(f2boptions['ban_time'])
BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
MAX_ATTEMPTS = int(f2boptions['max_attempts'])
RETRY_WINDOW = int(f2boptions['retry_window'])
NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6'])
ip = get_ip(address)
if not ip: return
address = str(ip)
self_network = ipaddress.ip_network(address)
with lock:
temp_whitelist = set(WHITELIST)
if temp_whitelist:
for wl_key in temp_whitelist:
wl_net = ipaddress.ip_network(wl_key, False)
if wl_net.overlaps(self_network):
logger.logInfo('Address %s is whitelisted by rule %s' % (self_network, wl_net))
return
net = ipaddress.ip_network((address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False)
net = str(net)
if not net in bans:
bans[net] = {'attempts': 0, 'last_attempt': 0, 'ban_counter': 0}
current_attempt = time.time()
if current_attempt - bans[net]['last_attempt'] > RETRY_WINDOW:
bans[net]['attempts'] = 0
bans[net]['attempts'] += 1
bans[net]['last_attempt'] = current_attempt
if bans[net]['attempts'] >= MAX_ATTEMPTS:
cur_time = int(round(time.time()))
NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
if type(ip) is ipaddress.IPv4Address:
with lock:
tables.banIPv4(net)
else:
with lock:
tables.banIPv6(net)
r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME)
else:
logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net))
def unban(net):
global lock
if not net in bans:
logger.logInfo('%s is not banned, skipping unban and deleting from queue (if any)' % net)
r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
return
logger.logInfo('Unbanning %s' % net)
if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network:
with lock:
tables.unbanIPv4(net)
else:
with lock:
tables.unbanIPv6(net)
r.hdel('F2B_ACTIVE_BANS', '%s' % net)
r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
if net in bans:
bans[net]['attempts'] = 0
bans[net]['ban_counter'] += 1
def permBan(net, unban=False):
global lock
is_unbanned = False
is_banned = False
if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network:
with lock:
if unban:
is_unbanned = tables.unbanIPv4(net)
else:
is_banned = tables.banIPv4(net)
else:
with lock:
if unban:
is_unbanned = tables.unbanIPv6(net)
else:
is_banned = tables.banIPv6(net)
if is_unbanned:
r.hdel('F2B_PERM_BANS', '%s' % net)
logger.logCrit('Removed host/network %s from blacklist' % net)
elif is_banned:
r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time())))
logger.logCrit('Added host/network %s to blacklist' % net)
def clear():
global lock
logger.logInfo('Clearing all bans')
for net in bans.copy():
unban(net)
with lock:
tables.clearIPv4Table()
tables.clearIPv6Table()
r.delete('F2B_ACTIVE_BANS')
r.delete('F2B_PERM_BANS')
pubsub.unsubscribe()
def watch():
logger.logInfo('Watching Redis channel F2B_CHANNEL')
pubsub.subscribe('F2B_CHANNEL')
global quit_now
global exit_code
while not quit_now:
try:
for item in pubsub.listen():
refreshF2bregex()
for rule_id, rule_regex in f2bregex.items():
if item['data'] and item['type'] == 'message':
try:
result = re.search(rule_regex, item['data'])
except re.error:
result = False
if result:
addr = result.group(1)
ip = ipaddress.ip_address(addr)
if ip.is_private or ip.is_loopback:
continue
logger.logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data']))
ban(addr)
except Exception as ex:
logger.logWarn('Error reading log line from pubsub: %s' % ex)
quit_now = True
exit_code = 2
def snat4(snat_target):
global lock
global quit_now
while not quit_now:
time.sleep(10)
with lock:
tables.snat4(snat_target, os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24')
def snat6(snat_target):
global lock
global quit_now
while not quit_now:
time.sleep(10)
with lock:
tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
def autopurge():
while not quit_now:
time.sleep(10)
refreshF2boptions()
BAN_TIME = int(f2boptions['ban_time'])
MAX_BAN_TIME = int(f2boptions['max_ban_time'])
BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
MAX_ATTEMPTS = int(f2boptions['max_attempts'])
QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
if QUEUE_UNBAN:
for net in QUEUE_UNBAN:
unban(str(net))
for net in bans.copy():
if bans[net]['attempts'] >= MAX_ATTEMPTS:
NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
TIME_SINCE_LAST_ATTEMPT = time.time() - bans[net]['last_attempt']
if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME or TIME_SINCE_LAST_ATTEMPT > MAX_BAN_TIME:
unban(net)
def mailcowChainOrder():
global lock
global quit_now
global exit_code
while not quit_now:
time.sleep(10)
with lock:
quit_now, exit_code = tables.checkIPv4ChainOrder()
if quit_now: return
quit_now, exit_code = tables.checkIPv6ChainOrder()
def isIpNetwork(address):
try:
ipaddress.ip_network(address, False)
except ValueError:
return False
return True
def genNetworkList(list):
resolver = dns.resolver.Resolver()
hostnames = []
networks = []
for key in list:
if isIpNetwork(key):
networks.append(key)
else:
hostnames.append(key)
for hostname in hostnames:
hostname_ips = []
for rdtype in ['A', 'AAAA']:
try:
answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3)
except dns.exception.Timeout:
logger.logInfo('Hostname %s timedout on resolve' % hostname)
break
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
continue
except dns.exception.DNSException as dnsexception:
logger.logInfo('%s' % dnsexception)
continue
for rdata in answer:
hostname_ips.append(rdata.to_text())
networks.extend(hostname_ips)
return set(networks)
def whitelistUpdate():
global lock
global quit_now
global WHITELIST
while not quit_now:
start_time = time.time()
list = r.hgetall('F2B_WHITELIST')
new_whitelist = []
if list:
new_whitelist = genNetworkList(list)
with lock:
if Counter(new_whitelist) != Counter(WHITELIST):
WHITELIST = new_whitelist
logger.logInfo('Whitelist was changed, it has %s entries' % len(WHITELIST))
time.sleep(60.0 - ((time.time() - start_time) % 60.0))
def blacklistUpdate():
global quit_now
global BLACKLIST
while not quit_now:
start_time = time.time()
list = r.hgetall('F2B_BLACKLIST')
new_blacklist = []
if list:
new_blacklist = genNetworkList(list)
if Counter(new_blacklist) != Counter(BLACKLIST):
addban = set(new_blacklist).difference(BLACKLIST)
delban = set(BLACKLIST).difference(new_blacklist)
BLACKLIST = new_blacklist
logger.logInfo('Blacklist was changed, it has %s entries' % len(BLACKLIST))
if addban:
for net in addban:
permBan(net=net)
if delban:
for net in delban:
permBan(net=net, unban=True)
time.sleep(60.0 - ((time.time() - start_time) % 60.0))
def quit(signum, frame):
global quit_now
quit_now = True
if __name__ == '__main__':
# In case a previous session was killed without cleanup
clear()
# Reinit MAILCOW chain
# Is called before threads start, no locking
logger.logInfo("Initializing mailcow netfilter chain")
tables.initChainIPv4()
tables.initChainIPv6()
watch_thread = Thread(target=watch)
watch_thread.daemon = True
watch_thread.start()
if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n':
try:
snat_ip = os.getenv('SNAT_TO_SOURCE')
snat_ipo = ipaddress.ip_address(snat_ip)
if type(snat_ipo) is ipaddress.IPv4Address:
snat4_thread = Thread(target=snat4,args=(snat_ip,))
snat4_thread.daemon = True
snat4_thread.start()
except ValueError:
print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address')
if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n':
try:
snat_ip = os.getenv('SNAT6_TO_SOURCE')
snat_ipo = ipaddress.ip_address(snat_ip)
if type(snat_ipo) is ipaddress.IPv6Address:
snat6_thread = Thread(target=snat6,args=(snat_ip,))
snat6_thread.daemon = True
snat6_thread.start()
except ValueError:
print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address')
autopurge_thread = Thread(target=autopurge)
autopurge_thread.daemon = True
autopurge_thread.start()
mailcowchainwatch_thread = Thread(target=mailcowChainOrder)
mailcowchainwatch_thread.daemon = True
mailcowchainwatch_thread.start()
blacklistupdate_thread = Thread(target=blacklistUpdate)
blacklistupdate_thread.daemon = True
blacklistupdate_thread.start()
whitelistupdate_thread = Thread(target=whitelistUpdate)
whitelistupdate_thread.daemon = True
whitelistupdate_thread.start()
signal.signal(signal.SIGTERM, quit)
atexit.register(clear)
while not quit_now:
time.sleep(0.5)
sys.exit(exit_code)
#!/usr/bin/env python3
import re
import os
import sys
import time
import atexit
import signal
import ipaddress
from collections import Counter
from random import randint
from threading import Thread
from threading import Lock
import redis
import json
import dns.resolver
import dns.exception
import uuid
from modules.Logger import Logger
from modules.IPTables import IPTables
from modules.NFTables import NFTables
# connect to redis
while True:
try:
redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
if "".__eq__(redis_slaveof_ip):
r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
else:
r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
r.ping()
except Exception as ex:
print('%s - trying again in 3 seconds' % (ex))
time.sleep(3)
else:
break
pubsub = r.pubsub()
# rename fail2ban to netfilter
if r.exists('F2B_LOG'):
r.rename('F2B_LOG', 'NETFILTER_LOG')
# globals
WHITELIST = []
BLACKLIST= []
bans = {}
quit_now = False
exit_code = 0
lock = Lock()
# init Logger
logger = Logger(r)
# init backend
backend = sys.argv[1]
if backend == "nftables":
logger.logInfo('Using NFTables backend')
tables = NFTables("MAILCOW", logger)
else:
logger.logInfo('Using IPTables backend')
tables = IPTables("MAILCOW", logger)
def refreshF2boptions():
global f2boptions
global quit_now
global exit_code
f2boptions = {}
if not r.get('F2B_OPTIONS'):
f2boptions['ban_time'] = r.get('F2B_BAN_TIME')
f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME')
f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT')
f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS')
f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW')
f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4')
f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6')
else:
try:
f2boptions = json.loads(r.get('F2B_OPTIONS'))
except ValueError:
logger.logCrit('Error loading F2B options: F2B_OPTIONS is not json')
quit_now = True
exit_code = 2
verifyF2boptions(f2boptions)
r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
def verifyF2boptions(f2boptions):
verifyF2boption(f2boptions,'ban_time', 1800)
verifyF2boption(f2boptions,'max_ban_time', 10000)
verifyF2boption(f2boptions,'ban_time_increment', True)
verifyF2boption(f2boptions,'max_attempts', 10)
verifyF2boption(f2boptions,'retry_window', 600)
verifyF2boption(f2boptions,'netban_ipv4', 32)
verifyF2boption(f2boptions,'netban_ipv6', 128)
verifyF2boption(f2boptions,'banlist_id', str(uuid.uuid4()))
verifyF2boption(f2boptions,'manage_external', 0)
def verifyF2boption(f2boptions, f2boption, f2bdefault):
f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault
def refreshF2bregex():
global f2bregex
global quit_now
global exit_code
if not r.get('F2B_REGEX'):
f2bregex = {}
f2bregex[1] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
f2bregex[2] = 'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
f2bregex[3] = 'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
f2bregex[4] = 'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
f2bregex[5] = 'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
f2bregex[6] = '-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),'
f2bregex[7] = '-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
f2bregex[8] = '-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
f2bregex[9] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
f2bregex[10] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
else:
try:
f2bregex = {}
f2bregex = json.loads(r.get('F2B_REGEX'))
except ValueError:
logger.logCrit('Error loading F2B options: F2B_REGEX is not json')
quit_now = True
exit_code = 2
def get_ip(address):
ip = ipaddress.ip_address(address)
if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped:
ip = ip.ipv4_mapped
if ip.is_private or ip.is_loopback:
return False
return ip
def ban(address):
global f2boptions
global lock
refreshF2boptions()
BAN_TIME = int(f2boptions['ban_time'])
BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
MAX_ATTEMPTS = int(f2boptions['max_attempts'])
RETRY_WINDOW = int(f2boptions['retry_window'])
NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6'])
ip = get_ip(address)
if not ip: return
address = str(ip)
self_network = ipaddress.ip_network(address)
with lock:
temp_whitelist = set(WHITELIST)
if temp_whitelist:
for wl_key in temp_whitelist:
wl_net = ipaddress.ip_network(wl_key, False)
if wl_net.overlaps(self_network):
logger.logInfo('Address %s is whitelisted by rule %s' % (self_network, wl_net))
return
net = ipaddress.ip_network((address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False)
net = str(net)
if not net in bans:
bans[net] = {'attempts': 0, 'last_attempt': 0, 'ban_counter': 0}
current_attempt = time.time()
if current_attempt - bans[net]['last_attempt'] > RETRY_WINDOW:
bans[net]['attempts'] = 0
bans[net]['attempts'] += 1
bans[net]['last_attempt'] = current_attempt
if bans[net]['attempts'] >= MAX_ATTEMPTS:
cur_time = int(round(time.time()))
NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1:
with lock:
tables.banIPv4(net)
elif int(f2boptions['manage_external']) != 1:
with lock:
tables.banIPv6(net)
r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME)
else:
logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net))
def unban(net):
global lock
if not net in bans:
logger.logInfo('%s is not banned, skipping unban and deleting from queue (if any)' % net)
r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
return
logger.logInfo('Unbanning %s' % net)
if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network:
with lock:
tables.unbanIPv4(net)
else:
with lock:
tables.unbanIPv6(net)
r.hdel('F2B_ACTIVE_BANS', '%s' % net)
r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
if net in bans:
bans[net]['attempts'] = 0
bans[net]['ban_counter'] += 1
def permBan(net, unban=False):
global f2boptions
global lock
is_unbanned = False
is_banned = False
if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network:
with lock:
if unban:
is_unbanned = tables.unbanIPv4(net)
elif int(f2boptions['manage_external']) != 1:
is_banned = tables.banIPv4(net)
else:
with lock:
if unban:
is_unbanned = tables.unbanIPv6(net)
elif int(f2boptions['manage_external']) != 1:
is_banned = tables.banIPv6(net)
if is_unbanned:
r.hdel('F2B_PERM_BANS', '%s' % net)
logger.logCrit('Removed host/network %s from blacklist' % net)
elif is_banned:
r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time())))
logger.logCrit('Added host/network %s to blacklist' % net)
def clear():
global lock
logger.logInfo('Clearing all bans')
for net in bans.copy():
unban(net)
with lock:
tables.clearIPv4Table()
tables.clearIPv6Table()
r.delete('F2B_ACTIVE_BANS')
r.delete('F2B_PERM_BANS')
pubsub.unsubscribe()
def watch():
logger.logInfo('Watching Redis channel F2B_CHANNEL')
pubsub.subscribe('F2B_CHANNEL')
global quit_now
global exit_code
while not quit_now:
try:
for item in pubsub.listen():
refreshF2bregex()
for rule_id, rule_regex in f2bregex.items():
if item['data'] and item['type'] == 'message':
try:
result = re.search(rule_regex, item['data'])
except re.error:
result = False
if result:
addr = result.group(1)
ip = ipaddress.ip_address(addr)
if ip.is_private or ip.is_loopback:
continue
logger.logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data']))
ban(addr)
except Exception as ex:
logger.logWarn('Error reading log line from pubsub: %s' % ex)
quit_now = True
exit_code = 2
def snat4(snat_target):
global lock
global quit_now
while not quit_now:
time.sleep(10)
with lock:
tables.snat4(snat_target, os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24')
def snat6(snat_target):
global lock
global quit_now
while not quit_now:
time.sleep(10)
with lock:
tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
def autopurge():
while not quit_now:
time.sleep(10)
refreshF2boptions()
BAN_TIME = int(f2boptions['ban_time'])
MAX_BAN_TIME = int(f2boptions['max_ban_time'])
BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
MAX_ATTEMPTS = int(f2boptions['max_attempts'])
QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
if QUEUE_UNBAN:
for net in QUEUE_UNBAN:
unban(str(net))
for net in bans.copy():
if bans[net]['attempts'] >= MAX_ATTEMPTS:
NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
TIME_SINCE_LAST_ATTEMPT = time.time() - bans[net]['last_attempt']
if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME or TIME_SINCE_LAST_ATTEMPT > MAX_BAN_TIME:
unban(net)
def mailcowChainOrder():
global lock
global quit_now
global exit_code
while not quit_now:
time.sleep(10)
with lock:
quit_now, exit_code = tables.checkIPv4ChainOrder()
if quit_now: return
quit_now, exit_code = tables.checkIPv6ChainOrder()
def isIpNetwork(address):
try:
ipaddress.ip_network(address, False)
except ValueError:
return False
return True
def genNetworkList(list):
resolver = dns.resolver.Resolver()
hostnames = []
networks = []
for key in list:
if isIpNetwork(key):
networks.append(key)
else:
hostnames.append(key)
for hostname in hostnames:
hostname_ips = []
for rdtype in ['A', 'AAAA']:
try:
answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3)
except dns.exception.Timeout:
logger.logInfo('Hostname %s timedout on resolve' % hostname)
break
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
continue
except dns.exception.DNSException as dnsexception:
logger.logInfo('%s' % dnsexception)
continue
for rdata in answer:
hostname_ips.append(rdata.to_text())
networks.extend(hostname_ips)
return set(networks)
def whitelistUpdate():
global lock
global quit_now
global WHITELIST
while not quit_now:
start_time = time.time()
list = r.hgetall('F2B_WHITELIST')
new_whitelist = []
if list:
new_whitelist = genNetworkList(list)
with lock:
if Counter(new_whitelist) != Counter(WHITELIST):
WHITELIST = new_whitelist
logger.logInfo('Whitelist was changed, it has %s entries' % len(WHITELIST))
time.sleep(60.0 - ((time.time() - start_time) % 60.0))
def blacklistUpdate():
global quit_now
global BLACKLIST
while not quit_now:
start_time = time.time()
list = r.hgetall('F2B_BLACKLIST')
new_blacklist = []
if list:
new_blacklist = genNetworkList(list)
if Counter(new_blacklist) != Counter(BLACKLIST):
addban = set(new_blacklist).difference(BLACKLIST)
delban = set(BLACKLIST).difference(new_blacklist)
BLACKLIST = new_blacklist
logger.logInfo('Blacklist was changed, it has %s entries' % len(BLACKLIST))
if addban:
for net in addban:
permBan(net=net)
if delban:
for net in delban:
permBan(net=net, unban=True)
time.sleep(60.0 - ((time.time() - start_time) % 60.0))
def quit(signum, frame):
global quit_now
quit_now = True
if __name__ == '__main__':
refreshF2boptions()
# In case a previous session was killed without cleanup
clear()
# Reinit MAILCOW chain
# Is called before threads start, no locking
logger.logInfo("Initializing mailcow netfilter chain")
tables.initChainIPv4()
tables.initChainIPv6()
watch_thread = Thread(target=watch)
watch_thread.daemon = True
watch_thread.start()
if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n':
try:
snat_ip = os.getenv('SNAT_TO_SOURCE')
snat_ipo = ipaddress.ip_address(snat_ip)
if type(snat_ipo) is ipaddress.IPv4Address:
snat4_thread = Thread(target=snat4,args=(snat_ip,))
snat4_thread.daemon = True
snat4_thread.start()
except ValueError:
print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address')
if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n':
try:
snat_ip = os.getenv('SNAT6_TO_SOURCE')
snat_ipo = ipaddress.ip_address(snat_ip)
if type(snat_ipo) is ipaddress.IPv6Address:
snat6_thread = Thread(target=snat6,args=(snat_ip,))
snat6_thread.daemon = True
snat6_thread.start()
except ValueError:
print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address')
autopurge_thread = Thread(target=autopurge)
autopurge_thread.daemon = True
autopurge_thread.start()
mailcowchainwatch_thread = Thread(target=mailcowChainOrder)
mailcowchainwatch_thread.daemon = True
mailcowchainwatch_thread.start()
blacklistupdate_thread = Thread(target=blacklistUpdate)
blacklistupdate_thread.daemon = True
blacklistupdate_thread.start()
whitelistupdate_thread = Thread(target=whitelistUpdate)
whitelistupdate_thread.daemon = True
whitelistupdate_thread.start()
signal.signal(signal.SIGTERM, quit)
atexit.register(clear)
while not quit_now:
time.sleep(0.5)
sys.exit(exit_code)

View File

@ -85,6 +85,8 @@ $cors_settings = cors('get');
$cors_settings['allowed_origins'] = str_replace(", ", "\n", $cors_settings['allowed_origins']);
$cors_settings['allowed_methods'] = explode(", ", $cors_settings['allowed_methods']);
$f2b_data = fail2ban('get');
$template = 'admin.twig';
$template_data = [
'tfa_data' => $tfa_data,
@ -101,7 +103,8 @@ $template_data = [
'domains' => $domains,
'all_domains' => $all_domains,
'mailboxes' => $mailboxes,
'f2b_data' => fail2ban('get'),
'f2b_data' => $f2b_data,
'f2b_banlist_url' => getBaseUrl() . "/api/v1/get/fail2ban/banlist/" . $f2b_data['banlist_id'],
'q_data' => quarantine('settings'),
'qn_data' => quota_notification('get'),
'rsettings_map' => file_get_contents('http://nginx:8081/settings.php'),
@ -113,6 +116,7 @@ $template_data = [
'password_complexity' => password_complexity('get'),
'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'],
'cors_settings' => $cors_settings,
'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on',
'lang_admin' => json_encode($lang['admin']),
'lang_datatables' => json_encode($lang['datatables'])
];

View File

@ -1,5 +1,5 @@
<?php
function fail2ban($_action, $_data = null) {
function fail2ban($_action, $_data = null, $_extra = null) {
global $redis;
$_data_log = $_data;
switch ($_action) {
@ -247,6 +247,7 @@ function fail2ban($_action, $_data = null) {
$netban_ipv6 = intval((isset($_data['netban_ipv6'])) ? $_data['netban_ipv6'] : $is_now['netban_ipv6']);
$wl = (isset($_data['whitelist'])) ? $_data['whitelist'] : $is_now['whitelist'];
$bl = (isset($_data['blacklist'])) ? $_data['blacklist'] : $is_now['blacklist'];
$manage_external = (isset($_data['manage_external'])) ? intval($_data['manage_external']) : 0;
}
else {
$_SESSION['return'][] = array(
@ -266,6 +267,8 @@ function fail2ban($_action, $_data = null) {
$f2b_options['netban_ipv6'] = ($netban_ipv6 > 128) ? 128 : $netban_ipv6;
$f2b_options['max_attempts'] = ($max_attempts < 1) ? 1 : $max_attempts;
$f2b_options['retry_window'] = ($retry_window < 1) ? 1 : $retry_window;
$f2b_options['banlist_id'] = $is_now['banlist_id'];
$f2b_options['manage_external'] = ($manage_external > 0) ? 1 : 0;
try {
$redis->Set('F2B_OPTIONS', json_encode($f2b_options));
$redis->Del('F2B_WHITELIST');
@ -329,5 +332,71 @@ function fail2ban($_action, $_data = null) {
'msg' => 'f2b_modified'
);
break;
case 'banlist':
try {
$f2b_options = json_decode($redis->Get('F2B_OPTIONS'), true);
}
catch (RedisException $e) {
$_SESSION['return'][] = array(
'type' => 'danger',
'log' => array(__FUNCTION__, $_action, $_data_log, $_extra),
'msg' => array('redis_error', $e)
);
http_response_code(500);
return false;
}
if (is_array($_extra)) {
$_extra = $_extra[0];
}
if ($_extra != $f2b_options['banlist_id']){
http_response_code(404);
return false;
}
switch ($_data) {
case 'get':
try {
$bl = $redis->hKeys('F2B_BLACKLIST');
$active_bans = $redis->hKeys('F2B_ACTIVE_BANS');
}
catch (RedisException $e) {
$_SESSION['return'][] = array(
'type' => 'danger',
'log' => array(__FUNCTION__, $_action, $_data_log, $_extra),
'msg' => array('redis_error', $e)
);
http_response_code(500);
return false;
}
$banlist = implode("\n", array_merge($bl, $active_bans));
return $banlist;
break;
case 'refresh':
if ($_SESSION['mailcow_cc_role'] != "admin") {
return false;
}
$f2b_options['banlist_id'] = uuid4();
try {
$redis->Set('F2B_OPTIONS', json_encode($f2b_options));
}
catch (RedisException $e) {
$_SESSION['return'][] = array(
'type' => 'danger',
'log' => array(__FUNCTION__, $_action, $_data_log, $_extra),
'msg' => array('redis_error', $e)
);
return false;
}
$_SESSION['return'][] = array(
'type' => 'success',
'log' => array(__FUNCTION__, $_action, $_data_log, $_extra),
'msg' => 'f2b_banlist_refreshed'
);
return true;
break;
}
break;
}
}

View File

@ -2246,6 +2246,21 @@ function cors($action, $data = null) {
break;
}
}
function getBaseURL() {
$protocol = isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on' ? 'https' : 'http';
$host = $_SERVER['HTTP_HOST'];
$base_url = $protocol . '://' . $host;
return $base_url;
}
function uuid4() {
$data = openssl_random_pseudo_bytes(16);
$data[6] = chr(ord($data[6]) & 0x0f | 0x40);
$data[8] = chr(ord($data[8]) & 0x3f | 0x80);
return vsprintf('%s%s-%s-%s-%s-%s%s%s', str_split(bin2hex($data), 4));
}
function get_logs($application, $lines = false) {
if ($lines === false) {

View File

@ -70,6 +70,8 @@ try {
}
}
catch (Exception $e) {
// Stop when redis is not available
http_response_code(500);
?>
<center style='font-family:sans-serif;'>Connection to Redis failed.<br /><br />The following error was reported:<br/><?=$e->getMessage();?></center>
<?php
@ -98,6 +100,7 @@ try {
}
catch (PDOException $e) {
// Stop when SQL connection fails
http_response_code(500);
?>
<center style='font-family:sans-serif;'>Connection to database failed.<br /><br />The following error was reported:<br/> <?=$e->getMessage();?></center>
<?php
@ -105,6 +108,7 @@ exit;
}
// Stop when dockerapi is not available
if (fsockopen("tcp://dockerapi", 443, $errno, $errstr) === false) {
http_response_code(500);
?>
<center style='font-family:sans-serif;'>Connection to dockerapi container failed.<br /><br />The following error was reported:<br/><?=$errno;?> - <?=$errstr;?></center>
<?php

View File

@ -391,3 +391,11 @@ function addTag(tagAddElem, tag = null){
$(tagValuesElem).val(JSON.stringify(value_tags));
$(tagInputElem).val('');
}
function copyToClipboard(id) {
var copyText = document.getElementById(id);
copyText.select();
copyText.setSelectionRange(0, 99999);
// only works with https connections
navigator.clipboard.writeText(copyText.value);
mailcow_alert_box(lang.copy_to_clipboard, "success");
}

View File

@ -503,6 +503,16 @@ if (isset($_GET['query'])) {
print(json_encode($getArgs));
$_SESSION['challenge'] = $WebAuthn->getChallenge();
return;
break;
case "fail2ban":
if (!isset($_SESSION['mailcow_cc_role'])){
switch ($object) {
case 'banlist':
header('Content-Type: text/plain');
echo fail2ban('banlist', 'get', $extra);
break;
}
}
break;
}
if (isset($_SESSION['mailcow_cc_role'])) {
@ -1324,6 +1334,10 @@ if (isset($_GET['query'])) {
break;
case "fail2ban":
switch ($object) {
case 'banlist':
header('Content-Type: text/plain');
echo fail2ban('banlist', 'get', $extra);
break;
default:
$data = fail2ban('get');
process_get_return($data);
@ -1943,7 +1957,14 @@ if (isset($_GET['query'])) {
process_edit_return(fwdhost('edit', array_merge(array('fwdhost' => $items), $attr)));
break;
case "fail2ban":
process_edit_return(fail2ban('edit', array_merge(array('network' => $items), $attr)));
switch ($object) {
case 'banlist':
process_edit_return(fail2ban('banlist', 'refresh', $items));
break;
default:
process_edit_return(fail2ban('edit', array_merge(array('network' => $items), $attr)));
break;
}
break;
case "ui_texts":
process_edit_return(customize('edit', 'ui_texts', $attr));

View File

@ -148,6 +148,7 @@
"change_logo": "Logo ändern",
"configuration": "Konfiguration",
"convert_html_to_text": "Konvertiere HTML zu reinem Text",
"copy_to_clipboard": "Text wurde in die Zwischenablage kopiert!",
"cors_settings": "CORS Einstellungen",
"credentials_transport_warning": "<b>Warnung</b>: Das Hinzufügen einer neuen Regel bewirkt die Aktualisierung der Authentifizierungsdaten aller vorhandenen Einträge mit identischem Next Hop.",
"customer_id": "Kunde",
@ -181,6 +182,8 @@
"f2b_blacklist": "Blacklist für Netzwerke und Hosts",
"f2b_filter": "Regex-Filter",
"f2b_list_info": "Ein Host oder Netzwerk auf der Blacklist wird immer eine Whitelist-Einheit überwiegen. <b>Die Aktualisierung der Liste dauert einige Sekunden.</b>",
"f2b_manage_external": "Fail2Ban extern verwalten",
"f2b_manage_external_info": "Fail2ban wird die Banlist weiterhin pflegen, jedoch werden keine aktiven Regeln zum blockieren gesetzt. Die unten generierte Banlist, kann verwendet werden, um den Datenverkehr extern zu blockieren.",
"f2b_max_attempts": "Max. Versuche",
"f2b_max_ban_time": "Maximale Bannzeit in Sekunden",
"f2b_netban_ipv4": "Netzbereich für IPv4-Banns (8-32)",
@ -1035,6 +1038,7 @@
"domain_removed": "Domain %s wurde entfernt",
"dovecot_restart_success": "Dovecot wurde erfolgreich neu gestartet",
"eas_reset": "ActiveSync Gerät des Benutzers %s wurde zurückgesetzt",
"f2b_banlist_refreshed": "Banlist ID wurde erfolgreich erneuert.",
"f2b_modified": "Änderungen an Fail2ban-Parametern wurden gespeichert",
"forwarding_host_added": "Weiterleitungs-Host %s wurde hinzugefügt",
"forwarding_host_removed": "Weiterleitungs-Host %s wurde entfernt",

View File

@ -154,6 +154,7 @@
"logo_dark_label": "Inverted for dark mode",
"configuration": "Configuration",
"convert_html_to_text": "Convert HTML to plain text",
"copy_to_clipboard": "Text copied to clipboard!",
"cors_settings": "CORS Settings",
"credentials_transport_warning": "<b>Warning</b>: Adding a new transport map entry will update the credentials for all entries with a matching next hop column.",
"customer_id": "Customer ID",
@ -187,6 +188,8 @@
"f2b_blacklist": "Blacklisted networks/hosts",
"f2b_filter": "Regex filters",
"f2b_list_info": "A blacklisted host or network will always outweigh a whitelist entity. <b>List updates will take a few seconds to be applied.</b>",
"f2b_manage_external": "Manage Fail2Ban externally",
"f2b_manage_external_info": "Fail2ban will still maintain the banlist, but it will not actively set rules to block traffic. Use the generated banlist below to externally block the traffic.",
"f2b_max_attempts": "Max. attempts",
"f2b_max_ban_time": "Max. ban time (s)",
"f2b_netban_ipv4": "IPv4 subnet size to apply ban on (8-32)",
@ -1046,6 +1049,7 @@
"domain_removed": "Domain %s has been removed",
"dovecot_restart_success": "Dovecot was restarted successfully",
"eas_reset": "ActiveSync devices for user %s were reset",
"f2b_banlist_refreshed": "Banlist ID has been successfully refreshed.",
"f2b_modified": "Changes to Fail2ban parameters have been saved",
"forwarding_host_added": "Forwarding host %s has been added",
"forwarding_host_removed": "Forwarding host %s has been removed",

View File

@ -42,6 +42,13 @@
<input type="number" class="form-control" id="f2b_netban_ipv6" name="netban_ipv6" value="{{ f2b_data.netban_ipv6 }}" required>
</div>
</div>
<div class="mb-4">
<div class="form-check form-switch">
<input class="form-check-input" type="checkbox" id="f2b_manage_external" value="1" name="manage_external" {% if f2b_data.manage_external == 1 %}checked{% endif %}>
<label class="form-check-label" for="f2b_manage_external">{{ lang.admin.f2b_manage_external }}</label>
</div>
<p class="text-muted">{{ lang.admin.f2b_manage_external_info }}</p>
</div>
<hr>
<p class="text-muted">{{ lang.admin.f2b_list_info|raw }}</p>
<div class="mb-2">
@ -90,6 +97,15 @@
{% if not f2b_data.active_bans and not f2b_data.perm_bans %}
<i>{{ lang.admin.no_active_bans }}</i>
{% endif %}
<form class="form-inline" data-id="f2b_banlist" role="form" method="post">
<div class="input-group mb-3">
<input type="text" class="form-control" aria-label="Banlist url" value="{{ f2b_banlist_url}}" id="banlist_url">
{% if is_https %}
<button class="btn btn-secondary" type="button" onclick="copyToClipboard('banlist_url')"><i class="bi bi-clipboard"></i></button>
{% endif %}
<button class="btn btn-secondary" type="button" data-action="edit_selected" data-item="{{ f2b_data.banlist_id }}" data-id="f2b_banlist" data-api-url='edit/fail2ban/banlist' data-api-attr='{}'><i class="bi bi-arrow-clockwise"></i></button>
</div>
</form>
{% for active_ban in f2b_data.active_bans %}
<p>
<span class="badge fs-7 bg-info d-block d-sm-inline-block">