From 5ed113c47ff4de0c0ab29fda2d95c51820bbf856 Mon Sep 17 00:00:00 2001 From: Kraeutergarten Date: Sun, 19 May 2019 09:48:10 +0200 Subject: [PATCH] resolving whitelist every minute --- data/Dockerfiles/netfilter/server.py | 127 ++++++++++++++++----------- 1 file changed, 76 insertions(+), 51 deletions(-) diff --git a/data/Dockerfiles/netfilter/server.py b/data/Dockerfiles/netfilter/server.py index 4fd1e3b0..18d98d81 100644 --- a/data/Dockerfiles/netfilter/server.py +++ b/data/Dockerfiles/netfilter/server.py @@ -6,6 +6,7 @@ import time import atexit import signal import ipaddress +from collections import Counter from random import randint from threading import Thread from threading import Lock @@ -38,18 +39,13 @@ RULES[5] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)' RULES[6] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+' #RULES[7] = '-login: Aborted login \(no auth .+\): user=.+, rip=([0-9a-f\.:]+), lip.+' +WHITELIST = [] + bans = {} log = {} quit_now = False lock = Lock() -def is_ip_network(address): - try: - ipaddress.ip_network(address.decode('ascii'), False) - except ValueError: - return False - return True - def refreshF2boptions(): global f2boptions global quit_now @@ -118,7 +114,6 @@ def ban(address): RETRY_WINDOW = int(f2boptions['retry_window']) NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4']) NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6']) - WHITELIST = r.hgetall('F2B_WHITELIST') ip = ipaddress.ip_address(address.decode('ascii')) if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped: @@ -128,50 +123,9 @@ def ban(address): return self_network = ipaddress.ip_network(address.decode('ascii')) - if WHITELIST: - wl_hostnames=[] - wl_networks=[] - - for wl_key in WHITELIST: - if is_ip_network(wl_key): - wl_networks.append(wl_key) - else: - wl_hostnames.append(wl_key) - for w1_hostname in wl_hostnames: - hostname_ips = [] - for rdtype in ['A', 'AAAA']: - try: - answer = resolver.query(qname=w1_hostname, rdtype=rdtype, lifetime=1) - except dns.exception.Timeout as timout: - log['time'] = int(round(time.time())) - log['priority'] = 'info' - log['message'] = 'Hostname %s timedout on resolve' % (w1_hostname) - r.lpush('NETFILTER_LOG', json.dumps(log, ensure_ascii=False)) - print 'Hostname %s timedout on resolve' % (w1_hostname) - break - except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): - continue - except dns.exception.DNSException as dnsexception: - log['time'] = int(round(time.time())) - log['priority'] = 'info' - log['message'] = '%s' % (dnsexception) - r.lpush('NETFILTER_LOG', json.dumps(log, ensure_ascii=False)) - print '%s' % (dnsexception) - continue - - for rdata in answer: - hostname_ips.append(rdata.to_text()) - - wl_networks.extend(hostname_ips) - - log['time'] = int(round(time.time())) - log['priority'] = 'info' - log['message'] = 'Hostname %s is resolved to %s' % (w1_hostname, hostname_ips) - r.lpush('NETFILTER_LOG', json.dumps(log, ensure_ascii=False)) - print 'Hostname %s is resolved to %s' % (w1_hostname, hostname_ips) - - for wl_key in wl_networks: + if WHITELIST: + for wl_key in WHITELIST: wl_net = ipaddress.ip_network(wl_key.decode('ascii'), False) if wl_net.overlaps(self_network): @@ -419,6 +373,73 @@ def autopurge(): if time.time() - bans[net]['last_attempt'] > BAN_TIME: unban(net) +def isIpNetwork(address): + try: + ipaddress.ip_network(address.decode('ascii'), False) + except ValueError: + return False + return True + + +def genNetworkList(list): + hostnames = [] + networks = [] + + for key in list: + if isIpNetwork(key): + networks.append(key.encode("utf-8")) + else: + hostnames.append(key) + + for hostname in hostnames: + hostname_ips = [] + for rdtype in ['A', 'AAAA']: + try: + answer = resolver.query(qname=hostname, rdtype=rdtype, lifetime=10) + except dns.exception.Timeout: + log['time'] = int(round(time.time())) + log['priority'] = 'info' + log['message'] = 'Hostname %s timedout on resolve' % (hostname) + r.lpush('NETFILTER_LOG', json.dumps(log, ensure_ascii=False)) + print 'Hostname %s timedout on resolve' % (hostname) + break + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + continue + except dns.exception.DNSException as dnsexception: + log['time'] = int(round(time.time())) + log['priority'] = 'info' + log['message'] = '%s' % (dnsexception) + r.lpush('NETFILTER_LOG', json.dumps(log, ensure_ascii=False)) + print '%s' % (dnsexception) + continue + + for rdata in answer: + hostname_ips.append(rdata.to_text().encode("utf-8")) + + networks.extend(hostname_ips) + + return networks + +def whitelistUpdate(): + global lock + global quit_now + global WHITELIST + + while not quit_now: + start_time = time.time() + list = r.hgetall('F2B_WHITELIST') + if list: + new_whitelist = genNetworkList(list) + if Counter(new_whitelist) != Counter(WHITELIST): + WHITELIST = new_whitelist + log['time'] = int(round(time.time())) + log['priority'] = 'info' + log['message'] = 'New entrys for whitelist %s' % (WHITELIST) + r.lpush('NETFILTER_LOG', json.dumps(log, ensure_ascii=False)) + print 'New entrys for whitelist %s' % (WHITELIST) + + time.sleep(60.0 - ((time.time() - start_time) % 60.0)) + def initChain(): # Is called before threads start, no locking print "Initializing mailcow netfilter chain" @@ -520,6 +541,10 @@ if __name__ == '__main__': mailcowchainwatch_thread.daemon = True mailcowchainwatch_thread.start() + whitelistupdate_thread = Thread(target=whitelistUpdate) + whitelistupdate_thread.daemon = True + whitelistupdate_thread.start() + signal.signal(signal.SIGTERM, quit) atexit.register(clear)