Merge pull request #5313 from mailcow/feat/f2b-banlist

[Web] add f2b_banlist endpoint

data/Dockerfiles/netfilter/main.py

@@ -15,6 +15,7 @@ import redis
 import json
 import dns.resolver
 import dns.exception
+import uuid
 from modules.Logger import Logger
 from modules.IPTables import IPTables
 from modules.NFTables import NFTables
@@ -97,6 +98,8 @@ def verifyF2boptions(f2boptions):
   verifyF2boption(f2boptions,'retry_window', 600)
   verifyF2boption(f2boptions,'netban_ipv4', 32)
   verifyF2boption(f2boptions,'netban_ipv6', 128)
+  verifyF2boption(f2boptions,'banlist_id', str(uuid.uuid4()))
+  verifyF2boption(f2boptions,'manage_external', 0)
 
 def verifyF2boption(f2boptions, f2boption, f2bdefault):
   f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault
@@ -137,6 +140,7 @@ def get_ip(address):
   return ip
 
 def ban(address):
+  global f2boptions
   global lock
 
   refreshF2boptions()
@@ -178,10 +182,10 @@ def ban(address):
     cur_time = int(round(time.time()))
     NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
     logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
-    if type(ip) is ipaddress.IPv4Address:
+    if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1:
       with lock:
         tables.banIPv4(net)
-    else:
+    elif int(f2boptions['manage_external']) != 1:
       with lock:
         tables.banIPv6(net)
 
@@ -212,6 +216,7 @@ def unban(net):
     bans[net]['ban_counter'] += 1
 
 def permBan(net, unban=False):
+  global f2boptions
   global lock
 
   is_unbanned = False
@@ -220,13 +225,13 @@ def permBan(net, unban=False):
     with lock:
       if unban:
         is_unbanned = tables.unbanIPv4(net)
-      else:
+      elif int(f2boptions['manage_external']) != 1:
         is_banned = tables.banIPv4(net)
   else:
     with lock:
       if unban:
         is_unbanned = tables.unbanIPv6(net)
-      else:
+      elif int(f2boptions['manage_external']) != 1:
         is_banned = tables.banIPv6(net)
 
 
@@ -404,6 +409,7 @@ def quit(signum, frame):
 
 
 if __name__ == '__main__':
+  refreshF2boptions()
   # In case a previous session was killed without cleanup
   clear()
   # Reinit MAILCOW chain

data/web/admin.php

@@ -85,6 +85,8 @@ $cors_settings = cors('get');
 $cors_settings['allowed_origins'] = str_replace(", ", "\n", $cors_settings['allowed_origins']);
 $cors_settings['allowed_methods'] = explode(", ", $cors_settings['allowed_methods']);
 
+$f2b_data = fail2ban('get');
+
 $template = 'admin.twig';
 $template_data = [
   'tfa_data' => $tfa_data,
@@ -101,7 +103,8 @@ $template_data = [
   'domains' => $domains,
   'all_domains' => $all_domains,
   'mailboxes' => $mailboxes,
-  'f2b_data' => fail2ban('get'),
+  'f2b_data' => $f2b_data,
+  'f2b_banlist_url' => getBaseUrl() . "/api/v1/get/fail2ban/banlist/" . $f2b_data['banlist_id'],
   'q_data' => quarantine('settings'),
   'qn_data' => quota_notification('get'),
   'rsettings_map' => file_get_contents('http://nginx:8081/settings.php'),
@@ -113,6 +116,7 @@ $template_data = [
   'password_complexity' => password_complexity('get'),
   'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'],
   'cors_settings' => $cors_settings,
+  'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on',
   'lang_admin' => json_encode($lang['admin']),
   'lang_datatables' => json_encode($lang['datatables'])
 ];

data/web/inc/functions.fail2ban.inc.php

@@ -1,5 +1,5 @@
 <?php
-function fail2ban($_action, $_data = null) {
+function fail2ban($_action, $_data = null, $_extra = null) {
   global $redis;
   $_data_log = $_data;
   switch ($_action) {
@@ -247,6 +247,7 @@ function fail2ban($_action, $_data = null) {
         $netban_ipv6 = intval((isset($_data['netban_ipv6'])) ? $_data['netban_ipv6'] : $is_now['netban_ipv6']);
         $wl = (isset($_data['whitelist'])) ? $_data['whitelist'] : $is_now['whitelist'];
         $bl = (isset($_data['blacklist'])) ? $_data['blacklist'] : $is_now['blacklist'];
+        $manage_external = (isset($_data['manage_external'])) ? intval($_data['manage_external']) : 0;
       }
       else {
         $_SESSION['return'][] = array(
@@ -266,6 +267,8 @@ function fail2ban($_action, $_data = null) {
       $f2b_options['netban_ipv6'] = ($netban_ipv6 > 128) ? 128 : $netban_ipv6;
       $f2b_options['max_attempts'] = ($max_attempts < 1) ? 1 : $max_attempts;
       $f2b_options['retry_window'] = ($retry_window < 1) ? 1 : $retry_window;
+      $f2b_options['banlist_id'] = $is_now['banlist_id'];
+      $f2b_options['manage_external'] = ($manage_external > 0) ? 1 : 0;
       try {
         $redis->Set('F2B_OPTIONS', json_encode($f2b_options));
         $redis->Del('F2B_WHITELIST');
@@ -329,5 +332,71 @@ function fail2ban($_action, $_data = null) {
         'msg' => 'f2b_modified'
       );
     break;
+    case 'banlist':
+      try {
+        $f2b_options = json_decode($redis->Get('F2B_OPTIONS'), true);
+      } 
+      catch (RedisException $e) {
+        $_SESSION['return'][] = array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__, $_action, $_data_log, $_extra),
+          'msg' => array('redis_error', $e)
+        );
+        http_response_code(500);
+        return false;
+      }
+      if (is_array($_extra)) {
+        $_extra = $_extra[0];
+      }
+      if ($_extra != $f2b_options['banlist_id']){
+        http_response_code(404);
+        return false;
+      }
+
+      switch ($_data) {
+        case 'get':
+          try {
+            $bl = $redis->hKeys('F2B_BLACKLIST');
+            $active_bans = $redis->hKeys('F2B_ACTIVE_BANS');
+          } 
+          catch (RedisException $e) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_data_log, $_extra),
+              'msg' => array('redis_error', $e)
+            );
+            http_response_code(500);
+            return false;
+          }
+          $banlist = implode("\n", array_merge($bl, $active_bans));
+          return $banlist;
+        break;
+        case 'refresh':
+          if ($_SESSION['mailcow_cc_role'] != "admin") {
+            return false;
+          }
+
+          $f2b_options['banlist_id'] = uuid4();
+          try {
+            $redis->Set('F2B_OPTIONS', json_encode($f2b_options));
+          } 
+          catch (RedisException $e) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_data_log, $_extra),
+              'msg' => array('redis_error', $e)
+            );
+            return false;
+          }
+
+          $_SESSION['return'][] = array(
+            'type' => 'success',
+            'log' => array(__FUNCTION__, $_action, $_data_log, $_extra),
+            'msg' => 'f2b_banlist_refreshed'
+          );
+          return true;
+        break;
+      }
+    break;
   }
 }

data/web/inc/functions.inc.php

@@ -2246,6 +2246,21 @@ function cors($action, $data = null) {
     break;
   }
 }
+function getBaseURL() {
+  $protocol = isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on' ? 'https' : 'http';
+  $host = $_SERVER['HTTP_HOST'];
+  $base_url = $protocol . '://' . $host;
+
+  return $base_url;
+}
+function uuid4() {
+  $data = openssl_random_pseudo_bytes(16);
+
+  $data[6] = chr(ord($data[6]) & 0x0f | 0x40);
+  $data[8] = chr(ord($data[8]) & 0x3f | 0x80);
+
+  return vsprintf('%s%s-%s-%s-%s-%s%s%s', str_split(bin2hex($data), 4));
+}
 
 function get_logs($application, $lines = false) {
   if ($lines === false) {

data/web/inc/prerequisites.inc.php

@@ -70,6 +70,8 @@ try {
   }
 }
 catch (Exception $e) {
+// Stop when redis is not available
+http_response_code(500);
 ?>
 <center style='font-family:sans-serif;'>Connection to Redis failed.<br /><br />The following error was reported:<br/><?=$e->getMessage();?></center>
 <?php
@@ -98,6 +100,7 @@ try {
 }
 catch (PDOException $e) {
 // Stop when SQL connection fails
+http_response_code(500);
 ?>
 <center style='font-family:sans-serif;'>Connection to database failed.<br /><br />The following error was reported:<br/>  <?=$e->getMessage();?></center>
 <?php
@@ -105,6 +108,7 @@ exit;
 }
 // Stop when dockerapi is not available
 if (fsockopen("tcp://dockerapi", 443, $errno, $errstr) === false) {
+  http_response_code(500);
 ?>
 <center style='font-family:sans-serif;'>Connection to dockerapi container failed.<br /><br />The following error was reported:<br/><?=$errno;?> - <?=$errstr;?></center>
 <?php

data/web/js/build/013-mailcow.js

@@ -391,3 +391,11 @@ function addTag(tagAddElem, tag = null){
   $(tagValuesElem).val(JSON.stringify(value_tags));
   $(tagInputElem).val('');
 }
+function copyToClipboard(id) {
+  var copyText = document.getElementById(id);
+  copyText.select();
+  copyText.setSelectionRange(0, 99999);
+  // only works with https connections
+  navigator.clipboard.writeText(copyText.value);
+  mailcow_alert_box(lang.copy_to_clipboard, "success");
+}

data/web/json_api.php

@@ -504,6 +504,16 @@ if (isset($_GET['query'])) {
           $_SESSION['challenge'] = $WebAuthn->getChallenge();
           return;
         break;          
+        case "fail2ban":
+          if (!isset($_SESSION['mailcow_cc_role'])){
+            switch ($object) {
+              case 'banlist':
+                header('Content-Type: text/plain');
+                echo fail2ban('banlist', 'get', $extra);
+              break;
+            }
+          }
+        break;
       }
       if (isset($_SESSION['mailcow_cc_role'])) {
         switch ($category) {
@@ -1324,6 +1334,10 @@ if (isset($_GET['query'])) {
           break;
           case "fail2ban":
             switch ($object) {
+              case 'banlist':
+                header('Content-Type: text/plain');
+                echo fail2ban('banlist', 'get', $extra);
+              break;
               default:
                 $data = fail2ban('get');
                 process_get_return($data);
@@ -1943,8 +1957,15 @@ if (isset($_GET['query'])) {
           process_edit_return(fwdhost('edit', array_merge(array('fwdhost' => $items), $attr)));
         break;
         case "fail2ban":
+          switch ($object) {
+            case 'banlist':
+              process_edit_return(fail2ban('banlist', 'refresh', $items));
+            break;
+            default:
               process_edit_return(fail2ban('edit', array_merge(array('network' => $items), $attr)));
             break;
+          }
+        break;
         case "ui_texts":
           process_edit_return(customize('edit', 'ui_texts', $attr));
         break;

data/web/lang/lang.de-de.json

@@ -148,6 +148,7 @@
         "change_logo": "Logo ändern",
         "configuration": "Konfiguration",
         "convert_html_to_text": "Konvertiere HTML zu reinem Text",
+        "copy_to_clipboard": "Text wurde in die Zwischenablage kopiert!",
         "cors_settings": "CORS Einstellungen",
         "credentials_transport_warning": "<b>Warnung</b>: Das Hinzufügen einer neuen Regel bewirkt die Aktualisierung der Authentifizierungsdaten aller vorhandenen Einträge mit identischem Next Hop.",
         "customer_id": "Kunde",
@@ -181,6 +182,8 @@
         "f2b_blacklist": "Blacklist für Netzwerke und Hosts",
         "f2b_filter": "Regex-Filter",
         "f2b_list_info": "Ein Host oder Netzwerk auf der Blacklist wird immer eine Whitelist-Einheit überwiegen. <b>Die Aktualisierung der Liste dauert einige Sekunden.</b>",
+        "f2b_manage_external": "Fail2Ban extern verwalten",
+        "f2b_manage_external_info": "Fail2ban wird die Banlist weiterhin pflegen, jedoch werden keine aktiven Regeln zum blockieren gesetzt. Die unten generierte Banlist, kann verwendet werden, um den Datenverkehr extern zu blockieren.",
         "f2b_max_attempts": "Max. Versuche",
         "f2b_max_ban_time": "Maximale Bannzeit in Sekunden",
         "f2b_netban_ipv4": "Netzbereich für IPv4-Banns (8-32)",
@@ -1035,6 +1038,7 @@
         "domain_removed": "Domain %s wurde entfernt",
         "dovecot_restart_success": "Dovecot wurde erfolgreich neu gestartet",
         "eas_reset": "ActiveSync Gerät des Benutzers %s wurde zurückgesetzt",
+        "f2b_banlist_refreshed": "Banlist ID wurde erfolgreich erneuert.",
         "f2b_modified": "Änderungen an Fail2ban-Parametern wurden gespeichert",
         "forwarding_host_added": "Weiterleitungs-Host %s wurde hinzugefügt",
         "forwarding_host_removed": "Weiterleitungs-Host %s wurde entfernt",

data/web/lang/lang.en-gb.json

@@ -154,6 +154,7 @@
         "logo_dark_label": "Inverted for dark mode",
         "configuration": "Configuration",
         "convert_html_to_text": "Convert HTML to plain text",
+        "copy_to_clipboard": "Text copied to clipboard!",
         "cors_settings": "CORS Settings",
         "credentials_transport_warning": "<b>Warning</b>: Adding a new transport map entry will update the credentials for all entries with a matching next hop column.",
         "customer_id": "Customer ID",
@@ -187,6 +188,8 @@
         "f2b_blacklist": "Blacklisted networks/hosts",
         "f2b_filter": "Regex filters",
         "f2b_list_info": "A blacklisted host or network will always outweigh a whitelist entity. <b>List updates will take a few seconds to be applied.</b>",
+        "f2b_manage_external": "Manage Fail2Ban externally",
+        "f2b_manage_external_info": "Fail2ban will still maintain the banlist, but it will not actively set rules to block traffic. Use the generated banlist below to externally block the traffic.",
         "f2b_max_attempts": "Max. attempts",
         "f2b_max_ban_time": "Max. ban time (s)",
         "f2b_netban_ipv4": "IPv4 subnet size to apply ban on (8-32)",
@@ -1046,6 +1049,7 @@
         "domain_removed": "Domain %s has been removed",
         "dovecot_restart_success": "Dovecot was restarted successfully",
         "eas_reset": "ActiveSync devices for user %s were reset",
+        "f2b_banlist_refreshed": "Banlist ID has been successfully refreshed.",
         "f2b_modified": "Changes to Fail2ban parameters have been saved",
         "forwarding_host_added": "Forwarding host %s has been added",
         "forwarding_host_removed": "Forwarding host %s has been removed",

data/web/templates/admin/tab-config-f2b.twig

@@ -42,6 +42,13 @@
             <input type="number" class="form-control" id="f2b_netban_ipv6" name="netban_ipv6" value="{{ f2b_data.netban_ipv6 }}" required>
           </div>
         </div>
+        <div class="mb-4">
+          <div class="form-check form-switch">
+            <input class="form-check-input" type="checkbox" id="f2b_manage_external" value="1" name="manage_external" {% if f2b_data.manage_external == 1 %}checked{% endif %}>
+            <label class="form-check-label" for="f2b_manage_external">{{ lang.admin.f2b_manage_external }}</label>
+          </div>
+          <p class="text-muted">{{ lang.admin.f2b_manage_external_info }}</p>
+        </div>
         <hr>
         <p class="text-muted">{{ lang.admin.f2b_list_info|raw }}</p>
         <div class="mb-2">
@@ -90,6 +97,15 @@
       {% if not f2b_data.active_bans and not f2b_data.perm_bans %}
         <i>{{ lang.admin.no_active_bans }}</i>
       {% endif %}
+      <form class="form-inline" data-id="f2b_banlist" role="form" method="post">
+        <div class="input-group mb-3">
+          <input type="text" class="form-control" aria-label="Banlist url" value="{{ f2b_banlist_url}}" id="banlist_url">
+          {% if is_https %}
+          <button class="btn btn-secondary" type="button" onclick="copyToClipboard('banlist_url')"><i class="bi bi-clipboard"></i></button>
+          {% endif %}
+          <button class="btn btn-secondary" type="button" data-action="edit_selected" data-item="{{ f2b_data.banlist_id }}" data-id="f2b_banlist" data-api-url='edit/fail2ban/banlist' data-api-attr='{}'><i class="bi bi-arrow-clockwise"></i></button>
+        </div>
+      </form>
       {% for active_ban in f2b_data.active_bans %}
         <p>
           <span class="badge fs-7 bg-info d-block d-sm-inline-block">