Merge pull request #4968 from felixoi/staging

Watchdog: Allow sending notifications via webhooks
2023-12-11 16:29:52 +01:00 · 2023-12-11 16:29:52 +01:00 · 996772a27d
commit 996772a27d
parent c2e5dfd933 7f4e9c1ad4
4 changed files with 92 additions and 44 deletions
--- a/data/Dockerfiles/watchdog/watchdog.sh
+++ b/data/Dockerfiles/watchdog/watchdog.sh
@ -19,9 +19,11 @@ fi

 if [[ "${WATCHDOG_VERBOSE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
  SMTP_VERBOSE="--verbose"
+  CURL_VERBOSE="--verbose"
  set -xv
 else
  SMTP_VERBOSE=""
+  CURL_VERBOSE=""
  exec 2>/dev/null
 fi

@ -97,7 +99,9 @@ log_msg() {
  echo $(date) $(printf '%s\n' "${1}")
 }

-function mail_error() {
+function notify_error() {
+  # Check if one of the notification options is enabled
+  [[ -z ${WATCHDOG_NOTIFY_EMAIL} ]] && [[ -z ${WATCHDOG_NOTIFY_WEBHOOK} ]] && return 0
  THROTTLE=
  [[ -z ${1} ]] && return 1
  # If exists, body will be the content of "/tmp/${1}", even if ${2} is set
@ -122,37 +126,57 @@ function mail_error() {
  else
    SUBJECT="${WATCHDOG_SUBJECT}: ${1}"
  fi
-  IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
-  for rcpt in "${MAIL_RCPTS[@]}"; do
-    RCPT_DOMAIN=
-    RCPT_MX=
-    RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
-    CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx)
-    if [[ -z ${CHECK_FOR_VALID_MX} ]]; then
-      log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
+
+  # Send mail notification if enabled
+  if [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]]; then
+    IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
+    for rcpt in "${MAIL_RCPTS[@]}"; do
+      RCPT_DOMAIN=
+      RCPT_MX=
+      RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
+      CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx)
+      if [[ -z ${CHECK_FOR_VALID_MX} ]]; then
+        log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
+        return 1
+      fi
+      [ -f "/tmp/${1}" ] && BODY="/tmp/${1}"
+      timeout 10s ./smtp-cli --missing-modules-ok \
+        "${SMTP_VERBOSE}" \
+        --charset=UTF-8 \
+        --subject="${SUBJECT}" \
+        --body-plain="${BODY}" \
+        --add-header="X-Priority: 1" \
+        --to=${rcpt} \
+        --from="watchdog@${MAILCOW_HOSTNAME}" \
+        --hello-host=${MAILCOW_HOSTNAME} \
+        --ipv4
+      if [[ $? -eq 1 ]]; then # exit code 1 is fine
+        log_msg "Sent notification email to ${rcpt}"
+      else
+        if [[ "${SMTP_VERBOSE}" == "" ]]; then
+          log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf."
+        else
+          log_msg "Error while sending notification email to ${rcpt}."
+        fi
+      fi
+    done
+  fi
+
+  # Send webhook notification if enabled
+  if [[ ! -z ${WATCHDOG_NOTIFY_WEBHOOK} ]]; then
+    if [[ -z ${WATCHDOG_NOTIFY_WEBHOOK_BODY} ]]; then
+      log_msg "No webhook body set, skipping webhook notification..."
      return 1
    fi
-    [ -f "/tmp/${1}" ] && BODY="/tmp/${1}"
-    timeout 10s ./smtp-cli --missing-modules-ok \
-      "${SMTP_VERBOSE}" \
-      --charset=UTF-8 \
-      --subject="${SUBJECT}" \
-      --body-plain="${BODY}" \
-      --add-header="X-Priority: 1" \
-      --to=${rcpt} \
-      --from="watchdog@${MAILCOW_HOSTNAME}" \
-      --hello-host=${MAILCOW_HOSTNAME} \
-      --ipv4
-    if [[ $? -eq 1 ]]; then # exit code 1 is fine
-      log_msg "Sent notification email to ${rcpt}"
-    else
-      if [[ "${SMTP_VERBOSE}" == "" ]]; then
-        log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf."
-      else
-        log_msg "Error while sending notification email to ${rcpt}."
-      fi
-    fi
-  done
+
+    # Replace subject and body placeholders
+    WEBHOOK_BODY=$(echo ${WATCHDOG_NOTIFY_WEBHOOK_BODY} | sed "s|\$SUBJECT\|\${SUBJECT}|$SUBJECT|g" | sed "s|\$BODY\|\${BODY}|$BODY|")
+    
+    # POST to webhook
+    curl -X POST -H "Content-Type: application/json" ${CURL_VERBOSE} -d "${WEBHOOK_BODY}" ${WATCHDOG_NOTIFY_WEBHOOK}
+
+    log_msg "Sent notification using webhook"
+  fi
 }

 get_container_ip() {
@ -197,7 +221,7 @@ get_container_ip() {
 # One-time check
 if grep -qi "$(echo ${IPV6_NETWORK} | cut -d: -f1-3)" <<< "$(ip a s)"; then
  if [[ -z "$(get_ipv6)" ]]; then
-    mail_error "ipv6-config" "enable_ipv6 is true in docker-compose.yml, but an IPv6 link could not be established. Please verify your IPv6 connection."
+    notify_error "ipv6-config" "enable_ipv6 is true in docker-compose.yml, but an IPv6 link could not be established. Please verify your IPv6 connection."
  fi
 fi

@ -746,9 +770,7 @@ olefy_checks() {
 }

 # Notify about start
-if [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]]; then
-  mail_error "watchdog-mailcow" "Watchdog started monitoring mailcow."
-fi
+notify_error "watchdog-mailcow" "Watchdog started monitoring mailcow."

 # Create watchdog agents

@ -1029,33 +1051,33 @@ while true; do
  fi
  if [[ ${com_pipe_answer} == "ratelimit" ]]; then
    log_msg "At least one ratelimit was applied"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}"
+    notify_error "${com_pipe_answer}"
  elif [[ ${com_pipe_answer} == "mail_queue_status" ]]; then
    log_msg "Mail queue status is critical"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}"
+    notify_error "${com_pipe_answer}"
  elif [[ ${com_pipe_answer} == "external_checks" ]]; then
    log_msg "Your mailcow is an open relay!"
    # Define $2 to override message text, else print service was restarted at ...
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please stop mailcow now and check your network configuration!"
+    notify_error "${com_pipe_answer}" "Please stop mailcow now and check your network configuration!"
  elif [[ ${com_pipe_answer} == "mysql_repl_checks" ]]; then
    log_msg "MySQL replication is not working properly"
    # Define $2 to override message text, else print service was restarted at ...
    # Once mail per 10 minutes
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check the SQL replication status" 600
+    notify_error "${com_pipe_answer}" "Please check the SQL replication status" 600
  elif [[ ${com_pipe_answer} == "dovecot_repl_checks" ]]; then
    log_msg "Dovecot replication is not working properly"
    # Define $2 to override message text, else print service was restarted at ...
    # Once mail per 10 minutes
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check the Dovecot replicator status" 600
+    notify_error "${com_pipe_answer}" "Please check the Dovecot replicator status" 600
  elif [[ ${com_pipe_answer} == "certcheck" ]]; then
    log_msg "Certificates are about to expire"
    # Define $2 to override message text, else print service was restarted at ...
    # Only mail once a day
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please renew your certificate" 86400
+    notify_error "${com_pipe_answer}" "Please renew your certificate" 86400
  elif [[ ${com_pipe_answer} == "acme-mailcow" ]]; then
    log_msg "acme-mailcow did not complete successfully"
    # Define $2 to override message text, else print service was restarted at ...
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
+    notify_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
  elif [[ ${com_pipe_answer} == "fail2ban" ]]; then
    F2B_RES=($(timeout 4s ${REDIS_CMDLINE} --raw GET F2B_RES 2> /dev/null))
    if [[ ! -z "${F2B_RES}" ]]; then
@ -1065,7 +1087,7 @@ while true; do
        log_msg "Banned ${host}"
        rm /tmp/fail2ban 2> /dev/null
        timeout 2s whois "${host}" > /tmp/fail2ban
-        [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && [[ ${WATCHDOG_NOTIFY_BAN} =~ ^([yY][eE][sS]|[yY])+$ ]] && mail_error "${com_pipe_answer}" "IP ban: ${host}"
+        [[ ${WATCHDOG_NOTIFY_BAN} =~ ^([yY][eE][sS]|[yY])+$ ]] && notify_error "${com_pipe_answer}" "IP ban: ${host}"
      done
    fi
  elif [[ ${com_pipe_answer} =~ .+-mailcow ]]; then
@ -1085,7 +1107,7 @@ while true; do
      else
        log_msg "Sending restart command to ${CONTAINER_ID}..."
        curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/restart
-        [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}"
+        notify_error "${com_pipe_answer}"
        log_msg "Wait for restarted container to settle and continue watching..."
        sleep 35
      fi
@ -1095,3 +1117,4 @@ while true; do
    kill -USR1 ${BACKGROUND_TASKS[*]}
  fi
 done
+
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -457,7 +457,7 @@ services:
        - /lib/modules:/lib/modules:ro

    watchdog-mailcow:
-      image: mailcow/watchdog:1.98
+      image: mailcow/watchdog:1.99
      dns:
        - ${IPV4_NETWORK:-172.22.1}.254
      tmpfs:
@ -487,6 +487,8 @@ services:
        - WATCHDOG_NOTIFY_EMAIL=${WATCHDOG_NOTIFY_EMAIL:-}
        - WATCHDOG_NOTIFY_BAN=${WATCHDOG_NOTIFY_BAN:-y}
        - WATCHDOG_SUBJECT=${WATCHDOG_SUBJECT:-Watchdog ALERT}
+        - WATCHDOG_NOTIFY_WEBHOOK=${WATCHDOG_NOTIFY_WEBHOOK}
+        - WATCHDOG_NOTIFY_WEBHOOK_BODY=${WATCHDOG_NOTIFY_WEBHOOK_BODY}
        - WATCHDOG_EXTERNAL_CHECKS=${WATCHDOG_EXTERNAL_CHECKS:-n}
        - WATCHDOG_MYSQL_REPLICATION_CHECKS=${WATCHDOG_MYSQL_REPLICATION_CHECKS:-n}
        - WATCHDOG_VERBOSE=${WATCHDOG_VERBOSE:-n}
--- a/generate_config.sh
+++ b/generate_config.sh
@ -398,6 +398,13 @@ USE_WATCHDOG=y
 #WATCHDOG_NOTIFY_EMAIL=a@example.com,b@example.com,c@example.com
 #WATCHDOG_NOTIFY_EMAIL=

+# Send notifications to a webhook URL that receives a POST request with the content type "application/json".
+# You can use this to send notifications to services like Discord, Slack and others.
+#WATCHDOG_NOTIFY_WEBHOOK=https://discord.com/api/webhooks/XXXXXXXXXXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+# JSON body included in the webhook POST request. Needs to be in single quotes.
+# Following variables are available: SUBJECT, BODY
+#WATCHDOG_NOTIFY_WEBHOOK_BODY='{"username": "mailcow Watchdog", "content": "**${SUBJECT}**\n${BODY}"}'
+
 # Notify about banned IP (includes whois lookup)
 WATCHDOG_NOTIFY_BAN=n

--- a/update.sh
+++ b/update.sh
@ -441,6 +441,8 @@ CONFIG_ARRAY=(
  "SKIP_SOGO"
  "USE_WATCHDOG"
  "WATCHDOG_NOTIFY_EMAIL"
+  "WATCHDOG_NOTIFY_WEBHOOK"
+  "WATCHDOG_NOTIFY_WEBHOOK_BODY"
  "WATCHDOG_NOTIFY_BAN"
  "WATCHDOG_EXTERNAL_CHECKS"
  "WATCHDOG_SUBJECT"
@ -623,6 +625,20 @@ for option in ${CONFIG_ARRAY[@]}; do
      echo "#MAILDIR_SUB=Maildir" >> mailcow.conf
      echo "MAILDIR_SUB=" >> mailcow.conf
    fi
+  elif [[ ${option} == "WATCHDOG_NOTIFY_WEBHOOK" ]]; then
+    if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
+      echo '# Send notifications to a webhook URL that receives a POST request with the content type "application/json".' >> mailcow.conf
+      echo '# You can use this to send notifications to services like Discord, Slack and others.' >> mailcow.conf
+      echo '#WATCHDOG_NOTIFY_WEBHOOK=https://discord.com/api/webhooks/XXXXXXXXXXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' >> mailcow.conf
+    fi
+  elif [[ ${option} == "WATCHDOG_NOTIFY_WEBHOOK_BODY" ]]; then
+    if ! grep -q ${option} mailcow.conf; then
+      echo "Adding new option \"${option}\" to mailcow.conf"
+      echo '# JSON body included in the webhook POST request. Needs to be in single quotes.' >> mailcow.conf
+      echo '# Following variables are available: SUBJECT, BODY' >> mailcow.conf
+      echo '#WATCHDOG_NOTIFY_WEBHOOK_BODY=\'{"username": "mailcow Watchdog", "content": "**${SUBJECT}**\n${BODY}"}\'' >> mailcow.conf
+    fi
  elif [[ ${option} == "WATCHDOG_NOTIFY_BAN" ]]; then
    if ! grep -q ${option} mailcow.conf; then
      echo "Adding new option \"${option}\" to mailcow.conf"