From f493d3a9576905371c09dbfb73c6d5b0ae32ab31 Mon Sep 17 00:00:00 2001 From: andryyy Date: Tue, 29 Jan 2019 00:11:12 +0100 Subject: [PATCH] [Dovecot] Simplify Docker image [Dovecot] Set Dovecot plugins dynamically via file and exclude Solr if not enabled [Dovecot] Add new quarantine notification script --- data/Dockerfiles/dovecot/Dockerfile | 55 +++------ data/Dockerfiles/dovecot/docker-entrypoint.sh | 45 ++++++- data/Dockerfiles/dovecot/imapsync_cron.pl | 8 +- data/Dockerfiles/dovecot/quarantine_notify.py | 110 ++++++++++++++++++ data/Dockerfiles/dovecot/trim_logs.sh | 16 +-- 5 files changed, 177 insertions(+), 57 deletions(-) create mode 100755 data/Dockerfiles/dovecot/quarantine_notify.py diff --git a/data/Dockerfiles/dovecot/Dockerfile b/data/Dockerfiles/dovecot/Dockerfile index 38687379..d8a61a06 100644 --- a/data/Dockerfiles/dovecot/Dockerfile +++ b/data/Dockerfiles/dovecot/Dockerfile @@ -64,6 +64,9 @@ RUN apt-get update && apt-get -y --no-install-recommends install \ libregexp-common-perl \ liburi-perl \ lzma-dev \ + python-redis \ + python-jinja2 \ + python-mysql.connector \ make \ mysql-client \ procps \ @@ -73,9 +76,8 @@ RUN apt-get update && apt-get -y --no-install-recommends install \ syslog-ng \ syslog-ng-core \ syslog-ng-mod-redis \ - && rm -rf /var/lib/apt/lists/* - -RUN curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz | tar xvz \ + && rm -rf /var/lib/apt/lists/* \ + && curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz | tar xvz \ && cd dovecot-$DOVECOT_VERSION \ && ./configure --with-solr --with-mysql --with-ldap --with-lzma --with-lz4 --with-ssl=openssl --with-notify=inotify --with-storages=mdbox,sdbox,maildir,mbox,imapc,pop3c --with-bzlib --with-zlib --enable-hardening \ && make -j3 \ @@ -89,15 +91,19 @@ RUN curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz | && make install \ && make clean \ && cd .. \ - && rm -rf dovecot-2.3-pigeonhole-$PIGEONHOLE_VERSION + && rm -rf dovecot-2.3-pigeonhole-$PIGEONHOLE_VERSION \ + && cpanm Data::Uniqid Mail::IMAPClient String::Util \ + && groupadd -g 5000 vmail \ + && groupadd -g 401 dovecot \ + && groupadd -g 402 dovenull \ + && useradd -g vmail -u 5000 vmail -d /var/vmail \ + && useradd -c "Dovecot unprivileged user" -d /dev/null -u 401 -g dovecot -s /bin/false dovecot \ + && useradd -c "Dovecot login user" -d /dev/null -u 402 -g dovenull -s /bin/false dovenull \ + && touch /etc/default/locale \ + && apt-get purge -y build-essential automake autotools-dev default-libmysqlclient-dev libbz2-dev libcurl4-openssl-dev libexpat1-dev liblz-dev liblz4-dev liblzma-dev libpam-dev libssl-dev lzma-dev \ + && apt-get autoremove --purge -y \ + && rm -rf /tmp/* /var/tmp/* -RUN cpanm Data::Uniqid Mail::IMAPClient String::Util -RUN echo '* * * * * root /usr/local/bin/imapsync_cron.pl 2>&1 | /usr/bin/logger' > /etc/cron.d/imapsync -RUN echo '30 3 * * * vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync -RUN echo '* * * * * vmail /usr/local/bin/trim_logs.sh >> /dev/console 2>&1' > /etc/cron.d/trim_logs -RUN echo '25 * * * * vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc -RUN echo '30 1 * * * root /usr/local/bin/sa-rules.sh >> /dev/console 2>&1' > /etc/cron.d/sa-rules -RUN echo '0 2 * * * root /usr/bin/curl http://solr:8983/solr/dovecot/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize COPY trim_logs.sh /usr/local/bin/trim_logs.sh COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf COPY imapsync /usr/local/bin/imapsync @@ -112,32 +118,7 @@ COPY maildir_gc.sh /usr/local/bin/maildir_gc.sh COPY docker-entrypoint.sh / COPY supervisord.conf /etc/supervisor/supervisord.conf COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh - -RUN chmod +x /usr/local/lib/dovecot/sieve/rspamd-pipe-ham \ - /usr/local/lib/dovecot/sieve/rspamd-pipe-spam \ - /usr/local/bin/imapsync_cron.pl \ - /usr/local/bin/postlogin.sh \ - /usr/local/bin/imapsync \ - /usr/local/bin/trim_logs.sh \ - /usr/local/bin/sa-rules.sh \ - /usr/local/bin/maildir_gc.sh \ - /usr/local/sbin/stop-supervisor.sh - -RUN groupadd -g 5000 vmail \ - && groupadd -g 401 dovecot \ - && groupadd -g 402 dovenull \ - && useradd -g vmail -u 5000 vmail -d /var/vmail \ - && useradd -c "Dovecot unprivileged user" -d /dev/null -u 401 -g dovecot -s /bin/false dovecot \ - && useradd -c "Dovecot login user" -d /dev/null -u 402 -g dovenull -s /bin/false dovenull - -RUN touch /etc/default/locale -RUN apt-get purge -y build-essential automake autotools-dev default-libmysqlclient-dev libbz2-dev libcurl4-openssl-dev libexpat1-dev liblz-dev liblz4-dev liblzma-dev libpam-dev libssl-dev lzma-dev \ - && apt-get autoremove --purge -y +COPY quarantine_notify.py /usr/local/bin/quarantine_notify.py ENTRYPOINT ["/docker-entrypoint.sh"] CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf - -RUN rm -rf \ - /tmp/* \ - /var/tmp/* - diff --git a/data/Dockerfiles/dovecot/docker-entrypoint.sh b/data/Dockerfiles/dovecot/docker-entrypoint.sh index 85ecdf51..7072c62e 100755 --- a/data/Dockerfiles/dovecot/docker-entrypoint.sh +++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh @@ -7,11 +7,16 @@ while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${D sleep 2 done -# Hard-code env vars to scripts due to cron not passing them to the perl script -sed -i "/^\$DBUSER/c\\\$DBUSER='${DBUSER}';" /usr/local/bin/imapsync_cron.pl -sed -i "/^\$DBPASS/c\\\$DBPASS='${DBPASS}';" /usr/local/bin/imapsync_cron.pl -sed -i "/^\$DBNAME/c\\\$DBNAME='${DBNAME}';" /usr/local/bin/imapsync_cron.pl -sed -i "s/LOG_LINES/${LOG_LINES}/g" /usr/local/bin/trim_logs.sh +# Hard-code env vars to scripts due to cron not passing them to the scripts +sed -i "s/__DBUSER__/${DBUSER}/g" /usr/local/bin/imapsync_cron.pl +sed -i "s/__DBPASS__/${DBPASS}/g" /usr/local/bin/imapsync_cron.pl +sed -i "s/__DBNAME__/${DBNAME}/g" /usr/local/bin/imapsync_cron.pl + +sed -i "s/__DBUSER__/${DBUSER}/g" /usr/local/bin/quarantine_notify.py +sed -i "s/__DBPASS__/${DBPASS}/g" /usr/local/bin/quarantine_notify.py +sed -i "s/__DBNAME__/${DBNAME}/g" /usr/local/bin/quarantine_notify.py + +sed -i "s/__LOG_LINES__/${LOG_LINES}/g" /usr/local/bin/trim_logs.sh # Create missing directories [[ ! -d /usr/local/etc/dovecot/sql/ ]] && mkdir -p /usr/local/etc/dovecot/sql/ @@ -87,7 +92,17 @@ EOF echo -n ${ACL_ANYONE} > /usr/local/etc/dovecot/acl_anyone -# Create userdb dict for Dovecot +if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then +echo -n 'quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify' > /usr/local/etc/dovecot/mail_plugins +echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify mail_log' > /usr/local/etc/dovecot/mail_plugins_imap +echo -n 'quota sieve acl zlib listescape mail_crypt mail_crypt_acl' > /usr/local/etc/dovecot/mail_plugins_lmtp +else +echo -n 'quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify fts fts_solr' > /usr/local/etc/dovecot/mail_plugins +echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify mail_log fts fts_solr' > /usr/local/etc/dovecot/mail_plugins_imap +echo -n 'quota sieve acl zlib listescape mail_crypt mail_crypt_acl fts fts_solr' > /usr/local/etc/dovecot/mail_plugins_lmtp +fi +chmod 644 /usr/local/etc/dovecot/mail_plugins /usr/local/etc/dovecot/mail_plugins_imap /usr/local/etc/dovecot/mail_plugins_lmtp /templates/quarantine.tpl + cat < /usr/local/etc/dovecot/sql/dovecot-dict-sql-userdb.conf driver = mysql connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}" @@ -142,6 +157,24 @@ chown -R vmail:vmail /var/vmail/sieve chown -R vmail:vmail /var/volatile adduser vmail tty chmod g+rw /dev/console +chmod +x /usr/local/lib/dovecot/sieve/rspamd-pipe-ham \ + /usr/local/lib/dovecot/sieve/rspamd-pipe-spam \ + /usr/local/bin/imapsync_cron.pl \ + /usr/local/bin/postlogin.sh \ + /usr/local/bin/imapsync \ + /usr/local/bin/trim_logs.sh \ + /usr/local/bin/sa-rules.sh \ + /usr/local/bin/maildir_gc.sh \ + /usr/local/sbin/stop-supervisor.sh + +# Setup cronjobs +echo '* * * * * root /usr/local/bin/imapsync_cron.pl 2>&1 | /usr/bin/logger' > /etc/cron.d/imapsync +echo '30 3 * * * vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync +echo '* * * * * vmail /usr/local/bin/trim_logs.sh >> /dev/console 2>&1' > /etc/cron.d/trim_logs +echo '25 * * * * vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc +echo '30 1 * * * root /usr/local/bin/sa-rules.sh >> /dev/console 2>&1' > /etc/cron.d/sa-rules +echo '0 2 * * * root /usr/bin/curl http://solr:8983/solr/dovecot/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize +echo '*/20 * * * * vmail /usr/local/bin/quarantine_notify.py >> /dev/console 2>&1' > /etc/cron.d/quarantine_notify # Fix more than 1 hardlink issue touch /etc/crontab /etc/cron.*/* diff --git a/data/Dockerfiles/dovecot/imapsync_cron.pl b/data/Dockerfiles/dovecot/imapsync_cron.pl index 1ce46c4c..4fad97ab 100755 --- a/data/Dockerfiles/dovecot/imapsync_cron.pl +++ b/data/Dockerfiles/dovecot/imapsync_cron.pl @@ -26,16 +26,12 @@ sub qqw($) { return @values } -$DBNAME = ''; -$DBUSER = ''; -$DBPASS = ''; - $run_dir="/tmp"; -$dsn = "DBI:mysql:database=" . $DBNAME . ";mysql_socket=/var/run/mysqld/mysqld.sock"; +$dsn = 'DBI:mysql:database=__DBNAME__;mysql_socket=/var/run/mysqld/mysqld.sock'; $lock_file = $run_dir . "/imapsync_busy"; $lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1); $lockmgr->lock($lock_file) || die "can't lock ${lock_file}"; -$dbh = DBI->connect($dsn, $DBUSER, $DBPASS, { +$dbh = DBI->connect($dsn, '__DBUSER__', '__DBPASS__', { mysql_auto_reconnect => 1, mysql_enable_utf8mb4 => 1 }); diff --git a/data/Dockerfiles/dovecot/quarantine_notify.py b/data/Dockerfiles/dovecot/quarantine_notify.py new file mode 100755 index 00000000..fd5a8072 --- /dev/null +++ b/data/Dockerfiles/dovecot/quarantine_notify.py @@ -0,0 +1,110 @@ +#!/usr/bin/python + +import smtplib +import os +import mysql.connector +from email.MIMEMultipart import MIMEMultipart +from email.MIMEText import MIMEText +from email.Utils import COMMASPACE, formatdate +import cgi +import jinja2 +from jinja2 import Template +import json +import redis +import time + +while True: + try: + r = redis.StrictRedis(host='redis', decode_responses=True, port=6379, db=0) + r.ping() + except Exception as ex: + print '%s - trying again...' % (ex) + time.sleep(3) + else: + break + +time_now = int(time.time()) + +def query_mysql(query, headers = True, update = False): + while True: + try: + cnx = mysql.connector.connect(unix_socket = '/var/run/mysqld/mysqld.sock', user='__DBUSER__', passwd='__DBPASS__', database='__DBNAME__', charset="utf8") + except Exception as ex: + print '%s - trying again...' % (ex) + time.sleep(3) + else: + break + cur = cnx.cursor() + cur.execute(query) + if not update: + result = [] + columns = tuple( [d[0].decode('utf8') for d in cur.description] ) + for row in cur: + if headers: + result.append(dict(zip(columns, row))) + else: + result.append(row) + cur.close() + cnx.close() + return result + else: + cnx.commit() + cur.close() + cnx.close() + +def notify_rcpt(rcpt, msg_count): + meta_query = query_mysql('SELECT id, subject, sender, created FROM quarantine WHERE notified = 0 AND rcpt = "%s"' % (rcpt)) + if r.get('Q_HTML'): + try: + template = Template(r.get('Q_HTML')) + except: + print "Error: Cannot parse quarantine template, falling back to default template." + with open('/templates/quarantine.tpl') as file_: + template = Template(file_.read()) + else: + with open('/templates/quarantine.tpl') as file_: + template = Template(file_.read()) + html = template.render(meta=meta_query, counter=msg_count) + count = 0 + while count < 15: + try: + server = smtplib.SMTP('postfix', 589, 'quarntine') + server.ehlo() + msg = MIMEMultipart('alternative') + msg['From'] = r.get('Q_SENDER') or "quarantine@localhost" + msg['Subject'] = r.get('Q_SUBJ') or "Spam Quarantine Notification" + msg['Date'] = formatdate(localtime = True) + text = "You have %d new items" % (msg_count) + text_part = MIMEText(text, 'plain') + html_part = MIMEText(html, 'html') + msg.attach(text_part) + msg.attach(html_part) + msg['To'] = str(rcpt) + text = msg.as_string() + server.sendmail(msg['From'], msg['To'], text) + server.quit() + for res in meta_query: + query_mysql('UPDATE quarantine SET notified = 1 WHERE id = "%d"' % (res['id']), update = True) + r.hset('Q_LAST_NOTIFIED', record['rcpt'], time_now) + break + except Exception as ex: + print '%s' % (ex) + time.sleep(3) + +records = query_mysql('SELECT count(id) AS counter, rcpt FROM quarantine WHERE notified = 0 GROUP BY rcpt') + +for record in records: + last_notification = int(r.hget('Q_LAST_NOTIFIED', record['rcpt'])) or 0 + attrs_json = query_mysql('SELECT attributes FROM mailbox WHERE username = "%s"' % (record['rcpt'])) + attrs = json.loads(str(attrs_json[0]['attributes'])) + if attrs['quarantine_notification'] == 'hourly': + if last_notification == 0 or (last_notification + 3600) > time_now: + notify_rcpt(record['rcpt'], record['counter']) + elif attrs['quarantine_notification'] == 'daily': + if last_notification == 0 or (last_notification + 86400) > time_now: + notify_rcpt(record['rcpt'], record['counter']) + elif attrs['quarantine_notification'] == 'weekly': + if last_notification == 0 or (last_notification + 604800) > time_now: + notify_rcpt(record['rcpt'], record['counter']) + else: + break diff --git a/data/Dockerfiles/dovecot/trim_logs.sh b/data/Dockerfiles/dovecot/trim_logs.sh index 7630b0ae..2fec55d3 100755 --- a/data/Dockerfiles/dovecot/trim_logs.sh +++ b/data/Dockerfiles/dovecot/trim_logs.sh @@ -7,12 +7,12 @@ catch_non_zero() { echo "Command ${CMD} failed to execute, exit code was ${EC}" fi } -catch_non_zero "/usr/bin/redis-cli -h redis LTRIM ACME_LOG 0 LOG_LINES" -catch_non_zero "/usr/bin/redis-cli -h redis LTRIM POSTFIX_MAILLOG 0 LOG_LINES" -catch_non_zero "/usr/bin/redis-cli -h redis LTRIM DOVECOT_MAILLOG 0 LOG_LINES" -catch_non_zero "/usr/bin/redis-cli -h redis LTRIM SOGO_LOG 0 LOG_LINES" -catch_non_zero "/usr/bin/redis-cli -h redis LTRIM NETFILTER_LOG 0 LOG_LINES" -catch_non_zero "/usr/bin/redis-cli -h redis LTRIM AUTODISCOVER_LOG 0 LOG_LINES" -catch_non_zero "/usr/bin/redis-cli -h redis LTRIM API_LOG 0 LOG_LINES" -catch_non_zero "/usr/bin/redis-cli -h redis LTRIM RL_LOG 0 LOG_LINES" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM ACME_LOG 0 __LOG_LINES__" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM POSTFIX_MAILLOG 0 __LOG_LINES__" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM DOVECOT_MAILLOG 0 __LOG_LINES__" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM SOGO_LOG 0 __LOG_LINES__" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM NETFILTER_LOG 0 __LOG_LINES__" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM AUTODISCOVER_LOG 0 __LOG_LINES__" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM API_LOG 0 __LOG_LINES__" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM RL_LOG 0 __LOG_LINES__"