amavisd-new-2.7.1/000751 000621 000620 00000000000 11747115633 013473 5ustar00markcmi000000 000000 amavisd-new-2.7.1/p0f-patch000640 000621 000620 00000002725 10770224160 015175 0ustar00markcmi000000 000000 --- p0f-query.h~ Mon Aug 21 16:11:27 2006 +++ p0f-query.h Mon Mar 10 18:12:33 2008 @@ -69,5 +69,5 @@ _u16 mflags; /* Masquerade flags (D_*) */ _s32 uptime; /* Uptime in hours (-1 = unknown) */ -}; +} __attribute__ ((packed)); @@ -86,5 +86,5 @@ _u32 cmisses; /* Total number of cache query misses */ _u32 uptime; /* Process uptime in seconds */ -}; +} __attribute__ ((packed)); /* --------------------------------------- */ --- p0f-query.c~ Wed Sep 6 14:24:41 2006 +++ p0f-query.c Mon Mar 10 18:19:46 2008 @@ -75,5 +75,5 @@ cur->ports = (sport << 16) + dport; - memset(sc,0,sizeof(sc)); + memset(sc,0,sizeof(*sc)); if (genre) { strncpy(sc->genre,genre,19); --- p0f.c~ Sun Mar 9 23:43:26 2008 +++ p0f.c Mon Mar 10 19:32:36 2008 @@ -1133,5 +1133,5 @@ if (use_cache || find_masq) p0f_addcache(src,dst,sp,dp,p->os,p->desc,(p->no_detail || fuzzy_now) ? - -1 : (p->ttl - ttl),p->no_detail ? 0 : lookup_link(mss,0), + -1 : (p->ttl - ttl),p->no_detail ? 0 : lookup_link(mss,1), tos_desc, orig_df ^ df, nat, !p->userland, mss, p-sig, tstamp ? tstamp / 360000 : -1); @@ -1232,5 +1232,5 @@ if (use_cache) - p0f_addcache(src,dst,sp,dp,0,0,-1,lookup_link(mss,0),tos_desc, + p0f_addcache(src,dst,sp,dp,0,0,-1,lookup_link(mss,1),tos_desc, 0,nat,0 /* not real, we're not sure */ ,mss,(_u32)-1, tstamp ? tstamp / 360000 : -1); amavisd-new-2.7.1/amavisd-new.spec000640 000621 000620 00000014404 11003706426 016554 0ustar00markcmi000000 000000 # Upstream: # # %define logmsg logger -t %{name}/rpm Summary: Mail virus-scanner Name: amavisd-new Version: 2.6.0 Release: 1 License: GPL Group: System Environment/Daemons URL: http://www.ijs.si/software/amavisd/ Packager: Marius Andreiana Vendor: Amavisd-new Source: http://www.ijs.si/software/amavisd/amavisd-new-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root Requires: arc >= 5.21e, nomarch >= 1.2, unrar >= 2.71, zoo >= 2.10 Requires: bzip2, cpio, file, freeze, lha, lzop, ncompress, unarj Requires: perl(Archive::Tar), perl(Archive::Zip), perl(Compress::Zlib) Requires: perl(Convert::TNEF), perl(Convert::UUlib), perl(IO::Stringy) Requires: perl(MIME::Base64), perl(MIME::Tools), perl(Unix::Syslog) Requires: perl(Time::HiRes), perl(Digest::MD5), perl(Digest::SHA1) Requires: perl(Digest::HMAC), perl(Net::DNS), perl(Mail::SpamAssassin) Requires: perl-MailTools, perl(Net::Server) >= 0.86, perl-HTML-Parser >= 3.24 Requires: perl(DB_File), perl(Mail::DKIM) >= 0.31 Obsoletes: amavisd %description AMaViS is a program that interfaces a mail transfer agent (MTA) with one or more virus scanners. Amavisd-new is a branch created by Mark Martinec that adds serveral performance and robustness features. It's partly based on work being done on the official amavisd branch. Please see the README.amavisd-new-RELNOTES file for a detailed description. %prep %setup -n amavisd-new-%{version} %{__cat} <<'EOF' >amavisd.sysconfig EOF %{__cat} <<'EOF' >amavisd.sysv #!/bin/bash # # Init script for AMaViS email virus scanner. # # Written by Dag Wieers . # Modified by Marius Andreiana. # # chkconfig: 2345 79 31 # description: AMaViS virus scanner. # # processname: amavisd # config: %{_sysconfdir}/amavisd.conf # pidfile: %{_localstatedir}/run/amavisd.pid source %{_initrddir}/functions [ -x %{_sbindir}/amavisd ] || exit 1 [ -r %{_sysconfdir}/amavisd.conf ] || exit 1 ### Default variables AMAVIS_ACCOUNT="amavis" SYSCONFIG="%{_sysconfdir}/sysconfig/amavisd" prog_config_file=%{_sysconfdir}/amavisd.conf ### Read configuration [ -r "$SYSCONFIG" ] && source "$SYSCONFIG" RETVAL=0 prog="amavisd" desc="Mail Virus Scanner" start() { echo -n $"Starting $desc ($prog): " daemon --user "$AMAVIS_ACCOUNT" %{_sbindir}/$prog -c $prog_config_file RETVAL=$? echo [ $RETVAL -eq 0 ] && touch %{_localstatedir}/lock/subsys/$prog return $RETVAL } stop() { echo -n $"Shutting down $desc ($prog): " su - $AMAVIS_ACCOUNT -c "%{_sbindir}/$prog -c $prog_config_file stop" RETVAL=$? echo [ $RETVAL -eq 0 ] && rm -f %{_localstatedir}/lock/subsys/$prog return $RETVAL } reload() { echo -n $"Reloading $desc ($prog): " su - $AMAVIS_ACCOUNT -c "%{_sbindir}/$prog -c $prog_config_file reload" RETVAL=$? echo return $RETVAL } restart() { stop start } case "$1" in start) start ;; stop) stop ;; restart) restart ;; reload) reload ;; condrestart) [ -e %{_localstatedir}/lock/subsys/$prog ] && restart RETVAL=$? ;; status) status $prog RETVAL=$? ;; *) echo $"Usage: $0 {start|stop|restart|reload|condrestart|status}" RETVAL=1 esac exit $RETVAL EOF %build %install %{__rm} -rf %{buildroot} %{__install} -d -m0755 %{buildroot}%{_sbindir} %{__perl} -pi.orig -e ' s|=\s*'\''vscan'\''|= "amavis"|; s|^#*(\$MYHOME)\s*=.*$|$1 = "%{_localstatedir}/spool/amavis";|; s|^#*(\$QUARANTINEDIR)\s*=.*$|$1 = "\$MYHOME/virusmails";| ' amavisd.conf %{__install} -d -m0700 %{buildroot}%{_localstatedir}/spool/amavis/virusmails/ %{__install} -d -m0700 %{buildroot}%{_localstatedir}/amavis/tmp %{__install} -d -m0700 %{buildroot}%{_localstatedir}/amavis/db %{__install} -D -m0755 amavisd %{buildroot}%{_sbindir}/amavisd %{__install} -D -m0755 amavisd-nanny %{buildroot}%{_sbindir}/amavisd-nanny %{__install} -D -m0755 amavisd-agent %{buildroot}%{_sbindir}/amavisd-agent %{__install} -D -m0755 p0f-analyzer.pl %{buildroot}%{_sbindir}/p0f-analyzer.pl %{__install} -D -m0755 amavisd.sysv %{buildroot}%{_initrddir}/amavisd %{__install} -D -m0700 amavisd.conf %{buildroot}%{_sysconfdir}/amavisd.conf %{__install} -D -m0644 LDAP.schema %{buildroot}%{_sysconfdir}/openldap/schema/amavisd-new.schema %{__install} -D -m0644 amavisd.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/amavisd %clean %{__rm} -rf %{buildroot} %pre /usr/sbin/useradd -c "AMaViS email scanner user" -M -s /bin/sh -r amavis \ -d "/var/spool/amavis" &>/dev/null || : %post /sbin/chkconfig --add amavisd if [ -r /etc/postfixes/aliases ]; then if ! grep -q "^virusalert:" /etc/postfix/aliases; then echo -e "virusalert:\troot" >> /etc/postfix/aliases if [ -x /usr/bin/newaliases ]; then /usr/bin/newaliases &>/dev/null else %logmsg "Cannot exec newaliases. Please run it manually." fi fi fi if [ -r /etc/mail/aliases ]; then if ! grep -q "^virusalert:" /etc/mail/aliases; then echo -e "virusalert:\troot" >> /etc/mail/aliases if [ -x /usr/bin/newaliases ]; then /usr/bin/newaliases &>/dev/null else %logmsg "Cannot exec newaliases. Please run it manually." fi fi fi %preun if [ $1 -eq 0 ] ; then /sbin/service amavisd stop &>/dev/null || : /sbin/chkconfig --del amavisd fi %postun if [ $1 -ne 0 ]; then /sbin/service amavisd condrestart &>/dev/null || : fi if [ "`getent passwd amavis`" ]; then echo -en "removing user amavis.\n" /usr/sbin/userdel "amavis" 2>/dev/null || : fi if [ "`getent group amavis`" ]; then echo -en "removing group amavis.\n" /usr/sbin/groupdel "amavis" 2>/dev/null || : fi %files %defattr(-, root, root, 0755) %doc AAAREADME.first LDAP.schema LICENSE MANIFEST RELEASE_NOTES README_FILES/* test-messages/ %config %{_initrddir}/amavisd %config %{_sysconfdir}/openldap/schema/*.schema #%{_sbindir}/amavis %{_sbindir}/amavisd %defattr(0640, root, amavis, 0755) %config(noreplace) %{_sysconfdir}/amavisd.conf %config(noreplace) %{_sysconfdir}/sysconfig/amavisd %defattr(0700, amavis, amavis, 0700) %dir %{_localstatedir}/spool/amavis/ %dir %{_localstatedir}/spool/amavis/virusmails/ %dir %{_localstatedir}/amavis %dir %{_localstatedir}/amavis/tmp %dir %{_localstatedir}/amavis/db %changelog * Mon Oct 06 2004 Marius Andreiana - Use amavisd's stop, reload, as Mark suggested - remove amavis user/group on uninstall - fix perms on /var/amavis * Mon Oct 04 2004 Marius Andreiana - Initial release, changed DAG's spec file amavisd-new-2.7.1/INSTALL000640 000621 000620 00000030437 11213462350 014520 0ustar00markcmi000000 000000 amavisd-new consists of a daemon 'amavisd', and (in some setups) a helper program, which is only needed with certain mail transport agents (MTA). For Postfix, Exim-V4, and dual-sendmail setups no helper program is needed for interfacing MTA with amavisd daemon . Obtaining the software: ======================= Fetch the tarball and unpack it: curl -O http://www.ijs.si/software/amavisd/amavisd-new-.tar.gz gzip -d -c amavisd-new-.tar.gz | tar xvf - cd amavisd-new- The most important files thus obtained are amavisd and amavisd.conf. Start reading with AAAREADME.first, then RELEASE_NOTES if upgrading, and INSTALL and README_FILES/ for new installations. Check also the on-line documentation at: http://www.ijs.si/software/amavisd/ and http://www.ijs.si/software/amavisd/amavisd-new-docs.html Prerequisites: ============== file(1) utility is required, the most recent version is heartly recommended (current version is 4.24 at the time of a release). There are a number of security and robustness problems with older versions. Archive::Zip (Archive-Zip-x.xx) (1.14 or later, currently 1.23) Compress::Zlib (Compress-Zlib-x.xx) (1.35 or later, currently 2.008) Compress::Raw::Zlib (Compress-Raw-Zlib) (2.017 or later) Convert::TNEF (Convert-TNEF-x.xx) Convert::UUlib (Convert-UUlib-x.xxx) (1.08 or later, stick to new versions!) MIME::Base64 (MIME-Base64-x.xx) MIME::Parser (MIME-Tools-x.xxxx) (latest version from CPAN - currently 5.425) Mail::Internet (MailTools-1.58 or later have workarounds for Perl 5.8.0 bugs) Net::Server (Net-Server-x.xx) (version 0.88 finally does setuid right) Digest::MD5 (Digest-MD5-x.xx) (2.22 or later) IO::Stringy (IO-stringy-x.xxx) Time::HiRes (Time-HiRes-x.xx) (use 1.49 or later, older can cause problems) Unix::Syslog (Unix-Syslog-x.xxx) BerkeleyDB with bdb library (preferably 4.4.20 or later) Mail::DKIM (Mail-DKIM-0.31 or later) The following external programs are used for decoding/dearchiving if they are available: compress, gzip, bzip2, nomarch (or arc), lha, arj (or unarj), rar (or unrar), unzoo (or zoo), pax, cpio, lzop, freeze (or unfreeze or melt), ripole, tnef, cabextract. Self-extracting archives (executables) can be of types zip, rar, lha or arj, and are only recognized when the corresponding dearchiver is available. optional Perl modules: Mail::SpamAssassin for doing spam scanning (latest version) DBI with appropriate DBD::* if using SQL lookups or SQL logging/quarantining Net::LDAP if using LDAP lookups Authen::SASL authenticating on mail forwarding and on submitting DSN Mail::ClamAV Perl module interface to ClamAV library SAVI Perl module interface to Sophos library (0.30 or later) optional, but usually desired: virus scanners external programs for doing virus scanning, like ClamAV Some external programs may already be provided with the system, but it is worth checking that their version is recent. The following lists the programs and their distribution sites (not necessarily the only or the official). The most crucial programs are marked with an asterisk: * file: ftp://ftp.astron.com/pub/file/ compress: ftp://ftp.warwick.ac.uk/pub/compression/ * gzip: http://www.gzip.org/ bzip2: http://www.bzip.org/ nomarch: http://rus.members.beeb.net/nomarch.html arc: ftp://ftp.kiarchive.ru/pub/unix/arcers/ lha: http://www2m.biglobe.ne.jp/~dolphin/lha/lha.htm 7z: http://p7zip.sourceforge.net/, http://www.7-zip.org/ unarj: ftp://ftp.kiarchive.ru/pub/unix/arcers/ arj: http://testcase.newmail.ru/files/ (arj is preferable to unarj) rar, unrar: http://www.rarsoft.com/, http://www.rarsoft.com/rar_add.htm, ftp://ftp.kiarchive.ru/pub/unix/arcers/ (rar preferred to unrar) unzoo: http://critical.ch/distfiles/ zoo: ftp://ftp.kiarchive.ru/pub/unix/arcers/ (zoo preferred to unzoo) lzop: http://www.lzop.org/download/ freeze: ftp://ftp.warwick.ac.uk/pub/compression/ ripOLE: http://www.pldaniels.com/ripole/ tnef: http://tnef.sourceforge.net/ * pax: http://www.gnu.org/software/paxutils/ or: http://heirloom.sourceforge.net/ cpio: http://www.gnu.org/software/cpio/ or: http://heirloom.sourceforge.net/ cabextract: http://www.kyz.uklinux.net/cabextract.php * ClamAV: http://clamav.elektrapro.com/ (open source virus scanner) SAVI: http://www.csupomona.edu/~henson/www/projects/SAVI-Perl/dist/ dspam: http://www.nuclearelephant.com/projects/dspam/ bdb: http://www.sleepycat.com/ (Berkeley db libr. used via BerkeleyDB) p0f: http://lcamtuf.coredump.cx/p0f.shtml Optional third-party utilities: MailZu: http://www.mailzu.net/ (quarantine management web UI) amavisd-milter: http://sourceforge.net/projects/amavisd-milter/ (alternative sendmail milter helper program supporting the new AM.PDP protocol) See also: http://www.ijs.si/software/amavisd/#contrib Installing the daemon: ====================== - Perl version 5.8.2 or later is recommended. While 5.6.1 may theoretically still be the lowest usable version, a bunch of problems were resolved in later Perl versions which were reported to show in certain environments. Some examples: taint bugs, socket descriptors not closed on exec, Net::Server looping waiting for a socket connect, problems with handling of UTF8/Unicode in Perl; - create (or choose) a Unix group dedicated to run amavisd daemon and possibly virus scanners. This should NOT be one of user or system groups and should NOT be shared with other applications such as mail or www (except possibly virus scanners). It is customary to name the group 'amavis' (or perhaps 'vscan' or 'sweep'); (edit /etc/group, or use system-specific tools, such as vigr) - create (or choose) a Unix account (username and its UID) dedicated to run amavisd daemon and possibly virus scanners. This should NOT be one of user or system accounts and should NOT be shared with other applications such as mail or www (except possibly virus scanners). Most certainly do NOT use "root", and do NOT use "nobody" nor an account used by mailer, such as "postfix", "mail", "smmsp" or "mailnull"). It is customary to name the user "amavis" or "vscan"; Choose a home directory (e.g. /var/amavis or /var/lib/amavis) for this user. (use vipw, or system-specific tools to add a user) Create its home directory, unless account creation procedure already did it: mkdir /var/amavis Create the following subdirectories: mkdir /var/amavis/tmp /var/amavis/var /var/amavis/db /var/amavis/home Check or set the ownership and protection of the directories to be readable and writable by the chosen UID, and not writable by other non-privileged users; chown -R amavis:amavis /var/amavis chmod -R 750 /var/amavis - unpack the amavisd-new source distribution (see 'Obtaining the software' above) wherever desired (/usr/local/src or elsewhere), and cd to that directory; - copy file amavisd to wherever you want it to reside, such as /usr/local/sbin, and make sure its protection setting allows it to be executed and read, but not overwritten by non-privileged users. This is a Perl source, so it is readable by any text viewer if needed. cp amavisd /usr/local/sbin/ chown root /usr/local/sbin/amavisd chmod 755 /usr/local/sbin/amavisd - copy file amavisd.conf to wherever you want it to reside such as /etc, and make sure it is not writable by non-privileged users, not even by amavis; cp amavisd.conf /etc/ chown root:amavis /etc/amavisd.conf chmod 640 /etc/amavisd.conf (if the file contains sensitive information like a password for accessing a SQL database, it should not be world-readable) Some sites prefer location /etc/amavis/ or /usr/local/etc/. If using a non-default location, one may use a command line option -c when starting the daemon to specify a non-default configuration file, or provide a soft link at the default location. Multiple -c options are permitted and enable splitting the config file into sections such as site-specific and general sections; - create a directory (e.g. /var/virusmails) to be used by amavisd-new as a quarantine area (if quarantining to files is desired). Set ownership and protection of the directory to be readable and writable by the chosen UID, and not writable by other non-privileged users; mkdir /var/virusmails chown amavis:amavis /var/virusmails chmod 750 /var/virusmails - edit file /etc/amavisd.conf and adjust variables $daemon_group and $daemon_user to match the chosen group and user name, adjust variables $MYHOME, $TEMPBASE, $db_home and $QUARANTINEDIR to match the directories just created, then check/adjust other variables, for example: $MYHOME = '/var/amavis'; $TEMPBASE = "$MYHOME/tmp"; $db_home = "$MYHOME/db"; Optionally, if $MYHOME is preferred uncluttered and for extra security owned by root (not modifyable by user amavis): $MYHOME = '/var/amavis'; $helpers_home = "$MYHOME/home"; $pid_file = "$helpers_home/amavisd.pid"; $lock_file = "$helpers_home/amavisd.lock"; in which case the ownership of /var/amavisd should be changed to root and ownership of /var/amavis/home must be amavis: chown root /var/amavis chown -R amavis:amavis /var/amavis/home chmod 750 /var/amavis /var/amavis/home If $TEMPBASE resides on a dedicated file system, it may be prudent to specify mount options: noexec,nosuid,nodev. - install virus scanners (if they are to be used), and Perl module Mail::SpamAssassin (if desired), and adjust variables in /etc/amavisd.conf. There are several other Perl modules needed by amavisd daemon (see 'Prerequisites') - if they are not yet installed, a list of missing modules will be logged when amavisd is started; - some virus scanners run as daemons or change UID when checking files. It is easiest to run such virus scanners under the same UID/GID (or at least within the same group) as amavisd, to avoid file permission problems when virus scanner reads files prepared for checking by amavisd daemon. Some virus scanners may require write permission to the $TEMPBASE directory to be able to create auxiliary files there. If a different UID is preferred for an AV scanner, a solution for ClamAV is to add user clamav to the amavis group (e.g.: vscan:*:110:clamav in a file /etc/group), and then add: AllowSupplementaryGroups yes to clamd.conf. - start the program 'amavisd', either as root (possibly with option -u user), or with su(1) as the user chosen above. It should start up, and (if root) change its GID/UID to the setting provided. It is wise to start it up for the first time with a 'debug' option: /usr/local/sbin/amavisd -u vscan debug or: /usr/local/sbin/amavisd debug When checking SpamAssassin operations, the following can be useful: /usr/local/sbin/amavisd debug-sa - later when everything has been tested and works, a shell script amavisd_init.sh or similar may be made to run at system startup/shutdown time; - depending on the mailer used, read the appropriate README.* file and follow instructions there. With some mailers (Postfix, Exim V4 or a dual-MTA setup with any SMTP-capable mailers, including sendmail) no helper program is needed. With some other mailers (sendmail milter, or historical sendmail invoking content filter via local delivery agent) one of the supplied helper programs is needed: amavisd-milter.c, or amavis.c respectively. These are available from the helper-progs subdirectory. The helper-progs/config.h.in may need to be adjusted to match the system and amavisd configuration settings. See also alternative sendmail milter supporting the new AM.PDP protocol at http://sourceforge.net/projects/amavisd-milter/ . NOTE: Check amavisd-new web page at http://www.ijs.si/software/amavisd/ if there are any patches needed for external components, such as Net::Server module or Razor agents. Testing the daemon: =================== Initial checkout is described in MTA-specific README.* file, please follow instructions there. The subdirectory test-messages contains a couple of sample mail messages, and brief instructions for testing are in file README there. amavisd-new-2.7.1/amavisd-release000751 000621 000620 00000022315 11561036651 016462 0ustar00markcmi000000 000000 #!/usr/bin/perl -T #------------------------------------------------------------------------------ # This is amavisd-release, an EXAMPLE quarantine release utility program. # It uses AM.PDP protocol to send a request to amavisd daemon to release # a quarantined mail message. # # Usage: # amavisd-release mail_file secret_id [alt_recip1 alt_recip2 ..] # # To be placed in amavisd.conf: # $interface_policy{'SOCK'} = 'AM.PDP'; # $policy_bank{'AM.PDP'} = { protocol=>'AM.PDP' }; # $unix_socketname = '/var/amavis/amavisd.sock'; #or: # $interface_policy{'9998'} = 'AM.PDP'; # $policy_bank{'AM.PDP'} = { protocol=>'AM.PDP' }; # $inet_socket_port = [10024,9998]; # # To obtain secret_id and quar_type belonging to some mail_id: # $ mysql mail_log -e \ # 'select secret_id,quar_type,content from msgs where mail_id="W+7uJyXUjw32"' # # If secret_id is not available, administrator may choose to skip checking # of secret_id in the amavisd daemon by setting a configuration variable # $auth_required_release to false (it defaults to true). If the release # client program specifies a nonempty secret_id in the request, the secret_id # will be validated and a request will fail if not valid, regardless of the # setting of $auth_required_release. Turning off a requirement for a valid # secret_id widens the right to release to anyone who can connect to amavisd # socket (Unix or inet). Access to the socket therefore needs to be restricted # using socket protection (unix socket) or @inet_acl (for inet socket). # # Author: Mark Martinec # Copyright (C) 2005-2011 Mark Martinec, All Rights Reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of the author, nor the name of the "Jozef Stefan" # Institute, nor the names of contributors may be used to endorse or # promote products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER # OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # #(the license above is the new BSD license, and pertains to this program only) # # Patches and problem reports are welcome. # The latest version of this program is available at: # http://www.ijs.si/software/amavisd/ #------------------------------------------------------------------------------ use warnings; use warnings FATAL => 'utf8'; no warnings 'uninitialized'; use strict; use re 'taint'; use IO::Socket; use Time::HiRes (); use vars qw($VERSION); $VERSION = 1.510; use vars qw($log_level $socketname); $log_level = 1; # $socketname = '127.0.0.1:9998'; $socketname = '/var/amavis/amavisd.sock'; sub sanitize_str { my($str, $keep_eol) = @_; my(%map) = ("\r" => '\\r', "\n" => '\\n', "\f" => '\\f', "\t" => '\\t', "\b" => '\\b', "\e" => '\\e', "\\" => '\\\\'); if ($keep_eol) { $str =~ s/([^\012\040-\133\135-\176])/ # and \240-\376 ? exists($map{$1}) ? $map{$1} : sprintf(ord($1)>255 ? '\\x{%04x}' : '\\%03o', ord($1))/eg; } else { $str =~ s/([^\040-\133\135-\176])/ # and \240-\376 ? exists($map{$1}) ? $map{$1} : sprintf(ord($1)>255 ? '\\x{%04x}' : '\\%03o', ord($1))/eg; } $str; } sub ll($) { my($level) = @_; $level <= $log_level; } sub do_log($$;@) { my($level, $errmsg, @args) = @_; $errmsg = sprintf($errmsg,@args) if @args; print STDERR sanitize_str($errmsg),"\n" if $level <= $log_level; } sub proto_decode($) { my($str) = @_; $str =~ s/%([0-9a-fA-F]{2})/pack("C",hex($1))/eg; $str; } sub proto_encode($@) { my($attribute_name,@strings) = @_; local($1); $attribute_name =~ # encode all but alfanumerics, '_' and '-' s/([^0-9a-zA-Z_-])/sprintf("%%%02x",ord($1))/eg; for (@strings) { # encode % and nonprintables s/([^\041-\044\046-\176])/sprintf("%%%02x",ord($1))/eg; } $attribute_name . '=' . join(' ',@strings); } sub ask_amavisd($$) { my($sock,$query_ref) = @_; my(@encoded_query) = map { /^([^=]+)=(.*)\z/s; proto_encode($1,$2) } @$query_ref; do_log(2,'> '.$_) for @encoded_query; $sock->print( map { $_."\015\012" } (@encoded_query,'') ) or die "Can't write response to socket: $!"; $sock->flush or die "Can't flush on socket: $!"; my(%attr); local($/) = "\015\012"; # set line terminator to CRLF # must not use \r and \n, which may not be \015 and \012 on certain platforms do_log(2,"waiting for response"); while(<$sock>) { last if /^\015\012\z/; # end of response if (/^ ([^=\000\012]*?) (=|:[ \t]*) ([^\012]*?) \015\012 \z/xsi) { my($attr_name) = proto_decode($1); my($attr_val) = proto_decode($3); if (!exists $attr{$attr_name}) { $attr{$attr_name} = [] } push(@{$attr{$attr_name}}, $attr_val); } } if (!defined($_) && $! != 0) { die "read from socket failed: $!" } \%attr; } sub release_file($$$@) { my($sock,$mail_file,$secret_id,@alt_recips) = @_; my($fn_path,$fn_prefix,$mail_id,$fn_suffix,$part_tag); local($1,$2,$3,$4); $part_tag = $1 if $mail_file =~ s/ \[ ( [^\]]* ) \] \z//xs; if ($mail_file =~ m{^ ([^/].*/)? ([A-Z0-9][A-Z0-9._-]*[_-])? ([A-Z0-9][A-Z0-9_+-]{10,14}[A-Z0-9]) (\.gz)? \z}xsi) { ($fn_path,$fn_prefix,$mail_id,$fn_suffix) = ($1,$2,$3,$4); } elsif ($mail_file =~ m{^ ([^/].*/)? () ([A-Za-z0-9$._=+-]+?) (\.gz)?\z}xs){ ($fn_path,$fn_prefix,$mail_id,$fn_suffix) = ($1,$2,$3,$4); # old style } else { usage("Invalid quarantine ID: $mail_file"); } my($quar_type) = $fn_suffix eq '.gz' ? 'Z' : $fn_path ne '' ? 'F' : ''; my($request_type) = $0 =~ /\breport\z/i ? 'report' : $0 =~ /\brequeue\z/i ? 'requeue' : 'release'; my(@query); push(@query, "request=$request_type"); push(@query, "mail_id=$mail_id"); push(@query, "quar_type=$quar_type") if $quar_type ne ''; push(@query, "secret_id=$secret_id") if $secret_id ne ''; push(@query, "mail_file=$mail_file"); # ignored with an SQL quarantine push(@query, "partition_tag=$part_tag") if $part_tag ne ''; if (@alt_recips) { # override original recipients if list is nonempty push(@query, map {"recipient=$_"} @alt_recips); } my($attr_ref) = ask_amavisd($sock,\@query); $attr_ref && %$attr_ref or die "Invalid response received"; if (ll(2)) { for my $attr_name (keys %$attr_ref) { for my $attr_val (@{$attr_ref->{$attr_name}}) { do_log(2,"< %s=%s", $attr_name,$attr_val); } } } do_log(0,$_) for (@{$attr_ref->{'setreply'}}); # may do another release request on the same session if needed ... } sub usage(;$) { my($msg) = @_; print STDERR $msg,"\n\n" if $msg ne ''; my($prog) = $0; $prog =~ s{^.*/(?=[^/]+\z)}{}; print STDERR "$prog version $VERSION\n"; die "Usage: \$ $prog mail_file [secret_id [alt_recip1 alt_recip2 ...]]\n". " or to read request lines from stdin: \$ $prog -\n"; } # Main program starts here my($sock,$mail_file,$secret_id); @ARGV >= 1 or usage("Not enough arguments"); $mail_file = shift(@ARGV); # quarantine file id or '-' if ($mail_file eq '-') { @ARGV==0 or usage("Extra arguments after '-'") } my($is_inet) = $socketname=~m{^/} ? 0 : 1; # simpleminded: unix vs. inet sock if ($is_inet) { # inet socket $sock = IO::Socket::INET->new($socketname) or die "Can't connect to INET socket $socketname: $!"; } else { # unix socket $sock = IO::Socket::UNIX->new(Type => SOCK_STREAM) or die "Can't create UNIX socket: $!"; $sock->connect( pack_sockaddr_un($socketname) ) or die "Can't connect to UNIX socket $socketname: $!"; } if ($mail_file eq '-') { undef $!; while (<>) { # read from STDIN: mail_file [secret_id [alt_recips...]] chomp; next if /^[ \t]*(#.*)?$/; # skip empty lines or comments my($mail_file, $secret_id, @alt_recips) = split(' '); release_file($sock,$mail_file,$secret_id,@alt_recips); } $!==0 or die "Error reading from STDIN: $!"; } else { # assume empty secret_id if the second arg looks like e-mail addr $secret_id = $ARGV[0]=~/\@/ ? '' : shift(@ARGV); release_file($sock,$mail_file,$secret_id,@ARGV); } $sock->close or die "Error closing socket: $!"; close(STDIN) or die "Error closing STDIN: $!"; amavisd-new-2.7.1/TODO000640 000621 000620 00000011313 11216470420 014147 0ustar00markcmi000000 000000 SMTP, LMTP, mail address handling - split_localpart(): check 'owner-special' handling, e.g. foo-request-spam@ - separate the application from SMTP protocol handling - fallback relays, MX backups? - one_response_for_all: report (propagate) all MTA SMTP responses if different MAIL MODIFICATIONS, EXPAND, NOTIFICATIONS: - make possible to use proper MIME content structure in notifications; - optionally attach (chopped?) mail body to DSN? - use modified headers (and body?) as provided by SA ? - macro expander: do not replace formal arguments % which are within quoted replacement text (in the regexp macro as well); - use macro expander instead of simple string replacements in Subject template edits; - provide per-recipient 'header_edits' property so that custom hooks will be able to add per-recipient headers; suggested by Jacob Leaver; - per-virus notifications, e.g. include URL or instructions in DSN (could make use of associative arrays in EXPAND if available); - don't send notifications if notifications text turns out to be empty, making it possible for a macro expansion during customization to turn off DSN (Florian Effenberger); - choose the admin sender address and From header in DSN based on recipient address/domain (how to handle multiple recipient mail?); VIRUS AND OTHER MALWARE SCANNING: - some virus scanners need option '--mbox' when given full mail file, but not on already decoded parts (Michael Boelen); - some scanners need proper file name extension to be able to recognize and decode a file correctly; - viruses_that_recips_dont_care_about; - optimize-away banned checks when bypassing banned checks is requested and $banned_namepath_re is used; - keep consulting a blacklist even if spam checking is bypassed; MAIL DECODING/DE-ARCHIVING: - unwrap pgp/gpg armoring, especially on signed-only messages; - file(1) is unable to differentiate or recognize various types of pgp/gpg mail (signed/encrypted/armored/signature/key); - per-recipient bypass_decoding; - yEnc encoding www.yenc.org (NNTP); Appledouble encoding, Macintosh StuffIt; - store_mgr: stop_expensive_decoding_at=n ? - seek some solution to prevent decoder from attempting to create files outside of its designated directory; - provide separate failure modes of expected (I/O, system) and unexpected errors when decoding; INTERNALS, CODING, ... - amavis-milter.c: be able to approve locally originating DSN without calling amavisd to avoid deadlock (or the need to force '-odd'); - use timers in a manner providing some resiliency to clock jumps; - use multi-timers Perl module? make timer independent of its use by SA; QUARANTINE - provide per-recipient headers such as X-Spam* scores in the quarantined msg; - do the quarantining later in the flow (after forwarding) to be able to do the correct intersect between per-recipient quarantine_to and per-recipient kill level and other blockings; - disable quarantine (and virus admin notifications) based on virus name; - automatically turn off quarantinining when mail is to be delivered; - update msgrcpt.rs field after a quarantine release; - support quarantining by MTA (milter, HOLD); - invoke sa-learn when releasing false positives from quarantine; GENERAL, NEW FEATURES - provide a LDAP and SQL proxy (or make use of the Postfix proxy_read_maps) to reduce the number of sessions to LDAP and SQL server; - "reverse AM.PDP" - lookups on external information sources or provide information to external plugins; - should bypass* implicitly turn on *lovers? - do not notify recipient when they will receive mail anyway (D_PASS,*_lovers) - notify when placing on HOLD at MTA ? - configurable action on HOLD ? - timeout -> HOLD ? - notify at double bounce? (treat/describe as DISCARD) - store information about detected virus names to SQL when logging to SQL - AM.PDP should be able to split mail by discarding a request and explicitly re-sending it - README.sql-pg: change SQL datatype VARCHAR into VARBINARY for data fields mail_id, secret_id and quar_loc, and CHAR into BINARY for msgs.content and msgs.quar_type to preserve case sensitivity on string comparison operators (same change have already been done in README.sql-mysql, but the data type in amavisd calls have not yet been changed); DOCUMENTATION - documentation, documentation, documentation; - malformed mail project and the like: tests, write FAQ; - cleaner web page; SOME OF THE MORE SELF-CONTAINED PROJECTS - Net::Server IPv6 support; - write a Perl module for interfacing with libarchive (http://people.freebsd.org/~kientzle/libarchive/), which is an ambitious (currently primarily FreeBSD) programming library that can create and read several streaming archive formats, including most popular tar variants and the POSIX cpio format. amavisd-new-2.7.1/LDAP.ldif000640 000621 000620 00000075031 11325352320 015045 0ustar00markcmi000000 000000 #-------------------------------------------------------------------------- # LDAP Schema for amavisd-new Jacques Supcik, PhD #----------------------------- IP-Plus Internet Services # Release 1.2.2 Swisscom Enterprise Solutions Ltd # 30 May 2004 3050 Bern - Switzerland #-------------------------------------------------------------------------- # Copyright (c) 2004 Jacques Supcik, Swisscom Enterprise Solutions Ltd. # Permission is granted to copy, distribute and/or modify this document # under the terms of the GNU Free Documentation License, Version 1.2 # or any later version published by the Free Software Foundation; # with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. # A copy of the license is included in the section entitled "GNU # Free Documentation License". #-------------------------------------------------------------------------- # Changes made to LDAP Schema to make it import and play nicely with # Novell NDS - Michael Tracey, SONOPRESS USA, LLC April 07 2005 # ( uncomment each dn:, changetype:, add:, add X-NDS-NAME attribute, replace # "attributetype" by "attributetypes:" and "objectclasse" by "objectclasses:" # (plural,colon), and unwrap each attributetypes: and objectclasses: ) #-------------------------------------------------------------------------- # 1.3.6.1.4.1.15312 Jozef Stefan Institute's OID # 1.3.6.1.4.1.15312.2 amavisd-new # 1.3.6.1.4.1.15312.2.2 amavisd-new LDAP Elements # 1.3.6.1.4.1.15312.2.2.1 AttributeTypes # 1.3.6.1.4.1.15312.2.2.2 ObjectClasses # 1.3.6.1.4.1.15312.2.2.3 Syntax Definitions #-------------------------------------------------------------------------- # # Attribute Types #----------------- # dn: cn=amavisd,cn=schema,cn=config objectClass: olcSchemaConfig cn: amavisd olcAttributeTypes: {0}( 1.3.6.1.4.1.15312.2.2.1.1 NAME 'amavisVirusLover' DESC 'Virus Lover' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SING LE-VALUE ) olcAttributeTypes: {1}( 1.3.6.1.4.1.15312.2.2.1.2 NAME 'amavisBannedFilesLover ' DESC 'Banned Files Lover' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115 .121.1.7 SINGLE-VALUE ) olcAttributeTypes: {2}( 1.3.6.1.4.1.15312.2.2.1.3 NAME 'amavisBypassVirusCheck s' DESC 'Bypass Virus Check' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.11 5.121.1.7 SINGLE-VALUE ) olcAttributeTypes: {3}( 1.3.6.1.4.1.15312.2.2.1.4 NAME 'amavisBypassSpamChecks ' DESC 'Bypass Spam Check' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115. 121.1.7 SINGLE-VALUE ) olcAttributeTypes: {4}( 1.3.6.1.4.1.15312.2.2.1.5 NAME 'amavisSpamTagLevel' DE SC 'Spam Tag Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5Substring sMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {5}( 1.3.6.1.4.1.15312.2.2.1.6 NAME 'amavisSpamTag2Level' D ESC 'Spam Tag2 Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5Substri ngsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {6}( 1.3.6.1.4.1.15312.2.2.1.7 NAME 'amavisSpamKillLevel' D ESC 'Spam Kill Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5Substri ngsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {7}( 1.3.6.1.4.1.15312.2.2.1.8 NAME 'amavisSpamModifiesSubj ' DESC 'Modifies Subject on spam' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.14 66.115.121.1.7 SINGLE-VALUE ) olcAttributeTypes: {8}( 1.3.6.1.4.1.15312.2.2.1.9 NAME 'amavisWhitelistSender' DESC 'White List Sender' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5Sub stringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) olcAttributeTypes: {9}( 1.3.6.1.4.1.15312.2.2.1.10 NAME 'amavisBlacklistSender ' DESC 'Black List Sender' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5Su bstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) olcAttributeTypes: {10}( 1.3.6.1.4.1.15312.2.2.1.11 NAME 'amavisSpamQuarantine To' DESC 'Spam Quarantine to' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA 5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {11}( 1.3.6.1.4.1.15312.2.2.1.12 NAME 'amavisSpamLover' DES C 'Spam Lover' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SING LE-VALUE ) olcAttributeTypes: {12}( 1.3.6.1.4.1.15312.2.2.1.13 NAME 'amavisBadHeaderLover ' DESC 'Bad Header Lover' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.1 21.1.7 SINGLE-VALUE ) olcAttributeTypes: {13}( 1.3.6.1.4.1.15312.2.2.1.14 NAME 'amavisBypassBannedCh ecks' DESC 'Bypass Banned Files Check' EQUALITY booleanMatch SYNTAX 1.3.6.1.4 .1.1466.115.121.1.7 SINGLE-VALUE ) olcAttributeTypes: {14}( 1.3.6.1.4.1.15312.2.2.1.15 NAME 'amavisBypassHeaderCh ecks' DESC 'Bypass Header Check' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.146 6.115.121.1.7 SINGLE-VALUE ) olcAttributeTypes: {15}( 1.3.6.1.4.1.15312.2.2.1.16 NAME 'amavisVirusQuarantin eTo' DESC 'Virus quarantine location' EQUALITY caseIgnoreIA5Match SUBSTR case IgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VAL UE ) olcAttributeTypes: {16}( 1.3.6.1.4.1.15312.2.2.1.17 NAME 'amavisBannedQuaranti neTo' DESC 'Banned Files quarantine location' EQUALITY caseIgnoreIA5Match SUB STR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SI NGLE-VALUE ) olcAttributeTypes: {17}( 1.3.6.1.4.1.15312.2.2.1.18 NAME 'amavisBadHeaderQuara ntineTo' DESC 'Bad Header quarantine location' EQUALITY caseIgnoreIA5Match SU BSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} S INGLE-VALUE ) olcAttributeTypes: {18}( 1.3.6.1.4.1.15312.2.2.1.19 NAME 'amavisLocal' DESC 'I s user considered local' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.12 1.1.7 SINGLE-VALUE ) olcAttributeTypes: {19}( 1.3.6.1.4.1.15312.2.2.1.20 NAME 'amavisMessageSizeLim it' DESC 'Message size limit' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA 5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {20}( 1.3.6.1.4.1.15312.2.2.1.21 NAME 'amavisWarnVirusRecip ' DESC 'Notify virus recipients' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.146 6.115.121.1.7 SINGLE-VALUE ) olcAttributeTypes: {21}( 1.3.6.1.4.1.15312.2.2.1.22 NAME 'amavisWarnBannedReci p' DESC 'Notify banned file recipients' EQUALITY booleanMatch SYNTAX 1.3.6.1. 4.1.1466.115.121.1.7 SINGLE-VALUE ) olcAttributeTypes: {22}( 1.3.6.1.4.1.15312.2.2.1.23 NAME 'amavisWarnBadHeaderR ecip' DESC 'Notify bad header recipients' EQUALITY booleanMatch SYNTAX 1.3.6. 1.4.1.1466.115.121.1.7 SINGLE-VALUE ) olcAttributeTypes: {23}( 1.3.6.1.4.1.15312.2.2.1.24 NAME 'amavisVirusAdmin' DE SC 'Virus admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMa tch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {24}( 1.3.6.1.4.1.15312.2.2.1.25 NAME 'amavisNewVirusAdmin' DESC 'New virus admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5Subst ringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {25}( 1.3.6.1.4.1.15312.2.2.1.26 NAME 'amavisSpamAdmin' DES C 'Spam admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatc h SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {26}( 1.3.6.1.4.1.15312.2.2.1.27 NAME 'amavisBannedAdmin' D ESC 'Banned file admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5Subst ringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {27}( 1.3.6.1.4.1.15312.2.2.1.28 NAME 'amavisBadHeaderAdmin ' DESC 'Bad header admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5Sub stringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {28}( 1.3.6.1.4.1.15312.2.2.1.29 NAME 'amavisBannedRuleName s' DESC 'Banned rule names' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5S ubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {29}( 1.3.6.1.4.1.15312.2.2.1.30 NAME 'amavisSpamDsnCutoffL evel' DESC 'Spam DSN Cutoff Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgn oreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {30}( 1.3.6.1.4.1.15312.2.2.1.31 NAME 'amavisSpamQuarantine CutoffLevel' DESC 'Spam Quarantine Cutoff Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {31}( 1.3.6.1.4.1.15312.2.2.1.32 NAME 'amavisSpamSubjectTag ' DESC 'Spam Subject Tag' EQUALITY caseExactIA5Match SUBSTR caseExactSubstrin gsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {32}( 1.3.6.1.4.1.15312.2.2.1.33 NAME 'amavisSpamSubjectTag 2' DESC 'Spam Subject Tag2' EQUALITY caseExactIA5Match SUBSTR caseExactSubstr ingsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {33}( 1.3.6.1.4.1.15312.2.2.1.34 NAME 'amavisArchiveQuarant ineTo' DESC 'Archive quarantine location' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE -VALUE ) olcAttributeTypes: {34}( 1.3.6.1.4.1.15312.2.2.1.35 NAME 'amavisAddrExtensionV irus' DESC 'Spam Subject Tag' EQUALITY caseExactIA5Match SUBSTR caseExactSubs tringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {35}( 1.3.6.1.4.1.15312.2.2.1.36 NAME 'amavisAddrExtensionS pam' DESC 'Spam Subject Tag' EQUALITY caseExactIA5Match SUBSTR caseExactSubst ringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {36}( 1.3.6.1.4.1.15312.2.2.1.37 NAME 'amavisAddrExtensionB anned' DESC 'Spam Subject Tag' EQUALITY caseExactIA5Match SUBSTR caseExactSub stringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcAttributeTypes: {37}( 1.3.6.1.4.1.15312.2.2.1.38 NAME 'amavisAddrExtensionB adHeader' DESC 'Spam Subject Tag' EQUALITY caseExactIA5Match SUBSTR caseExact SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) olcObjectClasses: {0}( 1.3.6.1.4.1.15312.2.2.2.1 NAME 'amavisAccount' DESC 'Am avisd Account' SUP top AUXILIARY MAY ( amavisVirusLover $ amavisBypassVirusCh ecks $ amavisSpamLover $ amavisBypassSpamChecks $ amavisBannedFilesLover $ am avisBypassBannedChecks $ amavisBadHeaderLover $ amavisBypassHeaderChecks $ am avisSpamTagLevel $ amavisSpamTag2Level $ amavisSpamKillLevel $ amavisWhitelis tSender $ amavisBlacklistSender $ amavisSpamQuarantineTo $ amavisVirusQuarant ineTo $ amavisBannedQuarantineTo $ amavisBadHeaderQuarantineTo $ amavisArchiv eQuarantineTo $ amavisSpamModifiesSubj $ amavisLocal $ amavisMessageSizeLimit $ amavisWarnVirusRecip $ amavisWarnBannedRecip $ amavisWarnBadHeaderRecip $ amavisVirusAdmin $ amavisNewVirusAdmin $ amavisSpamAdmin $ amavisBannedAdmin $ amavisBadHeaderAdmin $ amavisBannedRuleNames $ amavisSpamDsnCutoffLevel $ a mavisSpamQuarantineCutoffLevel $ amavisSpamSubjectTag $ amavisSpamSubjectTag2 $ amavisAddrExtensionVirus $ amavisAddrExtensionSpam $ amavisAddrExtensionBa nned $ amavisAddrExtensionBadHeader $ cn $ description ) ) structuralObjectClass: olcSchemaConfig entryUUID: 89780868-59f2-102d-9b64-5d22657ff500 creatorsName: cn=config createTimestamp: 20081209040645Z entryCSN: 20081209040645.022675Z#000000#000#000000 modifiersName: cn=config modifyTimestamp: 20081209040645Z #-------------------------------------------------------------------------- # # GNU Free Documentation License # Version 1.2, November 2002 # # # Copyright (C) 2000,2001,2002 Free Software Foundation, Inc. # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Everyone is permitted to copy and distribute verbatim copies # of this license document, but changing it is not allowed. # # # 0. PREAMBLE # # The purpose of this License is to make a manual, textbook, or other # functional and useful document "free" in the sense of freedom: to # assure everyone the effective freedom to copy and redistribute it, # with or without modifying it, either commercially or noncommercially. # Secondarily, this License preserves for the author and publisher a way # to get credit for their work, while not being considered responsible # for modifications made by others. # # This License is a kind of "copyleft", which means that derivative # works of the document must themselves be free in the same sense. It # complements the GNU General Public License, which is a copyleft # license designed for free software. # # We have designed this License in order to use it for manuals for free # software, because free software needs free documentation: a free # program should come with manuals providing the same freedoms that the # software does. But this License is not limited to software manuals; # it can be used for any textual work, regardless of subject matter or # whether it is published as a printed book. We recommend this License # principally for works whose purpose is instruction or reference. # # # 1. APPLICABILITY AND DEFINITIONS # # This License applies to any manual or other work, in any medium, that # contains a notice placed by the copyright holder saying it can be # distributed under the terms of this License. Such a notice grants a # world-wide, royalty-free license, unlimited in duration, to use that # work under the conditions stated herein. The "Document", below, # refers to any such manual or work. Any member of the public is a # licensee, and is addressed as "you". You accept the license if you # copy, modify or distribute the work in a way requiring permission # under copyright law. # # A "Modified Version" of the Document means any work containing the # Document or a portion of it, either copied verbatim, or with # modifications and/or translated into another language. # # A "Secondary Section" is a named appendix or a front-matter section of # the Document that deals exclusively with the relationship of the # publishers or authors of the Document to the Document's overall subject # (or to related matters) and contains nothing that could fall directly # within that overall subject. (Thus, if the Document is in part a # textbook of mathematics, a Secondary Section may not explain any # mathematics.) The relationship could be a matter of historical # connection with the subject or with related matters, or of legal, # commercial, philosophical, ethical or political position regarding # them. # # The "Invariant Sections" are certain Secondary Sections whose titles # are designated, as being those of Invariant Sections, in the notice # that says that the Document is released under this License. If a # section does not fit the above definition of Secondary then it is not # allowed to be designated as Invariant. The Document may contain zero # Invariant Sections. If the Document does not identify any Invariant # Sections then there are none. # # The "Cover Texts" are certain short passages of text that are listed, # as Front-Cover Texts or Back-Cover Texts, in the notice that says that # the Document is released under this License. A Front-Cover Text may # be at most 5 words, and a Back-Cover Text may be at most 25 words. # # A "Transparent" copy of the Document means a machine-readable copy, # represented in a format whose specification is available to the # general public, that is suitable for revising the document # straightforwardly with generic text editors or (for images composed of # pixels) generic paint programs or (for drawings) some widely available # drawing editor, and that is suitable for input to text formatters or # for automatic translation to a variety of formats suitable for input # to text formatters. A copy made in an otherwise Transparent file # format whose markup, or absence of markup, has been arranged to thwart # or discourage subsequent modification by readers is not Transparent. # An image format is not Transparent if used for any substantial amount # of text. A copy that is not "Transparent" is called "Opaque". # # Examples of suitable formats for Transparent copies include plain # ASCII without markup, Texinfo input format, LaTeX input format, SGML # or XML using a publicly available DTD, and standard-conforming simple # HTML, PostScript or PDF designed for human modification. Examples of # transparent image formats include PNG, XCF and JPG. Opaque formats # include proprietary formats that can be read and edited only by # proprietary word processors, SGML or XML for which the DTD and/or # processing tools are not generally available, and the # machine-generated HTML, PostScript or PDF produced by some word # processors for output purposes only. # # The "Title Page" means, for a printed book, the title page itself, # plus such following pages as are needed to hold, legibly, the material # this License requires to appear in the title page. For works in # formats which do not have any title page as such, "Title Page" means # the text near the most prominent appearance of the work's title, # preceding the beginning of the body of the text. # # A section "Entitled XYZ" means a named subunit of the Document whose # title either is precisely XYZ or contains XYZ in parentheses following # text that translates XYZ in another language. (Here XYZ stands for a # specific section name mentioned below, such as "Acknowledgements", # "Dedications", "Endorsements", or "History".) To "Preserve the Title" # of such a section when you modify the Document means that it remains a # section "Entitled XYZ" according to this definition. # # The Document may include Warranty Disclaimers next to the notice which # states that this License applies to the Document. These Warranty # Disclaimers are considered to be included by reference in this # License, but only as regards disclaiming warranties: any other # implication that these Warranty Disclaimers may have is void and has # no effect on the meaning of this License. # # # 2. VERBATIM COPYING # # You may copy and distribute the Document in any medium, either # commercially or noncommercially, provided that this License, the # copyright notices, and the license notice saying this License applies # to the Document are reproduced in all copies, and that you add no other # conditions whatsoever to those of this License. You may not use # technical measures to obstruct or control the reading or further # copying of the copies you make or distribute. However, you may accept # compensation in exchange for copies. If you distribute a large enough # number of copies you must also follow the conditions in section 3. # # You may also lend copies, under the same conditions stated above, and # you may publicly display copies. # # # 3. COPYING IN QUANTITY # # If you publish printed copies (or copies in media that commonly have # printed covers) of the Document, numbering more than 100, and the # Document's license notice requires Cover Texts, you must enclose the # copies in covers that carry, clearly and legibly, all these Cover # Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on # the back cover. Both covers must also clearly and legibly identify # you as the publisher of these copies. The front cover must present # the full title with all words of the title equally prominent and # visible. You may add other material on the covers in addition. # Copying with changes limited to the covers, as long as they preserve # the title of the Document and satisfy these conditions, can be treated # as verbatim copying in other respects. # # If the required texts for either cover are too voluminous to fit # legibly, you should put the first ones listed (as many as fit # reasonably) on the actual cover, and continue the rest onto adjacent # pages. # # If you publish or distribute Opaque copies of the Document numbering # more than 100, you must either include a machine-readable Transparent # copy along with each Opaque copy, or state in or with each Opaque copy # a computer-network location from which the general network-using # public has access to download using public-standard network protocols # a complete Transparent copy of the Document, free of added material. # If you use the latter option, you must take reasonably prudent steps, # when you begin distribution of Opaque copies in quantity, to ensure # that this Transparent copy will remain thus accessible at the stated # location until at least one year after the last time you distribute an # Opaque copy (directly or through your agents or retailers) of that # edition to the public. # # It is requested, but not required, that you contact the authors of the # Document well before redistributing any large number of copies, to give # them a chance to provide you with an updated version of the Document. # # # 4. MODIFICATIONS # # You may copy and distribute a Modified Version of the Document under # the conditions of sections 2 and 3 above, provided that you release # the Modified Version under precisely this License, with the Modified # Version filling the role of the Document, thus licensing distribution # and modification of the Modified Version to whoever possesses a copy # of it. In addition, you must do these things in the Modified Version: # # A. Use in the Title Page (and on the covers, if any) a title distinct # from that of the Document, and from those of previous versions # (which should, if there were any, be listed in the History section # of the Document). You may use the same title as a previous version # if the original publisher of that version gives permission. # B. List on the Title Page, as authors, one or more persons or entities # responsible for authorship of the modifications in the Modified # Version, together with at least five of the principal authors of the # Document (all of its principal authors, if it has fewer than five), # unless they release you from this requirement. # C. State on the Title page the name of the publisher of the # Modified Version, as the publisher. # D. Preserve all the copyright notices of the Document. # E. Add an appropriate copyright notice for your modifications # adjacent to the other copyright notices. # F. Include, immediately after the copyright notices, a license notice # giving the public permission to use the Modified Version under the # terms of this License, in the form shown in the Addendum below. # G. Preserve in that license notice the full lists of Invariant Sections # and required Cover Texts given in the Document's license notice. # H. Include an unaltered copy of this License. # I. Preserve the section Entitled "History", Preserve its Title, and add # to it an item stating at least the title, year, new authors, and # publisher of the Modified Version as given on the Title Page. If # there is no section Entitled "History" in the Document, create one # stating the title, year, authors, and publisher of the Document as # given on its Title Page, then add an item describing the Modified # Version as stated in the previous sentence. # J. Preserve the network location, if any, given in the Document for # public access to a Transparent copy of the Document, and likewise # the network locations given in the Document for previous versions # it was based on. These may be placed in the "History" section. # You may omit a network location for a work that was published at # least four years before the Document itself, or if the original # publisher of the version it refers to gives permission. # K. For any section Entitled "Acknowledgements" or "Dedications", # Preserve the Title of the section, and preserve in the section all # the substance and tone of each of the contributor acknowledgements # and/or dedications given therein. # L. Preserve all the Invariant Sections of the Document, # unaltered in their text and in their titles. Section numbers # or the equivalent are not considered part of the section titles. # M. Delete any section Entitled "Endorsements". Such a section # may not be included in the Modified Version. # N. Do not retitle any existing section to be Entitled "Endorsements" # or to conflict in title with any Invariant Section. # O. Preserve any Warranty Disclaimers. # # If the Modified Version includes new front-matter sections or # appendices that qualify as Secondary Sections and contain no material # copied from the Document, you may at your option designate some or all # of these sections as invariant. To do this, add their titles to the # list of Invariant Sections in the Modified Version's license notice. # These titles must be distinct from any other section titles. # # You may add a section Entitled "Endorsements", provided it contains # nothing but endorsements of your Modified Version by various # parties--for example, statements of peer review or that the text has # been approved by an organization as the authoritative definition of a # standard. # # You may add a passage of up to five words as a Front-Cover Text, and a # passage of up to 25 words as a Back-Cover Text, to the end of the list # of Cover Texts in the Modified Version. Only one passage of # Front-Cover Text and one of Back-Cover Text may be added by (or # through arrangements made by) any one entity. If the Document already # includes a cover text for the same cover, previously added by you or # by arrangement made by the same entity you are acting on behalf of, # you may not add another; but you may replace the old one, on explicit # permission from the previous publisher that added the old one. # # The author(s) and publisher(s) of the Document do not by this License # give permission to use their names for publicity for or to assert or # imply endorsement of any Modified Version. # # # 5. COMBINING DOCUMENTS # # You may combine the Document with other documents released under this # License, under the terms defined in section 4 above for modified # versions, provided that you include in the combination all of the # Invariant Sections of all of the original documents, unmodified, and # list them all as Invariant Sections of your combined work in its # license notice, and that you preserve all their Warranty Disclaimers. # # The combined work need only contain one copy of this License, and # multiple identical Invariant Sections may be replaced with a single # copy. If there are multiple Invariant Sections with the same name but # different contents, make the title of each such section unique by # adding at the end of it, in parentheses, the name of the original # author or publisher of that section if known, or else a unique number. # Make the same adjustment to the section titles in the list of # Invariant Sections in the license notice of the combined work. # # In the combination, you must combine any sections Entitled "History" # in the various original documents, forming one section Entitled # "History"; likewise combine any sections Entitled "Acknowledgements", # and any sections Entitled "Dedications". You must delete all sections # Entitled "Endorsements". # # # 6. COLLECTIONS OF DOCUMENTS # # You may make a collection consisting of the Document and other documents # released under this License, and replace the individual copies of this # License in the various documents with a single copy that is included in # the collection, provided that you follow the rules of this License for # verbatim copying of each of the documents in all other respects. # # You may extract a single document from such a collection, and distribute # it individually under this License, provided you insert a copy of this # License into the extracted document, and follow this License in all # other respects regarding verbatim copying of that document. # # # 7. AGGREGATION WITH INDEPENDENT WORKS # # A compilation of the Document or its derivatives with other separate # and independent documents or works, in or on a volume of a storage or # distribution medium, is called an "aggregate" if the copyright # resulting from the compilation is not used to limit the legal rights # of the compilation's users beyond what the individual works permit. # When the Document is included in an aggregate, this License does not # apply to the other works in the aggregate which are not themselves # derivative works of the Document. # # If the Cover Text requirement of section 3 is applicable to these # copies of the Document, then if the Document is less than one half of # the entire aggregate, the Document's Cover Texts may be placed on # covers that bracket the Document within the aggregate, or the # electronic equivalent of covers if the Document is in electronic form. # Otherwise they must appear on printed covers that bracket the whole # aggregate. # # # 8. TRANSLATION # # Translation is considered a kind of modification, so you may # distribute translations of the Document under the terms of section 4. # Replacing Invariant Sections with translations requires special # permission from their copyright holders, but you may include # translations of some or all Invariant Sections in addition to the # original versions of these Invariant Sections. You may include a # translation of this License, and all the license notices in the # Document, and any Warranty Disclaimers, provided that you also include # the original English version of this License and the original versions # of those notices and disclaimers. In case of a disagreement between # the translation and the original version of this License or a notice # or disclaimer, the original version will prevail. # # If a section in the Document is Entitled "Acknowledgements", # "Dedications", or "History", the requirement (section 4) to Preserve # its Title (section 1) will typically require changing the actual # title. # # # 9. TERMINATION # # You may not copy, modify, sublicense, or distribute the Document except # as expressly provided for under this License. Any other attempt to # copy, modify, sublicense or distribute the Document is void, and will # automatically terminate your rights under this License. However, # parties who have received copies, or rights, from you under this # License will not have their licenses terminated so long as such # parties remain in full compliance. # # # 10. FUTURE REVISIONS OF THIS LICENSE # # The Free Software Foundation may publish new, revised versions # of the GNU Free Documentation License from time to time. Such new # versions will be similar in spirit to the present version, but may # differ in detail to address new problems or concerns. See # http://www.gnu.org/copyleft/. # # Each version of the License is given a distinguishing version number. # If the Document specifies that a particular numbered version of this # License "or any later version" applies to it, you have the option of # following the terms and conditions either of that specified version or # of any later version that has been published (not as a draft) by the # Free Software Foundation. If the Document does not specify a version # number of this License, you may choose any version ever published (not # as a draft) by the Free Software Foundation. amavisd-new-2.7.1/amavisd-snmp-subagent000751 000621 000620 00000150765 11517620656 017645 0ustar00markcmi000000 000000 #!/usr/bin/perl -T #------------------------------------------------------------------------------ # This program implements a SNMP AgentX (RFC 2741) subagent for amavisd-new. # # Author: Mark Martinec # Copyright (C) 2009,2010,2011 Mark Martinec, All Rights Reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of the author, nor the name of the "Jozef Stefan" # Institute, nor the names of contributors may be used to endorse or # promote products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER # OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # #(the license above is the new BSD license, and pertains to this program only) # # Patches and problem reports are welcome. # The latest version of this program is available at: # http://www.ijs.si/software/amavisd/ #------------------------------------------------------------------------------ package AmavisAgent; use strict; use re 'taint'; use warnings FATAL => 'utf8'; use Errno qw(ESRCH ENOENT EACCES EEXIST); use POSIX (); use Time::HiRes (); use IO::File qw(O_RDONLY O_WRONLY O_RDWR O_CREAT O_EXCL); use Unix::Syslog qw(:macros :subs); use BerkeleyDB; use vars qw($VERSION); $VERSION = 1.005; use vars qw($myversion $myproduct_name $myversion_id $myversion_date); $myproduct_name = 'amavis-agentx'; $myversion_id = '1.5'; $myversion_date = '20110117'; $myversion = "$myproduct_name-$myversion_id ($myversion_date)"; my($agent_name) = $myproduct_name; use vars qw($syslog_ident $syslog_facility); $syslog_ident = $myproduct_name; $syslog_facility = LOG_MAIL; my($db_home) = # DB databases directory defined $ENV{'AMAVISD_DB_HOME'} ? $ENV{'AMAVISD_DB_HOME'} : '/var/amavis/db'; my($mta_queue_dir); my($top) = '1.3.6.1.4.1.15312.2.1'; my(@databases) = ( { root_oid_str => "$top.1", name => 'am.snmp', file => 'snmp.db' }, { root_oid_str => "$top.2", name => 'am.nanny', file => 'nanny.db' }, { root_oid_str => "$top.3.1.1", name => 'pf.maildrop', file => 'maildrop', ttl => 18 }, { root_oid_str => "$top.3.1.2", name => 'pf.incoming', file => 'incoming', ttl => 18 }, { root_oid_str => "$top.3.1.3", name => 'pf.active', file => 'active', ttl => 18 }, { root_oid_str => "$top.3.1.4", name => 'pf.deferred', file => 'deferred', ttl => 18 }, ); # 1.3.6.1.4.1.15312 enterprises . Jozef Stefan Institute # 1.3.6.1.4.1.15312.2 amavisd-new # 1.3.6.1.4.1.15312.2.1 amavisd-new SNMP # 1.3.6.1.4.1.15312.2.1.1 amavisd-new Statistics # 1.3.6.1.4.1.15312.2.1.2 amavisd-new Process status # 1.3.6.1.4.1.15312.2.1.3 amavisd-new (a view into MTA queue sizes) # 1.3.6.1.4.1.15312.2.2 amavisd-new LDAP Elements my($log_level) = 0; my($daemonize) = 1; my($pid_filename); # e.g. "/var/run/amavisd-snmp-subagent.pid"; my($pid_file_created) = 0; my($syslog_open) = 0; my($num_proc_gone) = 0; # geometic progression, rounded, # common ratio = exp((ln(60)-ln(1))/6) = 1.97860 my(@age_slots) = ( 0.1, 0.2, 0.5, 1, 2, 4, 8, 15, 30, # seconds 1*60, 2*60, 4*60, 8*60, 15*60, 30*60, # minutes 1*3600, 2*3600, 4*3600, 8*3600, 15*3600, 30*3600); # hours package AmavisVariable; sub new { my($class) = @_; bless [(undef) x 7], $class } sub oid { my($self)=shift; !@_ ? $self->[0] : ($self->[0]=shift) } sub oidstr { my($self)=shift; !@_ ? $self->[1] : ($self->[1]=shift) } sub name { my($self)=shift; !@_ ? $self->[2] : ($self->[2]=shift) } sub type { my($self)=shift; !@_ ? $self->[3] : ($self->[3]=shift) } sub suffix { my($self)=shift; !@_ ? $self->[4] : ($self->[4]=shift) } sub value { my($self)=shift; !@_ ? $self->[5] : ($self->[5]=shift) } sub next { my($self)=shift; !@_ ? $self->[6] : ($self->[6]=shift) } package AmavisAgent; use NetSNMP::OID; use NetSNMP::ASN qw(:all); use NetSNMP::agent qw(:all); use NetSNMP::default_store qw(:all); my(%oidstr_to_obj); my(@oid_sorted_list); my($keep_running) = 1; my(%variables); my(%asn_name_to_type) = ( 'C32' => ASN_COUNTER, 'C64' => ASN_COUNTER64, 'G32' => ASN_GAUGE, 'INT' => ASN_INTEGER, 'I64' => ASN_INTEGER64, 'U32' => ASN_UNSIGNED, 'U64' => ASN_UNSIGNED64, 'STR' => ASN_OCTET_STR, 'OID' => ASN_OBJECT_ID, 'TIM' => ASN_TIMETICKS, ); my(%asn_type_to_full_name) = ( ASN_COUNTER, 'Counter32', ASN_COUNTER64, 'Counter64', ASN_GAUGE, 'Gauge32', ASN_INTEGER, 'Integer32', # 'IpAddress', ASN_INTEGER64, 'Integer64', ASN_UNSIGNED, 'Unsigned32', ASN_UNSIGNED64, 'Unsigned64', ASN_OCTET_STR, 'DisplayString', ASN_OBJECT_ID, 'OBJECT IDENTIFIER', ASN_TIMETICKS, 'TimeTicks', ); sub do_log($$;@) { my($level,$errmsg,@args) = @_; if ($level <= $log_level) { # treat $errmsg as sprintf format string if additional arguments provided if (@args) { $errmsg = sprintf($errmsg,@args) } if (!$syslog_open) { print STDERR $errmsg."\n"; # ignoring I/O status } else { my($prio) = $level <= -2 ? LOG_ERR : $level <= -1 ? LOG_WARNING : $level <= 0 ? LOG_NOTICE : $level <= 1 ? LOG_INFO : LOG_DEBUG; syslog(LOG_INFO, "%s", $errmsg); } } } # Returns the smallest defined number from the list, or undef sub min(@) { my($r) = @_ == 1 && ref($_[0]) ? $_[0] : \@_; # accept list, or a list ref my($m); for (@$r) { $m = $_ if defined $_ && (!defined $m || $_ < $m) } $m; } # Returns the largest defined number from the list, or undef sub max(@) { my($r) = @_ == 1 && ref($_[0]) ? $_[0] : \@_; # accept list, or a list ref my($m); for (@$r) { $m = $_ if defined $_ && (!defined $m || $_ > $m) } $m; } # Return untainted copy of a string (argument can be a string or a string ref) sub untaint($) { no re 'taint'; my($str); if (defined($_[0])) { local($1); # avoid Perl taint bug: tainted global $1 propagates taintedness $str = $1 if (ref($_[0]) ? ${$_[0]} : $_[0]) =~ /^(.*)\z/s; } $str; } sub declare_variable($$;$$$) { my($oid_str,$name, $typename,$instance_lo,$instance_hi) = @_; $typename = 'C32' if !defined $typename; $instance_lo = 0 if !defined $instance_lo; $instance_hi = $instance_lo if !defined $instance_hi; $instance_hi = $instance_lo if $instance_hi < $instance_lo; my($type) = $asn_name_to_type{$typename}; for my $ind ($instance_lo .. $instance_hi) { my($full_oid_str) = sprintf("%s.%d", $oid_str,$ind); # do_log(5, "declaring variable %s, %s", $full_oid_str, $name); my($var) = AmavisVariable->new; $var->oidstr($full_oid_str); # $var->oid(NetSNMP::OID->new($full_oid_str)); # later $var->type($type); # $var->suffix($suffix); $var->name("$name.$ind"); if (!exists $variables{"$name.$ind"}) { $variables{"$name.$ind"} = $var; } else { # allow an amavisd variable name to map to multiple SNMP variables if (ref $variables{"$name.$ind"} ne 'ARRAY') { $variables{"$name.$ind"} = [ $variables{"$name.$ind"} ]; } push(@{$variables{"$name.$ind"}}, $var); } } } sub set_variable_value($$) { my($name, $value) = @_; my($instance); local($1,$2); if ($name =~ /^(.*)\.(\d+)/) { $name = $1; $instance = $2 } $instance = 0 if !defined $instance; my($v) = $variables{"$name.$instance"}; if (!defined($v)) { do_log(5, "No such variable %s.%s", $name,$instance); } else { my(@var) = ref $v eq 'ARRAY' ? @$v : $v; for my $var (@var) { my($type) = $var->type; if ($name =~ /^TimeElapsed/) { $value = $value/10 } # ms -> 0.01s ticks elsif ($type == ASN_COUNTER || $type == ASN_GAUGE || $type == ASN_INTEGER || $type == ASN_UNSIGNED || $type == ASN_TIMETICKS) { $value = 0+$value } elsif ($type == ASN_COUNTER64 || $type == ASN_INTEGER64 || $type == ASN_UNSIGNED64) { $value = sprintf("%1.0f",$value) } elsif ($type == ASN_OCTET_STR) { $value = "$value" } $var->value($value); } } } sub reset_all_variable_values($) { my($root_oid_str) = @_; while (my($key,$v) = each(%variables)) { my(@var) = ref $v eq 'ARRAY' ? @$v : $v; for my $var (@var) { if (!defined($root_oid_str) || $var->oidstr =~ /^\Q$root_oid_str\E\./) { $var->value(undef); } } } } sub dump_variables() { for my $oid (@oid_sorted_list) { my(@oidlist) = $oid->to_array; my($oidstr) = join('.', @oidlist); my($descr) = ""; my($suffix_sp) = join(' ', @oidlist[9 .. ($#oidlist-1)]); my($var) = $oidstr_to_obj{$oidstr}; my($mib_type_name) = $asn_type_to_full_name{$var->type}; my($name) = $var->name; $name =~ s/\.0\z//; printf STDERR (<<'END', $name, $mib_type_name, $descr, $suffix_sp); %s OBJECT-TYPE SYNTAX %s MAX-ACCESS read-only STATUS current DESCRIPTION "%s" ::= { amavis %s } END } } sub init_data() { while (my($name,$v) = each(%variables)) { my(@var) = ref $v eq 'ARRAY' ? @$v : $v; for my $var (@var) { my($oidstr) = $var->oidstr; $var->oid(NetSNMP::OID->new($oidstr)); $oidstr_to_obj{$oidstr} = $var; } } @oid_sorted_list = sort { snmp_oid_compare($a,$b) } map { $_->oid } map { ref $_ eq 'ARRAY' ? @$_ : $_ } values(%variables); # build a linked list of variable objects in OID sorted order # to speed up sequential MIB traversal by getnext my($prev_var); for my $oid (@oid_sorted_list) { my($oidstr) = join('.', $oid->to_array); my($var) = $oidstr_to_obj{$oidstr}; $prev_var->next($var) if defined $prev_var; $prev_var = $var; } } sub collect_all_db_data($$) { my($database,$values) = @_; my($dbfile) = $database->{file}; my(@dbstat) = stat("$db_home/$dbfile"); my($errn) = @dbstat ? 0 : 0+$!; $errn==0 || $errn==ENOENT or die "stat $db_home/$dbfile: $!"; if (defined $database->{db} && $database->{old_db_inode} != $dbstat[1]) { $database->{db}->db_close==0 or die "BDB db_close error: $BerkeleyDB::Error $!"; undef $database->{db}; do_log(1, "Reopening snmp database %s/%s", $db_home,$dbfile); } if (!defined $database->{db} && $errn==0) { reset_all_variable_values($database->{root_oid_str}); $database->{old_db_inode} = $dbstat[1]; $database->{env} = BerkeleyDB::Env->new( -Home => $db_home, -Flags => DB_INIT_CDB | DB_INIT_MPOOL, -ErrFile => \*STDOUT, -Verbose => 1); defined $database->{env} or die "BDB no env: $BerkeleyDB::Error $!"; $database->{db} = BerkeleyDB::Hash->new(-Filename => $dbfile, -Env => $database->{env}); defined $database->{db} or die "BDB no dbS 1: $BerkeleyDB::Error $!"; } my($eval_stat,$interrupt); $interrupt = ''; if (!defined $database->{db}) { do_log(1, "No snmp database %s/%s", $db_home,$dbfile); } else { my($stat,$key,$val); my($h1) = sub { $interrupt = $_[0] }; local(@SIG{qw(INT HUP TERM TSTP QUIT ALRM USR1 USR2)}) = ($h1) x 8; eval { # be as quick as possible while a database is locked by a cursor my($cursor) = $database->{db}->db_cursor; # obtain read lock $database->{cursor} = $cursor; defined $cursor or die "db_cursor error: $BerkeleyDB::Error"; while ( ($stat=$cursor->c_get($key,$val,DB_NEXT)) == 0 ) { $values->{$key} = $val; } $stat==DB_NOTFOUND or die "c_get: $BerkeleyDB::Error $!"; $cursor->c_close==0 or die "c_close error: $BerkeleyDB::Error"; undef $database->{cursor}; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; }; if (defined $database->{db}) { # unlock, ignoring status $database->{cursor}->c_close if defined $database->{cursor}; undef $database->{cursor}; } } if ($interrupt ne '') { kill($interrupt,$$) } # resignal elsif ($eval_stat ne '') { chomp($eval_stat); die "BDB $eval_stat\n" } } sub count_files_in_postfix_dir($$); # prototype sub count_files_in_postfix_dir($$) { my($dir,$deadline) = @_; local(*DIR); my($f); my($cnt) = 0; my($aborted) = 0; if (!opendir(DIR,$dir)) { do_log(-1, "Can't open directory %s: %s", $dir,$!); } else { while (defined($f = readdir(DIR))) { next if $f eq '.' || $f eq '..'; # Postfix uses one-character subdirs if (length($f) == 1 && -d "$dir/$f") { my($n,$abt) = count_files_in_postfix_dir("$dir/$f", $deadline); $cnt += $n; if ($abt) { $aborted = 1; last } } else { $cnt++; } if (defined $deadline && Time::HiRes::time > $deadline) { $aborted = 1; last; } } closedir(DIR) or die "Error closing directory $dir: $!"; } ($cnt,$aborted); } sub update_data($) { my($database) = @_; do_log(3, "updating variables from %s", $database->{name}); my($start_time) = Time::HiRes::time; if ($database->{name} =~ /^pf/) { # not really a database file, just a 'view' into an MTA spool directory my($dir) = $database->{file}; my($cnt,$aborted) = count_files_in_postfix_dir("$mta_queue_dir/$dir", $start_time + 5); my($var_name) = "MtaQueueEntries\u$dir"; set_variable_value($var_name, $cnt); do_log(3, "mta queue: %s %d", $var_name,$cnt); do_log(-1,"exceeded time limit on dir %s, aborted after %.1f s, ". "count so far: %d", $var_name, Time::HiRes::time - $start_time, $cnt) if $aborted; } elsif ($database->{file} eq 'snmp.db') { my(%values); collect_all_db_data($database,\%values); while (my($key,$val) = each(%values)) { next if $key =~ /\.byOS\./s; next if $key =~ /^virus\.byname\./s; next if $key =~ /^(?:OpsDecType|OpsDecBy|OpsSql)/s; next if $key =~ /^entropy/s; next if $key =~ /^ContentBadHdr/ && $key !~ /^ContentBadHdrMsgs/; local($1); if ($val =~ /^(?:C32|C64) (.*)\z/) { set_variable_value($key, 0+$1); } elsif ($val =~ /^STR (.*)\z/) { set_variable_value($key, "$1"); } elsif ($key eq 'sysUpTime' && $val =~ /^INT (.*)\z/) { my($uptime) = $start_time - $1; my($ticks) = int($uptime*100); set_variable_value($key, $ticks); } elsif ($val =~ /^(?:G32|INT|I64|U32|U64|TIM) (.*)\z/) { set_variable_value($key, 0+$1); } elsif ($val =~ /^OID (.*)\z/) { set_variable_value($key, $1); } else { # set_variable_value($key, $val); } } } elsif ($database->{file} eq 'nanny.db') { my(%values); collect_all_db_data($database,\%values); my(%proc_timestamp, %proc_state, %proc_task_id); while (my($key,$val) = each(%values)) { local($1,$2); if ($val !~ /^(\d+(?:\.\d*)?) (.*?) *\z/s) { do_log(0, "Bad %s db entry: %s, %s", $database->{file},$key,$val); } else { $proc_timestamp{$key} = $1; my($task_id) = $2; $proc_state{$key} = $1 if $task_id =~ s/^([^0-9])//; $proc_task_id{$key} = $task_id; } } my(@to_be_removed); my($num_proc_idle) = 0; my($num_proc_busy) = 0; my(@num_proc_busy_by_age); my(%num_proc_busy_by_activity); for my $pid (keys(%proc_timestamp)) { my($idling) = $proc_task_id{$pid} eq '' && $proc_state{$pid} =~ /^[. ]?\z/s; my($age) = $start_time - $proc_timestamp{$pid}; my($n) = kill(0,$pid); # test if the process is still there if ($n == 0 && $! != ESRCH) { do_log(-1, "Can't check the process %s: %s", $pid,$!); } elsif ($n == 0) { # ESRCH means there is no such process push(@to_be_removed, $pid); # process went away! } elsif ($idling) { $num_proc_idle++; } else { # busy for $age seconds $num_proc_busy++; $num_proc_busy_by_age[0]++; my($j) = 1; for my $t (@age_slots) { if ($age >= $t) { $num_proc_busy_by_age[$j]++ } $j++; } my($s) = $proc_state{$pid}; if ($s eq 'm' || $s eq 'd' || $s eq 'F') { $s = 'm' } elsif ($s eq 'D' || $s eq 'V' || $s eq 'S') { } else { $s = ' ' } $num_proc_busy_by_activity{$s}++; } } $num_proc_gone += scalar(@to_be_removed); # all are gauges, except ProcGone set_variable_value('ProcAll', $num_proc_idle+$num_proc_busy); set_variable_value('ProcIdle', $num_proc_idle); set_variable_value('ProcBusy', $num_proc_busy); set_variable_value('ProcGone', $num_proc_gone); # counter! for my $j (0..@age_slots) { # age_slots start at 1, zero is in addition set_variable_value("ProcBusy$j", $num_proc_busy_by_age[$j] || 0); } set_variable_value("ProcBusyTransfer",$num_proc_busy_by_activity{'m'}||0); set_variable_value("ProcBusyDecode", $num_proc_busy_by_activity{'D'}||0); set_variable_value("ProcBusyVirus", $num_proc_busy_by_activity{'V'}||0); set_variable_value("ProcBusySpam", $num_proc_busy_by_activity{'S'}||0); set_variable_value("ProcBusyOther", $num_proc_busy_by_activity{' '}||0); if (@to_be_removed) { # some processes no longer exist, update db my($eval_stat,$interrupt); $interrupt = ''; my($h1) = sub { $interrupt = $_[0] }; local(@SIG{qw(INT HUP TERM TSTP QUIT ALRM USR1 USR2)}) = ($h1) x 8; eval { # obtain a write lock my($cursor) = $database->{db}->db_cursor(DB_WRITECURSOR); $database->{cursor} = $cursor; defined $cursor or die "BDB db_cursor error: $BerkeleyDB::Error"; for my $key (@to_be_removed) { my($val); my($stat) = $cursor->c_get($key,$val,DB_SET); $stat==0 || $stat==DB_NOTFOUND or die "BDB c_get: $BerkeleyDB::Error, $!."; if ($stat==0) { # remove existing entry $cursor->c_del==0 or die "c_del: $BerkeleyDB::Error, $!."; } } $cursor->c_close==0 or die "c_close error: $BerkeleyDB::Error"; undef $database->{cursor}; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; }; if (defined $database->{db}) { # unlock, ignoring status $database->{cursor}->c_close if defined $database->{cursor}; undef $database->{cursor}; } } } my($now) = Time::HiRes::time; my($elapsed) = $now - $start_time; $elapsed = 0 if $elapsed < 0; # clock jump? my($ll) = $elapsed >= 30 ? -1 : $elapsed >= 5 ? 0 : $elapsed >= 1 ? 2 : 3; do_log($ll, "updating %s took %.3f s", $database->{name}, $elapsed); my($ttl_lower_bound) = 8*$elapsed; # don't be a hog! my($since_query) = $database->{last_query_timestamp}; $since_query = $now - $since_query if defined $since_query; if (defined $since_query && $elapsed > 4) { # there is a chance that a SNMP client timed out on this query; # stretch the next update period to allow one quick next response # from cached data, assuming queries are at about regular intervals $ttl_lower_bound = max($ttl_lower_bound, 1.5 * $since_query); } $ttl_lower_bound = min($ttl_lower_bound, 20*60); # cap at 20 minutes my($ttl) = $database->{ttl}; $ttl = 4 if !defined $ttl || $ttl <= 0; if ($ttl < $ttl_lower_bound) { $ttl = $ttl_lower_bound; do_log(3, "postponing refresh on %s for another %.1f s%s", $database->{name}, $ttl, !defined $since_query ? '' : sprintf(", %.1f s since query",$since_query) ); } $database->{last_refreshed} = $now; $database->{update_due_at} = $now + $ttl; } sub find_next_gt($$) { my($x, $a_ref) = @_; my($l, $u) = (0, $#$a_ref); my($j); while ($l <= $u) { $j = int(($l + $u)/2); if ($a_ref->[$j] > $x) { $u = $j-1 } else { $l = $j+1 } } $l > $#$a_ref ? -1 : $l; } my($fast_poll) = 0; my($last_query_timestamp) = 0; sub snmp_handler($$$$) { my($handler, $registration_info, $request_info, $requests) = @_; my($now) = Time::HiRes::time; my($dt) = $now - $last_query_timestamp; if ($dt < 1.5) { $fast_poll = 1 } elsif ($dt > 4) { $fast_poll = 0 } $last_query_timestamp = $now; my($mode) = $request_info->getMode; for (my $req=$requests; $req; $req=$req->next) { my($oid_in_request) = $req->getOID; # OID from a request my($actual_oid); my($err); my($eom) = 0; if ($mode == MODE_GET) { $actual_oid = $oid_in_request; do_log(5, "Get %s", $oid_in_request); } elsif ($mode == MODE_GETBULK) { # never happens, not registered for getbulk do_log(2, "GetBulk %s", $oid_in_request); } elsif ($mode == MODE_GETNEXT) { if (!@oid_sorted_list) { $eom = 1; # end of MIB } elsif ($oid_in_request < $oid_sorted_list[0]) { $actual_oid = $oid_sorted_list[0]; $req->setOID($actual_oid); do_log(4, "First: %s -> %s", $oid_in_request,$actual_oid); } elsif ($oid_in_request > $oid_sorted_list[-1]) { $eom = 1; # end of MIB do_log(4, "Last: %s", $oid_in_request); } else { # check first for a sequential traversal, likely faster my($var) = $oidstr_to_obj{join('.', $oid_in_request->to_array)}; if ($var) { my($next_var) = $var->next; if (!$next_var) { $eom = 1; # end of MIB } else { $actual_oid = $next_var->oid; $req->setOID($actual_oid); } } if (!$err && !defined $actual_oid) { # fall back to a binary search do_log(5, "Using a binary search for %s", $oid_in_request); my($ind) = find_next_gt($oid_in_request, \@oid_sorted_list); if ($ind < 0) { $eom = 1; # end of MIB } else { $actual_oid = $oid_sorted_list[$ind]; $req->setOID($actual_oid); } } } do_log(5, "GetNext %s -> %s", $oid_in_request, !defined $actual_oid ? 'undef' : $actual_oid); } else { do_log(0, "Unknown request %s", $oid_in_request); $req->setError($request_info, SNMP_ERR_NOTWRITABLE); $err = 1; } if ($err) { # already dealt with } elsif ($eom || !defined $actual_oid) { # end of MIB # just silently not provide a value do_log(5, "No more MIB beyond %s", $oid_in_request); } else { my($oid_str) = join('.', $actual_oid->to_array); my($var) = $oidstr_to_obj{$oid_str}; if (!$var) { $req->setError($request_info, SNMP_ERR_NOSUCHNAME); } else { # find out under which OID root the query falls for my $database (@databases) { next if !$database->{registered}; my($root_oid_str) = $database->{root_oid_str}; if ($oid_str =~ /^\Q$root_oid_str\E\./) { $database->{last_query_timestamp} = $now; if (!defined($database->{update_due_at}) || Time::HiRes::time >= $database->{update_due_at} + ($fast_poll ? 4 : 0) ) { # fast polling stretches time-to-update a bit, increasing # chances of collecting consistent data from the same moment update_data($database); # stale MIB, needs updating } } } my($type, $value, $name) = ($var->type, $var->value, $var->name); if (!defined $type) { $req->setError($request_info, SNMP_ERR_BADVALUE); } else { if (!defined $value) { if ($type == ASN_OCTET_STR) { $value = "" } elsif ($type == ASN_OBJECT_ID) { $value = "0" } elsif ($type == ASN_COUNTER64 || $type == ASN_INTEGER64 || $type == ASN_UNSIGNED64) { $value = "0" } else { $value = 0 } } my($status) = $req->setValue($type,$value); if (!$status) { do_log(0, "setValue error: %s, %s, %s", $type,$name,$value); $req->setError($request_info, SNMP_ERR_BADVALUE); } } } } } 1; } sub daemonize() { my($pid); closelog(); $syslog_open = 0; STDOUT->autoflush(1); STDERR->autoflush(1); # the first fork allows the shell to return and allows doing a setsid eval { $pid = fork(); 1 } or do { my($eval_stat) = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die "Error forking #1: $eval_stat"; }; defined $pid or die "Can't fork #1: $!"; if ($pid) { # parent process terminates here POSIX::_exit(0); # avoid END and destructor processing } # disassociate from a controlling terminal my($pgid) = POSIX::setsid(); defined $pgid && $pgid >= 0 or die "Can't start a new session: $!"; # We are now a session leader. As a session leader, opening a file # descriptor that is a terminal will make it our controlling terminal. # The second fork makes us NOT a session leader. Only session leaders # can acquire a controlling terminal, so we may now open up any file # we wish without worrying that it will become a controlling terminal. # second fork prevents from accidentally reacquiring a controlling terminal eval { $pid = fork(); 1 } or do { my($eval_stat) = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die "Error forking #2: $eval_stat"; }; defined $pid or die "Can't fork #2: $!"; if ($pid) { # parent process terminates here POSIX::_exit(0); # avoid END and destructor processing } # a daemonized child process, live long and prosper... do_log(2, "Daemonized as process [%s]", $$); chdir('/') or die "Can't chdir to '/': $!"; openlog($syslog_ident, LOG_PID | LOG_NDELAY, $syslog_facility); $syslog_open = 1; close(STDIN) or die "Can't close STDIN: $!"; close(STDOUT) or die "Can't close STDOUT: $!"; open(STDIN, '/dev/null') or die "Can't open /dev/null: $!"; close(STDERR) or die "Can't close STDERR: $!"; open(STDERR, '>&STDOUT') or die "Can't dup STDOUT: $!"; } sub usage() { return <<"EOD"; Usage: $0 [options] Options: -V show version, then exit -h show help, then exit -f stay in foreground -d log_level debugging level, 0..5, default 0 -P pid_file a file name to receive a PID of a damonized process -D db_home_dir amavis database directory ($db_home), default AMAVISD_DB_HOME or /var/amavis/db EOD } # main program starts here delete @ENV{'PATH', 'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; $SIG{INT} = sub { die "interrupted\n" }; # do the END code block $SIG{TERM} = sub { die "terminated\n" }; # do the END code block $SIG{PIPE} = 'IGNORE'; # don't signal on a write to a widowed pipe while (@ARGV >= 2 && $ARGV[0] =~ /^-[dDP]\z/ || @ARGV >= 1 && $ARGV[0] =~ /^-[hVf-]\z/) { my($opt,$val); $opt = shift @ARGV; $val = shift @ARGV if $opt !~ /^-[hVf-]\z/; # these take no arguments if ($opt eq '--') { last; } elsif ($opt eq '-h') { # -h (help) die "$myversion\n\n" . usage(); } elsif ($opt eq '-V') { # -V (version) die "$myversion\n"; } elsif ($opt eq '-f') { # -f (foreground) $daemonize = 0; } elsif ($opt eq '-d') { # -d log_level $log_level = 0+$val; } elsif ($opt eq '-D') { # -D db_home_dir, empty string turns off db use $db_home = untaint($val) if $val ne ''; } elsif ($opt eq '-P') { # -P pid_file $pid_filename = untaint($val) if $val ne ''; } else { die "Error in parsing command line options: $opt\n\n" . usage(); } } !@ARGV or die "Unprocessed command line options: $ARGV[0]\n\n" . usage(); if (!defined $mta_queue_dir) { # test for access to Postfix queue directory local($ENV{PATH}) = '/usr/sbin:/usr/local/sbin:/opt/postfix/sbin'; $! = 0; $mta_queue_dir = qx(postconf -h queue_directory); if (!defined $mta_queue_dir) { if ($! != 0) { do_log(1, "no postfix (unable to run postconf command): $!"); } else { do_log(1, "failed to execute \"postconf queue_directory\": $?"); } } else { chomp $mta_queue_dir; if ($mta_queue_dir =~ /^\s*\z/) { do_log(1, "unknown Postfix queue directory"); undef $mta_queue_dir; } else { do_log(2, "got a Postfix queue directory: %s", $mta_queue_dir); my($dir) = "$mta_queue_dir/active"; local(*DIR); if (!opendir(DIR,$dir)) { # testing access do_log(1, "can't open directory %s: %s", $dir,$!); undef $mta_queue_dir; } else { closedir(DIR) or die "Error closing directory $dir: $!"; } } } } { # amavisd statistics MIB my($r) = $databases[0]->{root_oid_str}; declare_variable("$r.1.1", 'sysDescr', 'STR'); declare_variable("$r.1.2", 'sysObjectID', 'OID'); declare_variable("$r.1.3", 'sysUpTime', 'TIM'); declare_variable("$r.1.4", 'sysContact', 'STR'); declare_variable("$r.1.5", 'sysName', 'STR'); declare_variable("$r.1.6", 'sysLocation', 'STR'); declare_variable("$r.1.7", 'sysServices', 'INT'); declare_variable("$r.2.1", 'InMsgs'); # orig=x locl=x declare_variable("$r.2.2", 'InMsgsInbound'); # orig=0 locl=1 declare_variable("$r.2.3", 'InMsgsOutbound'); # orig=1 locl=0 declare_variable("$r.2.4", 'InMsgsInternal'); # orig=1 locl=1 declare_variable("$r.2.5", 'InMsgsOriginating'); # orig=1 locl=x declare_variable("$r.2.6", 'InMsgsOpenRelay'); # orig=0 locl=0 # these have duplicates at $r.{19..26}.1, except InMsgsStatusRelayed declare_variable("$r.2.7", 'InMsgsStatusAccepted'); # 2xx, AM.PDP declare_variable("$r.2.8", 'InMsgsStatusRelayed'); # 2xx, forward declare_variable("$r.2.9", 'InMsgsStatusDiscarded'); # 2xx, no DSN declare_variable("$r.2.10", 'InMsgsStatusNoBounce'); # 2xx, no DSN declare_variable("$r.2.11", 'InMsgsStatusBounced'); # 2xx, DSN sent declare_variable("$r.2.12", 'InMsgsStatusRejected'); # 5xx declare_variable("$r.2.13", 'InMsgsStatusTempFailed'); # 4xx declare_variable("$r.3.1", 'InMsgsSize', 'C64'); declare_variable("$r.3.2", 'InMsgsSizeInbound', 'C64'); declare_variable("$r.3.3", 'InMsgsSizeOutbound', 'C64'); declare_variable("$r.3.4", 'InMsgsSizeInternal', 'C64'); declare_variable("$r.3.5", 'InMsgsSizeOriginating', 'C64'); declare_variable("$r.3.6", 'InMsgsSizeOpenRelay', 'C64'); declare_variable("$r.4.1", 'InMsgsRecips'); # orig=x locl=x declare_variable("$r.4.2", 'InMsgsRecipsInbound'); # orig=0 locl=1 declare_variable("$r.4.3", 'InMsgsRecipsOutbound'); # orig=1 locl=0 declare_variable("$r.4.4", 'InMsgsRecipsInternal'); # orig=1 locl=1 declare_variable("$r.4.5", 'InMsgsRecipsOriginating'); # orig=1 locl=x declare_variable("$r.4.6", 'InMsgsRecipsOpenRelay'); # orig=0 locl=0 declare_variable("$r.4.7", 'InMsgsRecipsLocal'); # orig=x locl=1 declare_variable("$r.5.1", 'InMsgsBounce'); declare_variable("$r.5.2", 'InMsgsBounceNullRPath'); declare_variable("$r.5.3", 'InMsgsBounceKilled'); declare_variable("$r.5.4", 'InMsgsBounceUnverifiable'); declare_variable("$r.5.5", 'InMsgsBounceRescuedByDomain'); declare_variable("$r.5.6", 'InMsgsBounceRescuedByOriginating'); declare_variable("$r.5.7", 'InMsgsBounceRescuedByPenPals'); declare_variable("$r.6.1", 'OutMsgs'); declare_variable("$r.6.2", 'OutMsgsRelay'); declare_variable("$r.6.3", 'OutMsgsSubmit'); declare_variable("$r.6.4", 'OutMsgsSubmitQuar'); declare_variable("$r.6.5", 'OutMsgsSubmitDsn'); declare_variable("$r.6.6", 'OutMsgsSubmitNotif'); declare_variable("$r.6.7", 'OutMsgsSubmitAV'); declare_variable("$r.6.8", 'OutMsgsSubmitArf'); declare_variable("$r.6.9", 'OutMsgsProtoLocal'); declare_variable("$r.6.10", 'OutMsgsProtoLocalRelay'); declare_variable("$r.6.11", 'OutMsgsProtoLocalSubmit'); declare_variable("$r.6.12", 'OutMsgsProtoSMTP'); declare_variable("$r.6.13", 'OutMsgsProtoSMTPRelay'); declare_variable("$r.6.14", 'OutMsgsProtoSMTPSubmit'); declare_variable("$r.6.15", 'OutMsgsProtoLMTP'); declare_variable("$r.6.16", 'OutMsgsProtoLMTPRelay'); declare_variable("$r.6.17", 'OutMsgsProtoLMTPSubmit'); declare_variable("$r.6.18", 'OutMsgsProtoBSMTP'); declare_variable("$r.6.19", 'OutMsgsProtoBSMTPRelay'); declare_variable("$r.6.20", 'OutMsgsProtoBSMTPSubmit'); declare_variable("$r.6.21", 'OutMsgsProtoPipe'); declare_variable("$r.6.22", 'OutMsgsProtoPipeRelay'); declare_variable("$r.6.23", 'OutMsgsProtoPipeSubmit'); declare_variable("$r.6.24", 'OutMsgsProtoSQL'); declare_variable("$r.6.25", 'OutMsgsProtoSQLRelay'); declare_variable("$r.6.26", 'OutMsgsProtoSQLSubmit'); declare_variable("$r.6.27", 'OutMsgsDelivers'); # 2xx declare_variable("$r.6.28", 'OutMsgsAttemptFails'); # 4xx declare_variable("$r.6.29", 'OutMsgsRejects'); # 5xx declare_variable("$r.7.1", 'OutMsgsSize'); declare_variable("$r.7.2", 'OutMsgsSizeRelay'); declare_variable("$r.7.3", 'OutMsgsSizeSubmit'); declare_variable("$r.7.4", 'OutMsgsSizeSubmitQuar'); declare_variable("$r.7.5", 'OutMsgsSizeSubmitDsn'); declare_variable("$r.7.6", 'OutMsgsSizeSubmitNotif'); declare_variable("$r.7.7", 'OutMsgsSizeSubmitAV'); declare_variable("$r.7.8", 'OutMsgsSizeSubmitArf'); declare_variable("$r.7.9", 'OutMsgsSizeProtoLocal'); declare_variable("$r.7.10", 'OutMsgsSizeProtoLocalRelay'); declare_variable("$r.7.11", 'OutMsgsSizeProtoLocalSubmit'); declare_variable("$r.7.12", 'OutMsgsSizeProtoSMTP'); declare_variable("$r.7.13", 'OutMsgsSizeProtoSMTPRelay'); declare_variable("$r.7.14", 'OutMsgsSizeProtoSMTPSubmit'); declare_variable("$r.7.15", 'OutMsgsSizeProtoLMTP'); declare_variable("$r.7.16", 'OutMsgsSizeProtoLMTPRelay'); declare_variable("$r.7.17", 'OutMsgsSizeProtoLMTPSubmit'); declare_variable("$r.7.18", 'OutMsgsSizeProtoBSMTP'); declare_variable("$r.7.19", 'OutMsgsSizeProtoBSMTPRelay'); declare_variable("$r.7.20", 'OutMsgsSizeProtoBSMTPSubmit'); declare_variable("$r.7.21", 'OutMsgsSizeProtoPipe'); declare_variable("$r.7.22", 'OutMsgsSizeProtoPipeRelay'); declare_variable("$r.7.23", 'OutMsgsSizeProtoPipeSubmit'); declare_variable("$r.7.24", 'OutMsgsSizeProtoSQL'); declare_variable("$r.7.25", 'OutMsgsSizeProtoSQLRelay'); declare_variable("$r.7.26", 'OutMsgsSizeProtoSQLSubmit'); declare_variable("$r.8.1", 'QuarMsgs'); declare_variable("$r.8.2", 'QuarMsgsArch'); declare_variable("$r.8.3", 'QuarMsgsClean'); declare_variable("$r.8.4", 'QuarMsgsMtaFailed'); declare_variable("$r.8.5", 'QuarMsgsOversized'); declare_variable("$r.8.6", 'QuarMsgsBadHdr'); declare_variable("$r.8.7", 'QuarMsgsSpammy'); declare_variable("$r.8.8", 'QuarMsgsSpam'); declare_variable("$r.8.9", 'QuarMsgsUnchecked'); declare_variable("$r.8.10", 'QuarMsgsBanned'); declare_variable("$r.8.11", 'QuarMsgsVirus'); declare_variable("$r.8.12", 'QuarAttemptTempFails'); declare_variable("$r.8.13", 'QuarAttemptFails'); declare_variable("$r.9.1", 'QuarMsgsSize', 'C64'); declare_variable("$r.9.2", 'QuarMsgsSizeArch', 'C64'); declare_variable("$r.9.3", 'QuarMsgsSizeClean', 'C64'); declare_variable("$r.9.4", 'QuarMsgsSizeMtaFailed', 'C64'); declare_variable("$r.9.5", 'QuarMsgsSizeOversized', 'C64'); declare_variable("$r.9.6", 'QuarMsgsSizeBadHdr', 'C64'); declare_variable("$r.9.7", 'QuarMsgsSizeSpammy', 'C64'); declare_variable("$r.9.8", 'QuarMsgsSizeSpam', 'C64'); declare_variable("$r.9.9", 'QuarMsgsSizeUnchecked', 'C64'); declare_variable("$r.9.10", 'QuarMsgsSizeBanned', 'C64'); declare_variable("$r.9.11", 'QuarMsgsSizeVirus', 'C64'); declare_variable("$r.10.1.1", 'ContentCleanMsgs'); declare_variable("$r.10.1.2", 'ContentCleanMsgsInbound'); declare_variable("$r.10.1.3", 'ContentCleanMsgsOutbound'); declare_variable("$r.10.1.4", 'ContentCleanMsgsInternal'); declare_variable("$r.10.1.5", 'ContentCleanMsgsOriginating'); declare_variable("$r.10.1.6", 'ContentCleanMsgsOpenRelay'); declare_variable("$r.10.2.1", 'ContentMtaFailedMsgs'); declare_variable("$r.10.2.2", 'ContentMtaFailedMsgsInbound'); declare_variable("$r.10.2.3", 'ContentMtaFailedMsgsOutbound'); declare_variable("$r.10.2.4", 'ContentMtaFailedMsgsInternal'); declare_variable("$r.10.2.5", 'ContentMtaFailedMsgsOriginating'); declare_variable("$r.10.2.6", 'ContentMtaFailedMsgsOpenRelay'); declare_variable("$r.10.3.1", 'ContentOversizedMsgs'); declare_variable("$r.10.3.2", 'ContentOversizedMsgsInbound'); declare_variable("$r.10.3.3", 'ContentOversizedMsgsOutbound'); declare_variable("$r.10.3.4", 'ContentOversizedMsgsInternal'); declare_variable("$r.10.3.5", 'ContentOversizedMsgsOriginating'); declare_variable("$r.10.3.6", 'ContentOversizedMsgsOpenRelay'); declare_variable("$r.10.4.1", 'ContentBadHdrMsgs'); declare_variable("$r.10.4.2", 'ContentBadHdrMsgsInbound'); declare_variable("$r.10.4.3", 'ContentBadHdrMsgsOutbound'); declare_variable("$r.10.4.4", 'ContentBadHdrMsgsInternal'); declare_variable("$r.10.4.5", 'ContentBadHdrMsgsOriginating'); declare_variable("$r.10.4.6", 'ContentBadHdrMsgsOpenRelay'); declare_variable("$r.10.5.1", 'ContentSpammyMsgs'); declare_variable("$r.10.5.2", 'ContentSpammyMsgsInbound'); declare_variable("$r.10.5.3", 'ContentSpammyMsgsOutbound'); declare_variable("$r.10.5.4", 'ContentSpammyMsgsInternal'); declare_variable("$r.10.5.5", 'ContentSpammyMsgsOriginating'); declare_variable("$r.10.5.6", 'ContentSpammyMsgsOpenRelay'); declare_variable("$r.10.6.1", 'ContentSpamMsgs'); declare_variable("$r.10.6.2", 'ContentSpamMsgsInbound'); declare_variable("$r.10.6.3", 'ContentSpamMsgsOutbound'); declare_variable("$r.10.6.4", 'ContentSpamMsgsInternal'); declare_variable("$r.10.6.5", 'ContentSpamMsgsOriginating'); declare_variable("$r.10.6.6", 'ContentSpamMsgsOpenRelay'); declare_variable("$r.10.7.1", 'ContentUncheckedMsgs'); declare_variable("$r.10.7.2", 'ContentUncheckedMsgsInbound'); declare_variable("$r.10.7.3", 'ContentUncheckedMsgsOutbound'); declare_variable("$r.10.7.4", 'ContentUncheckedMsgsInternal'); declare_variable("$r.10.7.5", 'ContentUncheckedMsgsOriginating'); declare_variable("$r.10.7.6", 'ContentUncheckedMsgsOpenRelay'); declare_variable("$r.10.8.1", 'ContentBannedMsgs'); declare_variable("$r.10.8.2", 'ContentBannedMsgsInbound'); declare_variable("$r.10.8.3", 'ContentBannedMsgsOutbound'); declare_variable("$r.10.8.4", 'ContentBannedMsgsInternal'); declare_variable("$r.10.8.5", 'ContentBannedMsgsOriginating'); declare_variable("$r.10.8.6", 'ContentBannedMsgsOpenRelay'); declare_variable("$r.10.9.1", 'ContentVirusMsgs'); declare_variable("$r.10.9.2", 'ContentVirusMsgsInbound'); declare_variable("$r.10.9.3", 'ContentVirusMsgsOutbound'); declare_variable("$r.10.9.4", 'ContentVirusMsgsInternal'); declare_variable("$r.10.9.5", 'ContentVirusMsgsOriginating'); declare_variable("$r.10.9.6", 'ContentVirusMsgsOpenRelay'); declare_variable("$r.11.1", 'CacheAttempts'); declare_variable("$r.11.2", 'CacheMisses'); declare_variable("$r.11.3", 'CacheHits'); declare_variable("$r.11.4", 'CacheHitsVirusCheck'); declare_variable("$r.11.5", 'CacheHitsVirusMsgs'); declare_variable("$r.11.6", 'OutConnNew'); declare_variable("$r.11.7", 'OutConnQuit'); declare_variable("$r.11.8", 'OutConnTransact'); declare_variable("$r.11.9", 'OutConnReuseFail'); declare_variable("$r.11.10", 'OutConnReuseRecent'); declare_variable("$r.11.11", 'OutConnReuseRefreshed'); declare_variable("$r.12.1", 'OpsDec'); declare_variable("$r.12.2", 'OpsSpamCheck'); declare_variable("$r.12.3", 'OpsVirusCheck'); declare_variable("$r.13.1", 'PenPalsAttempts'); declare_variable("$r.13.2", 'PenPalsAttemptsRid'); declare_variable("$r.13.3", 'PenPalsAttemptsMid'); declare_variable("$r.13.4", 'PenPalsMisses'); declare_variable("$r.13.5", 'PenPalsHits'); declare_variable("$r.13.6", 'PenPalsHitsRid'); declare_variable("$r.13.7", 'PenPalsHitsMid'); declare_variable("$r.13.8", 'PenPalsHitsMidRid'); declare_variable("$r.13.9", 'PenPalsSavedFromTag2'); declare_variable("$r.13.10", 'PenPalsSavedFromTag3'); declare_variable("$r.13.11", 'PenPalsSavedFromKill'); declare_variable("$r.14.1", 'SqlAddrSenderAttempts'); declare_variable("$r.14.2", 'SqlAddrSenderMisses'); declare_variable("$r.14.3", 'SqlAddrSenderHits'); declare_variable("$r.14.4", 'SqlAddrRecipAttempts'); declare_variable("$r.14.5", 'SqlAddrRecipMisses'); declare_variable("$r.14.6", 'SqlAddrRecipHits'); declare_variable("$r.15.1", 'LogEntries', 'C64'); declare_variable("$r.15.2", 'LogEntriesEmerg', 'C64'); declare_variable("$r.15.3", 'LogEntriesAlert', 'C64'); declare_variable("$r.15.4", 'LogEntriesCrit', 'C64'); # lvl le -3 declare_variable("$r.15.5", 'LogEntriesErr', 'C64'); # lvl le -2 declare_variable("$r.15.6", 'LogEntriesWarning', 'C64'); # lvl le -1 declare_variable("$r.15.7", 'LogEntriesNotice', 'C64'); # lvl le 0 declare_variable("$r.15.8", 'LogEntriesInfo', 'C64'); # lvl le 1 declare_variable("$r.15.9", 'LogEntriesDebug', 'C64'); # lvl le 2 declare_variable("$r.15.10", 'LogEntriesLevel0', 'C64'); # le 0 declare_variable("$r.15.11", 'LogEntriesLevel1', 'C64'); # eq 1 declare_variable("$r.15.12", 'LogEntriesLevel2', 'C64'); # eq 2 declare_variable("$r.15.13", 'LogEntriesLevel3', 'C64'); # eq 3 declare_variable("$r.15.14", 'LogEntriesLevel4', 'C64'); # eq 4 declare_variable("$r.15.15", 'LogEntriesLevel5', 'C64'); # ge 5 declare_variable("$r.15.16", 'LogLines', 'C64'); declare_variable("$r.15.17", 'LogRetries', 'C64'); declare_variable("$r.16.1", 'TimeElapsedTotal', 'INT'); declare_variable("$r.16.2", 'TimeElapsedReceiving', 'INT'); declare_variable("$r.16.3", 'TimeElapsedSending', 'INT'); declare_variable("$r.16.4", 'TimeElapsedDecoding', 'INT'); declare_variable("$r.16.5", 'TimeElapsedPenPals', 'INT'); declare_variable("$r.16.6", 'TimeElapsedVirusCheck','INT'); declare_variable("$r.16.7", 'TimeElapsedSpamCheck', 'INT'); declare_variable("$r.17.1", 'UserCounter1', 'C64'); declare_variable("$r.17.2", 'UserCounter2', 'C64'); declare_variable("$r.17.3", 'UserCounter3', 'C64'); declare_variable("$r.17.4", 'UserCounter4', 'C64'); declare_variable("$r.17.5", 'UserCounter5', 'C64'); declare_variable("$r.17.6", 'UserCounter6', 'C64'); declare_variable("$r.17.7", 'UserCounter7', 'C64'); declare_variable("$r.17.8", 'UserCounter8', 'C64'); declare_variable("$r.17.9", 'UserCounter9', 'C64'); declare_variable("$r.17.10", 'UserCounter10', 'C64'); declare_variable("$r.18.1", 'UserGauge1', 'G32'); declare_variable("$r.18.2", 'UserGauge2', 'G32'); declare_variable("$r.18.3", 'UserGauge3', 'G32'); declare_variable("$r.18.4", 'UserGauge4', 'G32'); declare_variable("$r.18.5", 'UserGauge5', 'G32'); declare_variable("$r.18.6", 'UserGauge6', 'G32'); declare_variable("$r.18.7", 'UserGauge7', 'G32'); declare_variable("$r.18.8", 'UserGauge8', 'G32'); declare_variable("$r.18.9", 'UserGauge9', 'G32'); declare_variable("$r.18.10", 'UserGauge10', 'G32'); declare_variable("$r.19.1", 'InMsgsStatusAccepted'); # 2xx, AM.PDP declare_variable("$r.19.2", 'InMsgsStatusAcceptedInbound'); declare_variable("$r.19.3", 'InMsgsStatusAcceptedOutbound'); declare_variable("$r.19.4", 'InMsgsStatusAcceptedInternal'); declare_variable("$r.19.5", 'InMsgsStatusAcceptedOriginating'); declare_variable("$r.19.6", 'InMsgsStatusAcceptedOpenRelay'); declare_variable("$r.20.1", 'InMsgsStatusRelayedUntagged'); # 2xx, fwd declare_variable("$r.20.2", 'InMsgsStatusRelayedUntaggedInbound'); declare_variable("$r.20.3", 'InMsgsStatusRelayedUntaggedOutbound'); declare_variable("$r.20.4", 'InMsgsStatusRelayedUntaggedInternal'); declare_variable("$r.20.5", 'InMsgsStatusRelayedUntaggedOriginating'); declare_variable("$r.20.6", 'InMsgsStatusRelayedUntaggedOpenRelay'); declare_variable("$r.21.1", 'InMsgsStatusRelayedTagged'); # 2xx, forward declare_variable("$r.21.2", 'InMsgsStatusRelayedTaggedInbound'); declare_variable("$r.21.3", 'InMsgsStatusRelayedTaggedOutbound'); declare_variable("$r.21.4", 'InMsgsStatusRelayedTaggedInternal'); declare_variable("$r.21.5", 'InMsgsStatusRelayedTaggedOriginating'); declare_variable("$r.21.6", 'InMsgsStatusRelayedTaggedOpenRelay'); declare_variable("$r.22.1", 'InMsgsStatusDiscarded'); # 2xx, no DSN declare_variable("$r.22.2", 'InMsgsStatusDiscardedInbound'); declare_variable("$r.22.3", 'InMsgsStatusDiscardedOutbound'); declare_variable("$r.22.4", 'InMsgsStatusDiscardedInternal'); declare_variable("$r.22.5", 'InMsgsStatusDiscardedOriginating'); declare_variable("$r.22.6", 'InMsgsStatusDiscardedOpenRelay'); declare_variable("$r.23.1", 'InMsgsStatusNoBounce'); # 2xx, no DSN declare_variable("$r.23.2", 'InMsgsStatusNoBounceInbound'); declare_variable("$r.23.3", 'InMsgsStatusNoBounceOutbound'); declare_variable("$r.23.4", 'InMsgsStatusNoBounceInternal'); declare_variable("$r.23.5", 'InMsgsStatusNoBounceOriginating'); declare_variable("$r.23.6", 'InMsgsStatusNoBounceOpenRelay'); declare_variable("$r.24.1", 'InMsgsStatusBounced'); # 2xx, DSN sent declare_variable("$r.24.2", 'InMsgsStatusBouncedInbound'); declare_variable("$r.24.3", 'InMsgsStatusBouncedOutbound'); declare_variable("$r.24.4", 'InMsgsStatusBouncedInternal'); declare_variable("$r.24.5", 'InMsgsStatusBouncedOriginating'); declare_variable("$r.24.6", 'InMsgsStatusBouncedOpenRelay'); declare_variable("$r.25.1", 'InMsgsStatusRejected'); # 5xx declare_variable("$r.25.2", 'InMsgsStatusRejectedInbound'); declare_variable("$r.25.3", 'InMsgsStatusRejectedOutbound'); declare_variable("$r.25.4", 'InMsgsStatusRejectedInternal'); declare_variable("$r.25.5", 'InMsgsStatusRejectedOriginating'); declare_variable("$r.25.6", 'InMsgsStatusRejectedOpenRelay'); declare_variable("$r.26.1", 'InMsgsStatusTempFailed'); # 4xx declare_variable("$r.26.2", 'InMsgsStatusTempFailedInbound'); declare_variable("$r.26.3", 'InMsgsStatusTempFailedOutbound'); declare_variable("$r.26.4", 'InMsgsStatusTempFailedInternal'); declare_variable("$r.26.5", 'InMsgsStatusTempFailedOriginating'); declare_variable("$r.26.6", 'InMsgsStatusTempFailedOpenRelay'); } { # amavisd child processes MIB my($r) = $databases[1]->{root_oid_str}; declare_variable("$r.1.1", 'ProcGone'); # counter! declare_variable("$r.1.2", 'ProcAll', 'G32'); declare_variable("$r.1.3", 'ProcIdle', 'G32'); declare_variable("$r.1.4", 'ProcBusy', 'G32'); declare_variable("$r.1.5", 'ProcBusyTransfer', 'G32'); declare_variable("$r.1.6", 'ProcBusyDecode', 'G32'); declare_variable("$r.1.7", 'ProcBusyVirus', 'G32'); declare_variable("$r.1.8", 'ProcBusySpam', 'G32'); declare_variable("$r.1.9", 'ProcBusyOther', 'G32'); declare_variable(sprintf("%s.2.%d", $r,$_+1), 'ProcBusy'.$_, 'G32') for (0..@age_slots); } if (defined $mta_queue_dir) { # Postfix queue size MIB declare_variable($databases[2]->{root_oid_str}, 'MtaQueueEntriesMaildrop', 'G32'); declare_variable($databases[3]->{root_oid_str}, 'MtaQueueEntriesIncoming', 'G32'); declare_variable($databases[4]->{root_oid_str}, 'MtaQueueEntriesActive', 'G32'); declare_variable($databases[5]->{root_oid_str}, 'MtaQueueEntriesDeferred', 'G32'); } if (!$daemonize) { do_log(0,"%s starting in foreground, perl %s", $myversion,$]); } else { # daemonize $SIG{'__WARN__'} = # log warnings sub { my($m) = @_; chomp($m); do_log(-1,"_WARN: %s",$m) }; $SIG{'__DIE__' } = # log uncaught errors sub { if (!$^S) { my($m) = @_; chomp($m); do_log(-2,"_DIE: %s",$m) } }; openlog($syslog_ident, LOG_PID | LOG_NDELAY, $syslog_facility); $syslog_open = 1; do_log(2,"to be daemonized"); daemonize(); do_log(0,"%s starting. daemonized as PID [%s], perl %s", $myversion,$$,$]); if (defined $pid_filename && $pid_filename ne '') { my($pidf) = IO::File->new; my($stat) = $pidf->open($pid_filename, O_CREAT|O_EXCL|O_RDWR, 0640); if (!$stat && $! == EEXIST) { do_log(0,"PID file %s exists, overwriting", $pid_filename); $stat = $pidf->open($pid_filename, O_CREAT|O_RDWR, 0640); } $stat or die "Can't create file $pid_filename: $!"; $pid_file_created = 1; $pidf->print("$$\n") or die "Can't write to $pid_filename: $!"; $pidf->close or die "Can't close $pid_filename: $!"; } } #netsnmp_ds_set_boolean(NETSNMP_DS_APPLICATION_ID, # NETSNMP_DS_LIB_DONT_READ_CONFIGS, 1); my($agent) = NetSNMP::agent->new('Name' => $agent_name, 'AgentX' => 1) or die "Can't create a SNMP agent $agent_name"; init_data(); # must come *after* NetSNMP::agent->new # dump_variables(); for my $database (@databases) { my($root_oid_str) = $database->{root_oid_str}; my($db_name) = $database->{name}; if ($db_name =~ /^pf/ && !defined $mta_queue_dir) { do_log(2, "not registering root OID %s for %s", $root_oid_str,$db_name); } else { do_log(2, "registering root OID %s for %s", $root_oid_str,$db_name); $root_oid_str = '.' . $root_oid_str; $agent->register($agent_name, $root_oid_str, \&snmp_handler) or die "Can't register a SNMP agent $agent_name under $root_oid_str"; $database->{registered} = 1; } } while ($keep_running) { $agent->agent_check_and_process(1); } exit; END { if (defined $agent) { eval { $agent->shutdown }; # ignoring status } for my $database (@databases) { eval { if (defined $database->{db}) { if (defined $database->{cursor}) { $database->{cursor}->c_close; # close database, ignoring status undef $database->{cursor}; } $database->{db}->db_close == 0 or warn(sprintf("BDB db_close error on a %s file: %s %s", $database->{db}, $BerkeleyDB::Error, $!)); } }; # ignoring status } if ($pid_file_created) { unlink($pid_filename) or eval { do_log(0, "Can't remove file %s: %s", $pid_filename,$!) }; } eval { do_log(2, "%s shutting down", $myproduct_name) }; if ($syslog_open) { eval { closelog() }; $syslog_open = 0; } } amavisd-new-2.7.1/amavisd-custom.conf000640 000621 000620 00000032352 11747105736 017306 0ustar00markcmi000000 000000 package Amavis::Custom; use strict; use warnings; no warnings qw(uninitialized redefine); # Example use of custom hooks, available since amavisd-new-2.5.0 # This code can be placed directly at end of file amavisd.conf, # or invoked from there by a call to include_config_files such as: # include_config_files('/etc/amavisd-custom.conf'); # or specified on amavisd command line by using additional -c options. # # It replaces dummy hooks in package Amavis::Custom (in file amavisd) # with replacement subroutines of the same name, and thus enable them. # # The code below demonstrates obtaining and displaying some of the more # interesting information on each passing mail, and inserting some custom # header fields in passed mail. # The example below also illustrates how to use existing code in amavisd # to interface with a SQL database server (e.g. MySQL or PostgreSQL), # allowing for persistent connections and automatic reconnect in case # of a connection failure. # # Modifying recipient address, sending a copy to a mailbox quarantine, # or creating and sending a short notification alert is illustrated. #testing database: # $ mysqladmin create user_presence # $ mysql user_presence # CREATE TABLE users ( # email varchar(255) NOT NULL UNIQUE, # present char(1) # ); # INSERT INTO users VALUES ('test@example.com', 'Y'); # INSERT INTO users VALUES ('absent@example.com', 'N'); # INSERT INTO users VALUES ('postmaster@example.com', 'Y'); # replaces placeholder routines in Amavis::Custom with actual code use DBI qw(:sql_types); use DBD::mysql; BEGIN { import Amavis::Conf qw(:platform :confvars c cr ca $myhostname); import Amavis::Util qw(do_log untaint safe_encode safe_decode); import Amavis::rfc2821_2822_Tools; import Amavis::Notify qw(build_mime_entity); } # MAIL PROCESSING SEQUENCE: # # child process initialization # loop for each mail: # receive mail, parse and make available some basic information # *custom hook: new() - may inspect info, may load policy banks # mail checking and collecting results # *custom hook: checks() - called after virus and spam checks but before # taking decisions what to do with mail; may inspect or modify results # deciding mail fate (lookup on *_lovers, thresholds, ...) # quarantining # sending notifications (to admin and recip) # *custom hook: before_send() - may send other notif., quarantine, modify mail # forwarding (unless blocked) # *custom hook: after_send() - may suppress DSN, send reports, quarantine # sending delivery status notification (if needed) # issue main log entry, manage statistics (timing, counters, nanny) # *custom hook: mail_done() - may inspect results # endloop after $max_requests or earlier # invoked at child process creation time; # return an object, or just undef when custom checks are not needed sub new { my($class,$conn,$msginfo) = @_; my($self) = bless {}, $class; my($conn_h) = Amavis::Out::SQL::Connection->new( ['DBI:mysql:database=user_presence;host=127.0.0.1', 'user1', 'passwd1'] ); $self->{'conn_h'} = $conn_h; $self; # returning an object activates further callbacks, # returning undef disables them } #sub checks { # may be left out if not needed # my($self,$conn,$msginfo) = @_; #} sub before_send { my($self,$conn,$msginfo) = @_; # $self ... whatever was returned by new() # $conn ... object with information about a SMTP connection # $msginfo ... object with info. about a mail message being processed my($ll) = 2; # log level (0 is the most important level, 1, 2,... 5 less so) do_log($ll,"CUSTOM: new message"); # examine some data pertaining to the SMTP connection from client # See methods in Amavis::In::Connection for the full set of available data. # # SMTP client's IP address as a string (IPv4 or IPv6) my($client_ip) = $msginfo->client_addr; # does client IP address match @mynetworks_maps? (boolean) my($is_client_ip_internal) = $msginfo->client_addr_mynets; do_log($ll,"CUSTOM: [%s], is internal IP: %s, %s", $client_ip, $is_client_ip_internal ? 'YES' : 'NO', $msginfo->originating ? 'ORIGINATING' : 'incoming'); # examine some data pertaining to the message as a whole (not per-pecipient) # See methods in Amavis::In::Message for the full set of available data. # my($log_id) = $msginfo->log_id; # log ID string, e.g. '48262-21-2' my($mail_id) = $msginfo->mail_id; # long-term unique id, e.g. 'yxqmZgS+M09R' my($sender) = $msginfo->sender; # envelope sender address, e.g. 'usr@e.com' my($mail_size) = $msginfo->msg_size; # mail size in bytes my($spam_level)= $msginfo->spam_level; # spam level (without per-recip boost) do_log($ll,"CUSTOM: %d bytes, score: %.2f", $log_id,$mail_id,$mail_size,$spam_level); do_log($ll,"CUSTOM: Return-Path (env. sender): <%s>", $sender); my($sigs_ref) = $msginfo->dkim_signatures_valid; do_log($ll,"CUSTOM: dkim valid, d=%s", join(',', map {$_->domain} @$sigs_ref) ) if defined $sigs_ref && @$sigs_ref; # full mail is only stored in file, which may be read if desired (see below); # full mail header is available in ->orig_header; # some mail header fields are available through $msginfo->orig_header_fields # these may be multiline, may contain folding whitespace or comments; # alternatively, the whole original mail header is available in ->orig_header my($m_id) = $msginfo->get_header_field_body('message-id'); # e.g. <12@e.n> my($subj) = $msginfo->get_header_field_body('subject'); my($from) = $msginfo->get_header_field_body('from'); # e.g.: "=?ISO-8859-1?Q?Ren=E9_van_den_Berg?=" my($is_bulk) = $msginfo->orig_header_fields->{'precedence'}; # e.g. List $is_bulk = $is_bulk=~/^[ \t]*(bulk|list|junk)\b/i ? $1 : undef; for ($m_id,$from,$subj) { # RFC2047-decode char. sets in some header fields local($1); chomp; my($str); s/\n([ \t])/$1/sg; s/^[ \t]+//s; s/[ \t]+\z//s; # unfold, trim eval { $str = safe_decode('MIME-Header',$_) }; # to string of logical chr $_ = $str if $@ eq ''; # replace if all ok, otherwise keep unchanged } # $m_id, $from, and $subj are now ready for examination - Perl logical chars do_log($ll,"CUSTOM: Subject: %s",safe_encode('iso-8859-1',$subj)); #as Latin1 do_log($ll,"CUSTOM: From: %s", safe_encode('iso-8859-1',$from)); # as Latin1 # NOTE: rfc2822 allows multiple addresses in the From field! my($rfc2822_sender) = $msginfo->rfc2822_sender; # undef or scalar my(@rfc2822_from) = do { my $f = $msginfo->rfc2822_from; ref $f ? @$f : $f }; do_log($ll,"CUSTOM: From (parsed): %s", join(', ',@rfc2822_from)); do_log($ll,"CUSTOM: Sender: %s", $rfc2822_sender) if defined $rfc2822_sender; my($tempdir) = $msginfo->mail_tempdir; # working directory for this process # $tempdir/parts/ is a directory where mail parts were extracted to my($mail_file_name) = $msginfo->mail_text_fn; # filename of the original mail, normally $tempdir/email.txt do_log($ll,"CUSTOM: temp.dir: %s", $tempdir); do_log($ll,"CUSTOM: filename: %s", $mail_file_name); # full mail header is available in ->orig_header; # some individual header fields are quickly accessible ->orig_header_fields # mail body is only stored in file, which may be read if desired my($fh) = $msginfo->mail_text; # file handle of our original mail my($line); my($line_cnt) = 0; # $fh->seek(0,0) or die "Can't rewind mail file: $!"; # for ($! = 0; defined($line = $fh->getline); $! = 0) { # $line_cnt++; # # examine one $line at a time; (or read by blocks for speed) # } # defined $line || $!==0 or die "Error reading mail file: $!"; # do_log($ll,"CUSTOM: %d lines", $line_cnt); my($all_local) = !grep { !$_->recip_is_local } @{$msginfo->per_recip_data}; if ($all_local) { my($hdr_edits) = $msginfo->header_edits; my($rly_country) = $msginfo->supplementary_info('RELAYCOUNTRY'); $hdr_edits->add_header('X-Relay-Countries', $rly_country) if defined $rly_country && $rly_country ne ''; my($languages) = $msginfo->supplementary_info('LANGUAGES'); $hdr_edits->add_header('X-Spam-Languages', $languages) if defined $languages && $languages ne ''; } # examine some data pertaining to the each recipient of the message # See methods in Amavis::In::Message::PerRecip for the full set of data. # my($any_passed) = 0; for my $r (@{$msginfo->per_recip_data}) { # $r contains per-recipient data next if $r->recip_done; # skip recipient that won't receive a message # if all recipients have ->recip_done true, mail will not be passed at all $any_passed++; my($recip) = $r->recip_addr; # recipient envelope address, e.g. rc@ex.com my($is_local) = $r->recip_is_local; # recipient matches @local_domains_maps my($localpart,$domain) = split_address($recip); my($spam_level_boost) = $r->recip_score_boost; # per-recip score contrib. # $spam_level + $spam_level_boost is the actual per-recipient spam score my($do_tag) = $r->is_in_contents_category(CC_CLEAN,1); # >= tag_level my($do_tag2) = $r->is_in_contents_category(CC_SPAMMY); # >= tag2_level my($do_kill) = $r->is_in_contents_category(CC_SPAM); # >= kill_level do_log($ll,"CUSTOM: recip: %s, score: %.2f, %s, %s, %s, %s", $recip, $spam_level+$spam_level_boost, $is_local ? 'IS LOCAL' : 'not local', $do_tag ? 'tag' : 'no-tag', $do_tag2 ? 'tag2' : 'no-tag2', $do_kill ? 'kill' : 'no-kill'); # don't bother with outgoing mail! next if !$is_local; # do a SQL lookup my($conn_h) = $self->{'conn_h'}; $conn_h->begin_work_nontransaction; # (re)connect if not connected # my($select_clause) = 'SELECT present,email FROM users WHERE users.email=?'; # list of actual arguments replacing '?' placeholders my(@pos_args) = ( lc(untaint($recip)) ); $conn_h->execute($select_clause,@pos_args); # do the query # my($a_ref); my($user_is_offline); while ( defined($a_ref=$conn_h->fetchrow_arrayref($select_clause)) ) { do_log($ll,"CUSTOM: SQL fields %s", join(", ", @$a_ref)); $user_is_offline = 1 if $a_ref->[0] =~ /^(0|N)$/i; } $conn_h->finish($select_clause) if defined $a_ref; # only if not all read if ($user_is_offline) { # we have three choices of alerting the recipient: # - redirect his mail to dedicated e-mail address; # - use quarantining code to deliver a copy of the message to # a dedicated address; # - construct and send a notification to a dedicated address # my($choice) = 0; if ($choice == 0) { # ignore } elsif ($choice == 1) { # rewrite address and deliver normally my($new_addr) = $localpart . '+redirect' . $domain; $r->recip_addr_modified($new_addr); # replaces delivery address! } elsif ($choice == 2) { # quarantine (i.e. send a mail copy) to a dedicated mailbox # in addition to delivering normally my($new_addr) = 'alert+' . $localpart . $domain; Amavis::do_quarantine($conn, $msginfo, undef, [$new_addr], 'local:all-%m'); } elsif ($choice == 3) { # construct and send a short notification, # in addition to delivering normally my($when) = rfc2822_timestamp($msginfo->rx_time); my($text) = <<"EOD"; From: Alerting Service To: <$recip> Subject: New message from $sender Message-ID: A new message just arrived on $when from $from (return-path <$sender>) Subject: $subj EOD my($notification) = Amavis::In::Message->new; $notification->rx_time($msginfo->rx_time); # copy the reception time $notification->log_id($log_id); # copy log id $notification->delivery_method(c('notify_method')); $notification->sender(''); # use null return path to avoid loops $notification->sender_smtp('<>'); my($new_addr) = 'alert+' . $localpart . $domain; $notification->recips([$new_addr]); # character set is controlled through $hdr_encoding and $bdy_encoding # config variables, defaults to 'iso-8859-1' $notification->mail_text( string_to_mime_entity(\$text, $msginfo, undef,0,0)); Amavis::mail_dispatch($conn, $notification, 1, 0); my($n_smtp_resp, $n_exit_code, $n_dsn_needed) = one_response_for_all($notification, 0); # check status if ($n_smtp_resp =~ /^2/ && !$n_dsn_needed) { # ok } elsif ($n_smtp_resp =~ /^4/) { die "temporarily unable to alert recipient: $n_smtp_resp"; } else { do_log(-1, "FAILED to alert recipient: %s", $n_smtp_resp); } } } } if (!$any_passed) { do_log($ll,"CUSTOM: mail is blocked for all recipients"); } else { # will do delivery do_log($ll,"CUSTOM: being delivered to %d recips", $any_passed); # add a custom header field if desired (for all recipients of this message) # $msginfo->header_edits->add_header('X-Amavis-Example', # sprintf("a custom header field, mail contains %d lines",$line_cnt) ); } do_log($ll,"CUSTOM: done"); }; #sub after_send { # may be left out if not needed # my($self,$conn,$msginfo) = @_; #} #sub mail_done { # may be left out if not needed # my($self,$conn,$msginfo) = @_; #} 1; # insure a defined return # vacation: see RFC 3834 amavisd-new-2.7.1/amavisd.conf000640 000621 000620 00000106367 11747105744 016005 0ustar00markcmi000000 000000 use strict; # a minimalistic configuration file for amavisd-new with all necessary settings # # see amavisd.conf-default for a list of all variables with their defaults; # for more details see documentation in INSTALL, README_FILES/* # and at http://www.ijs.si/software/amavisd/amavisd-new-docs.html # COMMONLY ADJUSTED SETTINGS: # @bypass_virus_checks_maps = (1); # controls running of anti-virus code # @bypass_spam_checks_maps = (1); # controls running of anti-spam code # $bypass_decode_parts = 1; # controls running of decoders&dearchivers $max_servers = 2; # num of pre-forked children (2..30 is common), -m $daemon_user = 'vscan'; # (no default; customary: vscan or amavis), -u $daemon_group = 'vscan'; # (no default; customary: vscan or amavis), -g $mydomain = 'example.com'; # a convenient default for other settings # $MYHOME = '/var/amavis'; # a convenient default for other settings, -H $TEMPBASE = "$MYHOME/tmp"; # working directory, needs to exist, -T $ENV{TMPDIR} = $TEMPBASE; # environment variable TMPDIR, used by SA, etc. $QUARANTINEDIR = '/var/virusmails'; # -Q # $quarantine_subdir_levels = 1; # add level of subdirs to disperse quarantine # $release_format = 'resend'; # 'attach', 'plain', 'resend' # $report_format = 'arf'; # 'attach', 'plain', 'resend', 'arf' # $daemon_chroot_dir = $MYHOME; # chroot directory or undef, -R # $db_home = "$MYHOME/db"; # dir for bdb nanny/cache/snmp databases, -D # $helpers_home = "$MYHOME/var"; # working directory for SpamAssassin, -S # $lock_file = "$MYHOME/var/amavisd.lock"; # -L # $pid_file = "$MYHOME/var/amavisd.pid"; # -P #NOTE: create directories $MYHOME/tmp, $MYHOME/var, $MYHOME/db manually $log_level = 0; # verbosity 0..5, -d $log_recip_templ = undef; # disable by-recipient level-0 log entries $do_syslog = 1; # log via syslogd (preferred) $syslog_facility = 'mail'; # Syslog facility as a string # e.g.: mail, daemon, user, local0, ... local7 $enable_db = 1; # enable use of BerkeleyDB/libdb (SNMP and nanny) $nanny_details_level = 2; # nanny verbosity: 1: traditional, 2: detailed $enable_dkim_verification = 1; # enable DKIM signatures verification $enable_dkim_signing = 1; # load DKIM signing code, keys defined by dkim_key @local_domains_maps = ( [".$mydomain"] ); # list of all local domains @mynetworks = qw( 127.0.0.0/8 [::1] [FE80::]/10 [FEC0::]/10 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 ); $unix_socketname = "$MYHOME/amavisd.sock"; # amavisd-release or amavis-milter # option(s) -p overrides $inet_socket_port and $unix_socketname $inet_socket_port = 10024; # listen on this local TCP port(s) # $inet_socket_port = [10024,10026]; # listen on multiple TCP ports $policy_bank{'MYNETS'} = { # mail originating from @mynetworks originating => 1, # is true in MYNETS by default, but let's make it explicit os_fingerprint_method => undef, # don't query p0f for internal clients }; # it is up to MTA to re-route mail from authenticated roaming users or # from internal hosts to a dedicated TCP port (such as 10026) for filtering $interface_policy{'10026'} = 'ORIGINATING'; $policy_bank{'ORIGINATING'} = { # mail supposedly originating from our users originating => 1, # declare that mail was submitted by our smtp client allow_disclaimers => 1, # enables disclaimer insertion if available # notify administrator of locally originating malware virus_admin_maps => ["virusalert\@$mydomain"], spam_admin_maps => ["virusalert\@$mydomain"], warnbadhsender => 1, # forward to a smtpd service providing DKIM signing service forward_method => 'smtp:[127.0.0.1]:10027', # force MTA conversion to 7-bit (e.g. before DKIM signing) smtpd_discard_ehlo_keywords => ['8BITMIME'], bypass_banned_checks_maps => [1], # allow sending any file names and types terminate_dsn_on_notify_success => 0, # don't remove NOTIFY=SUCCESS option }; $interface_policy{'SOCK'} = 'AM.PDP-SOCK'; # only applies with $unix_socketname # Use with amavis-release over a socket or with Petr Rehor's amavis-milter.c # (with amavis-milter.c from this package or old amavis.c client use 'AM.CL'): $policy_bank{'AM.PDP-SOCK'} = { protocol => 'AM.PDP', auth_required_release => 0, # do not require secret_id for amavisd-release }; $sa_tag_level_deflt = 2.0; # add spam info headers if at, or above that level $sa_tag2_level_deflt = 6.2; # add 'spam detected' headers at that level $sa_kill_level_deflt = 6.9; # triggers spam evasive actions (e.g. blocks mail) $sa_dsn_cutoff_level = 10; # spam level beyond which a DSN is not sent $sa_crediblefrom_dsn_cutoff_level = 18; # likewise, but for a likely valid From # $sa_quarantine_cutoff_level = 25; # spam level beyond which quarantine is off $penpals_bonus_score = 8; # (no effect without a @storage_sql_dsn database) $penpals_threshold_high = $sa_kill_level_deflt; # don't waste time on hi spam $bounce_killer_score = 100; # spam score points to add for joe-jobbed bounces $sa_mail_body_size_limit = 400*1024; # don't waste time on SA if mail is larger $sa_local_tests_only = 0; # only tests which do not require internet access? # @lookup_sql_dsn = # ( ['DBI:mysql:database=mail;host=127.0.0.1;port=3306', 'user1', 'passwd1'], # ['DBI:mysql:database=mail;host=host2', 'username2', 'password2'], # ["DBI:SQLite:dbname=$MYHOME/sql/mail_prefs.sqlite", '', ''] ); # @storage_sql_dsn = @lookup_sql_dsn; # none, same, or separate database # $timestamp_fmt_mysql = 1; # if using MySQL *and* msgs.time_iso is TIMESTAMP; # defaults to 0, which is good for non-MySQL or if msgs.time_iso is CHAR(16) $virus_admin = "virusalert\@$mydomain"; # notifications recip. $mailfrom_notify_admin = "virusalert\@$mydomain"; # notifications sender $mailfrom_notify_recip = "virusalert\@$mydomain"; # notifications sender $mailfrom_notify_spamadmin = "spam.police\@$mydomain"; # notifications sender $mailfrom_to_quarantine = ''; # null return path; uses original sender if undef @addr_extension_virus_maps = ('virus'); @addr_extension_banned_maps = ('banned'); @addr_extension_spam_maps = ('spam'); @addr_extension_bad_header_maps = ('badh'); # $recipient_delimiter = '+'; # undef disables address extensions altogether # when enabling addr extensions do also Postfix/main.cf: recipient_delimiter=+ $path = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/usr/bin:/bin'; # $dspam = 'dspam'; $MAXLEVELS = 14; $MAXFILES = 1500; $MIN_EXPANSION_QUOTA = 100*1024; # bytes (default undef, not enforced) $MAX_EXPANSION_QUOTA = 300*1024*1024; # bytes (default undef, not enforced) $sa_spam_subject_tag = '***Spam*** '; $defang_virus = 1; # MIME-wrap passed infected mail $defang_banned = 1; # MIME-wrap passed mail containing banned name # for defanging bad headers only turn on certain minor contents categories: $defang_by_ccat{CC_BADH.",3"} = 1; # NUL or CR character in header $defang_by_ccat{CC_BADH.",5"} = 1; # header line longer than 998 characters $defang_by_ccat{CC_BADH.",6"} = 1; # header field syntax error # OTHER MORE COMMON SETTINGS (defaults may suffice): # $myhostname = 'host.example.com'; # must be a fully-qualified domain name! # $notify_method = 'smtp:[127.0.0.1]:10025'; # $forward_method = 'smtp:[127.0.0.1]:10025'; # set to undef with milter! # $final_virus_destiny = D_DISCARD; # $final_banned_destiny = D_DISCARD; # $final_spam_destiny = D_PASS; #!!! D_DISCARD / D_REJECT # $final_bad_header_destiny = D_PASS; # $bad_header_quarantine_method = undef; # $os_fingerprint_method = 'p0f:*:2345'; # to query p0f-analyzer.pl ## hierarchy by which a final setting is chosen: ## policy bank (based on port or IP address) -> *_by_ccat ## *_by_ccat (based on mail contents) -> *_maps ## *_maps (based on recipient address) -> final configuration value # SOME OTHER VARIABLES WORTH CONSIDERING (see amavisd.conf-default for all) # $warnbadhsender, # $warnvirusrecip, $warnbannedrecip, $warnbadhrecip, (or @warn*recip_maps) # # @bypass_virus_checks_maps, @bypass_spam_checks_maps, # @bypass_banned_checks_maps, @bypass_header_checks_maps, # # @virus_lovers_maps, @spam_lovers_maps, # @banned_files_lovers_maps, @bad_header_lovers_maps, # # @blacklist_sender_maps, @score_sender_maps, # # $clean_quarantine_method, $virus_quarantine_to, $banned_quarantine_to, # $bad_header_quarantine_to, $spam_quarantine_to, # # $defang_bad_header, $defang_undecipherable, $defang_spam # REMAINING IMPORTANT VARIABLES ARE LISTED HERE BECAUSE OF LONGER ASSIGNMENTS @keep_decoded_original_maps = (new_RE( qr'^MAIL$', # retain full original message for virus checking qr'^MAIL-UNDECIPHERABLE$', # recheck full mail if it contains undecipherables qr'^(ASCII(?! cpio)|text|uuencoded|xxencoded|binhex)'i, # qr'^Zip archive data', # don't trust Archive::Zip )); $banned_filename_re = new_RE( ### BLOCKED ANYWHERE # qr'^UNDECIPHERABLE$', # is or contains any undecipherable components qr'^\.(exe-ms|dll)$', # banned file(1) types, rudimentary # qr'^\.(exe|lha|cab|dll)$', # banned file(1) types ### BLOCK THE FOLLOWING, EXCEPT WITHIN UNIX ARCHIVES: # [ qr'^\.(gz|bz2)$' => 0 ], # allow any in gzip or bzip2 [ qr'^\.(rpm|cpio|tar)$' => 0 ], # allow any in Unix-type archives qr'.\.(pif|scr)$'i, # banned extensions - rudimentary # qr'^\.zip$', # block zip type ### BLOCK THE FOLLOWING, EXCEPT WITHIN ARCHIVES: # [ qr'^\.(zip|rar|arc|arj|zoo)$'=> 0 ], # allow any within these archives qr'^application/x-msdownload$'i, # block these MIME types qr'^application/x-msdos-program$'i, qr'^application/hta$'i, # qr'^message/partial$'i, # rfc2046 MIME type # qr'^message/external-body$'i, # rfc2046 MIME type # qr'^(application/x-msmetafile|image/x-wmf)$'i, # Windows Metafile MIME type # qr'^\.wmf$', # Windows Metafile file(1) type # block certain double extensions in filenames qr'^(?!cid:).*\.[^./]*[A-Za-z][^./]*\.\s*(exe|vbs|pif|scr|bat|cmd|com|cpl|dll)[.\s]*$'i, # qr'\{[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}\}?'i, # Class ID CLSID, strict # qr'\{[0-9a-z]{4,}(-[0-9a-z]{4,}){0,7}\}?'i, # Class ID extension CLSID, loose qr'.\.(exe|vbs|pif|scr|cpl)$'i, # banned extension - basic # qr'.\.(exe|vbs|pif|scr|cpl|bat|cmd|com)$'i, # banned extension - basic+cmd # qr'.\.(ade|adp|app|bas|bat|chm|cmd|com|cpl|crt|emf|exe|fxp|grp|hlp|hta| # inf|ini|ins|isp|js|jse|lib|lnk|mda|mdb|mde|mdt|mdw|mdz|msc|msi| # msp|mst|ocx|ops|pcd|pif|prg|reg|scr|sct|shb|shs|sys|vb|vbe|vbs|vxd| # wmf|wsc|wsf|wsh)$'ix, # banned extensions - long # qr'.\.(asd|asf|asx|url|vcs|wmd|wmz)$'i, # consider also # qr'.\.(ani|cur|ico)$'i, # banned cursors and icons filename # qr'^\.ani$', # banned animated cursor file(1) type # qr'.\.(mim|b64|bhx|hqx|xxe|uu|uue)$'i, # banned extension - WinZip vulnerab. ); # See http://support.microsoft.com/default.aspx?scid=kb;EN-US;q262631 # and http://www.cknow.com/vtutor/vtextensions.htm # ENVELOPE SENDER SOFT-WHITELISTING / SOFT-BLACKLISTING @score_sender_maps = ({ # a by-recipient hash lookup table, # results from all matching recipient tables are summed # ## per-recipient personal tables (NOTE: positive: black, negative: white) # 'user1@example.com' => [{'bla-mobile.press@example.com' => 10.0}], # 'user3@example.com' => [{'.ebay.com' => -3.0}], # 'user4@example.com' => [{'cleargreen@cleargreen.com' => -7.0, # '.cleargreen.com' => -5.0}], ## site-wide opinions about senders (the '.' matches any recipient) '.' => [ # the _first_ matching sender determines the score boost new_RE( # regexp-type lookup table, just happens to be all soft-blacklist [qr'^(bulkmail|offers|cheapbenefits|earnmoney|foryou)@'i => 5.0], [qr'^(greatcasino|investments|lose_weight_today|market\.alert)@'i=> 5.0], [qr'^(money2you|MyGreenCard|new\.tld\.registry|opt-out|opt-in)@'i=> 5.0], [qr'^(optin|saveonlsmoking2002k|specialoffer|specialoffers)@'i => 5.0], [qr'^(stockalert|stopsnoring|wantsome|workathome|yesitsfree)@'i => 5.0], [qr'^(your_friend|greatoffers)@'i => 5.0], [qr'^(inkjetplanet|marketopt|MakeMoney)\d*@'i => 5.0], ), # read_hash("/var/amavis/sender_scores_sitewide"), { # a hash-type lookup table (associative array) 'nobody@cert.org' => -3.0, 'cert-advisory@us-cert.gov' => -3.0, 'owner-alert@iss.net' => -3.0, 'slashdot@slashdot.org' => -3.0, 'securityfocus.com' => -3.0, 'ntbugtraq@listserv.ntbugtraq.com' => -3.0, 'security-alerts@linuxsecurity.com' => -3.0, 'mailman-announce-admin@python.org' => -3.0, 'amavis-user-admin@lists.sourceforge.net'=> -3.0, 'amavis-user-bounces@lists.sourceforge.net' => -3.0, 'spamassassin.apache.org' => -3.0, 'notification-return@lists.sophos.com' => -3.0, 'owner-postfix-users@postfix.org' => -3.0, 'owner-postfix-announce@postfix.org' => -3.0, 'owner-sendmail-announce@lists.sendmail.org' => -3.0, 'sendmail-announce-request@lists.sendmail.org' => -3.0, 'donotreply@sendmail.org' => -3.0, 'ca+envelope@sendmail.org' => -3.0, 'noreply@freshmeat.net' => -3.0, 'owner-technews@postel.acm.org' => -3.0, 'ietf-123-owner@loki.ietf.org' => -3.0, 'cvs-commits-list-admin@gnome.org' => -3.0, 'rt-users-admin@lists.fsck.com' => -3.0, 'clp-request@comp.nus.edu.sg' => -3.0, 'surveys-errors@lists.nua.ie' => -3.0, 'emailnews@genomeweb.com' => -5.0, 'yahoo-dev-null@yahoo-inc.com' => -3.0, 'returns.groups.yahoo.com' => -3.0, 'clusternews@linuxnetworx.com' => -3.0, lc('lvs-users-admin@LinuxVirtualServer.org') => -3.0, lc('owner-textbreakingnews@CNNIMAIL12.CNN.COM') => -5.0, # soft-blacklisting (positive score) 'sender@example.net' => 3.0, '.example.net' => 1.0, }, ], # end of site-wide tables }); @decoders = ( ['mail', \&do_mime_decode], # ['asc', \&do_ascii], # ['uue', \&do_ascii], # ['hqx', \&do_ascii], # ['ync', \&do_ascii], ['F', \&do_uncompress, ['unfreeze','freeze -d','melt','fcat'] ], ['Z', \&do_uncompress, ['uncompress','gzip -d','zcat'] ], ['gz', \&do_uncompress, 'gzip -d'], ['gz', \&do_gunzip], ['bz2', \&do_uncompress, 'bzip2 -d'], ['xz', \&Amavis::Unpackers::do_uncompress, ['xzdec', 'xz -dc', 'unxz -c', 'xzcat'] ], ['lzma', \&Amavis::Unpackers::do_uncompress, ['lzmadec', 'xz -dc --format=lzma', 'lzma -dc', 'unlzma -c', 'lzcat', 'lzmadec'] ], ['lzo', \&do_uncompress, 'lzop -d'], ['rpm', \&do_uncompress, ['rpm2cpio.pl','rpm2cpio'] ], ['cpio', \&do_pax_cpio, ['pax','gcpio','cpio'] ], ['tar', \&do_pax_cpio, ['pax','gcpio','cpio'] ], ['deb', \&do_ar, 'ar'], # ['a', \&do_ar, 'ar'], # unpacking .a seems an overkill ['zip', \&do_unzip], ['7z', \&do_7zip, ['7zr','7za','7z'] ], ['rar', \&do_unrar, ['rar','unrar'] ], ['arj', \&do_unarj, ['arj','unarj'] ], ['arc', \&do_arc, ['nomarch','arc'] ], ['zoo', \&do_zoo, ['zoo','unzoo'] ], # ['lha', \&do_lha, 'lha'], # unmaintained - security risk # ['doc', \&do_ole, 'ripole'], ['cab', \&do_cabextract, 'cabextract'], ['tnef', \&do_tnef_ext, 'tnef'], ['tnef', \&do_tnef], # ['sit', \&do_unstuff, 'unstuff'], # broken/unsafe decoder ['exe', \&do_executable, ['rar','unrar'], 'lha', ['arj','unarj'] ], ); @av_scanners = ( # ### http://www.sophos.com/ # ['Sophos-SSSP', # \&ask_daemon, ["{}", 'sssp:/var/run/savdi/sssp.sock'], # # or: ["{}", 'sssp:[127.0.0.1]:4010'], # qr/^DONE OK\b/m, qr/^VIRUS\b/m, qr/^VIRUS\s*(\S*)/m ], # ### http://www.clanfield.info/sophie/ (http://www.vanja.com/tools/sophie/) # ['Sophie', # \&ask_daemon, ["{}/\n", 'sophie:/var/run/sophie'], # qr/(?x)^ 0+ ( : | [\000\r\n]* $)/, qr/(?x)^ 1 ( : | [\000\r\n]* $)/, # qr/(?x)^ [-+]? \d+ : (.*?) [\000\r\n]* $/m ], # ### http://www.csupomona.edu/~henson/www/projects/SAVI-Perl/ # ['Sophos SAVI', \&ask_daemon, ['{}','savi-perl:'] ], # ['Avira SAVAPI', # \&ask_daemon, ["*", 'savapi:/var/tmp/.savapi3', 'product-id'], # qr/^(200|210)/m, qr/^(310|420|319)/m, # qr/^(?:310|420)[,\s]*(?:.* <<< )?(.+?)(?: ; |$)/m ], # settings for the SAVAPI3.conf: ArchiveScan=1, HeurLevel=2, MailboxScan=1 # ### http://www.clamav.net/ # ['ClamAV-clamd', # \&ask_daemon, ["CONTSCAN {}\n", "/var/run/clamav/clamd.sock"], # qr/\bOK$/m, qr/\bFOUND$/m, # qr/^.*?: (?!Infected Archive)(.*) FOUND$/m ], # # NOTE: run clamd under the same user as amavisd - or run it under its own # # uid such as clamav, add user clamav to the amavis group, and then add # # AllowSupplementaryGroups to clamd.conf; # # NOTE: match socket name (LocalSocket) in clamav.conf to the socket name in # # this entry; when running chrooted one may prefer a socket under $MYHOME. # ### http://www.clamav.net/ and CPAN (memory-hungry! clamd is preferred) # # note that Mail::ClamAV requires perl to be build with threading! # ['Mail::ClamAV', \&ask_daemon, ['{}','clamav-perl:'], # [0], [1], qr/^INFECTED: (.+)/m], # ### http://www.openantivirus.org/ # ['OpenAntiVirus ScannerDaemon (OAV)', # \&ask_daemon, ["SCAN {}\n", '127.0.0.1:8127'], # qr/^OK/m, qr/^FOUND: /m, qr/^FOUND: (.+)/m ], # ### http://www.vanja.com/tools/trophie/ # ['Trophie', # \&ask_daemon, ["{}/\n", 'trophie:/var/run/trophie'], # qr/(?x)^ 0+ ( : | [\000\r\n]* $)/m, qr/(?x)^ 1 ( : | [\000\r\n]* $)/m, # qr/(?x)^ [-+]? \d+ : (.*?) [\000\r\n]* $/m ], # ### http://www.grisoft.com/ # ['AVG Anti-Virus', # \&ask_daemon, ["SCAN {}\n", '127.0.0.1:55555'], # qr/^200/m, qr/^403/m, qr/^403 .*?: ([^\r\n]+)/m ], # ### http://www.f-prot.com/ # ['F-Prot fpscand', # F-PROT Antivirus for BSD/Linux/Solaris, version 6 # \&ask_daemon, # ["SCAN FILE {}/*\n", '127.0.0.1:10200'], # qr/^(0|8|64) /m, # qr/^([1235679]|1[01345]) |<[^>:]*(?i)(infected|suspicious|unwanted)/m, # qr/(?i)<[^>:]*(?:infected|suspicious|unwanted)[^>:]*: ([^>]*)>/m ], # ### http://www.f-prot.com/ # ['F-Prot f-protd', # old version # \&ask_daemon, # ["GET {}/*?-dumb%20-archive%20-packed HTTP/1.0\r\n\r\n", # ['127.0.0.1:10200', '127.0.0.1:10201', '127.0.0.1:10202', # '127.0.0.1:10203', '127.0.0.1:10204'] ], # qr/(?i)]*>clean<\/summary>/m, # qr/(?i)]*>infected<\/summary>/m, # qr/(?i)(.+)<\/name>/m ], # ### http://www.sald.com/, http://www.dials.ru/english/, http://www.drweb.ru/ # ['DrWebD', \&ask_daemon, # DrWebD 4.31 or later # [pack('N',1). # DRWEBD_SCAN_CMD # pack('N',0x00280001). # DONT_CHANGEMAIL, IS_MAIL, RETURN_VIRUSES # pack('N', # path length # length("$TEMPBASE/amavis-yyyymmddTHHMMSS-xxxxx/parts/pxxx")). # '{}/*'. # path # pack('N',0). # content size # pack('N',0), # '/var/drweb/run/drwebd.sock', # # '/var/amavis/var/run/drwebd.sock', # suitable for chroot # # '/usr/local/drweb/run/drwebd.sock', # FreeBSD drweb ports default # # '127.0.0.1:3000', # or over an inet socket # ], # qr/\A\x00[\x10\x11][\x00\x10]\x00/sm, # IS_CLEAN,EVAL_KEY; SKIPPED # qr/\A\x00[\x00\x01][\x00\x10][\x20\x40\x80]/sm,# KNOWN_V,UNKNOWN_V,V._MODIF # qr/\A.{12}(?:infected with )?([^\x00]+)\x00/sm, # ], # # NOTE: If using amavis-milter, change length to: # # length("$TEMPBASE/amavis-milter-xxxxxxxxxxxxxx/parts/pxxx"). ### http://www.kaspersky.com/ (kav4mailservers) ['KasperskyLab AVP - aveclient', ['/usr/local/kav/bin/aveclient','/usr/local/share/kav/bin/aveclient', '/opt/kav/5.5/kav4mailservers/bin/aveclient','aveclient'], '-p /var/run/aveserver -s {}/*', [0,3,6,8], qr/\b(INFECTED|SUSPICION|SUSPICIOUS)\b/m, qr/(?:INFECTED|WARNING|SUSPICION|SUSPICIOUS) (.+)/m, ], # NOTE: one may prefer [0],[2,3,4,5], depending on how suspicious, # currupted or protected archives are to be handled ### http://www.kaspersky.com/ ['KasperskyLab AntiViral Toolkit Pro (AVP)', ['avp'], '-* -P -B -Y -O- {}', [0,3,6,8], [2,4], # any use for -A -K ? qr/infected: (.+)/m, sub {chdir('/opt/AVP') or die "Can't chdir to AVP: $!"}, sub {chdir($TEMPBASE) or die "Can't chdir back to $TEMPBASE $!"}, ], ### The kavdaemon and AVPDaemonClient have been removed from Kasperky ### products and replaced by aveserver and aveclient ['KasperskyLab AVPDaemonClient', [ '/opt/AVP/kavdaemon', 'kavdaemon', '/opt/AVP/AvpDaemonClient', 'AvpDaemonClient', '/opt/AVP/AvpTeamDream', 'AvpTeamDream', '/opt/AVP/avpdc', 'avpdc' ], "-f=$TEMPBASE {}", [0,8], [3,4,5,6], qr/infected: ([^\r\n]+)/m ], # change the startup-script in /etc/init.d/kavd to: # DPARMS="-* -Y -dl -f=/var/amavis /var/amavis" # (or perhaps: DPARMS="-I0 -Y -* /var/amavis" ) # adjusting /var/amavis above to match your $TEMPBASE. # The '-f=/var/amavis' is needed if not running it as root, so it # can find, read, and write its pid file, etc., see 'man kavdaemon'. # defUnix.prf: there must be an entry "*/var/amavis" (or whatever # directory $TEMPBASE specifies) in the 'Names=' section. # cd /opt/AVP/DaemonClients; configure; cd Sample; make # cp AvpDaemonClient /opt/AVP/ # su - vscan -c "${PREFIX}/kavdaemon ${DPARMS}" ### http://www.centralcommand.com/ ['CentralCommand Vexira (new) vascan', ['vascan','/usr/lib/Vexira/vascan'], "-a s --timeout=60 --temp=$TEMPBASE -y $QUARANTINEDIR ". "--log=/var/log/vascan.log {}", [0,3], [1,2,5], qr/(?x)^\s* (?:virus|iworm|macro|mutant|sequence|trojan)\ found:\ ( [^\]\s']+ )\ \.\.\.\ /m ], # Adjust the path of the binary and the virus database as needed. # 'vascan' does not allow to have the temp directory to be the same as # the quarantine directory, and the quarantine option can not be disabled. # If $QUARANTINEDIR is not used, then another directory must be specified # to appease 'vascan'. Move status 3 to the second list if password # protected files are to be considered infected. ### http://www.avira.com/ ### old Avira AntiVir 2.x (ex H+BEDV) or old CentralCommand Vexira Antivirus ['Avira AntiVir', ['antivir','vexira'], '--allfiles -noboot -nombr -rs -s -z {}', [0], qr/ALERT:|VIRUS:/m, qr/(?x)^\s* (?: ALERT: \s* (?: \[ | [^']* ' ) | (?i) VIRUS:\ .*?\ virus\ '?) ( [^\]\s']+ )/m ], # NOTE: if you only have a demo version, remove -z and add 214, as in: # '--allfiles -noboot -nombr -rs -s {}', [0,214], qr/ALERT:|VIRUS:/, ### http://www.avira.com/ ### Avira for UNIX 3.x ['Avira AntiVir', ['avscan'], '-s --batch --alert-action=none {}', [0,4], qr/(?:ALERT|FUND):/m, qr/(?:ALERT|FUND): (?:.* <<< )?(.+?)(?: ; |$)/m ], ### http://www.commandsoftware.com/ ['Command AntiVirus for Linux', 'csav', '-all -archive -packed {}', [50], [51,52,53], qr/Infection: (.+)/m ], ### http://www.symantec.com/ ['Symantec CarrierScan via Symantec CommandLineScanner', 'cscmdline', '-a scan -i 1 -v -s 127.0.0.1:7777 {}', qr/^Files Infected:\s+0$/m, qr/^Infected\b/m, qr/^(?:Info|Virus Name):\s+(.+)/m ], ### http://www.symantec.com/ ['Symantec AntiVirus Scan Engine', 'savsecls', '-server 127.0.0.1:7777 -mode scanrepair -details -verbose {}', [0], qr/^Infected\b/m, qr/^(?:Info|Virus Name):\s+(.+)/m ], # NOTE: check options and patterns to see which entry better applies # ### http://www.f-secure.com/products/anti-virus/ version 4.65 # ['F-Secure Antivirus for Linux servers', # ['/opt/f-secure/fsav/bin/fsav', 'fsav'], # '--delete=no --disinf=no --rename=no --archive=yes --auto=yes '. # '--dumb=yes --list=no --mime=yes {}', [0], [3,6,8], # qr/(?:infection|Infected|Suspected): (.+)/m ], ### http://www.f-secure.com/products/anti-virus/ version 5.52 ['F-Secure Antivirus for Linux servers', ['/opt/f-secure/fsav/bin/fsav', 'fsav'], '--virus-action1=report --archive=yes --auto=yes '. '--dumb=yes --list=no --mime=yes {}', [0], [3,4,6,8], qr/(?:infection|Infected|Suspected|Riskware): (.+)/m ], # NOTE: internal archive handling may be switched off by '--archive=no' # to prevent fsav from exiting with status 9 on broken archives # ### http://www.avast.com/ # ['avast! Antivirus daemon', # \&ask_daemon, # greets with 220, terminate with QUIT # ["SCAN {}\015\012QUIT\015\012", '/var/run/avast4/mailscanner.sock'], # qr/\t\[\+\]/m, qr/\t\[L\]\t/m, qr/\t\[L\]\t([^[ \t\015\012]+)/m ], # ### http://www.avast.com/ # ['avast! Antivirus - Client/Server Version', 'avastlite', # '-a /var/run/avast4/mailscanner.sock -n {}', [0], [1], # qr/\t\[L\]\t([^[ \t\015\012]+)/m ], ['CAI InoculateIT', 'inocucmd', # retired product '-sec -nex {}', [0], [100], qr/was infected by virus (.+)/m ], # see: http://www.flatmtn.com/computer/Linux-Antivirus_CAI.html ### http://www3.ca.com/Solutions/Product.asp?ID=156 (ex InoculateIT) ['CAI eTrust Antivirus', 'etrust-wrapper', '-arc -nex -spm h {}', [0], [101], qr/is infected by virus: (.+)/m ], # NOTE: requires suid wrapper around inocmd32; consider flag: -mod reviewer # see http://marc.theaimsgroup.com/?l=amavis-user&m=109229779912783 ### http://mks.com.pl/english.html ['MkS_Vir for Linux (beta)', ['mks32','mks'], '-s {}/*', [0], [1,2], qr/--[ \t]*(.+)/m ], ### http://mks.com.pl/english.html ['MkS_Vir daemon', 'mksscan', '-s -q {}', [0], [1..7], qr/^... (\S+)/m ], # ### http://www.nod32.com/, version v2.52 (old) # ['ESET NOD32 for Linux Mail servers', # ['/opt/eset/nod32/bin/nod32cli', 'nod32cli'], # '--subdir --files -z --sfx --rtp --adware --unsafe --pattern --heur '. # '-w -a --action-on-infected=accept --action-on-uncleanable=accept '. # '--action-on-notscanned=accept {}', # [0,3], [1,2], qr/virus="([^"]+)"/m ], # ### http://www.eset.com/, version v2.7 (old) # ['ESET NOD32 Linux Mail Server - command line interface', # ['/usr/bin/nod32cli', '/opt/eset/nod32/bin/nod32cli', 'nod32cli'], # '--subdir {}', [0,3], [1,2], qr/virus="([^"]+)"/m ], # ### http://www.eset.com/, version 2.71.12 # ['ESET Software ESETS Command Line Interface', # ['/usr/bin/esets_cli', 'esets_cli'], # '--subdir {}', [0], [1,2,3], qr/virus="([^"]+)"/m ], ### http://www.eset.com/, version 3.0 ['ESET Software ESETS Command Line Interface', ['/usr/bin/esets_cli', 'esets_cli'], '--subdir {}', [0], [1,2,3], qr/:\s*action="(?!accepted)[^"]*"\n.*:\s*virus="([^"]*)"/m ], ## http://www.nod32.com/, NOD32LFS version 2.5 and above ['ESET NOD32 for Linux File servers', ['/opt/eset/nod32/sbin/nod32','nod32'], '--files -z --mail --sfx --rtp --adware --unsafe --pattern --heur '. '-w -a --action=1 -b {}', [0], [1,10], qr/^object=.*, virus="(.*?)",/m ], # Experimental, based on posting from Rado Dibarbora (Dibo) on 2002-05-31 # ['ESET Software NOD32 Client/Server (NOD32SS)', # \&ask_daemon2, # greets with 200, persistent, terminate with QUIT # ["SCAN {}/*\r\n", '127.0.0.1:8448' ], # qr/^200 File OK/m, qr/^201 /m, qr/^201 (.+)/m ], ### http://www.norman.com/products_nvc.shtml ['Norman Virus Control v5 / Linux', 'nvcc', '-c -l:0 -s -u -temp:$TEMPBASE {}', [0,10,11], [1,2,14], qr/(?i).* virus in .* -> \'(.+)\'/m ], ### http://www.pandasoftware.com/ ['Panda CommandLineSecure 9 for Linux', ['/opt/pavcl/usr/bin/pavcl','pavcl'], '-auto -aex -heu -cmp -nbr -nor -nos -eng -nob {}', qr/Number of files infected[ .]*: 0+(?!\d)/m, qr/Number of files infected[ .]*: 0*[1-9]/m, qr/Found virus :\s*(\S+)/m ], # NOTE: for efficiency, start the Panda in resident mode with 'pavcl -tsr' # before starting amavisd - the bases are then loaded only once at startup. # To reload bases in a signature update script: # /opt/pavcl/usr/bin/pavcl -tsr -ulr; /opt/pavcl/usr/bin/pavcl -tsr # Please review other options of pavcl, for example: # -nomalw, -nojoke, -nodial, -nohackt, -nospyw, -nocookies # ### http://www.pandasoftware.com/ # ['Panda Antivirus for Linux', ['pavcl'], # '-TSR -aut -aex -heu -cmp -nbr -nor -nso -eng {}', # [0], [0x10, 0x30, 0x50, 0x70, 0x90, 0xB0, 0xD0, 0xF0], # qr/Found virus :\s*(\S+)/m ], # GeCAD AV technology is acquired by Microsoft; RAV has been discontinued. # Check your RAV license terms before fiddling with the following two lines! # ['GeCAD RAV AntiVirus 8', 'ravav', # '--all --archive --mail {}', [1], [2,3,4,5], qr/Infected: (.+)/m ], # # NOTE: the command line switches changed with scan engine 8.5 ! # # (btw, assigning stdin to /dev/null causes RAV to fail) ### http://www.nai.com/ ['NAI McAfee AntiVirus (uvscan)', 'uvscan', '--secure -rv --mime --summary --noboot - {}', [0], [13], qr/(?x) Found (?: \ the\ (.+)\ (?:virus|trojan) | \ (?:virus|trojan)\ or\ variant\ ([^ ]+) | :\ (.+)\ NOT\ a\ virus)/m, # sub {$ENV{LD_PRELOAD}='/lib/libc.so.6'}, # sub {delete $ENV{LD_PRELOAD}}, ], # NOTE1: with RH9: force the dynamic linker to look at /lib/libc.so.6 before # anything else by setting environment variable LD_PRELOAD=/lib/libc.so.6 # and then clear it when finished to avoid confusing anything else. # NOTE2: to treat encrypted files as viruses replace the [13] with: # qr/^\s{5,}(Found|is password-protected|.*(virus|trojan))/ ### http://www.virusbuster.hu/en/ ['VirusBuster', ['vbuster', 'vbengcl'], "{} -ss -i '*' -log=$MYHOME/vbuster.log", [0], [1], qr/: '(.*)' - Virus/m ], # VirusBuster Ltd. does not support the daemon version for the workstation # engine (vbuster-eng-1.12-linux-i386-libc6.tgz) any longer. The names of # binaries, some parameters AND return codes have changed (from 3 to 1). # See also the new Vexira entry 'vascan' which is possibly related. # ### http://www.virusbuster.hu/en/ # ['VirusBuster (Client + Daemon)', 'vbengd', # '-f -log scandir {}', [0], [3], # qr/Virus found = (.*);/m ], # # HINT: for an infected file it always returns 3, # # although the man-page tells a different story ### http://www.cyber.com/ ['CyberSoft VFind', 'vfind', '--vexit {}/*', [0], [23], qr/##==>>>> VIRUS ID: CVDL (.+)/m, # sub {$ENV{VSTK_HOME}='/usr/lib/vstk'}, ], ### http://www.avast.com/ ['avast! Antivirus', ['/usr/bin/avastcmd','avastcmd'], '-a -i -n -t=A {}', [0], [1], qr/\binfected by:\s+([^ \t\n\[\]]+)/m ], ### http://www.ikarus-software.com/ ['Ikarus AntiVirus for Linux', 'ikarus', '{}', [0], [40], qr/Signature (.+) found/m ], ### http://www.bitdefender.com/ ['BitDefender', 'bdscan', # new version '--action=ignore --no-list {}', qr/^Infected files\s*:\s*0+(?!\d)/m, qr/^(?:Infected files|Identified viruses|Suspect files)\s*:\s*0*[1-9]/m, qr/(?:suspected|infected)\s*:\s*(.*)(?:\033|$)/m ], ### http://www.bitdefender.com/ ['BitDefender', 'bdc', # old version '--arc --mail {}', qr/^Infected files *:0+(?!\d)/m, qr/^(?:Infected files|Identified viruses|Suspect files) *:0*[1-9]/m, qr/(?:suspected|infected): (.*)(?:\033|$)/m ], # consider also: --all --nowarn --alev=15 --flev=15. The --all argument may # not apply to your version of bdc, check documentation and see 'bdc --help' ### ArcaVir for Linux and Unix http://www.arcabit.pl/ ['ArcaVir for Linux', ['arcacmd','arcacmd.static'], '-v 1 -summary 0 -s {}', [0], [1,2], qr/(?:VIR|WIR):[ \t]*(.+)/m ], # ### a generic SMTP-client interface to a SMTP-based virus scanner # ['av_smtp', \&ask_av_smtp, # ['{}', 'smtp:[127.0.0.1]:5525', 'dummy@localhost'], # qr/^2/, qr/^5/, qr/^\s*(.*?)\s*$/m ], # ['File::Scan', sub {Amavis::AV::ask_av(sub{ # use File::Scan; my($fn)=@_; # my($f)=File::Scan->new(max_txt_size=>0, max_bin_size=>0); # my($vname) = $f->scan($fn); # $f->error ? (2,"Error: ".$f->error) # : ($vname ne '') ? (1,"$vname FOUND") : (0,"Clean")}, @_) }, # ["{}/*"], [0], [1], qr/^(.*) FOUND$/m ], # ### fully-fledged checker for JPEG marker segments of invalid length # ['check-jpeg', # sub { use JpegTester (); Amavis::AV::ask_av(\&JpegTester::test_jpeg, @_) }, # ["{}/*"], undef, [1], qr/^(bad jpeg: .*)$/m ], # # NOTE: place file JpegTester.pm somewhere where Perl can find it, # # for example in /usr/local/lib/perl5/site_perl ); @av_scanners_backup = ( ### http://www.clamav.net/ - backs up clamd or Mail::ClamAV ['ClamAV-clamscan', 'clamscan', "--stdout --no-summary -r --tempdir=$TEMPBASE {}", [0], qr/:.*\sFOUND$/m, qr/^.*?: (?!Infected Archive)(.*) FOUND$/m ], # ### http://www.clamav.net/ - using remote clamd scanner as a backup # ['ClamAV-clamdscan', 'clamdscan', # "--stdout --no-summary --config-file=/etc/clamd-client.conf {}", # [0], qr/:.*\sFOUND$/m, qr/^.*?: (?!Infected Archive)(.*) FOUND$/m ], # ['ClamAV-clamd-stream', # \&ask_daemon, ["*", 'clamd:/var/run/clamav/clamd.sock'], # qr/\bOK$/m, qr/\bFOUND$/m, # qr/^.*?: (?!Infected Archive)(.*) FOUND$/m ], ### http://www.f-prot.com/ - backs up F-Prot Daemon, V6 ['F-PROT Antivirus for UNIX', ['fpscan'], '--report --mount --adware {}', # consider: --applications -s 4 -u 3 -z 10 [0,8,64], [1,2,3, 4+1,4+2,4+3, 8+1,8+2,8+3, 12+1,12+2,12+3], qr/^\[Found\s+[^\]]*\]\s+<([^ \t(>]*)/m ], ### http://www.f-prot.com/ - backs up F-Prot Daemon (old) ['FRISK F-Prot Antivirus', ['f-prot','f-prot.sh'], '-dumb -archive -packed {}', [0,8], [3,6], # or: [0], [3,6,8], qr/(?:Infection:|security risk named) (.+)|\s+contains\s+(.+)$/m ], ### http://www.trendmicro.com/ - backs up Trophie ['Trend Micro FileScanner', ['/etc/iscan/vscan','vscan'], '-za -a {}', [0], qr/Found virus/m, qr/Found virus (.+) in/m ], ### http://www.sald.com/, http://drweb.imshop.de/ - backs up DrWebD ['drweb - DrWeb Antivirus', # security LHA hole in Dr.Web 4.33 and earlier ['/usr/local/drweb/drweb', '/opt/drweb/drweb', 'drweb'], '-path={} -al -go -ot -cn -upn -ok-', [0,32], [1,9,33], qr' infected (?:with|by)(?: virus)? (.*)$'m ], ### http://www.kaspersky.com/ ['Kaspersky Antivirus v5.5', ['/opt/kaspersky/kav4fs/bin/kav4fs-kavscanner', '/opt/kav/5.5/kav4unix/bin/kavscanner', '/opt/kav/5.5/kav4mailservers/bin/kavscanner', 'kavscanner'], '-i0 -xn -xp -mn -R -ePASBME {}/*', [0,10,15], [5,20,21,25], qr/(?:INFECTED|WARNING|SUSPICION|SUSPICIOUS) (.*)/m, # sub {chdir('/opt/kav/bin') or die "Can't chdir to kav: $!"}, # sub {chdir($TEMPBASE) or die "Can't chdir back to $TEMPBASE $!"}, ], # Commented out because the name 'sweep' clashes with Debian and FreeBSD # package/port of an audio editor. Make sure the correct 'sweep' is found # in the path when enabling. # # ### http://www.sophos.com/ - backs up Sophie or SAVI-Perl # ['Sophos Anti Virus (sweep)', 'sweep', # '-nb -f -all -rec -ss -sc -archive -cab -mime -oe -tnef '. # '--no-reset-atime {}', # [0,2], qr/Virus .*? found/m, # qr/^>>> Virus(?: fragment)? '?(.*?)'? found/m, # ], # # other options to consider: -idedir=/usr/local/sav # Always succeeds and considers mail clean. # Potentially useful when all other scanners fail and it is desirable # to let mail continue to flow with no virus checking (when uncommented). # ['always-clean', sub {0}], ); 1; # insure a defined return value amavisd-new-2.7.1/amavisd000751 000621 000620 00004765144 11747105723 015071 0ustar00markcmi000000 000000 #!/usr/bin/perl -T #------------------------------------------------------------------------------ # This is amavisd-new. # It is an interface between a message transfer agent (MTA) and virus # scanners and/or spam scanners, functioning as a mail content filter. # # It is a performance-enhanced and feature-enriched version of amavisd # (which in turn is a daemonized version of AMaViS), initially based # on amavisd-snapshot-20020300). # # All work since amavisd-snapshot-20020300: # Copyright (C) 2002-2012 Mark Martinec, # All Rights Reserved. # with contributions from the amavis-user mailing list and individuals, # as acknowledged in the release notes. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Author: Mark Martinec # Patches and problem reports are welcome. # # The latest version of this program is available at: # http://www.ijs.si/software/amavisd/ #------------------------------------------------------------------------------ # Here is a boilerplate from the amavisd(-snapshot) version, # which is the version that served as a base code for the initial # version of amavisd-new. License terms were the same: # # Author: Chris Mason # Current maintainer: Lars Hecking # Based on work by: # Mogens Kjaer, Carlsberg Laboratory, # Juergen Quade, Softing GmbH, # Christian Bricart # Rainer Link # This script is part of the AMaViS package. For more information see: # http://amavis.org/ # Copyright (C) 2000 - 2002 the people mentioned above # This software is licensed under the GNU General Public License (GPL) # See: http://www.gnu.org/copyleft/gpl.html #------------------------------------------------------------------------------ #------------------------------------------------------------------------------ #Index of packages in this file # Amavis::Boot # Amavis::Conf # Amavis::Log # Amavis::Timing # Amavis::Util # Amavis::ProcControl # Amavis::rfc2821_2822_Tools # Amavis::Lookup::RE # Amavis::Lookup::IP # Amavis::Lookup::Opaque # Amavis::Lookup::OpaqueRef # Amavis::Lookup::Label # Amavis::Lookup::SQLfield (just the new() method) # Amavis::Lookup::LDAPattr (just the new() method) # Amavis::Lookup # Amavis::Expand # Amavis::TempDir # Amavis::IO::FileHandle # Amavis::IO::Zlib # Amavis::IO::RW # Amavis::In::Connection # Amavis::In::Message::PerRecip # Amavis::In::Message # Amavis::Out::EditHeader # Amavis::Out # Amavis::UnmangleSender # Amavis::Unpackers::NewFilename # Amavis::Unpackers::Part # Amavis::Unpackers::OurFiler # Amavis::Unpackers::Validity # Amavis::Unpackers::MIME # Amavis::Notify # Amavis::Custom # Amavis #optionally compiled-in packages: --------------------------------------------- # Amavis::DB::SNMP # Amavis::DB # Amavis::Lookup::SQLfield (the rest) # Amavis::Lookup::SQL # Amavis::LDAP::Connection # Amavis::Lookup::LDAP # Amavis::Lookup::LDAPattr (the rest) # Amavis::In::AMPDP # Amavis::In::SMTP #( Amavis::In::Courier ) # Amavis::Out::SMTP::Protocol # Amavis::Out::SMTP::Session # Amavis::Out::SMTP # Amavis::Out::Pipe # Amavis::Out::BSMTP # Amavis::Out::Local # Amavis::OS_Fingerprint # Amavis::Out::SQL::Connection # Amavis::Out::SQL::Log # Amavis::IO::SQL # Amavis::Out::SQL::Quarantine # Amavis::AV # Amavis::SpamControl # Amavis::SpamControl::ExtProg # Amavis::SpamControl::SpamdClient # Mail::SpamAssassin::Logger::Amavislog # Amavis::SpamControl::SpamAssassin # Amavis::Unpackers # Amavis::DKIM::CustomSigner # Amavis::DKIM # Amavis::Tools #------------------------------------------------------------------------------ use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; # package Amavis::Boot; use strict; use re 'taint'; use Errno qw(ENOENT EACCES); # replacement for a 'require' with a more informative error handling #sub my_require($) { # my($filename) = @_; # my($result); # if (exists $INC{$filename} && !$INC{$filename}) { # die "Compilation failed in require\n"; # } elsif (exists $INC{$filename}) { # $result = 1; # already loaded # } else { # my($found) = 0; # for my $prefix (@INC) { # my($full_fname) = "$prefix/$filename"; # my(@stat_list) = stat($full_fname); # symlinks-friendly # my($errn) = @stat_list ? 0 : 0+$!; # if ($errn != ENOENT) { # $found = 1; # $INC{$filename} = $full_fname; # my($owner_uid) = $stat_list[4]; # my($msg); # if ($errn) { $msg = "is inaccessible: $!" } # elsif (-d _) { $msg = "is a directory" } # elsif (!-f _) { $msg = "is not a regular file" } # elsif ($> && -o _) { $msg = "should not be owned by EUID $>" } # elsif ($> && -w _) { $msg = "is writable by EUID $>, EGID $)" } # elsif ($owner_uid) { $msg = "should be owned by root (uid 0) "} # !defined($msg) or die "Requiring $full_fname, file $msg,\n"; # $! = 0; # $result = do $full_fname; # if (!defined($result) && $@ ne '') { # undef $INC{$filename}; chomp($@); # die "Error in file $full_fname: $@\n"; # } elsif (!defined($result) && $! != 0) { # undef $INC{$filename}; # die "Error reading file $full_fname: $!\n"; # } elsif (!$result) { # undef $INC{$filename}; # die "Module $full_fname did not return a true value\n"; # } # last; # } # } # die sprintf("my_require: Can't locate %s in \@INC (\@INC contains: %s)\n", # $filename, join(' ',@INC)) if !$found; # } # $result; #} # Fetch all required modules (or nicely report missing ones), and compile them # once-and-for-all at the parent process, so that forked children can inherit # and share already compiled code in memory. Children will still need to 'use' # modules if they want to inherit from their name space. # sub fetch_modules($$@) { my($reason, $required, @modules) = @_; my($have_sawampersand) = Devel::SawAmpersand->UNIVERSAL::can('sawampersand'); my($amp) = $have_sawampersand && Devel::SawAmpersand::sawampersand() ? 1 : 0; warn 'fetch_modules: PL_sawampersand flag was already turned on' if $amp; my(@missing); for my $m (@modules) { local($_) = $m; $_ .= /^auto::/ ? '.al' : '.pm' if !m{^/} && !m{\.(pm|pl|al|ix)\z}; s{::}{/}g; # eval { my_require $_ } #more informative on err, but some problems reported eval { require $_ } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; push(@missing,$m); $eval_stat =~ s/^/ /mgs; # indent printf STDERR ("fetch_modules: error loading %s module %s:\n%s\n", $required ? 'required' : 'optional', $_, $eval_stat) if $eval_stat !~ /\bCan't locate \Q$_\E in \@INC\b/; }; if ($have_sawampersand && !$amp && Devel::SawAmpersand::sawampersand()) { $amp = 1; warn "Loading of module $m turned on PL_sawampersand flag" } } die "ERROR: MISSING $reason:\n" . join('', map(" $_\n", @missing)) if $required && @missing; \@missing; } BEGIN { if ($] <= 5.008) { # deal with a glob() taint bug (perl 5.6.1, 5.8.0) fetch_modules('REQUIRED BASIC MODULES', 1, qw(File::Glob)); File::Glob->import(':globally'); # use the same module as Perl 5.8 uses } fetch_modules('REQUIRED BASIC MODULES', 1, qw( Exporter POSIX Fcntl Socket Errno Carp Time::HiRes IO::Handle IO::File IO::Socket IO::Socket::UNIX IO::Socket::INET IO::Stringy Digest::MD5 Unix::Syslog File::Basename Compress::Zlib MIME::Base64 MIME::QuotedPrint MIME::Words MIME::Head MIME::Body MIME::Entity MIME::Parser MIME::Decoder MIME::Decoder::Base64 MIME::Decoder::Binary MIME::Decoder::QuotedPrint MIME::Decoder::NBit MIME::Decoder::UU MIME::Decoder::Gzip64 Net::Server Net::Server::PreFork )); # with earlier versions of Perl one may need to add additional modules # to the list, such as: auto::POSIX::setgid auto::POSIX::setuid ... fetch_modules('OPTIONAL BASIC MODULES', 0, qw( PerlIO PerlIO::scalar Unix::Getrusage Carp::Heavy auto::POSIX::setgid auto::POSIX::setuid auto::POSIX::SigAction::new auto::POSIX::SigAction::safe MIME::Decoder::BinHex )); } 1; # package Amavis::Conf; use strict; use re 'taint'; # constants; intentionally leave value -1 unassigned for compatibility use constant D_TEMPFAIL => -4; use constant D_REJECT => -3; use constant D_BOUNCE => -2; use constant D_DISCARD => 0; use constant D_PASS => 1; # major contents_category constants, in increasing order of importance use constant CC_CATCHALL => 0; use constant CC_CLEAN => 1; # tag_level = "CC_CLEAN,1" use constant CC_MTA => 2; # trouble passing mail back to MTA use constant CC_OVERSIZED => 3; use constant CC_BADH => 4; use constant CC_SPAMMY => 5; # tag2_level (and: tag3_level = CC_SPAMMY,1) use constant CC_SPAM => 6; # kill_level use constant CC_UNCHECKED => 7; use constant CC_BANNED => 8; use constant CC_VIRUS => 9; # # in other words: major_ccat minor_ccat %subject_tag_maps_by_ccat ## if score >= kill level => CC_SPAM 0 ## elsif score >= tag3 level => CC_SPAMMY 1 @spam_subject_tag3_maps ## elsif score >= tag2 level => CC_SPAMMY 0 @spam_subject_tag2_maps ## elsif score >= tag level => CC_CLEAN 1 @spam_subject_tag_maps ## else => CC_CLEAN 0 BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); %EXPORT_TAGS = ( 'dynamic_confvars' => # per- policy bank settings [qw( $child_timeout $smtpd_timeout $policy_bank_name $protocol @inet_acl $myhostname $myauthservid $snmp_contact $snmp_location $myprogram_name $syslog_ident $syslog_facility $log_level $log_templ $log_recip_templ $forward_method $notify_method $resend_method $report_format $release_method $requeue_method $release_format $attachment_password $attachment_email_name $attachment_outer_name $os_fingerprint_method $os_fingerprint_dst_ip_and_port $originating @smtpd_discard_ehlo_keywords $soft_bounce $propagate_dsn_if_possible $terminate_dsn_on_notify_success $amavis_auth_user $amavis_auth_pass $auth_reauthenticate_forwarded $auth_required_out $auth_required_inp $auth_required_release @auth_mech_avail $tls_security_level_in $tls_security_level_out $local_client_bind_address $smtpd_message_size_limit $localhost_name $smtpd_greeting_banner $smtpd_quit_banner $mailfrom_to_quarantine $warn_offsite $bypass_decode_parts @decoders @av_scanners @av_scanners_backup @spam_scanners $first_infected_stops_scan $virus_scanners_failure_is_fatal $sa_spam_level_char $sa_mail_body_size_limit $penpals_bonus_score $penpals_halflife $bounce_killer_score $reputation_factor $undecipherable_subject_tag $localpart_is_case_sensitive $recipient_delimiter $replace_existing_extension $hdr_encoding $bdy_encoding $hdr_encoding_qb $allow_disclaimers $prepend_header_fields_hdridx $allow_fixing_improper_header $allow_fixing_improper_header_folding $allow_fixing_long_header_lines %allowed_added_header_fields %prefer_our_added_header_fields %allowed_header_tests $X_HEADER_TAG $X_HEADER_LINE $remove_existing_x_scanned_headers $remove_existing_spam_headers %sql_clause $partition_tag %local_delivery_aliases $banned_namepath_re $per_recip_whitelist_sender_lookup_tables $per_recip_blacklist_sender_lookup_tables @anomy_sanitizer_args @altermime_args_defang @altermime_args_disclaimer @disclaimer_options_bysender_maps %signed_header_fields @dkim_signature_options_bysender_maps $enable_dkim_verification $enable_dkim_signing $dkim_signing_service @local_domains_maps @mynetworks_maps @client_ipaddr_policy @forward_method_maps @newvirus_admin_maps @banned_filename_maps @spam_quarantine_bysender_to_maps @spam_tag_level_maps @spam_tag2_level_maps @spam_tag3_level_maps @spam_kill_level_maps @spam_subject_tag_maps @spam_subject_tag2_maps @spam_subject_tag3_maps @spam_dsn_cutoff_level_maps @spam_dsn_cutoff_level_bysender_maps @spam_crediblefrom_dsn_cutoff_level_maps @spam_crediblefrom_dsn_cutoff_level_bysender_maps @spam_quarantine_cutoff_level_maps @spam_notifyadmin_cutoff_level_maps @whitelist_sender_maps @blacklist_sender_maps @score_sender_maps @author_to_policy_bank_maps @signer_reputation_maps @message_size_limit_maps @debug_sender_maps @debug_recipient_maps @bypass_virus_checks_maps @bypass_spam_checks_maps @bypass_banned_checks_maps @bypass_header_checks_maps @viruses_that_fake_sender_maps @virus_name_to_spam_score_maps @virus_name_to_policy_bank_maps @remove_existing_spam_headers_maps @sa_userconf_maps @sa_username_maps %final_destiny_by_ccat %forward_method_maps_by_ccat %lovers_maps_by_ccat %defang_maps_by_ccat %subject_tag_maps_by_ccat %quarantine_method_by_ccat %quarantine_to_maps_by_ccat %notify_admin_templ_by_ccat %notify_recips_templ_by_ccat %notify_sender_templ_by_ccat %notify_autoresp_templ_by_ccat %notify_release_templ_by_ccat %notify_report_templ_by_ccat %warnsender_by_ccat %hdrfrom_notify_admin_by_ccat %mailfrom_notify_admin_by_ccat %hdrfrom_notify_recip_by_ccat %mailfrom_notify_recip_by_ccat %hdrfrom_notify_sender_by_ccat %hdrfrom_notify_release_by_ccat %hdrfrom_notify_report_by_ccat %admin_maps_by_ccat %warnrecip_maps_by_ccat %always_bcc_by_ccat %dsn_bcc_by_ccat %addr_extension_maps_by_ccat %addr_rewrite_maps_by_ccat %smtp_reason_by_ccat )], 'confvars' => # global settings (not per-policy, not per-recipient) [qw( $myproduct_name $myversion_id $myversion_id_numeric $myversion_date $myversion $instance_name @additional_perl_modules $MYHOME $TEMPBASE $QUARANTINEDIR $quarantine_subdir_levels $daemonize $courierfilter_shutdown $pid_file $lock_file $db_home $enable_db $mail_id_size_bits $daemon_user $daemon_group $daemon_chroot_dir $path $DEBUG $do_syslog $logfile $allow_preserving_evidence $log_short_templ $log_verbose_templ $logline_maxlen $nanny_details_level $max_servers $max_requests $min_servers $min_spare_servers $max_spare_servers %current_policy_bank %policy_bank %interface_policy @listen_sockets $inet_socket_port $inet_socket_bind $listen_queue_size $unix_socketname $unix_socket_mode $smtp_connection_cache_on_demand $smtp_connection_cache_enable $smtpd_recipient_limit $smtpd_tls_cert_file $smtpd_tls_key_file $enforce_smtpd_message_size_limit_64kb_min $MAXLEVELS $MAXFILES $MIN_EXPANSION_QUOTA $MIN_EXPANSION_FACTOR $MAX_EXPANSION_QUOTA $MAX_EXPANSION_FACTOR $lookup_maps_imply_sql_and_ldap @lookup_sql_dsn @storage_sql_dsn $sql_schema_version $timestamp_fmt_mysql $sql_quarantine_chunksize_max $sql_allow_8bit_address $sql_lookups_no_at_means_domain $ldap_lookups_no_at_means_domain $sql_store_info_for_all_msgs $trim_trailing_space_in_lookup_result_fields $enable_ldap $default_ldap @keep_decoded_original_maps @map_full_type_to_short_type_maps %banned_rules $penpals_threshold_low $penpals_threshold_high %dkim_signing_keys_by_domain @dkim_signing_keys_list @dkim_signing_keys_storage $file $altermime $enable_anomy_sanitizer )], 'sa' => # global SpamAssassin settings [qw( $spamcontrol_obj $sa_num_instances $helpers_home $sa_configpath $sa_siteconfigpath $sa_local_tests_only $sa_timeout $sa_debug $dspam $sa_spawned )], 'platform' => [qw( $can_truncate $unicode_aware $my_pid &D_TEMPFAIL &D_REJECT &D_BOUNCE &D_DISCARD &D_PASS &CC_CATCHALL &CC_CLEAN &CC_MTA &CC_OVERSIZED &CC_BADH &CC_SPAMMY &CC_SPAM &CC_UNCHECKED &CC_BANNED &CC_VIRUS %ccat_display_names %ccat_display_names_major )], # other variables settable by user in amavisd.conf, # but not directly accessible to the program 'hidden_confvars' => [qw( $mydomain )], 'legacy_dynamic_confvars' => # the rest of the program does not use these settings directly and they # should not be visible in, or imported to other modules, but may be # referenced indirectly through *_by_ccat variables for compatibility [qw( $final_virus_destiny $final_banned_destiny $final_unchecked_destiny $final_spam_destiny $final_bad_header_destiny @virus_lovers_maps @spam_lovers_maps @unchecked_lovers_maps @banned_files_lovers_maps @bad_header_lovers_maps $always_bcc $dsn_bcc $mailfrom_notify_sender $mailfrom_notify_recip $mailfrom_notify_admin $mailfrom_notify_spamadmin $hdrfrom_notify_sender $hdrfrom_notify_recip $hdrfrom_notify_admin $hdrfrom_notify_spamadmin $hdrfrom_notify_release $hdrfrom_notify_report $notify_virus_admin_templ $notify_spam_admin_templ $notify_virus_recips_templ $notify_spam_recips_templ $notify_virus_sender_templ $notify_spam_sender_templ $notify_sender_templ $notify_release_templ $notify_report_templ $notify_autoresp_templ $warnbannedsender $warnbadhsender $defang_virus $defang_banned $defang_spam $defang_bad_header $defang_undecipherable $defang_all $virus_quarantine_method $banned_files_quarantine_method $unchecked_quarantine_method $spam_quarantine_method $bad_header_quarantine_method $clean_quarantine_method $archive_quarantine_method @virus_quarantine_to_maps @banned_quarantine_to_maps @unchecked_quarantine_to_maps @spam_quarantine_to_maps @bad_header_quarantine_to_maps @clean_quarantine_to_maps @archive_quarantine_to_maps @virus_admin_maps @banned_admin_maps @spam_admin_maps @bad_header_admin_maps @spam_modifies_subj_maps @warnvirusrecip_maps @warnbannedrecip_maps @warnbadhrecip_maps @addr_extension_virus_maps @addr_extension_spam_maps @addr_extension_banned_maps @addr_extension_bad_header_maps )], 'legacy_confvars' => # legacy variables, predeclared for compatibility of amavisd.conf # The rest of the program does not use them directly and they should # not be visible in other modules, but may be referenced through # @*_maps variables for backwards compatibility [qw( %local_domains @local_domains_acl $local_domains_re @mynetworks %bypass_virus_checks @bypass_virus_checks_acl $bypass_virus_checks_re %bypass_spam_checks @bypass_spam_checks_acl $bypass_spam_checks_re %bypass_banned_checks @bypass_banned_checks_acl $bypass_banned_checks_re %bypass_header_checks @bypass_header_checks_acl $bypass_header_checks_re %virus_lovers @virus_lovers_acl $virus_lovers_re %spam_lovers @spam_lovers_acl $spam_lovers_re %banned_files_lovers @banned_files_lovers_acl $banned_files_lovers_re %bad_header_lovers @bad_header_lovers_acl $bad_header_lovers_re %virus_admin %spam_admin $newvirus_admin $virus_admin $banned_admin $bad_header_admin $spam_admin $warnvirusrecip $warnbannedrecip $warnbadhrecip $virus_quarantine_to $banned_quarantine_to $unchecked_quarantine_to $spam_quarantine_to $spam_quarantine_bysender_to $bad_header_quarantine_to $clean_quarantine_to $archive_quarantine_to $keep_decoded_original_re $map_full_type_to_short_type_re $banned_filename_re $viruses_that_fake_sender_re $sa_tag_level_deflt $sa_tag2_level_deflt $sa_tag3_level_deflt $sa_kill_level_deflt $sa_quarantine_cutoff_level @spam_notifyadmin_cutoff_level_maps $sa_dsn_cutoff_level $sa_crediblefrom_dsn_cutoff_level $sa_spam_modifies_subj $sa_spam_subject_tag1 $sa_spam_subject_tag %whitelist_sender @whitelist_sender_acl $whitelist_sender_re %blacklist_sender @blacklist_sender_acl $blacklist_sender_re $addr_extension_virus $addr_extension_spam $addr_extension_banned $addr_extension_bad_header $sql_select_policy $sql_select_white_black_list $gets_addr_in_quoted_form @debug_sender_acl $arc $bzip2 $lzop $lha $unarj $gzip $uncompress $unfreeze $unrar $zoo $pax $cpio $ar $rpm2cpio $cabextract $ripole $tnef $gunzip $bunzip2 $unlzop $unstuff $SYSLOG_LEVEL $syslog_priority $append_header_fields_to_bottom $insert_received_line $notify_xmailer_header $relayhost_is_client $sa_spam_report_header $sa_auto_whitelist $warnvirussender $warnspamsender $enable_global_cache $virus_check_negative_ttl $virus_check_positive_ttl $spam_check_negative_ttl $spam_check_positive_ttl )], ); Exporter::export_tags qw(dynamic_confvars confvars sa platform hidden_confvars legacy_dynamic_confvars legacy_confvars); } # BEGIN use POSIX (); use Carp (); use Errno qw(ENOENT EACCES EBADF); use vars @EXPORT; sub c($); sub cr($); sub ca($); sub dkim_key($$$;@); # prototypes use subs qw(c cr ca dkim_key); # access subroutines to config vars and keys BEGIN { push(@EXPORT,qw(c cr ca dkim_key)) } # access to dynamic config variables, returns a scalar config variable value; # one level of indirection is allowed # sub c($) { my($name) = @_; if (!exists $current_policy_bank{$name}) { Carp::croak(sprintf('No entry "%s" in policy bank "%s"', $name, $current_policy_bank{'policy_bank_name'})); } my $var = $current_policy_bank{$name}; my $r = ref $var; !$r ? $var : $r eq 'SCALAR' || $r eq 'REF' ? $$var : $var; } # return a ref to a config variable value, or undef if var is undefined # sub cr($) { my($name) = @_; if (!exists $current_policy_bank{$name}) { Carp::croak(sprintf('No entry "%s" in policy bank "%s"', $name, $current_policy_bank{'policy_bank_name'})); } my $var = $current_policy_bank{$name}; !defined $var ? undef : !ref $var ? \$var : $var; } # return a ref to a config variable value (which is supposed to be an array), # converting undef to an empty array, and a scalar to a one-element array # if necessary # sub ca($) { my($name) = @_; if (!exists $current_policy_bank{$name}) { Carp::croak(sprintf('No entry "%s" in policy bank "%s"', $name, $current_policy_bank{'policy_bank_name'})); } my $var = $current_policy_bank{$name}; !defined $var ? [] : !ref $var ? [$var] : $var; } sub deprecate_var($$$) { my($data_type, $var_name, $init_value) = @_; my($code) = <<'EOD'; tie(%n, '%p', %v) or die 'Tieing a variable %n failed'; package %p; use strict; use Carp (); sub TIESCALAR { my($class,$val) = @_; bless \$val, $class } sub FETCH { my $self = shift; $$self } sub STORE { my($self,$newv) = @_; my($oldv) = $$self; if ((defined $oldv || defined $newv) && (%t)) { Carp::carp('Variable %n was retired, changing its value has no effect.' . " See release notes.\n"); } $$self = $newv; } 1; EOD if ($data_type eq 'bool') { $code =~ s{%t}'($oldv ? 1 : 0) != ($newv ? 1 : 0)'g; } elsif ($data_type eq 'num') { $code =~ s{%t}'!defined $oldv || !defined $newv || $oldv != $newv'g; } elsif ($data_type eq 'str') { $code =~ s{%t}'!defined $oldv || !defined $newv || $oldv ne $newv'g; } else { die "Error deprecating a variable $var_name: bad type $data_type"; } $code =~ s/%n/$var_name/g; $code =~ s/%v/\$init_value/g; my($barename) = $var_name; $barename =~ s/^[\$\@%&]//; $code =~ s/%p/Amavis::Deprecate::$barename/g; eval $code or do { chomp $@; die "Error deprecating a variable $var_name: $@" }; } # Store a private DKIM signing key for a given domain and selector. # The argument $key can be a Mail::DKIM::PrivateKey object or a file # name containing a key in a PEM format (e.g. as generated by openssl). # For compatibility with dkim_milter the signing domain can include a '*' # as a wildcard - this is not recommended as this way amavisd could produce # signatures which have no corresponding public key published in DNS. # The proper way is to have one dkim_key entry for each published DNS RR. # Optional arguments can provide additional information about the resource # record (RR) of a public key, i.e. its options according to RFC 4871. # The subroutine is typically called from a configuration file, once for # each signing key available. # sub dkim_key($$$;@) { my($domain,$selector,$key) = @_; shift; shift; shift; @_%2 == 0 or die "dkim_key: a list of key/value pairs expected as options\n"; my(%key_options) = @_; # remaining args are options from a public key RR defined $domain && $domain ne '' or die "dkim_key: domain must not be empty: ($domain,$selector,$key)"; defined $selector && $selector ne '' or die "dkim_key: selector must not be empty: ($domain,$selector,$key)"; my($key_storage_ind); if (ref $key) { # key already preprocessed and provided as an object push(@dkim_signing_keys_storage, [$key]); $key_storage_ind = $#dkim_signing_keys_storage; } else { # assume a name of a file containing a private key in PEM format my($fname) = $key; my($pem_fh) = IO::File->new; # open a file with a private key $pem_fh->open($fname,'<') or die "Can't open PEM file $fname: $!"; my(@stat_list) = stat($pem_fh); # soft-link friendly @stat_list or warn "Error on accessing $fname: $!"; my($dev,$inode) = @stat_list; if ($dev && $inode) { for my $j (0..$#dkim_signing_keys_storage) { # same file reused? my($k,$dv,$in,$fn) = @{$dkim_signing_keys_storage[$j]}; if ($dv == $dev && $in == $inode) { $key_storage_ind = $j; last } } } if (!defined($key_storage_ind)) { # read file and store its contents as a new entry my($nbytes,$buff); $key = ''; while (($nbytes=$pem_fh->read($buff,16384)) > 0) { $key .= $buff } defined $nbytes or die "Error reading key from file $fname: $!"; push(@dkim_signing_keys_storage, [$key,$dev,$inode,$fname]); $key_storage_ind = $#dkim_signing_keys_storage; undef $buff; # release storage } $pem_fh->close or die "Error closing file $fname: $!"; $key_options{k} = 'rsa' if defined $key_options{k}; # force RSA } $domain = lc($domain) if !ref($domain); # possibly a regexp $selector = lc($selector); $key_options{domain} = $domain; $key_options{selector} = $selector; $key_options{key_storage_ind} = $key_storage_ind; if (@dkim_signing_keys_list > 100) { # sorry, skip the test to avoid slow O(n^2) searches } else { !grep($_->{domain} eq $domain && $_->{selector} eq $selector, @dkim_signing_keys_list) or die "dkim_key: selector $selector for domain $domain already in use\n"; } $key_options{key_ind} = $#dkim_signing_keys_list + 1; push(@dkim_signing_keys_list, \%key_options); # using a list preserves order } # essential initializations, right at the program start time, may run as root! # use vars qw($read_config_files_depth @actual_config_files); BEGIN { # init_primary: version, $unicode_aware, base policy bank $myprogram_name = $0; # typically 'amavisd' $myproduct_name = 'amavisd-new'; $myversion_id = '2.7.1'; $myversion_date = '20120429'; $myversion = "$myproduct_name-$myversion_id ($myversion_date)"; $myversion_id_numeric = # x.yyyzzz, allows numerical compare, like Perl $] sprintf('%8.6f', $1 + ($2 + $3/1000)/1000) if $myversion_id =~ /^(\d+)(?:\.(\d*)(?:\.(\d*))?)?(.*)$/; $sql_schema_version = $myversion_id_numeric; $unicode_aware = $] >= 5.008 && length("\x{263a}")==1 && eval { require Encode }; $read_config_files_depth = 0; eval { require Devel::SawAmpersand }; # load if avail, don't bother otherwise # initialize policy bank hash to contain dynamic config settings for my $tag (@EXPORT_TAGS{'dynamic_confvars', 'legacy_dynamic_confvars'}) { for my $v (@$tag) { local($1,$2); if ($v !~ /^([%\$\@])(.*)\z/) { die "Unsupported variable type: $v" } else { no strict 'refs'; my($type,$name) = ($1,$2); $current_policy_bank{$name} = $type eq '$' ? \${"Amavis::Conf::$name"} : $type eq '@' ? \@{"Amavis::Conf::$name"} : $type eq '%' ? \%{"Amavis::Conf::$name"} : undef; } } } $current_policy_bank{'policy_bank_name'} = ''; # builtin policy $current_policy_bank{'policy_bank_path'} = ''; $policy_bank{''} = { %current_policy_bank }; # copy } # end BEGIN - init_primary # boot-time initializations of simple global settings, may run as root! # BEGIN { # serves only as a quick default for other configuration settings $MYHOME = '/var/amavis'; $mydomain = '!change-mydomain-variable!.example.com';#intentionally bad deflt # Create debugging output - true: log to stderr; false: log to syslog/file $DEBUG = 0; # In case of trouble, allow preserving temporary files for forensics $allow_preserving_evidence = 1; # Cause Net::Server parameters 'background' and 'setsid' to be set, # resulting in the program to detach itself from the terminal $daemonize = 1; # Net::Server pre-forking settings - defaults, overruled by amavisd.conf $max_servers = 2; # number of pre-forked children $max_requests = 20; # retire a child after that many accepts, 0=unlimited # timeout for our processing: $child_timeout = 8*60; # abort child if it does not complete a task in n sec # timeout for waiting on client input: $smtpd_timeout = 8*60; # disconnect session if client is idle for too long; # $smtpd_timeout should be higher than Postfix's max_idle (default 100s) # Assume STDIN is a courierfilter pipe and shutdown when it becomes readable $courierfilter_shutdown = 0; # Can file be truncated? # Set to 1 if 'truncate' works (it is XPG4-UNIX standard feature, # not required by Posix). # Things will go faster with SMTP-in, otherwise (e.g. with milter) # it makes no difference as file truncation will not be used. $can_truncate = 1; # Customizable notification messages, logging $syslog_ident = 'amavis'; $syslog_facility = 'mail'; $log_level = 0; # should be less than (1023 - prefix), i.e. 980, # to avoid syslog truncating lines; see sub write_log $logline_maxlen = 980; $enable_db = 0; # load optional modules Amavis::DB & Amavis::DB::SNMP $nanny_details_level = 1; # register_proc verbosity: 0, 1, 2 # $enable_dkim_signing = undef; # $enable_dkim_verification = undef; $reputation_factor = 0.2; # a value between 0 and 1, controlling the amount # of 'bending' of a calculated spam score towards a fixed score assigned # to a signing domain (its 'reputation') through @signer_reputation_maps; # the formula is: adjusted_spam_score = f*reputation + (1-f)*spam_score; # which has the same semantics as auto_whitelist_factor in SpamAssassin AWL $lookup_maps_imply_sql_and_ldap = 1; # set to 0 to disable # Where to find SQL server(s) and database to support SQL lookups? # A list of triples: (dsn,user,passw). Specify more than one # for multiple (backup) SQL servers. # #@storage_sql_dsn = #@lookup_sql_dsn = # ( ['DBI:mysql:mail:host1', 'some-username1', 'some-password1'], # ['DBI:mysql:mail:host2', 'some-username2', 'some-password2'] ); # Does a database mail address field with no '@' character represent a # local username or a domain name? By default it implies a username in # SQL and LDAP lookups (but represents a domain in hash and acl lookups), # so domain names in SQL and LDAP should be specified as '@domain'. # Setting these to true will cause 'xxx' to be interpreted as a domain # name, just like in hash or acl lookups. # $sql_lookups_no_at_means_domain = 0; $ldap_lookups_no_at_means_domain = 0; # Maximum size (in bytes) for data written to a field 'quarantine.mail_text' # when quarantining to SQL. Must not exceed size allowed for a data type # on a given SQL server. It also determines a buffer size in amavisd. # Too large a value may exceed process virtual memory limits or just waste # memory, too small a value splits large mail into too many chunks, which # may be less efficient to process. # $sql_quarantine_chunksize_max = 16384; $sql_allow_8bit_address = 0; # the length of mail_id in bits, must be an integral multiple of 24 # (i.e. divisible by 6 and 8); the mail_id is represented externally # as a base64url-encoded string of size $mail_id_size_bits / 6 # $mail_id_size_bits = 72; # 24, 48, 72, 96 $sql_store_info_for_all_msgs = 1; $penpals_bonus_score = undef; # maximal (positive) score value by which spam # score is lowered when sender is known to have previously received mail # from our local user from this mail system. Zero or undef disables # pen pals lookups in SQL tables msgs and msgrcpt, and is a default. $penpals_halflife = 7*24*60*60; # exponential decay time constant in seconds; # pen pal bonus is halved for each halflife period since the last mail # sent by a local user to a current message's sender $penpals_threshold_low = 1.0; # SA score below which pen pals lookups are # not performed to save time; undef lets the threshold be ignored; $penpals_threshold_high = undef; # when (SA_score - $penpals_bonus_score > $penpals_threshold_high) # pen pals lookup will not be performed to save time, as it could not # influence blocking of spam even at maximal penpals bonus (age=0); # usual choice for value would be kill level or other reasonably high # value; undef lets the threshold be ignored and is a default (useful # for testing and statistics gathering); $bounce_killer_score = 0; # # Receiving mail related # $unix_socketname = '/var/amavis/amavisd.sock'; # e.g. milter or release # $inet_socket_port = 10024; # accept SMTP on this TCP port # $inet_socket_port = [10024,10026,10027]; # ...possibly on more than one $inet_socket_bind = '127.0.0.1'; # bind socket to a loopback interface, IPv4 @inet_acl = qw( 127.0.0.1 [::1] ); # allow SMTP access only from localhost @mynetworks = qw( 127.0.0.0/8 [::1] [FE80::]/10 [FEC0::]/10 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 ); $originating = 0; # a boolean, initially reflects @mynetworks, # but may be modified later through a policy bank $notify_method = 'smtp:[127.0.0.1]:10025'; $forward_method = 'smtp:[127.0.0.1]:10025'; $resend_method = undef; # overrides $forward_method on defanging if nonempty $release_method = undef; # overrides $notify_method on releasing # from quarantine if nonempty $requeue_method = 'smtp:[127.0.0.1]:25'; # requeueing release from a quar. $release_format = 'resend'; # (dsn), (arf), attach, plain, resend $report_format = 'arf'; # (dsn), arf, attach, plain, resend # when $release_format is 'attach', the following control the attachment: $attachment_password = ''; # '': no pwd; undef: PIN; code ref; or static str $attachment_email_name = 'msg-%m.eml'; $attachment_outer_name = 'msg-%m.zip'; $virus_quarantine_method = 'local:virus-%m'; $banned_files_quarantine_method = 'local:banned-%m'; $spam_quarantine_method = 'local:spam-%m.gz'; $bad_header_quarantine_method = 'local:badh-%m'; $unchecked_quarantine_method = undef; # 'local:unchecked-%m'; $clean_quarantine_method = undef; # 'local:clean-%m'; $archive_quarantine_method = undef; # 'local:archive-%m.gz'; $prepend_header_fields_hdridx = 0; # normally 0, use 1 for co-existence # with signing DK and DKIM milters $remove_existing_x_scanned_headers = 0; $remove_existing_spam_headers = 1; # fix improper header fields in passed or released mail - this setting # is a pre-condition for $allow_fixing_improper_header_folding and similar # (future) fixups; (desirable, but may break DKIM validation of messages # with illegal header section) $allow_fixing_improper_header = 1; # fix improper folded header fields made up entirely of whitespace, by # removing all-whitespace lines ($allow_fixing_improper_header must be true) $allow_fixing_improper_header_folding = 1; # truncate header section lines longer than 998 characters as limited # by the RFC 5322 ($allow_fixing_improper_header must be true) $allow_fixing_long_header_lines = 1; # encoding (charset in MIME terminology) # to be used in RFC 2047-encoded ... # $hdr_encoding = 'iso-8859-1'; # ... header field bodies # $bdy_encoding = 'iso-8859-1'; # ... notification body text $hdr_encoding = 'UTF-8'; # ... header field bodies $bdy_encoding = 'UTF-8'; # ... notification body text # encoding (encoding in MIME terminology) $hdr_encoding_qb = 'Q'; # quoted-printable (default) #$hdr_encoding_qb = 'B'; # base64 (usual for far east charsets) $smtpd_recipient_limit = 1100; # max recipients (RCPT TO) - sanity limit # $myhostname is used by SMTP server module in the initial SMTP welcome line, # in inserted Received: lines, Message-ID in notifications, log entries, ... $myhostname = (POSIX::uname)[1]; # should be a FQDN ! $snmp_contact = ''; # a value of sysContact OID $snmp_location = ''; # a value of sysLocation OID $smtpd_greeting_banner = '${helo-name} ${protocol} ${product} service ready'; $smtpd_quit_banner = '${helo-name} ${product} closing transmission channel'; $enforce_smtpd_message_size_limit_64kb_min = 1; # $localhost_name is the name of THIS host running amavisd # (typically 'localhost'). It is used in HELO SMTP command # when reinjecting mail back to MTA via SMTP for final delivery, # and in inserted Received header field $localhost_name = 'localhost'; $propagate_dsn_if_possible = 1; # pass on DSN if MTA announces this # capability; useful to be turned off globally but enabled in # MYNETS policy bank to hide internal mail routing from outsiders $terminate_dsn_on_notify_success = 0; # when true=>handle DSN NOTIFY=SUCCESS # locally, do not let NOTIFY=SUCCESS propagate to MTA (but allow # other DSN options like NOTIFY=NEVER/FAILURE/DELAY, ORCPT, RET, # and ENVID to propagate if possible) #@auth_mech_avail = ('PLAIN','LOGIN'); # empty list disables incoming AUTH #$auth_required_inp = 1; # incoming SMTP authentication required by amavisd? #$auth_required_out = 1; # SMTP authentication required by MTA $auth_required_release = 1; # secret_id is required for a quarantine release $tls_security_level_in = undef; # undef, 'may', 'encrypt', ... $tls_security_level_out = undef; # undef, 'may', 'encrypt', ... $smtpd_tls_cert_file = undef; # e.g. "$MYHOME/cert/amavisd-cert.pem" $smtpd_tls_key_file = undef; # e.g. "$MYHOME/cert/amavisd-key.pem" # SMTP AUTH username and password for notification submissions # (and reauthentication of forwarded mail if requested) #$amavis_auth_user = undef; # perhaps: 'amavisd' #$amavis_auth_pass = undef; #$auth_reauthenticate_forwarded = undef; # supply our own credentials also # for forwarded (passed) mail $smtp_connection_cache_on_demand = 1; $smtp_connection_cache_enable = 1; # whom quarantined messages appear to be sent from (envelope sender) # $mailfrom_to_quarantine = undef; # orig. sender if undef, or set explicitly # where to send quarantined malware - specify undef to disable, or an # e-mail address containing '@', or just a local part, which will be # mapped by %local_delivery_aliases into local mailbox name or directory. # The lookup key is a recipient address $virus_quarantine_to = 'virus-quarantine'; $banned_quarantine_to = 'banned-quarantine'; $unchecked_quarantine_to = 'unchecked-quarantine'; $spam_quarantine_to = 'spam-quarantine'; $bad_header_quarantine_to = 'bad-header-quarantine'; $clean_quarantine_to = 'clean-quarantine'; $archive_quarantine_to = 'archive-quarantine'; # similar to $spam_quarantine_to, but the lookup key is the sender address: $spam_quarantine_bysender_to = undef; # dflt: no by-sender spam quarantine # quarantine directory or mailbox file or empty # (only used if $*_quarantine_to specifies direct local delivery) $QUARANTINEDIR = undef; # no quarantine unless overridden by config $undecipherable_subject_tag = '***UNCHECKED*** '; # NOTE: all entries can accept mail_body_size_limit and score_factor options @spam_scanners = ( ['SpamAssassin', 'Amavis::SpamControl::SpamAssassin' ], # ['SpamdClient', 'Amavis::SpamControl::SpamdClient' ], # ['DSPAM', 'Amavis::SpamControl::ExtProg', $dspam, # [ qw(--stdout --classify --deliver=innocent,spam # --mode=toe --feature noise # --user), $daemon_user ], # mail_body_size_limit => 65000, score_factor => 1.0, # ], # ['CRM114', 'Amavis::SpamControl::ExtProg', 'crm', # [ qw(-u /var/amavis/home/.crm114 mailreaver.crm # --dontstore --report_only --stats_only # --good_threshold=10 --spam_threshold=-10) ], # mail_body_size_limit => 65000, score_factor => -0.20, # ], ); $sa_spawned = 0; # true: run SA in a subprocess; false: call SA directly # string to prepend to Subject header field when message qualifies as spam # $sa_spam_subject_tag1 = undef; # example: '***Possible Spam*** ' # $sa_spam_subject_tag = undef; # example: '***Spam*** ' $sa_spam_level_char = '*'; # character to be used in X-Spam-Level bar; # empty or undef disables adding this header field $sa_num_instances = 1; # number of SA instances, # usually 1, memory-expensive, keep small $sa_local_tests_only = 0; $sa_debug = undef; $sa_timeout = 30; # no longer used since 2.6.5 $file = 'file'; # path to the file(1) utility for classifying contents $altermime = 'altermime'; # path to the altermime utility (optional) @altermime_args_defang = qw(--verbose --removeall); @altermime_args_disclaimer = qw(--disclaimer=/etc/altermime-disclaimer.txt); # @altermime_args_disclaimer = # qw(--disclaimer=/etc/_OPTION_.txt --disclaimer-html=/etc/_OPTION_.html); # @disclaimer_options_bysender_maps = ( 'altermime-disclaimer' ); $MIN_EXPANSION_FACTOR = 5; # times original mail size $MAX_EXPANSION_FACTOR = 500; # times original mail size # See amavisd.conf and README.lookups for details. # What to do with the message (this is independent of quarantining): # Reject: tell MTA to generate a non-delivery notification, MTA gets 5xx # Bounce: generate a non-delivery notification by ourselves, MTA gets 250 # Discard: drop the message and pretend it was delivered, MTA gets 250 # Pass: accept/forward a message, MTA gets 250 # TempFail: temporary failure, client should retry, MTA gets 4xx # # COMPATIBILITY NOTE: the separation of *_destiny values into # D_BOUNCE, D_REJECT, D_DISCARD and D_PASS made settings $warn*sender only # still useful with D_PASS. The combination of D_DISCARD + $warn*sender=1 # is mapped into D_BOUNCE for compatibility. # The following symbolic constants can be used in *destiny settings: # # D_PASS mail will pass to recipients, regardless of contents; # # D_DISCARD mail will not be delivered to its recipients, sender will NOT be # notified. Effectively we lose mail (but it will be quarantined # unless disabled). # # D_BOUNCE mail will not be delivered to its recipients, a non-delivery # notification (bounce) will be sent to the sender by amavisd-new # (unless suppressed). Bounce (DSN) will not be sent if a virus # name matches $viruses_that_fake_sender_maps, or to messages # from mailing lists (Precedence: bulk|list|junk), or for spam # exceeding spam_dsn_cutoff_level # # D_REJECT mail will not be delivered to its recipients, amavisd will # return a 5xx status response. Depending on an MTA/amavisd stup # this will result either in a reject status passed back to a # connecting SMTP client (in a pre-queue setup: proxy or milter), # or an MTA will generate a bounce in a post-queue setup. # If not all recipients agree on rejecting a message (like when # different recipients have different thresholds on bad mail # contents and LMTP is not used) amavisd sends a bounce by itself # (same as D_BOUNCE). # # D_TEMPFAIL indicates a temporary failure, mail will not be delivered to # its recipients, sender should retry the operation later. # # Notes: # D_REJECT and D_BOUNCE are similar,the difference is in who is responsible # for informing the sender about non-delivery, and how informative # the notification can be (amavisd-new knows more than MTA); # With D_REJECT, MTA may reject original SMTP, or send DSN (delivery status # notification, colloquially called 'bounce') - depending on MTA # and its interface to a content checker; best suited for sendmail # milter or other pre-queue filtering setups # With D_BOUNCE, amavisd-new (not MTA) sends DSN (can better explain the # reason for mail non-delivery but unable to reject the original # SMTP session, and is in position to suppress DSN if considered # unsuitable). Best suited for Postfix and other dual-MTA setups. # Exceeded spam cutoff limit or faked virus sender implicitly # turns D_BOUNCE into a D_DISCARD; # D_REJECT, D_BOUNCE, D_DISCARD, D_PASS, D_TEMPFAIL $final_virus_destiny = D_DISCARD; $final_banned_destiny = D_DISCARD; $final_unchecked_destiny = D_PASS; $final_spam_destiny = D_PASS; $final_bad_header_destiny = D_PASS; # If decided to pass viruses (or spam) to certain recipients using # %lovers_maps_by_ccat, or by %final_destiny_by_ccat resulting in D_PASS, # one may set the corresponding %addr_extension_maps_by_ccat to some string, # and the recipient address will have this string appended as an address # extension to a local-part (mailbox part) of the address. This extension # can be used by a final local delivery agent for example to place such mail # in different folder. Leaving this variable undefined or an empty string # prevents appending address extension. Recipients which do not match # @local_domains_maps are not affected (i.e. non-local recipients (=outbound # mail) do not get address extension appended). # # LDAs usually default to stripping away address extension if no special # handling for it is specified, so having this option enabled normally # does no harm, provided the $recipients_delimiter character matches # the setting at the final MTA's local delivery agent (LDA). # # $addr_extension_virus = 'virus'; # for example # $addr_extension_spam = 'spam'; # $addr_extension_banned = 'banned'; # $addr_extension_bad_header = 'badh'; # Delimiter between local part of the recipient address and address extension # (which can optionally be added, see variable %addr_extension_maps_by_ccat. # E.g. recipient address gets # changed to . # # Delimiter should match an equivalent (final) MTA delimiter setting. # (e.g. for Postfix add 'recipient_delimiter = +' to main.cf). # Setting it to an empty string or to undef disables this feature # regardless of %addr_extension_maps_by_ccat setting. # $recipient_delimiter = '+'; $replace_existing_extension = 1; # true: replace ext; false: append ext # Affects matching of localpart of e-mail addresses (left of '@') # in lookups: true = case sensitive, false = case insensitive $localpart_is_case_sensitive = 0; # Trim trailing whitespace from SQL fields, LDAP attribute values # and hash righthand-sides as read by read_hash(); disabled by default; # turn it on for compatibility with pre-2.4.0 versions. $trim_trailing_space_in_lookup_result_fields = 0; # since 2.7.0: deprecated some old variables: # deprecate_var('bool', '$insert_received_line', 1); deprecate_var('bool', '$relayhost_is_client', undef); deprecate_var('bool', '$warnvirussender', undef); deprecate_var('bool', '$warnspamsender', undef); deprecate_var('bool', '$sa_spam_report_header', undef); deprecate_var('bool', '$sa_spam_modifies_subj', 1); deprecate_var('bool', '$sa_auto_whitelist', undef); deprecate_var('num', '$sa_timeout', 30); deprecate_var('str', '$syslog_priority', 'debug'); deprecate_var('str', '$SYSLOG_LEVEL', 'mail.debug'); deprecate_var('str', '$notify_xmailer_header', undef); # deprecate_var('array','@spam_modifies_subj_maps'); } # end BEGIN - init_secondary # init structured variables like %sql_clause, $map_full_type_to_short_type_re, # %ccat_display_names, @decoders, build default maps; may run as root! # BEGIN { $allowed_added_header_fields{lc($_)} = 1 for qw( Received DKIM-Signature Authentication-Results VBR-Info X-Quarantine-ID X-Amavis-Alert X-Amavis-Hold X-Amavis-Modified X-Amavis-PenPals X-Amavis-OS-Fingerprint X-Amavis-PolicyBank X-Spam-Status X-Spam-Level X-Spam-Flag X-Spam-Score X-Spam-Report X-Spam-Checker-Version X-Spam-Tests X-CRM114-Status X-CRM114-CacheID X-CRM114-Notice X-CRM114-Action X-DSPAM-Result X-DSPAM-Class X-DSPAM-Signature X-DSPAM-Processed X-DSPAM-Confidence X-DSPAM-Probability X-DSPAM-User X-DSPAM-Factors ); $allowed_added_header_fields{lc('X-Spam-Report')} = 0; $allowed_added_header_fields{lc('X-Spam-Checker-Version')} = 0; # $allowed_added_header_fields{lc(c(lc $X_HEADER_TAG))}=1; #later:read_config # even though SpamAssassin does provide the following header fields, we # prefer to provide our own version (per-recipient scores, version hiding); # our own non- "X-Spam" header fields are always preferred and need not # be listed here $prefer_our_added_header_fields{lc($_)} = 1 for qw( X-Spam-Status X-Spam-Level X-Spam-Flag X-Spam-Score X-Spam-Report X-Spam-Checker-Version X-CRM114-Status X-CRM114-CacheID X-DSPAM-Result X-DSPAM-Signature ); # controls which header section tests are performed in check_header_validity, # keys correspond to minor contents categories for CC_BADH $allowed_header_tests{lc($_)} = 1 for qw( other mime 8bit control empty long syntax missing multiple); # RFC 4871 standard set of header fields to be signed: my(@sign_headers) = qw(From Sender Reply-To Subject Date Message-ID To Cc In-Reply-To References MIME-Version Content-Type Content-Transfer-Encoding Content-ID Content-Description Resent-Date Resent-From Resent-Sender Resent-To Resent-Cc Resent-Message-ID List-Id List-Post List-Owner List-Subscribe List-Unsubscribe List-Help List-Archive); # additional header fields considered appropriate, see also RFC 4021 # and IANA registry "Permanent Message Header Field Names"; # see RFC 3834 for Auto-Submitted; RFC 5518 for VBR-Info (Vouch By Reference) push(@sign_headers, qw(Received Precedence Original-Message-ID Message-Context PICS-Label Sensitivity Solicitation Content-Location Content-Features Content-Disposition Content-Language Content-Alternative Content-Base Content-MD5 Content-Duration Content-Class Accept-Language Auto-Submitted Archived-At VBR-Info)); # note that we are signing Received despite the advise in RFC 4871; # some additional nonstandard header fields: push(@sign_headers, qw(Organization Organisation User-Agent X-Mailer)); $signed_header_fields{lc($_)} = 1 for @sign_headers; # Excluded: # DKIM-Signature DomainKey-Signature Authentication-Results # Keywords Comments Errors-To X-Virus-Scanned X-Archived-At X-No-Archive # Some MTAs are dropping Disposition-Notification-To, exclude: # Disposition-Notification-To Disposition-Notification-Options # Some mail scanners are dropping Return-Receipt-To, exclude it. # Signing a 'Sender' may not be a good idea because when such mail is sent # through a mailing list, this header field is usually replaced by a new one, # invalidating a signature. Long To and Cc address lists are often mangled, # especially when containing non-encoded display names. Off: Sender, To, Cc $signed_header_fields{lc($_)} = 0 for qw(Sender To Cc); # # a value greater than 1 causes signing of one additional null instance of # a header field, thus prohibiting prepending additional occurences of such # header field without breaking a signature $signed_header_fields{lc($_)} = 2 for qw(From Date Subject Content-Type); # provide names for content categories - to be used only for logging, # SNMP counter names and display purposes %ccat_display_names = ( CC_CATCHALL, 'CatchAll', # last resort, should not normally appear CC_CLEAN, 'Clean', CC_CLEAN.',1', 'CleanTag', # tag_level CC_MTA, 'MtaFailed', # unable to forward (general) CC_MTA.',1', 'MtaTempFailed', # MTA response was 4xx CC_MTA.',2', 'MtaRejected', # MTA response was 5xx CC_OVERSIZED, 'Oversized', CC_BADH, 'BadHdr', CC_BADH.',1', 'BadHdrMime', CC_BADH.',2', 'BadHdr8bit', CC_BADH.',3', 'BadHdrChar', CC_BADH.',4', 'BadHdrSpace', CC_BADH.',5', 'BadHdrLong', CC_BADH.',6', 'BadHdrSyntax', CC_BADH.',7', 'BadHdrMissing', CC_BADH.',8', 'BadHdrDupl', CC_SPAMMY, 'Spammy', # tag2_level CC_SPAMMY.',1','Spammy3', # tag3_level CC_SPAM, 'Spam', # kill_level CC_UNCHECKED, 'Unchecked', CC_BANNED, 'Banned', CC_VIRUS, 'Virus', ); # provide names for content categories - to be used only for logging, # SNMP counter names and display purposes, similar to %ccat_display_names # but only major contents category names are listed %ccat_display_names_major = ( CC_CATCHALL, 'CatchAll', # last resort, should not normally appear CC_CLEAN, 'Clean', CC_MTA, 'MtaFailed', # unable to forward CC_OVERSIZED, 'Oversized', CC_BADH, 'BadHdr', CC_SPAMMY, 'Spammy', # tag2_level CC_SPAM, 'Spam', # kill_level CC_UNCHECKED, 'Unchecked', CC_BANNED, 'Banned', CC_VIRUS, 'Virus', ); # $partition_tag is a user-specified SQL field value in tables maddr, msgs, # msgrcpt and quarantine, inserted into new records, but can be useful even # without SQL, accessible through a macro %P and in quarantine templates. # It is usually an integer, but depending on a schema may be of other data # type e.g. a string. May be used to speed up purging of old records by using # partitioned tables (MySQL 5.1+, PostgreSQL 8.1+). A possible usage can # be a week-of-a-year, or some other slowly changing value, allowing to # quickly drop old table partitions without wasting time on deleting # individual records. Mail addresses in table maddr are self-contained # within a partition tag, which means that the same mail address may # appear in more than one maddr partition (using different 'id's), and # that tables msgs and msgrcpt are guaranteed to reference a maddr.id # within their own partition tag. The $partition_tag may be a scalar # (an integer or a string), or a reference to a subroutine, which will be # called with an object of type Amavis::In::Message as argument, and its # result will be used as a partition tag value. Possible usage: # # $partition_tag = # sub { my($msginfo)=@_; iso8601_week($msginfo->rx_time) }; #or: # $partition_tag = # sub { my($msginfo)=@_; iso8601_yearweek($msginfo->rx_time) }; # # $spam_quarantine_method = 'local:W%P/spam/%m.gz'; # quar dir by week num # The SQL select clause to fetch per-recipient policy settings. # The %k will be replaced by a comma-separated list of query addresses # for a recipient (e.g. a full address, domain only, catchall), %a will be # replaced by an exact recipient address (same as the first entry in %k, # suitable for pattern matching), %l by a full unmodified localpart, %u by # a lowercased username (a localpart without extension), %e by lowercased # addr extension (which includes a delimiter), and %d for lowercased domain. # Use ORDER if there is a chance that multiple records will match - the # first match wins (i.e. the first returned record). If field names are # not unique (e.g. 'id'), the later field overwrites the earlier in a hash # returned by lookup, which is why we use 'users.*, policy.*, users.id', # i.e. the id is repeated at the end. # This is a legacy variable for upwards compatibility, now only referenced # by the program through a %sql_clause entry 'sel_policy' - newer config # files may assign directly to $sql_clause{'sel_policy'} if preferred. # $sql_select_policy = 'SELECT users.*, policy.*, users.id'. ' FROM users LEFT JOIN policy ON users.policy_id=policy.id'. ' WHERE users.email IN (%k) ORDER BY users.priority DESC'; # Btw, MySQL and PostgreSQL are happy with 'SELECT *, users.id', # but Oracle wants 'SELECT users.*, policy.*, users.id', which is # also acceptable to MySQL and PostgreSQL. # The SQL select clause to check sender in per-recipient whitelist/blacklist. # The first SELECT argument '?' will be users.id from recipient SQL lookup, # the %k will be replaced by a comma-separated list of query addresses # for a sender (e.g. a full address, domain only, catchall), %a will be # replaced by an exact sender address (same as the first entry in %k, # suitable for pattern matching), %l by a full unmodified localpart, %u by # a lowercased username (a localpart without extension), %e by lowercased # addr extension (which includes a delimiter), and %d for lowercased domain. # Only the first occurrence of '?' will be replaced by users.id, # subsequent occurrences of '?' will see empty string as an argument. # There can be zero or more occurrences of each %k, %a, %l, %u, %e, %d, # lookup keys will be replicated accordingly. # This is a separate legacy variable for upwards compatibility, now only # referenced by the program through %sql_clause entry 'sel_wblist' - newer # config files may assign directly to $sql_clause{'sel_wblist'} if preferred. # $sql_select_white_black_list = 'SELECT wb FROM wblist JOIN mailaddr ON wblist.sid=mailaddr.id'. ' WHERE wblist.rid=? AND mailaddr.email IN (%k)'. ' ORDER BY mailaddr.priority DESC'; %sql_clause = ( 'sel_policy' => \$sql_select_policy, 'sel_wblist' => \$sql_select_white_black_list, 'sel_adr' => 'SELECT id FROM maddr WHERE partition_tag=? AND email=?', 'ins_adr' => 'INSERT INTO maddr (partition_tag, email, domain) VALUES (?,?,?)', 'ins_msg' => 'INSERT INTO msgs (partition_tag, mail_id, secret_id, am_id,'. ' time_num, time_iso, sid, policy, client_addr, size, host)'. ' VALUES (?,?,?,?,?,?,?,?,?,?,?)', 'upd_msg' => 'UPDATE msgs SET content=?, quar_type=?, quar_loc=?, dsn_sent=?,'. ' spam_level=?, message_id=?, from_addr=?, subject=?, client_addr=?,'. ' originating=?'. ' WHERE partition_tag=? AND mail_id=?', 'ins_rcp' => 'INSERT INTO msgrcpt (partition_tag, mail_id, rseqnum, rid, is_local,'. ' content, ds, rs, bl, wl, bspam_level, smtp_resp)'. ' VALUES (?,?,?,?,?,?,?,?,?,?,?,?)', 'ins_quar' => 'INSERT INTO quarantine (partition_tag, mail_id, chunk_ind, mail_text)'. ' VALUES (?,?,?,?)', 'sel_msg' => # obtains partition_tag if missing in a release request 'SELECT partition_tag FROM msgs WHERE mail_id=?', 'sel_quar' => 'SELECT mail_text FROM quarantine'. ' WHERE partition_tag=? AND mail_id=?'. ' ORDER BY chunk_ind', 'sel_penpals' => # no message-id references list "SELECT msgs.time_num, msgs.mail_id, subject". " FROM msgs JOIN msgrcpt USING (partition_tag,mail_id)". " WHERE sid=? AND rid=? AND msgs.content!='V' AND ds='P'". " ORDER BY msgs.time_num DESC", # LIMIT 1 'sel_penpals_msgid' => # with a nonempty list of message-id references "SELECT msgs.time_num, msgs.mail_id, subject, message_id, rid". " FROM msgs JOIN msgrcpt USING (partition_tag,mail_id)". " WHERE sid=? AND msgs.content!='V' AND ds='P' AND message_id IN (%m)". " AND rid!=sid". " ORDER BY rid=? DESC, msgs.time_num DESC", # LIMIT 1 ); # NOTE on $sql_clause{'upd_msg'}: MySQL clobbers timestamp on update # (unless DEFAULT 0 is used) setting it to current local time and # losing the cherishly preserved and prepared time of mail reception. # From the MySQL 4.1 documentation: # * With neither DEFAULT nor ON UPDATE clauses, it is the same as # DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP. # * suppress the automatic initialization and update behaviors for the first # TIMESTAMP column by explicitly assigning it a constant DEFAULT value # (for example, DEFAULT 0) # * The first TIMESTAMP column in table row automatically is updated to # the current timestamp when the value of any other column in the row is # changed, unless the TIMESTAMP column explicitly is assigned a value # other than NULL. # maps full string as returned by a file(1) utility into a short string; # first match wins, more specific entries should precede general ones! # the result may be a string or a ref to a list of strings; # see also sub decompose_part() # prepare an arrayref, later to be converted to an Amavis::Lookup::RE object $map_full_type_to_short_type_re = [ [qr/^empty\z/ => 'empty'], [qr/^directory\z/ => 'dir'], [qr/^can't (stat|read)\b/ => 'dat'], # file(1) diagnostics [qr/^cannot open\b/ => 'dat'], # file(1) diagnostics [qr/^ERROR:/ => 'dat'], # file(1) diagnostics [qr/can't read magic file|couldn't find any magic files/ => 'dat'], [qr/^data\z/ => 'dat'], [qr/^ISO-8859.*\btext\b/ => 'txt'], [qr/^Non-ISO.*ASCII\b.*\btext\b/ => 'txt'], [qr/^Unicode\b.*\btext\b/i => 'txt'], [qr/^UTF.* Unicode text\b/i => 'txt'], [qr/^'diff' output text\b/ => 'txt'], [qr/^GNU message catalog\b/ => 'mo'], [qr/^PGP encrypted data\b/ => 'pgp'], [qr/^PGP armored data( signed)? message\b/ => ['pgp','pgp.asc'] ], [qr/^PGP armored\b/ => ['pgp','pgp.asc'] ], ### 'file' is a bit too trigger happy to claim something is 'mail text' # [qr/^RFC 822 mail text\b/ => 'mail'], [qr/^(ASCII|smtp|RFC 822) mail text\b/ => 'txt'], [qr/^JPEG image data\b/ => ['image','jpg'] ], [qr/^GIF image data\b/ => ['image','gif'] ], [qr/^PNG image data\b/ => ['image','png'] ], [qr/^TIFF image data\b/ => ['image','tif'] ], [qr/^PCX\b.*\bimage data\b/ => ['image','pcx'] ], [qr/^PC bitmap data\b/ => ['image','bmp'] ], [qr/^SVG Scalable Vector Graphics image\b/ => ['image','svg'] ], [qr/^MP2\b/ => ['audio','mpa','mp2'] ], [qr/^MP3\b/ => ['audio','mpa','mp3'] ], [qr/\bMPEG ADTS, layer III\b/ => ['audio','mpa','mp3'] ], [qr/^ISO Media, MPEG v4 system\b/ => ['audio','mpa','m4a','m4b'] ], [qr/^FLAC audio bitstream data\b/ => ['audio','flac'] ], [qr/^Ogg data, FLAC audio\b/ => ['audio','oga'] ], [qr/^Ogg data\b/ => ['audio','ogg'] ], [qr/^MPEG video stream data\b/ => ['movie','mpv'] ], [qr/^MPEG system stream data\b/ => ['movie','mpg'] ], [qr/^MPEG\b/ => ['movie','mpg'] ], [qr/^Matroska data\b/ => ['movie','mkv'] ], [qr/^Microsoft ASF\b/ => ['movie','wmv'] ], [qr/^RIFF\b.*\bAVI\b/ => ['movie','avi'] ], [qr/^RIFF\b.*\banimated cursor\b/ => ['movie','ani'] ], [qr/^RIFF\b.*\bWAVE audio\b/ => ['audio','wav'] ], [qr/^Macromedia Flash data\b/ => 'swf'], [qr/^HTML document text\b/ => 'html'], [qr/^XML document text\b/ => 'xml'], [qr/^exported SGML document text\b/ => 'sgml'], [qr/^PostScript document text\b/ => 'ps'], [qr/^PDF document\b/ => 'pdf'], [qr/^Rich Text Format data\b/ => 'rtf'], [qr/^Microsoft Office Document\b/i => 'doc'], # OLE2: doc, ppt, xls, ... [qr/^Microsoft Installer\b/i => 'doc'], # file(1) may misclassify [qr/^ms-windows meta(file|font)\b/i => 'wmf'], [qr/^LaTeX\b.*\bdocument text\b/ => 'lat'], [qr/^TeX DVI file\b/ => 'dvi'], [qr/\bdocument text\b/ => 'txt'], [qr/^compiled Java class data\b/ => 'java'], [qr/^MS Windows 95 Internet shortcut text\b/ => 'url'], [qr/^Compressed Google KML Document\b/ => 'kmz'], [qr/^frozen\b/ => 'F'], [qr/^gzip compressed\b/ => 'gz'], [qr/^bzip compressed\b/ => 'bz'], [qr/^bzip2 compressed\b/ => 'bz2'], [qr/^xz compressed\b/ => 'xz'], [qr/^lzma compressed\b/ => 'lzma'], [qr/^lzop compressed\b/ => 'lzo'], [qr/^compress'd/ => 'Z'], [qr/^Zip archive\b/i => 'zip'], [qr/^7-zip archive\b/i => '7z'], [qr/^RAR archive\b/i => 'rar'], [qr/^LHa.*\barchive\b/i => 'lha'], # (also known as .lzh) [qr/^ARC archive\b/i => 'arc'], [qr/^ARJ archive\b/i => 'arj'], [qr/^Zoo archive\b/i => 'zoo'], [qr/^(\S+\s+)?tar archive\b/i => 'tar'], [qr/^(\S+\s+)?cpio archive\b/i => 'cpio'], [qr/^StuffIt Archive\b/i => 'sit'], [qr/^Debian binary package\b/i => 'deb'], # standard Unix archive (ar) [qr/^current ar archive\b/i => 'a'], # standard Unix archive (ar) [qr/^RPM\b/ => 'rpm'], [qr/^(Transport Neutral Encapsulation Format|TNEF)\b/i => 'tnef'], [qr/^Microsoft Cabinet (file|archive)\b/i => 'cab'], [qr/^InstallShield Cabinet file\b/ => 'installshield'], [qr/^(uuencoded|xxencoded)\b/i => 'uue'], [qr/^binhex\b/i => 'hqx'], [qr/^(ASCII|text)\b/i => 'asc'], [qr/^Emacs.*byte-compiled Lisp data/i => 'asc'], # BinHex with empty line [qr/\bscript text executable\b/ => 'txt'], [qr/^MS Windows\b.*\bDLL\b/ => ['exe','dll'] ], [qr/\bexecutable for MS Windows\b.*\bDLL\b/ => ['exe','dll'] ], [qr/^MS-DOS executable \(built-in\)/ => 'asc'], # starts with LZ [qr/^(MS-)?DOS executable\b.*\bDLL\b/ => ['exe','dll'] ], [qr/^MS Windows\b.*\bexecutable\b/ => ['exe','exe-ms'] ], [qr/\bexecutable for MS Windows\b/ => ['exe','exe-ms'] ], [qr/^COM executable for DOS\b/ => 'asc'], # misclassified? [qr/^DOS executable \(COM\)/ => 'asc'], # misclassified? [qr/^(MS-)?DOS executable\b(?!.*\(COM\))/ => ['exe','exe-ms'] ], [qr/^PA-RISC.*\bexecutable\b/ => ['exe','exe-unix'] ], [qr/^ELF .*\bexecutable\b/ => ['exe','exe-unix'] ], [qr/^COFF format .*\bexecutable\b/ => ['exe','exe-unix'] ], [qr/^executable \(RISC System\b/ => ['exe','exe-unix'] ], [qr/^VMS\b.*\bexecutable\b/ => ['exe','exe-vms'] ], [qr/\bexecutable\b/i => 'exe'], [qr/\bshared object, /i => 'so'], [qr/\brelocatable, /i => 'o'], [qr/\btext\b/i => 'asc'], [qr/^/ => 'dat'], # catchall ]; # MS Windows PE 32-bit Intel 80386 GUI executable not relocatable # MS-DOS executable (EXE), OS/2 or MS Windows # MS-DOS executable PE for MS Windows (DLL) (GUI) Intel 80386 32-bit # MS-DOS executable PE for MS Windows (DLL) (GUI) Alpha 32-bit # MS-DOS executable, NE for MS Windows 3.x (driver) # MS-DOS executable (built-in) (any file starting with LZ!) # PE executable for MS Windows (DLL) (GUI) Intel 80386 32-bit # PE executable for MS Windows (GUI) Intel 80386 32-bit # NE executable for MS Windows 3.x # PA-RISC1.1 executable dynamically linked # PA-RISC1.1 shared executable dynamically linked # ELF 64-bit LSB executable, Alpha (unofficial), version 1 (FreeBSD), # for FreeBSD 5.0.1, dynamically linked (uses shared libs), stripped # ELF 64-bit LSB executable, Alpha (unofficial), version 1 (SYSV), # for GNU/Linux 2.2.5, dynamically linked (uses shared libs), stripped # ELF 64-bit MSB executable, SPARC V9, version 1 (FreeBSD), # for FreeBSD 5.0, dynamically linked (uses shared libs), stripped # ELF 64-bit MSB shared object, SPARC V9, version 1 (FreeBSD), stripped # ELF 32-bit LSB executable, Intel 80386, version 1, dynamically` # ELF 32-bit MSB executable, SPARC, version 1, dynamically linke` # COFF format alpha executable paged stripped - version 3.11-10 # COFF format alpha executable paged dynamically linked stripped` # COFF format alpha demand paged executable or object module # stripped - version 3.11-10 # COFF format alpha paged dynamically linked not stripped shared` # executable (RISC System/6000 V3.1) or obj module # VMS VAX executable # A list of pairs or n-tuples: [short-type, code_ref, optional-args...]. # Maps short types to a decoding routine, the first match wins. # Arguments beyond the first two can be program path string (or a listref of # paths to be searched) or a reference to a variable containing such a path, # which allows for lazy evaluation, making possible to assign values to # legacy configuration variables even after the assignment to @decoders. @decoders = ( ['mail', \&Amavis::Unpackers::do_mime_decode], # ['asc', \&Amavis::Unpackers::do_ascii], # ['uue', \&Amavis::Unpackers::do_ascii], # ['hqx', \&Amavis::Unpackers::do_ascii], # ['ync', \&Amavis::Unpackers::do_ascii], ['F', \&Amavis::Unpackers::do_uncompress, \$unfreeze], ['Z', \&Amavis::Unpackers::do_uncompress, \$uncompress], ['gz', \&Amavis::Unpackers::do_gunzip], ['gz', \&Amavis::Unpackers::do_uncompress, \$gunzip], ['bz2', \&Amavis::Unpackers::do_uncompress, \$bunzip2], ['xz', \&Amavis::Unpackers::do_uncompress, ['xzdec', 'xz -dc', 'unxz -c', 'xzcat'] ], ['lzma', \&Amavis::Unpackers::do_uncompress, ['lzmadec', 'xz -dc --format=lzma', 'lzma -dc', 'unlzma -c', 'lzcat', 'lzmadec'] ], ['lzo', \&Amavis::Unpackers::do_uncompress, \$unlzop], ['rpm', \&Amavis::Unpackers::do_uncompress, \$rpm2cpio], ['cpio', \&Amavis::Unpackers::do_pax_cpio, \$pax], ['cpio', \&Amavis::Unpackers::do_pax_cpio, \$cpio], ['tar', \&Amavis::Unpackers::do_pax_cpio, \$pax], ['tar', \&Amavis::Unpackers::do_pax_cpio, \$cpio], # ['tar', \&Amavis::Unpackers::do_tar], # no longer supported ['deb', \&Amavis::Unpackers::do_ar, \$ar], # ['a', \&Amavis::Unpackers::do_ar, \$ar], #unpacking .a seems an overkill ['zip', \&Amavis::Unpackers::do_unzip], ['kmz', \&Amavis::Unpackers::do_unzip], ['7z', \&Amavis::Unpackers::do_7zip, ['7zr','7za','7z'] ], ['rar', \&Amavis::Unpackers::do_unrar, \$unrar], ['arj', \&Amavis::Unpackers::do_unarj, \$unarj], ['arc', \&Amavis::Unpackers::do_arc, \$arc], ['zoo', \&Amavis::Unpackers::do_zoo, \$zoo], # ['lha', \&Amavis::Unpackers::do_lha, \$lha], # not safe ['doc', \&Amavis::Unpackers::do_ole, \$ripole], ['cab', \&Amavis::Unpackers::do_cabextract, \$cabextract], ['tnef', \&Amavis::Unpackers::do_tnef_ext, \$tnef], ['tnef', \&Amavis::Unpackers::do_tnef], # ['sit', \&Amavis::Unpackers::do_unstuff, \$unstuff], # not safe ['exe', \&Amavis::Unpackers::do_executable, \$unrar,\$lha,\$unarj], ); # build_default_maps @local_domains_maps = ( \%local_domains, \@local_domains_acl, \$local_domains_re); @mynetworks_maps = (\@mynetworks); @client_ipaddr_policy = map(($_,'MYNETS'), @mynetworks_maps); @bypass_virus_checks_maps = ( \%bypass_virus_checks, \@bypass_virus_checks_acl, \$bypass_virus_checks_re); @bypass_spam_checks_maps = ( \%bypass_spam_checks, \@bypass_spam_checks_acl, \$bypass_spam_checks_re); @bypass_banned_checks_maps = ( \%bypass_banned_checks, \@bypass_banned_checks_acl, \$bypass_banned_checks_re); @bypass_header_checks_maps = ( \%bypass_header_checks, \@bypass_header_checks_acl, \$bypass_header_checks_re); @virus_lovers_maps = ( \%virus_lovers, \@virus_lovers_acl, \$virus_lovers_re); @spam_lovers_maps = ( \%spam_lovers, \@spam_lovers_acl, \$spam_lovers_re); @banned_files_lovers_maps = ( \%banned_files_lovers, \@banned_files_lovers_acl, \$banned_files_lovers_re); @bad_header_lovers_maps = ( \%bad_header_lovers, \@bad_header_lovers_acl, \$bad_header_lovers_re); # @unchecked_lovers_maps = (); # empty, new setting, no need for backw compat. @warnvirusrecip_maps = (\$warnvirusrecip); @warnbannedrecip_maps = (\$warnbannedrecip); @warnbadhrecip_maps = (\$warnbadhrecip); @newvirus_admin_maps = (\$newvirus_admin); @virus_admin_maps = (\%virus_admin, \$virus_admin); @banned_admin_maps = (\$banned_admin, \%virus_admin, \$virus_admin); @bad_header_admin_maps= (\$bad_header_admin); @spam_admin_maps = (\%spam_admin, \$spam_admin); @virus_quarantine_to_maps = (\$virus_quarantine_to); @banned_quarantine_to_maps = (\$banned_quarantine_to); @unchecked_quarantine_to_maps = (\$unchecked_quarantine_to); @spam_quarantine_to_maps = (\$spam_quarantine_to); @spam_quarantine_bysender_to_maps = (\$spam_quarantine_bysender_to); @bad_header_quarantine_to_maps = (\$bad_header_quarantine_to); @clean_quarantine_to_maps = (\$clean_quarantine_to); @archive_quarantine_to_maps = (\$archive_quarantine_to); @keep_decoded_original_maps = (\$keep_decoded_original_re); @map_full_type_to_short_type_maps = (\$map_full_type_to_short_type_re); # @banned_filename_maps = ( {'.' => [$banned_filename_re]} ); # @banned_filename_maps = ( {'.' => 'DEFAULT'} );#names mapped by %banned_rules @banned_filename_maps = ( 'DEFAULT' ); # same as previous, but shorter @viruses_that_fake_sender_maps = (\$viruses_that_fake_sender_re, 1); @spam_tag_level_maps = (\$sa_tag_level_deflt); # CC_CLEAN,1 @spam_tag2_level_maps = (\$sa_tag2_level_deflt); # CC_SPAMMY @spam_tag3_level_maps = (\$sa_tag3_level_deflt); # CC_SPAMMY,1 @spam_kill_level_maps = (\$sa_kill_level_deflt); # CC_SPAM @spam_dsn_cutoff_level_maps = (\$sa_dsn_cutoff_level); @spam_dsn_cutoff_level_bysender_maps = (\$sa_dsn_cutoff_level); @spam_crediblefrom_dsn_cutoff_level_maps = (\$sa_crediblefrom_dsn_cutoff_level); @spam_crediblefrom_dsn_cutoff_level_bysender_maps = (\$sa_crediblefrom_dsn_cutoff_level); @spam_quarantine_cutoff_level_maps = (\$sa_quarantine_cutoff_level); @spam_subject_tag_maps = (\$sa_spam_subject_tag1); # note: inconsistent name @spam_subject_tag2_maps = (\$sa_spam_subject_tag); # note: inconsistent name # @spam_subject_tag3_maps = (); # new variable, no backwards compatib. needed @whitelist_sender_maps = ( \%whitelist_sender, \@whitelist_sender_acl, \$whitelist_sender_re); @blacklist_sender_maps = ( \%blacklist_sender, \@blacklist_sender_acl, \$blacklist_sender_re); @addr_extension_virus_maps = (\$addr_extension_virus); @addr_extension_spam_maps = (\$addr_extension_spam); @addr_extension_banned_maps = (\$addr_extension_banned); @addr_extension_bad_header_maps = (\$addr_extension_bad_header); @debug_sender_maps = (\@debug_sender_acl); # @debug_recipient_maps = (); @remove_existing_spam_headers_maps = (\$remove_existing_spam_headers); # new variables, no backwards compatibility needed, empty by default # @score_sender_maps, @author_to_policy_bank_maps, @signer_reputation_maps, # @message_size_limit_maps # build backwards-compatible settings hashes %final_destiny_by_ccat = ( CC_VIRUS, sub { c('final_virus_destiny') }, CC_BANNED, sub { c('final_banned_destiny') }, CC_UNCHECKED, sub { c('final_unchecked_destiny') }, CC_SPAM, sub { c('final_spam_destiny') }, CC_BADH, sub { c('final_bad_header_destiny') }, CC_MTA.',1', D_TEMPFAIL, CC_MTA.',2', D_REJECT, CC_OVERSIZED, D_BOUNCE, CC_CATCHALL, D_PASS, ); %forward_method_maps_by_ccat = ( CC_CATCHALL, sub { ca('forward_method_maps') }, ); %smtp_reason_by_ccat = ( # currently only used for blocked messages only, status 5xx # a multiline message will produce a valid multiline SMTP response CC_VIRUS, 'id=%n - INFECTED: %V', CC_BANNED, 'id=%n - BANNED: %F', CC_UNCHECKED, 'id=%n - UNCHECKED', CC_SPAM, 'id=%n - spam', CC_SPAMMY.',1', 'id=%n - spammy (tag3)', CC_SPAMMY, 'id=%n - spammy', CC_BADH.',1', 'id=%n - BAD HEADER: MIME error', CC_BADH.',2', 'id=%n - BAD HEADER: nonencoded 8-bit character', CC_BADH.',3', 'id=%n - BAD HEADER: contains invalid control character', CC_BADH.',4', 'id=%n - BAD HEADER: line made up entirely of whitespace', CC_BADH.',5', 'id=%n - BAD HEADER: line longer than RFC 5322 limit', CC_BADH.',6', 'id=%n - BAD HEADER: syntax error', CC_BADH.',7', 'id=%n - BAD HEADER: missing required header field', CC_BADH.',8', 'id=%n - BAD HEADER: duplicate header field', CC_BADH, 'id=%n - BAD HEADER', CC_OVERSIZED, 'id=%n - Message size exceeds recipient\'s size limit', CC_MTA.',1', 'id=%n - Temporary MTA failure on relaying', CC_MTA.',2', 'id=%n - Rejected by next-hop MTA on relaying', CC_MTA, 'id=%n - Unable to relay message back to MTA', CC_CLEAN, 'id=%n - CLEAN', CC_CATCHALL, 'id=%n - OTHER', # should not happen ); %lovers_maps_by_ccat = ( CC_VIRUS, sub { ca('virus_lovers_maps') }, CC_BANNED, sub { ca('banned_files_lovers_maps') }, CC_UNCHECKED, sub { ca('unchecked_lovers_maps') }, CC_SPAM, sub { ca('spam_lovers_maps') }, CC_SPAMMY, sub { ca('spam_lovers_maps') }, CC_BADH, sub { ca('bad_header_lovers_maps') }, ); %defang_maps_by_ccat = ( CC_VIRUS, sub { c('defang_virus') }, CC_BANNED, sub { c('defang_banned') }, CC_UNCHECKED, sub { c('defang_undecipherable') }, CC_SPAM, sub { c('defang_spam') }, CC_SPAMMY, sub { c('defang_spam') }, # CC_BADH.',3', 1, # NUL or CR character in header section # CC_BADH.',5', 1, # header line longer than 998 characters # CC_BADH.',6', 1, # header field syntax error CC_BADH, sub { c('defang_bad_header') }, ); %subject_tag_maps_by_ccat = ( CC_VIRUS, [ '***INFECTED*** ' ], CC_BANNED, undef, CC_UNCHECKED, sub { [ c('undecipherable_subject_tag') ] }, # not by-recip CC_SPAM, undef, CC_SPAMMY.',1', sub { ca('spam_subject_tag3_maps') }, CC_SPAMMY, sub { ca('spam_subject_tag2_maps') }, CC_CLEAN.',1', sub { ca('spam_subject_tag_maps') }, ); %quarantine_method_by_ccat = ( CC_VIRUS, sub { c('virus_quarantine_method') }, CC_BANNED, sub { c('banned_files_quarantine_method') }, CC_UNCHECKED, sub { c('unchecked_quarantine_method') }, CC_SPAM, sub { c('spam_quarantine_method') }, CC_BADH, sub { c('bad_header_quarantine_method') }, CC_CLEAN, sub { c('clean_quarantine_method') }, ); %quarantine_to_maps_by_ccat = ( CC_VIRUS, sub { ca('virus_quarantine_to_maps') }, CC_BANNED, sub { ca('banned_quarantine_to_maps') }, CC_UNCHECKED, sub { ca('unchecked_quarantine_to_maps') }, CC_SPAM, sub { ca('spam_quarantine_to_maps') }, CC_BADH, sub { ca('bad_header_quarantine_to_maps') }, CC_CLEAN, sub { ca('clean_quarantine_to_maps') }, ); %admin_maps_by_ccat = ( CC_VIRUS, sub { ca('virus_admin_maps') }, CC_BANNED, sub { ca('banned_admin_maps') }, CC_UNCHECKED, sub { ca('virus_admin_maps') }, CC_SPAM, sub { ca('spam_admin_maps') }, CC_BADH, sub { ca('bad_header_admin_maps') }, ); %always_bcc_by_ccat = ( CC_CATCHALL, sub { c('always_bcc') }, ); %dsn_bcc_by_ccat = ( CC_CATCHALL, sub { c('dsn_bcc') }, ); %mailfrom_notify_admin_by_ccat = ( CC_SPAM, sub { c('mailfrom_notify_spamadmin') }, CC_CATCHALL, sub { c('mailfrom_notify_admin') }, ); %hdrfrom_notify_admin_by_ccat = ( CC_SPAM, sub { c('hdrfrom_notify_spamadmin') }, CC_CATCHALL, sub { c('hdrfrom_notify_admin') }, ); %mailfrom_notify_recip_by_ccat = ( CC_CATCHALL, sub { c('mailfrom_notify_recip') }, ); %hdrfrom_notify_recip_by_ccat = ( CC_CATCHALL, sub { c('hdrfrom_notify_recip') }, ); %hdrfrom_notify_sender_by_ccat = ( CC_CATCHALL, sub { c('hdrfrom_notify_sender') }, ); %hdrfrom_notify_release_by_ccat = ( CC_CATCHALL, sub { c('hdrfrom_notify_release') }, ); %hdrfrom_notify_report_by_ccat = ( CC_CATCHALL, sub { c('hdrfrom_notify_report') }, ); %notify_admin_templ_by_ccat = ( CC_SPAM, sub { cr('notify_spam_admin_templ') }, CC_CATCHALL, sub { cr('notify_virus_admin_templ') }, ); %notify_recips_templ_by_ccat = ( CC_SPAM, sub { cr('notify_spam_recips_templ') }, #usualy empty CC_CATCHALL, sub { cr('notify_virus_recips_templ') }, ); %notify_sender_templ_by_ccat = ( # bounce templates CC_VIRUS, sub { cr('notify_virus_sender_templ') }, CC_BANNED, sub { cr('notify_virus_sender_templ') }, #historical reason CC_SPAM, sub { cr('notify_spam_sender_templ') }, CC_CATCHALL, sub { cr('notify_sender_templ') }, ); %notify_release_templ_by_ccat = ( CC_CATCHALL, sub { cr('notify_release_templ') }, ); %notify_report_templ_by_ccat = ( CC_CATCHALL, sub { cr('notify_report_templ') }, ); %notify_autoresp_templ_by_ccat = ( CC_CATCHALL, sub { cr('notify_autoresp_templ') }, ); %warnsender_by_ccat = ( # deprecated use, except perhaps for CC_BADH CC_VIRUS, undef, CC_BANNED, sub { c('warnbannedsender') }, CC_SPAM, undef, CC_BADH, sub { c('warnbadhsender') }, ); %warnrecip_maps_by_ccat = ( CC_VIRUS, sub { ca('warnvirusrecip_maps') }, CC_BANNED, sub { ca('warnbannedrecip_maps') }, CC_SPAM, undef, CC_BADH, sub { ca('warnbadhrecip_maps') }, ); %addr_extension_maps_by_ccat = ( CC_VIRUS, sub { ca('addr_extension_virus_maps') }, CC_BANNED, sub { ca('addr_extension_banned_maps') }, CC_SPAM, sub { ca('addr_extension_spam_maps') }, CC_SPAMMY, sub { ca('addr_extension_spam_maps') }, CC_BADH, sub { ca('addr_extension_bad_header_maps') }, # CC_OVERSIZED, 'oversized'; ); %addr_rewrite_maps_by_ccat = ( ); } # end BEGIN - init_tertiary # prototypes sub Amavis::Unpackers::do_mime_decode($$); sub Amavis::Unpackers::do_ascii($$); sub Amavis::Unpackers::do_uncompress($$$); sub Amavis::Unpackers::do_gunzip($$); sub Amavis::Unpackers::do_pax_cpio($$$); #sub Amavis::Unpackers::do_tar($$); # no longer supported sub Amavis::Unpackers::do_ar($$$); sub Amavis::Unpackers::do_unzip($$;$$); sub Amavis::Unpackers::do_7zip($$$;$); sub Amavis::Unpackers::do_unrar($$$;$); sub Amavis::Unpackers::do_unarj($$$;$); sub Amavis::Unpackers::do_arc($$$); sub Amavis::Unpackers::do_zoo($$$); sub Amavis::Unpackers::do_lha($$$;$); sub Amavis::Unpackers::do_ole($$$); sub Amavis::Unpackers::do_cabextract($$$); sub Amavis::Unpackers::do_tnef($$); sub Amavis::Unpackers::do_tnef_ext($$$); sub Amavis::Unpackers::do_unstuff($$$); sub Amavis::Unpackers::do_executable($$@); no warnings 'once'; # Define alias names or shortcuts in this module to make it simpler # to call these routines from amavisd.conf *read_l10n_templates = \&Amavis::Util::read_l10n_templates; *read_text = \&Amavis::Util::read_text; *read_hash = \&Amavis::Util::read_hash; *read_array = \&Amavis::Util::read_array; *dump_hash = \&Amavis::Util::dump_hash; *dump_array = \&Amavis::Util::dump_array; *ask_daemon = \&Amavis::AV::ask_daemon; *ask_clamav = \&Amavis::AV::ask_clamav; # deprecated, use ask_daemon *do_mime_decode = \&Amavis::Unpackers::do_mime_decode; *do_ascii = \&Amavis::Unpackers::do_ascii; *do_uncompress = \&Amavis::Unpackers::do_uncompress; *do_gunzip = \&Amavis::Unpackers::do_gunzip; *do_pax_cpio = \&Amavis::Unpackers::do_pax_cpio; *do_tar = \&Amavis::Unpackers::do_tar; # no longer supported *do_ar = \&Amavis::Unpackers::do_ar; *do_unzip = \&Amavis::Unpackers::do_unzip; *do_unrar = \&Amavis::Unpackers::do_unrar; *do_7zip = \&Amavis::Unpackers::do_7zip; *do_unarj = \&Amavis::Unpackers::do_unarj; *do_arc = \&Amavis::Unpackers::do_arc; *do_zoo = \&Amavis::Unpackers::do_zoo; *do_lha = \&Amavis::Unpackers::do_lha; *do_ole = \&Amavis::Unpackers::do_ole; *do_cabextract = \&Amavis::Unpackers::do_cabextract; *do_tnef_ext = \&Amavis::Unpackers::do_tnef_ext; *do_tnef = \&Amavis::Unpackers::do_tnef; *do_unstuff = \&Amavis::Unpackers::do_unstuff; *do_executable = \&Amavis::Unpackers::do_executable; *iso8601_week = \&Amavis::rfc2821_2822_Tools::iso8601_week; *iso8601_yearweek = \&Amavis::rfc2821_2822_Tools::iso8601_yearweek; *iso8601_year_and_week = \&Amavis::rfc2821_2822_Tools::iso8601_year_and_week; *iso8601_timestamp = \&Amavis::rfc2821_2822_Tools::iso8601_timestamp; *iso8601_utc_timestamp = \&Amavis::rfc2821_2822_Tools::iso8601_utc_timestamp; sub new_RE { Amavis::Lookup::RE->new(@_) } # shorthand: construct a query object for an SQL field sub q_sql_s { Amavis::Lookup::SQLfield->new(undef, $_[0], 'S-') } # string sub q_sql_n { Amavis::Lookup::SQLfield->new(undef, $_[0], 'N-') } # numeric sub q_sql_b { Amavis::Lookup::SQLfield->new(undef, $_[0], 'B-') } # boolean # shorthand: construct a query object for an LDAP attribute sub q_ldap_s { Amavis::Lookup::LDAPattr->new(undef, $_[0], 'S-') } # string sub q_ldap_n { Amavis::Lookup::LDAPattr->new(undef, $_[0], 'N-') } # numeric sub q_ldap_b { Amavis::Lookup::LDAPattr->new(undef, $_[0], 'B-') } # boolean sub Opaque { Amavis::Lookup::Opaque->new(@_) } sub OpaqueRef { Amavis::Lookup::OpaqueRef->new(@_) } # # Opaque provides a wrapper to arbitrary data structures, allowing them to be # treated as 'constant' pseudo-lookups, i.e. preventing arrays and hashes from # being interpreted as lookup lists/tables. In case of $forward_method this # allows for a listref of failover methods. Without the protection of Opaque # the listref would be interpreted by a lookup() as an acl lookup type instead # of a match-always data structure. The Opaque subroutine is not yet available # during a BEGIN phase, so this assignment must come after compiling the rest # of the code. # # This is the only case where both an array @*_maps as well as its default # element are members of a policy bank. Use lazy evaluation through a sub # to make this work as expected. # # @forward_method_maps = ( OpaqueRef(\$forward_method) ); @forward_method_maps = ( sub { Opaque(c('forward_method')) } ); # compatibility with old names use vars qw(%defang_by_ccat $sql_partition_tag $DO_SYSLOG $LOGFILE); *defang_by_ccat = \%defang_maps_by_ccat; *sql_partition_tag = \$partition_tag; *DO_SYSLOG = \$do_syslog; *LOGFILE = \$logfile; @virus_name_to_spam_score_maps = (new_RE( # the order matters! [ qr'^Structured\.(SSN|CreditCardNumber)\b' => 0.1 ], [ qr'^(Heuristics\.)?Phishing\.' => 0.1 ], [ qr'^(Email|HTML)\.Phishing\.(?!.*Sanesecurity)' => 0.1 ], [ qr'^Sanesecurity\.(Malware|Rogue|Trojan)\.' => undef ],# keep as infected [ qr'^Sanesecurity\.' => 0.1 ], [ qr'^Sanesecurity_PhishBar_' => 0 ], [ qr'^Sanesecurity.TestSig_' => 0 ], [ qr'^Email\.Spam\.Bounce(\.[^., ]*)*\.Sanesecurity\.' => 0 ], [ qr'^Email\.Spammail\b' => 0.1 ], [ qr'^MSRBL-(Images|SPAM)\b' => 0.1 ], [ qr'^VX\.Honeypot-SecuriteInfo\.com\.Joke' => 0.1 ], [ qr'^VX\.not-virus_(Hoax|Joke)\..*-SecuriteInfo\.com(\.|\z)' => 0.1 ], [ qr'^Email\.Spam.*-SecuriteInfo\.com(\.|\z)' => 0.1 ], [ qr'^Safebrowsing\.' => 0.1 ], [ qr'^winnow\.(phish|spam)\.' => 0.1 ], [ qr'^INetMsg\.SpamDomain' => 0.1 ], [ qr'^Doppelstern\.(Scam4|Phishing)' => 0.1 ], [ qr'^ScamNailer\.' => 0.1 ], [ qr'^HTML/Bankish' => 0.1 ], # F-Prot [ qr'-SecuriteInfo\.com(\.|\z)' => undef ], # keep as infected [ qr'^MBL_NA\.UNOFFICIAL' => 0.1 ], # false positives [ qr'^MBL_' => undef ], # keep as infected )); # Sanesecurity http://www.sanesecurity.co.uk/ # MSRBL- http://www.msrbl.com/site/contact # MBL http://www.malware.com.br/index.shtml # -SecuriteInfo.com http://clamav.securiteinfo.com/malwares.html # prepend a lookup table label object for logging purposes # sub label_default_maps() { for my $varname (qw( @disclaimer_options_bysender_maps @dkim_signature_options_bysender_maps @local_domains_maps @mynetworks_maps @forward_method_maps @newvirus_admin_maps @banned_filename_maps @spam_quarantine_bysender_to_maps @spam_tag_level_maps @spam_tag2_level_maps @spam_tag3_level_maps @spam_kill_level_maps @spam_subject_tag_maps @spam_subject_tag2_maps @spam_subject_tag3_maps @spam_dsn_cutoff_level_maps @spam_dsn_cutoff_level_bysender_maps @spam_crediblefrom_dsn_cutoff_level_maps @spam_crediblefrom_dsn_cutoff_level_bysender_maps @spam_quarantine_cutoff_level_maps @spam_notifyadmin_cutoff_level_maps @whitelist_sender_maps @blacklist_sender_maps @score_sender_maps @author_to_policy_bank_maps @signer_reputation_maps @message_size_limit_maps @debug_sender_maps @debug_recipient_maps @bypass_virus_checks_maps @bypass_spam_checks_maps @bypass_banned_checks_maps @bypass_header_checks_maps @viruses_that_fake_sender_maps @virus_name_to_spam_score_maps @virus_name_to_policy_bank_maps @remove_existing_spam_headers_maps @sa_userconf_maps @sa_username_maps @keep_decoded_original_maps @map_full_type_to_short_type_maps @virus_lovers_maps @spam_lovers_maps @unchecked_lovers_maps @banned_files_lovers_maps @bad_header_lovers_maps @virus_quarantine_to_maps @banned_quarantine_to_maps @unchecked_quarantine_to_maps @spam_quarantine_to_maps @bad_header_quarantine_to_maps @clean_quarantine_to_maps @archive_quarantine_to_maps @virus_admin_maps @banned_admin_maps @spam_admin_maps @bad_header_admin_maps @warnvirusrecip_maps @warnbannedrecip_maps @warnbadhrecip_maps @addr_extension_virus_maps @addr_extension_spam_maps @addr_extension_banned_maps @addr_extension_bad_header_maps )) { my($g) = $varname; $g =~ s{\@}{Amavis::Conf::}; # qualified variable name my($label) = $varname; $label=~s/^\@//; $label=~s/_maps$//; { no strict 'refs'; unshift(@$g, # NOTE: a symbolic reference Amavis::Lookup::Label->new($label)) if @$g; # no label if empty } } } # return a list of actually read&evaluated configuration files sub get_config_files_read() { @actual_config_files } # read and evaluate a configuration file, some sanity checking and housekeeping # sub read_config_file($$) { my($config_file,$is_optional) = @_; my(@stat_list) = stat($config_file); # symlinks-friendly my($errn) = @stat_list ? 0 : 0+$!; if ($errn == ENOENT && $is_optional) { # don't complain if missing } else { my($owner_uid) = $stat_list[4]; my($msg); if ($errn == ENOENT) { $msg = "does not exist" } elsif ($errn) { $msg = "is inaccessible: $!" } elsif (-d _) { $msg = "is a directory" } elsif (!-f _) { $msg = "is not a regular file" } elsif ($> && -o _) { $msg = "should not be owned by EUID $>"} elsif ($> && -w _) { $msg = "is writable by EUID $>, EGID $)" } elsif ($owner_uid) { $msg = "should be owned by root (uid 0) "} if (defined $msg) { die "Config file \"$config_file\" $msg," } $read_config_files_depth++; push(@actual_config_files, $config_file); if ($read_config_files_depth >= 100) { print STDERR "read_config_files: recursion depth limit exceeded\n"; exit 1; # avoid unwinding deep recursion, abort right away } local($_, $1,$2,$3,$4,$5,$6,$7,$8,$9); local $/ = $/; # protect us from a potential change in a config file $! = 0; if (defined(do $config_file)) {} elsif ($@ ne '') { die "Error in config file \"$config_file\": $@" } elsif ($! != 0) { die "Error reading config file \"$config_file\": $!" } $read_config_files_depth-- if $read_config_files_depth > 0; } 1; } sub include_config_files(@) { read_config_file($_,0) for @_; 1 } sub include_optional_config_files(@) { read_config_file($_,1) for @_; 1 } # supply remaining defaults after config files have already been read/evaluated # sub supply_after_defaults() { $daemon_chroot_dir = '' if !defined $daemon_chroot_dir || $daemon_chroot_dir eq '/'; # provide some sensible defaults for essential settings (post-defaults) $TEMPBASE = $MYHOME if !defined $TEMPBASE; $helpers_home = $MYHOME if !defined $helpers_home; $db_home = "$MYHOME/db" if !defined $db_home; $pid_file = "$MYHOME/amavisd.pid" if !defined $pid_file; # just keep $lock_file undefined by default, a temp file (POSIX::tmpnam) will # be provided by Net::Server for 'flock' serialization on a socket accept() # $lock_file = "$MYHOME/amavisd.lock" if !defined $lock_file; local($1,$2); $X_HEADER_LINE= "$myproduct_name at $mydomain" if !defined $X_HEADER_LINE; $X_HEADER_TAG = 'X-Virus-Scanned' if !defined $X_HEADER_TAG; if ($X_HEADER_TAG =~ /^[!-9;-\176]+\z/) { # implicitly add to %allowed_added_header_fields for compatibility, # unless the hash entry already exists my($allowed_hdrs) = cr('allowed_added_header_fields'); $allowed_hdrs->{lc($X_HEADER_TAG)} = 1 if $allowed_hdrs && !exists($allowed_hdrs->{lc($X_HEADER_TAG)}); } $gunzip = "$gzip -d" if !defined $gunzip && $gzip ne ''; $bunzip2 = "$bzip2 -d" if !defined $bunzip2 && $bzip2 ne ''; $unlzop = "$lzop -d" if !defined $unlzop && $lzop ne ''; # substring ${myhostname} will be expanded later, just before use my($pname) = '"Content-filter at ${myhostname}"'; $hdrfrom_notify_sender = "$pname " if !defined $hdrfrom_notify_sender; $hdrfrom_notify_recip = $mailfrom_notify_recip ne '' ? "$pname <$mailfrom_notify_recip>" : $hdrfrom_notify_sender if !defined $hdrfrom_notify_recip; $hdrfrom_notify_admin = $mailfrom_notify_admin ne '' ? "$pname <$mailfrom_notify_admin>" : $hdrfrom_notify_sender if !defined $hdrfrom_notify_admin; $hdrfrom_notify_spamadmin = $mailfrom_notify_spamadmin ne '' ? "$pname <$mailfrom_notify_spamadmin>" : $hdrfrom_notify_sender if !defined $hdrfrom_notify_spamadmin; $hdrfrom_notify_release = $hdrfrom_notify_sender if !defined $hdrfrom_notify_release; $hdrfrom_notify_report = $hdrfrom_notify_sender if !defined $hdrfrom_notify_report; if ($final_banned_destiny == D_DISCARD && c('warnbannedsender') ) { $final_banned_destiny = D_BOUNCE } if ($final_bad_header_destiny == D_DISCARD && c('warnbadhsender') ) { $final_bad_header_destiny = D_BOUNCE } if (!%banned_rules) { # an associative array mapping a rule name # to a single 'banned names/types' lookup table %banned_rules = ('DEFAULT'=>$banned_filename_re); # backwards compatibile } 1; } 1; # package Amavis::Log; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&init &collect_log_stats &log_to_stderr &log_fd &write_log &open_log &close_log); import Amavis::Conf qw(:platform $DEBUG c cr ca $myversion $logline_maxlen $daemon_user); } use subs @EXPORT_OK; use POSIX qw(locale_h strftime); use Fcntl qw(LOCK_SH LOCK_EX LOCK_UN); use Unix::Syslog qw(:macros :subs); use IO::File qw(O_RDONLY O_WRONLY O_RDWR O_APPEND O_CREAT O_EXCL); use Time::HiRes (); use vars qw($loghandle); # log file handle use vars qw($log_to_stderr $log_to_syslog $logfile_name $within_write_log); use vars qw($current_syslog_ident $current_syslog_facility); use vars qw($log_lines $log_retries %log_entries_by_level %log_status_counts); sub init($$) { ($log_to_syslog, $logfile_name) = @_; $log_lines = 0; %log_entries_by_level = (); $log_retries = 0; %log_status_counts = (); open_log(); if (!$log_to_syslog && $logfile_name eq '') { print STDERR "Logging to STDERR (no \$logfile and no \$do_syslog)\n" } } sub collect_log_stats() { my(@result) = ($log_lines, {%log_entries_by_level}, $log_retries, {%log_status_counts}); $log_lines = 0; %log_entries_by_level = (); $log_retries = 0; %log_status_counts = (); @result; } # turn debug logging to STDERR on or off # sub log_to_stderr(;$) { $log_to_stderr = shift if @_ > 0; $log_to_stderr; } # try to obtain file descriptor used by write_log, undef if unknown # sub log_fd() { $log_to_stderr ? fileno(STDERR) : $log_to_syslog ? undef # how to obtain fd on syslog? : defined $loghandle ? $loghandle->fileno : fileno(STDERR); } sub open_log() { # don't bother to skip opening the log even if $log_to_stderr (debug) is true if ($log_to_syslog) { my($id) = c('syslog_ident'); my($fac) = c('syslog_facility'); $fac =~ /^[A-Za-z0-9_]+\z/ or die "Suspicious syslog facility name: $fac"; my($syslog_facility_num) = eval("LOG_\U$fac"); $syslog_facility_num =~ /^\d+\z/ or die "Unknown syslog facility name: $fac"; # man syslog(3) on Linux: The argument 'ident' in the call of openlog() # is probably stored as-is. Thus, if the string it points to is changed, # syslog() may start prepending the changed string, and if the string # it points to ceases to exist, the results are undefined. Most portable # is to use a string constant. (we use a static variable here) $current_syslog_ident = $id; $current_syslog_facility = $fac; openlog($current_syslog_ident, LOG_PID | LOG_NDELAY, $syslog_facility_num); } elsif ($logfile_name ne '') { $loghandle = IO::File->new; $loghandle->open($logfile_name, O_CREAT|O_APPEND|O_WRONLY, 0640) or die "Failed to open log file $logfile_name: $!"; binmode($loghandle,':bytes') or die "Can't cancel :utf8 mode: $!"; $loghandle->autoflush(1); if ($> == 0) { local($1); my($uid) = $daemon_user=~/^(\d+)$/ ? $1 : (getpwnam($daemon_user))[2]; if ($uid) { chown($uid,-1,$logfile_name) or die "Can't chown logfile $logfile_name to $uid: $!"; } } } else { # logging to STDERR STDERR->autoflush(1); # just in case } } sub close_log() { if ($log_to_syslog) { closelog(); $current_syslog_ident = $current_syslog_facility = undef; } elsif (defined($loghandle) && $logfile_name ne '') { $loghandle->close or die "Error closing log file $logfile_name: $!"; undef $loghandle; } } # Log either to syslog or to a file # sub write_log($$$;@) { my($level,$am_id,$errmsg,@args) = @_; return if $within_write_log; $within_write_log = 1; $am_id = !defined $am_id ? '' : "($am_id) "; # treat $errmsg as sprintf format string if additional arguments provided if (@args && index($errmsg,'%') >= 0) { $errmsg = sprintf($errmsg,@args) } $errmsg = Amavis::Util::sanitize_str($errmsg); # my($old_locale) = POSIX::setlocale(LC_TIME,'C'); # English dates required! # if (length($errmsg) > 2000) { # crop at some arbitrary limit (< LINE_MAX) # $errmsg = substr($errmsg,0,2000) . "..."; # } my($alert_mark) = $level >= 0 ? '' : $level >= -1 ? '(!)' : '(!!)'; # $alert_mark .= '*' if $> == 0; $log_entries_by_level{"$level"}++; if ($log_to_syslog && !$log_to_stderr) { my($prio); if ($level >= 3) { $prio = LOG_DEBUG } # most frequent elsif ($level >= 2) { $prio = LOG_INFO } elsif ($level >= 1) { $prio = LOG_INFO } elsif ($level >= 0) { $prio = LOG_NOTICE } elsif ($level >= -1) { $prio = LOG_WARNING } elsif ($level >= -2) { $prio = LOG_ERR } else { $prio = LOG_CRIT } if (c('syslog_ident') ne $current_syslog_ident || c('syslog_facility') ne $current_syslog_facility) { close_log() if defined $current_syslog_ident || defined $current_syslog_facility; open_log(); } my($pre) = $alert_mark; # $logline_maxlen should be less than (1023 - prefix) for a typical syslog, # 980 is a suitable length to avoid truncations by the syslogd daemon my($logline_size) = $logline_maxlen; $logline_size = 50 if $logline_size < 50; # let at least something out while (length($am_id)+length($pre)+length($errmsg) > $logline_size) { my($avail) = $logline_size - length($am_id . $pre . '...'); $log_lines++; $! = 0; syslog($prio, '%s', $am_id . $pre . substr($errmsg,0,$avail) . '...'); if ($! != 0) { $log_retries++; $log_status_counts{"$!"}++ } $pre = $alert_mark . '...'; $errmsg = substr($errmsg,$avail); } $log_lines++; $! = 0; syslog($prio, '%s', $am_id . $pre . $errmsg); if ($! != 0) { $log_retries++; $log_status_counts{"$!"}++ } } else { $log_lines++; if ($log_to_stderr || !defined $loghandle) { my($now) = Time::HiRes::time; my($prefix) = sprintf('%s:%06.3f %s %s[%s]: ', # syslog-like prefix strftime('%b %e %H:%M',localtime($now)), $now-int($now/60)*60, c('myhostname'), c('myprogram_name'), $$); # milliseconds in timestamp # avoid multiple calls to write(2), join the string first! my($s) = $prefix . $am_id . $alert_mark . $errmsg . "\n"; print STDERR ($s) or die "Error writing to STDERR: $!"; } else { my($prefix) = sprintf('%s %s %s[%s]: ', # prepare a syslog-like prefix strftime('%b %e %H:%M:%S',localtime), c('myhostname'), c('myprogram_name'), $$); my($s) = $prefix . $am_id . $alert_mark . $errmsg . "\n"; # NOTE: a lock is on a file, not on a file handle flock($loghandle,LOCK_EX) or die "Can't lock a log file: $!"; seek($loghandle,0,2) or die "Can't position log file to its tail: $!"; $loghandle->print($s) or die "Error writing to log file: $!"; flock($loghandle,LOCK_UN) or die "Can't unlock a log file: $!"; } } # POSIX::setlocale(LC_TIME, $old_locale); $within_write_log = 0; } 1; # package Amavis::Timing; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&init §ion_time &report &get_time_so_far); } use subs @EXPORT_OK; use vars qw(@timing); use Time::HiRes (); # clear array @timing and enter start time # sub init() { @timing = (); section_time('init'); } # enter current time reading into array @timing # sub section_time($) { push(@timing, shift, Time::HiRes::time); } # returns a string - a report of elapsed time by section # sub report() { section_time('rundown'); my($notneeded, $t0) = (shift(@timing), shift(@timing)); my($total) = $t0 <= 0 ? 0 : $timing[$#timing] - $t0; if ($total < 0.0000001) { $total = 0.0000001 } my(@sections); my($t00) = $t0; while (@timing) { my($section, $t) = (shift(@timing), shift(@timing)); my($dt) = $t <= $t0 ? 0 : $t-$t0; # handle possible clock jumps my($dt_c) = $t <= $t00 ? 0 : $t-$t00; # handle possible clock jumps my($dtp) = $dt >= $total ? 100 : $dt*100.0/$total; # this event my($dtp_c) = $dt_c >= $total ? 100 : $dt_c*100.0/$total; # cumulative push(@sections, sprintf('%s: %.0f (%.0f%%)%.0f', $section, $dt*1000, $dtp, $dtp_c)); $t0 = $t; } sprintf('TIMING [total %.0f ms] - %s', $total * 1000, join(', ',@sections)); } # returns value in seconds of elapsed time for processing of this mail so far # sub get_time_so_far() { my($notneeded, $t0) = @timing; my($total) = $t0 <= 0 ? 0 : Time::HiRes::time - $t0; $total < 0 ? 0 : $total; } use vars qw($t_was_busy $t_busy_cum $t_idle_cum $t0); sub idle_proc(@) { my($t1) = Time::HiRes::time; if (defined $t0) { ($t_was_busy ? $t_busy_cum : $t_idle_cum) += $t1 - $t0; Amavis::Util::ll(5) && Amavis::Util::do_log(5, 'idle_proc, %s: was %s, %.1f ms, total idle %.3f s, busy %.3f s', $_[0], $t_was_busy ? 'busy' : 'idle', 1000*($t1 - $t0), $t_idle_cum, $t_busy_cum); } $t0 = $t1; } sub go_idle(@) { if ($t_was_busy) { idle_proc(@_); $t_was_busy = 0 } } sub go_busy(@) { if (!$t_was_busy) { idle_proc(@_); $t_was_busy = 1 } } sub report_load() { $t_busy_cum + $t_idle_cum <= 0 ? undef : sprintf('load: %.0f %%, total idle %.3f s, busy %.3f s', 100*$t_busy_cum / ($t_busy_cum + $t_idle_cum), $t_idle_cum, $t_busy_cum); } 1; # package Amavis::Util; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&untaint &min &max &minmax &unique_list &unique_ref &safe_encode &safe_decode &q_encode &xtext_encode &xtext_decode &proto_encode &proto_decode &orcpt_encode &orcpt_decode &snmp_count &snmp_count64 &snmp_counters_init &snmp_counters_get &ll &do_log &debug_oneshot &update_current_log_level &am_id &new_am_id &stir_random &add_entropy &fetch_entropy_bytes &generate_mail_id &make_password &crunching_start_time &prolong_timer &get_deadline &waiting_for_client &switch_to_my_time &switch_to_client_time &sanitize_str &fmt_struct &freeze &thaw &ccat_split &ccat_maj &cmp_ccat &cmp_ccat_maj &setting_by_given_contents_category_all &setting_by_given_contents_category &rmdir_recursively &read_text &read_l10n_templates &read_hash &read_array &dump_hash &dump_array &dynamic_destination &collect_equal_delivery_recips); import Amavis::Conf qw(:platform $DEBUG c cr ca $mail_id_size_bits $trim_trailing_space_in_lookup_result_fields); import Amavis::Log qw(write_log); import Amavis::Timing qw(section_time); } use subs @EXPORT_OK; use Errno qw(ENOENT EACCES EAGAIN ESRCH EBADF); use IO::File (); use Digest::MD5; # 2.22 provides 'clone' method, no longer needed since 2.7.0 use MIME::Base64; use Encode; # Perl 5.8 UTF-8 support use Scalar::Util qw(tainted); use vars qw($enc_tainted); BEGIN { $enc_tainted = substr($ENV{PATH}.$ENV{HOME}, 0,0); # tainted empty string tainted($enc_tainted) or warn "Amavis::Util: can't obtain a tainted string"; } # Return untainted copy of a string (argument can be a string or a string ref) # sub untaint($) { no re 'taint'; my($str); if (defined($_[0])) { local($1); # avoid Perl taint bug: tainted global $1 propagates taintedness $str = $1 if (ref($_[0]) ? ${$_[0]} : $_[0]) =~ /^(.*)\z/s; } $str; } # Returns the smallest defined number from the list, or undef # sub min(@) { my($r) = @_ == 1 && ref($_[0]) ? $_[0] : \@_; # accept list, or a list ref my($m); for (@$r) { $m = $_ if defined $_ && (!defined $m || $_ < $m) } $m; } # Returns the largest defined number from the list, or undef # sub max(@) { my($r) = @_ == 1 && ref($_[0]) ? $_[0] : \@_; # accept list, or a list ref my($m); for (@$r) { $m = $_ if defined $_ && (!defined $m || $_ > $m) } $m; } # Returns a pair of the smallest and the largest defined number from the list # sub minmax(@) { my($r) = @_ == 1 && ref($_[0]) ? $_[0] : \@_; # accept list, or a list ref my($min,$max); for (@$r) { if (defined $_) { $min = $_ if !defined $min || $_ < $min; $max = $_ if !defined $max || $_ > $max; } } ($min,$max); } # Returns a sublist of the supplied list of elements in an unchanged order, # where only the first occurrence of each defined element is retained # and duplicates removed # sub unique_list(@) { my($r) = @_ == 1 && ref($_[0]) ? $_[0] : \@_; # accept list, or a list ref my(%seen); my(@result) = grep(defined($_) && !$seen{$_}++, @$r); @result; } # same as unique, except that it returns a ref to the resulting list # sub unique_ref(@) { my($r) = @_ == 1 && ref($_[0]) ? $_[0] : \@_; # accept list, or a list ref my(%seen); my(@result) = grep(defined($_) && !$seen{$_}++, @$r); \@result; } # A wrapper for Encode::encode, avoiding a bug in Perl 5.8.0 which causes # Encode::encode to loop and fill memory when given a tainted string. # Also works around a CPAN bug #64642 in module Encode: # Tainted values have the taint flag cleared when encoded (or decoded) # https://rt.cpan.org/Public/Bug/Display.html?id=64642 # (still unresolved with Encode as bundled with Perl 5.14.2) # sub safe_encode($$;$) { # my($encoding,$str,$check) = @_; my $encoding = shift; return undef if !defined $_[0]; # must return undef even in a list context! my $enc = Encode::find_encoding($encoding); $enc or warn "safe_encode: unknown encoding '$encoding'"; return $enc->encode(@_) if !tainted($_[0]); # propagate taintedness across taint-related bugs in module Encode $enc_tainted . $enc->encode(untaint($_[0]), $_[1]); } sub safe_decode($$;$) { # my($encoding,$str,$check) = @_; my $encoding = shift; return undef if !defined $_[0]; # must return undef even in a list context! my $enc = Encode::find_encoding($encoding); return $_[0] if !$enc; return $enc->decode(@_) if !tainted($_[0]); # propagate taintedness across taint-related bugs in module Encode $enc_tainted . $enc->decode(untaint($_[0]), $_[1]); } # Do the Q-encoding manually, the MIME::Words::encode_mimeword does not # encode spaces and does not limit to 75 ch, which violates the RFC 2047 # sub q_encode($$$) { my($octets,$encoding,$charset) = @_; my($prefix) = '=?' . $charset . '?' . $encoding . '?'; my($suffix) = '?='; local($1,$2,$3); # FWS | utext (= NO-WS-CTL|rest of US-ASCII) $octets =~ /^ ( [\001-\011\013\014\016-\177]* [ \t] )? (.*?) ( [ \t] [\001-\011\013\014\016-\177]* )? \z/sx; my($head,$rest,$tail) = ($1,$2,$3); # Q-encode $rest according to RFC 2047 (not for use in comments or phrase) $rest =~ s{([\000-\037\177\200-\377=?_])}{sprintf('=%02X',ord($1))}egs; $rest =~ tr/ /_/; # turn spaces into _ (RFC 2047 allows it) my($s) = $head; my($len) = 75 - (length($prefix)+length($suffix)) - 2; while ($rest ne '') { $s .= ' ' if $s !~ /[ \t]\z/; # encoded words must be separated by FWS $rest =~ /^ ( .{0,$len} [^=] (?: [^=] | \z ) ) (.*) \z/sx; $s .= $prefix.$1.$suffix; $rest = $2; } $s.$tail; } # encode "+", "=" and any character outside the range "!" (33) .. "~" (126) # sub xtext_encode($) { # RFC 3461 my($str) = @_; local($1); # avoid Encode::is_utf8 check, always false on tainted, Perl bug #32687 $str = safe_encode('UTF-8',$str); # if Encode::is_utf8($str); $str =~ s/([^\041-\052\054-\074\076-\176])/sprintf('+%02X',ord($1))/egs; $str; } # decode xtext-encoded string as per RFC 3461 # sub xtext_decode($) { my($str) = @_; local($1); $str =~ s/\+([0-9a-fA-F]{2})/pack('C',hex($1))/egs; $str; } sub proto_encode($@) { my($attribute_name,@strings) = @_; local($1); for ($attribute_name,@strings) { # just in case, handle non-octet characters: s/([^\000-\377])/sprintf('\\x{%04x}',ord($1))/egs and do_log(-1,'proto_encode: non-octet character encountered: %s', $_); } $attribute_name =~ # encode all but alfanumerics, . _ + - s/([^0-9a-zA-Z._+-])/sprintf('%%%02x',ord($1))/egs; for (@strings) { # encode % and nonprintables s/([^\041-\044\046-\176])/sprintf('%%%02x',ord($1))/egs; } $attribute_name . '=' . join(' ',@strings); } sub proto_decode($) { my($str) = @_; local($1); $str =~ s/%([0-9a-fA-F]{2})/pack('C',hex($1))/egs; $str; } # xtext_encode and prepend 'rfc822;' to form a string to be used as ORCPT # sub orcpt_encode($) { # RFC 3461 # RFC 3461: Due to limitations in the Delivery Status Notification format, # the value of the original recipient address prior to encoding as "xtext" # MUST consist entirely of printable (graphic and white space) characters # from the US-ASCII [4] repertoire. my($str) = @_; local($1); # argument should be SMTP-quoted address $str = $1 if $str =~ /^<(.*)>\z/s; # strip-off <> $str =~ s/[^\040-\176]/?/gs; 'rfc822;' . xtext_encode($str); } sub orcpt_decode($) { # RFC 3461 my($str) = @_; # argument should be RFC 3461 -encoded address my($addr_type,$orcpt); local($1,$2); if (defined $str) { if ($str =~ /^([^\000-\040\177()<>\[\]\@\\:;,."]*);(.*\z)/si){ # atom;xtext ($addr_type,$orcpt) = ($1,$2); } else { ($addr_type,$orcpt) = ('rfc822',$str); # RFC 3464 address-type } $orcpt = xtext_decode($orcpt); # decode $orcpt =~ s/[^\040-\176]/?/gs; # some minimal sanitation } # result in $orcpt is presumably an RFC 5322 -encoded addr, no angle brackets ($addr_type,$orcpt); } # Set or get Amavis internal task id (also called: log id). # This task id performs a similar function as queue-id in MTA responses. # It may only be used in generating text part of SMTP responses, # or in generating log entries. It is only unique within a limited timespan. use vars qw($amavis_task_id); # internal task id # (accessible via am_id() and later also as $msginfo->log_id) sub am_id(;$) { if (@_) { # set, if argument present $amavis_task_id = shift; $0 = c('myprogram_name') . " ($amavis_task_id)"; } $amavis_task_id; # return current value } sub new_am_id($;$$) { my($str, $cnt, $seq) = @_; my($id); $id = defined $str ? $str : sprintf('%05d', $$); $id .= sprintf('-%02d', $cnt) if defined $cnt; $id .= '-'.$seq if defined $seq && $seq > 1; am_id($id); } use vars qw($entropy); # MD5 ctx (128 bits, 32 hex digits or 22 base64 chars) sub add_entropy(@) { # arguments may be strings or array references $entropy = Digest::MD5->new if !defined $entropy; my($s) = join(',', map((!defined $_ ? 'U' : ref eq 'ARRAY' ? @$_ : $_), @_)); # do_log(5,'add_entropy: %s',$s); $entropy->add($s); } sub fetch_entropy_bytes($) { my($n) = @_; # number of bytes to collect my($result) = ''; for (; $n > 0; $n--) { # collect as few bits per MD5 iteration as possible (RFC 4086 sect 6.2.1) # let's settle for 8 bits for practical reasons; fewer would be better my($digest) = $entropy->digest; # 16 bytes; also destroys accumulator $result .= substr($digest,0,1); # take 1 byte $entropy->reset; $entropy->add($digest); # cycle it back } # ll(5) && do_log(5,'fetch_entropy_bytes %s', # join(' ', map(sprintf('%02x',$_), unpack('C*',$result)))); $result; } # read number of bytes from a /dev/urandom device # sub read_random($) { my($required_bytes) = @_; my($result) = ''; my($fname) = '/dev/urandom'; # nonblocking device! if ($required_bytes > 0) { my($fh) = IO::File->new; $fh->open($fname,'<') or die "Can't open $fname: $!"; binmode($fh,':bytes') or die "Can't cancel :utf8 mode: $!"; my($nbytes) = $fh->read($result, $required_bytes); defined $nbytes or die "Error reading from $fname: $!"; $nbytes >= $required_bytes or die "Less data than requested: $!"; $fh->close or die "Error closing $fname: $!"; } $result; } # stir/initialize perl's random generator and our entropy pool; # to be called at startup of the main process and each child processes # sub stir_random() { my($random_bytes); eval { $random_bytes = read_random(16); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(0, 'read_random error: %s', $eval_stat); }; srand(); # let perl give it a try first, then stir-in some additional bits add_entropy($random_bytes, Time::HiRes::gettimeofday, $$, rand()); # # must prevent all child processes working with the same inherited random # seed, otherwise modules like File::Temp will step on each other's toes my($r) = unpack('L', fetch_entropy_bytes(4)) ^ int(rand(0xffffffff)); srand($r & 0x7fffffff); } # generate a reasonably unique (long-term) id based on collected entropy. # The result is a pair of (mostly public) mail_id, and a secret id, # where mail_id == b64(md5(secret)). The secret id could be used to # authorize releasing quarantined mail. Both the mail_id and secret id are # strings of characters [A-Za-z0-9-_], with an additional restriction # for mail_id which must begin and end with an alphanumeric character. # The number of bits in a mail_id is configurable through $mail_id_size_bits # and defaults to 72, yielding a 12-character base64url-encoded string. # The number of bits must be an integral multiple of 24, so that no base64 # trailing padding characters '=' are needed (RFC 4648). # Note the difference in base64-like encodings: # amavisd almost-base64: 62 +, 63 - (old, no longer used since 2.7.0) # RFC 4648 base64: 62 +, 63 / (not used here) # RFC 4648 base64url: 62 -, 63 _ # Generally, RFC 5322 controls, SP and specials must be avoided: ()<>[]:;@\,." # With verision 2.7.0 of amavisd we switched from almost-base64 to base64url # to avoid having to quote a '+' in regular expressions and in URL. # sub generate_mail_id() { my($id_b64, $secret_bin); # 72 bits = 9 bytes = 12 b64 chars # 96 bits = 12 bytes = 16 b64 chars $mail_id_size_bits > 0 && $mail_id_size_bits == int $mail_id_size_bits && $mail_id_size_bits % 24 == 0 or die "\$mail_id_size_bits ($mail_id_size_bits) must be a multiple of 24"; for (my $j=0; $j<100; $j++) { # provide some sanity loop limit just in case $secret_bin = fetch_entropy_bytes($mail_id_size_bits/8); # mail_id is computed as md5(secret), rely on unidirectionality of md5 $id_b64 = Digest::MD5->new->add($secret_bin)->b64digest; # b64(md5(sec)) add_entropy($id_b64,$j); # fold it back into accumulator $id_b64 = substr($id_b64, 0, $mail_id_size_bits/6); # crop b64 chars # finished if it starts and ends with an alfanumeric character last if $id_b64 =~ /^[A-Za-z0-9].*[A-Za-z0-9]\z/s; # retry on less than 7% of cases do_log(5,'generate_mail_id retry: %s', $id_b64); } my($secret_b64) = encode_base64($secret_bin,''); # $mail_id_size_bits/6 chars $secret_bin = 'X' x length($secret_bin); # can't hurt to be conservative $id_b64 =~ tr{+/}{-_}; # base64 -> RFC 4648 base64url [A-Za-z0-9-_] $secret_b64 =~ tr{+/}{-_}; # base64 -> RFC 4648 base64url [A-Za-z0-9-_] # do_log(5,'generate_mail_id: %s %s', $id_b64, $secret_b64); ($id_b64, $secret_b64); } # Returns a password that may be used for scrambling of a message being # released from a quarantine or mangled, with intention of preventing an # automatic or undesired implicit opening of a potentially dangerous message. # The first argument may be: a plain string, which is simply passed on # to the result, or: a code reference (to be evaluated in a scalar context), # allowing for lazy evaluation of a supplied password generating code, # or: undef, which causes a generation of a simple 4-digit PIN-like random # password. The second argument is just passed on unchanged to the supplied # subroutine and is expected to be a $msginfo object. # sub make_password($$) { my($password,$msginfo) = @_; if (ref $password eq 'CODE') { eval { $password = &$password($msginfo); chomp $password; $password =~ s/^[ \t]+//; $password =~ s/[ \t]+\z//; $password = untaint($password) if $password =~ /^[A-Za-z0-9:._=+-]*\z/; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1, 'password generating subroutine failed, '. 'supplying a default: %s', $@); $password = undef; }; } if (!defined $password) { # create a 4-digit random string $password = sprintf('%04d', unpack('S',fetch_entropy_bytes(2)) % 10000); } $password; } use vars qw(@counter_names); # elements may be counter names (increment is 1), or pairs: [name,increment], # or triples: [name,value,type], where type can be: C32, C64, INT, TIN or OID sub snmp_counters_init() { @counter_names = () } sub snmp_count(@) { push(@counter_names, @_) } sub snmp_count64(@) { push(@counter_names, map(ref $_ ?$_ :[$_,1,'C64'], @_)) } sub snmp_counters_get() { \@counter_names } use vars qw($debug_oneshot); sub debug_oneshot(;$$) { if (@_) { my($new_debug_oneshot) = shift; if (($new_debug_oneshot ? 1 : 0) != ($debug_oneshot ? 1 : 0)) { do_log(0, 'DEBUG_ONESHOT: TURNED '.($new_debug_oneshot ? 'ON' : 'OFF')); do_log(0, shift) if @_; # caller-provided extra log entry, usually # the one that caused debug_oneshot call } $debug_oneshot = $new_debug_oneshot; } $debug_oneshot; } use vars qw($current_log_level); sub update_current_log_level() { $current_log_level = c('log_level') || 0; } # is message log level below the current log level (i.e. eligible for logging)? # sub ll($) { # optimized away: my($current_log_level) = c('log_level'); if ($DEBUG || $debug_oneshot) { my($level) = @_; $level = 0 if ($DEBUG || $debug_oneshot) && $level > 0; $level <= $current_log_level; } else { # optimized common case $_[0] <= $current_log_level; } } # write log entry # sub do_log($$;@) { my($level) = shift; # my($errmsg,@args) = @_; # duplicate code from ll() to avoid one subroutine call $level = 0 if ($DEBUG || $debug_oneshot) && $level > 0; write_log($level, am_id(), shift, @_) if $level <= $current_log_level; 1; } # $timestamp_of_last_reception: a Unix time stamp when an MTA client send the # last command to us, the most important of which is the reception of a final # dot in SMTP session, which is a time when a client started to wait for our # response; this timestamp, along with a c('child_timeout'), make a deadline # time for our processing # # $waiting_for_client: which timeout is running: # false: processing is in our courtyard, true: waiting for a client # use vars qw($timestamp_of_last_reception $waiting_for_client); sub waiting_for_client(;$) { $waiting_for_client = shift if @_; $waiting_for_client; } sub get_deadline(@) { my($which_section, $allowed_share, $reserve, $max_time) = @_; # $allowed_share ... factor between 0 and 1 of the remaining time till a # deadline, to be allocated to the task that follows # $reserve ... try finishing up $reserve seconds before the deadline; # $max_time ... upper limit in seconds for the timer interval my($timer_interval, $timer_deadline, $time_to_deadline); my($child_t_o) = c('child_timeout'); if (!$child_t_o) { do_log(2, 'get_deadline %s - ignored, child_timeout not set', $which_section); } elsif (!defined $timestamp_of_last_reception) { do_log(2, 'get_deadline %s - ignored, master deadline not known', $which_section); } else { my($now) = Time::HiRes::time; $time_to_deadline = $timestamp_of_last_reception + $child_t_o - $now; $timer_interval = $time_to_deadline; if (!defined $allowed_share) { $allowed_share = 0.7; $timer_interval *= $allowed_share; } elsif ($allowed_share <= 0) { $timer_interval = 0; } elsif ($allowed_share >= 1) { # leave it unchanged } else { $timer_interval *= $allowed_share; } $reserve = 3 if !defined $reserve; if ($reserve > 0 && $timer_interval > $time_to_deadline - $reserve) { $timer_interval = $time_to_deadline - $reserve; } if ($timer_interval < 8) { # try to be generous $timer_interval = max(4, min(8,$time_to_deadline)); } my($j) = int($timer_interval); $timer_interval = $timer_interval > $j ? $j+1 : $j; # ceiling if (defined $max_time && $max_time > 0 && $timer_interval > $max_time) { $timer_interval = $max_time; } do_log(5, 'get_deadline %s - deadline in %.1f s, set to %.3f s', $which_section,$time_to_deadline,$timer_interval); $timer_deadline = $now + $timer_interval; } !wantarray ? $timer_interval : ($timer_interval, $timer_deadline, $time_to_deadline); } sub prolong_timer($;$$$) { my($which_section, $allowed_share, $reserve, $max_time) = @_; my($timer_interval, $timer_deadline, $time_to_deadline) = get_deadline(@_); if (defined $timer_interval) { my($prev_timer) = alarm($timer_interval); # restart/prolong the timer do_log(5,'prolong_timer %s: timer %d, was %d, deadline in %.1f s', $which_section, $timer_interval, $prev_timer, $time_to_deadline); } !wantarray ? $timer_interval : ($timer_interval, $timer_deadline, $time_to_deadline); } sub switch_to_my_time($) { # processing is in our courtyard my($msg) = @_; $waiting_for_client = 0; $timestamp_of_last_reception = Time::HiRes::time; my($child_t_o) = c('child_timeout'); if (!$child_t_o) { alarm(0); } else { prolong_timer( 'switch_to_my_time(' . $msg . ')' ); } } sub switch_to_client_time($) { # processing is now in client's hands my($msg) = @_; my($interval) = c('smtpd_timeout'); $interval = 5 if $interval < 5; do_log(5, 'switch_to_client_time %d s, %s', $interval,$msg); undef $timestamp_of_last_reception; alarm($interval); $waiting_for_client = 1; } # Mostly for debugging and reporting purposes: # Convert nonprintable characters in the argument # to \[rnftbe], or \octal code, and '\' to '\\', # and Unicode characters to \x{xxxx}, returning the sanitized string. # use vars qw(%quote_controls_map); BEGIN { %quote_controls_map = ("\r" => '\\r', "\n" => '\\n', "\f" => '\\f', "\t" => '\\t', "\b" => '\\b', "\e" => '\\e', "\\" => '\\\\'); } sub sanitize_str { my($str, $keep_eol) = @_; # avoid Encode::is_utf8 check, always false on tainted, Perl bug #32687 $str = safe_encode('UTF-8',$str); # if Encode::is_utf8($str); local($1); if ($keep_eol) { $str =~ s/([^\012\040-\133\135-\176])/ # and \240-\376 ? exists($quote_controls_map{$1}) ? $quote_controls_map{$1} : sprintf(ord($1)>255 ? '\\x{%04x}' : '\\%03o', ord($1))/egs; } else { $str =~ s/([^\040-\133\135-\176])/ # and \240-\376 ? exists($quote_controls_map{$1}) ? $quote_controls_map{$1} : sprintf(ord($1)>255 ? '\\x{%04x}' : '\\%03o', ord($1))/egs; } $str; } # pretty-print a structure for logging purposes: returns a string # sub fmt_struct($); # prototype sub fmt_struct($) { my($arg) = @_; !defined($arg) ? 'undef' : !ref($arg) ? '"'.$arg.'"' : ref($arg) eq 'ARRAY' ? '[' . join(',', map(fmt_struct($_),@$arg)) . ']' : ref($arg) eq 'HASH' ? '{' . join(',', map($_.'=>'.fmt_struct($arg->{$_}),keys(%$arg))) . '}' : $arg; }; # used by freeze: protect % and ~, as well as NUL and \200 for good measure # sub st_encode($) { my($str) = @_; local($1); $str =~ s/([%~\000\200])/sprintf('%%%02X',ord($1))/egs; $str; } # simple Storable::freeze lookalike # sub freeze($); # prototype sub freeze($) { my($obj) = @_; my($ty) = ref($obj); if (!defined($obj)) { 'U' } elsif (!$ty) { join('~', '', st_encode($obj)) } # string elsif ($ty eq 'SCALAR') { join('~', 'S', st_encode(freeze($$obj))) } elsif ($ty eq 'REF') { join('~', 'R', st_encode(freeze($$obj))) } elsif ($ty eq 'ARRAY') { join('~', 'A', map(st_encode(freeze($_)),@$obj)) } elsif ($ty eq 'HASH') { join('~', 'H', map {(st_encode($_),st_encode(freeze($obj->{$_})))} sort keys %$obj) } else { die "Can't freeze object type $ty" } } # simple Storable::thaw lookalike # sub thaw($); # prototype sub thaw($) { my($str) = @_; return undef if !defined $str; my($ty,@val) = split(/~/,$str,-1); for (@val) { s/%([0-9a-fA-F]{2})/pack('C',hex($1))/egs } if ($ty eq 'U') { undef } elsif ($ty eq '') { $val[0] } elsif ($ty eq 'S') { my($obj)=thaw($val[0]); \$obj } elsif ($ty eq 'R') { my($obj)=thaw($val[0]); \$obj } elsif ($ty eq 'A') { [map(thaw($_),@val)] } elsif ($ty eq 'H') { my($hr) = {}; while (@val) { my($k) = shift @val; $hr->{$k} = thaw(shift @val) } $hr; } else { die "Can't thaw object type $ty" } } # accepts either a single contents category (a string: "maj,min" or "maj"), # or a list of contents categories, in which case only the first element # is considered; returns a passed pair: (major_ccat, minor_ccat) # sub ccat_split($) { my($ccat) = @_; my($major,$minor); $ccat = $ccat->[0] if ref $ccat; # pick the first element if a list ($major,$minor) = split(/,/,$ccat,-1) if defined $ccat; !wantarray ? $major : ($major,$minor); } # accepts either a single contents category (a string: "maj,min" or "maj"), # or a list of contents categories, in which case only the first element # is considered; returns major_ccat # sub ccat_maj($) { my($ccat) = @_; my($major,$minor); $ccat = $ccat->[0] if ref $ccat; # pick the first element if a list ($major,$minor) = split(/,/,$ccat,-1) if defined $ccat; $major; } # compare numerically two strings of the form "maj,min" or just "maj", where # maj and min are numbers, representing major and minor contents categery # sub cmp_ccat($$) { my($a_maj,$a_min) = split(/,/, shift, -1); my($b_maj,$b_min) = split(/,/, shift, -1); $a_maj == $b_maj ? $a_min <=> $b_min : $a_maj <=> $b_maj; } # similar to cmp_ccat, but consider only the major category of both arguments # sub cmp_ccat_maj($$) { my($a_maj,$a_min) = split(/,/, shift, -1); my($b_maj,$b_min) = split(/,/, shift, -1); $a_maj <=> $b_maj; } # get a list of settings corresponding to all listed contents categories, # ordered from the most important category to the least; @ccat is a list of # relevant contents categories for which a query is made, it MUST already be # sorted in descending order; this is a classical subroutine, not a method! # sub setting_by_given_contents_category_all($@) { my($ccat,@settings_href_list) = @_; my(@r); if (!@settings_href_list) {} # no settings provided else { for my $e ((!defined($ccat) ?() :ref($ccat) ?@$ccat :$ccat), CC_CATCHALL) { if (grep(defined($_) && exists($_->{$e}), @settings_href_list)) { # supports lazy evaluation (setting may be a subroutine) my(@slist) = map { !defined($_) || !exists($_->{$e}) ? undef : do {my($s)=$_->{$e}; ref($s) eq 'CODE' ? &$s : $s} } @settings_href_list; push(@r, [$e,@slist]); # a tuple: [corresponding ccat, settings list] } } } @r; # a list of tuples } # similar to setting_by_given_contents_category_all(), but only the first # (the most relevant) setting is returned, without a corresponding ccat # sub setting_by_given_contents_category($@) { my($ccat,@settings_href_list) = @_; my(@slist); if (!@settings_href_list) {} # no settings provided else { for my $e ((!defined($ccat) ?() :ref($ccat) ?@$ccat :$ccat), CC_CATCHALL) { if (grep(defined($_) && exists($_->{$e}), @settings_href_list)) { # supports lazy evaluation (setting may be a subroutine) @slist = map { !defined($_) || !exists($_->{$e}) ? undef : do {my($s)=$_->{$e}; ref($s) eq 'CODE' ? &$s : $s} } @settings_href_list; last; } } } !wantarray ? $slist[0] : @slist; # only the first entry } # Removes a directory, along with its contents # # readdir is entitled to fail if the directory changes underneath, so do # the deletions by chunks: read a limited set of filenames into memory, # close directory, delete these files, and repeat # sub rmdir_recursively($;$); # prototype sub rmdir_recursively($;$) { my($dir, $exclude_itself) = @_; do_log(4,'rmdir_recursively: %s, excl=%s', $dir,$exclude_itself); my($f, @rmfiles, @rmdirs); my($more) = 1; my($dir_chmoded) = 0; while ($more) { local(*DIR); $more = 0; my($errn) = opendir(DIR,$dir) ? 0 : 0+$!; if ($errn == EACCES && !$dir_chmoded) { # relax protection on directory, then try again do_log(3,'rmdir_recursively: enabling read access to directory %s',$dir); chmod(0750,$dir) or do_log(-1, "Can't change protection-1 on dir %s: %s", $dir, $!); $dir_chmoded = 1; $errn = opendir(DIR,$dir) ? 0 : 0+$!; # try again } if ($errn) { die "Can't open directory $dir: $!" } my($cnt) = 0; # avoid slurping the whole directory contents into memory while (defined($f = readdir(DIR))) { next if $f eq '.' || $f eq '..'; my($fname) = $dir . '/' . $f; $errn = lstat($fname) ? 0 : 0+$!; if ($errn == EACCES && !$dir_chmoded) { # relax protection on the directory and retry do_log(3,'rmdir_recursively: enabling access to files in dir %s',$dir); chmod(0750,$dir) or do_log(-1, "Can't change protection-2 on dir %s: %s", $dir, $!); $dir_chmoded = 1; $errn = lstat($fname) ? 0 : 0+$!; # try again } if ($errn) { do_log(-1, "Can't access file \"%s\": $!", $fname,$!) } if (-d _) { push(@rmdirs,$f) } else { push(@rmfiles,$f) } $cnt++; if ($cnt >= 1000) { do_log(3,'rmdir_recursively: doing %d files and %d dirs for now in %s', scalar(@rmfiles), scalar(@rmdirs), $dir); $more = 1; last; } } closedir(DIR) or die "Error closing directory $dir: $!"; my($cntf) = scalar(@rmfiles); for my $f (@rmfiles) { my($fname) = $dir . '/' . untaint($f); if (unlink($fname)) { # ok } elsif ($! == EACCES && !$dir_chmoded) { # relax protection on the directory, then try again do_log(3,'rmdir_recursively: enabling write access to dir %s',$dir); my($what) = -l _ ? 'symlink' :-d _ ? 'directory' :'non-regular file'; chmod(0750,$dir) or do_log(-1, "Can't change protection-3 on dir %s: %s", $dir, $!); $dir_chmoded = 1; unlink($fname) or die "Can't remove $what $fname: $!"; } } undef @rmfiles; section_time("unlink-$cntf-files") if $cntf > 0; for my $d (@rmdirs) { rmdir_recursively($dir.'/'.untaint($d)); } undef @rmdirs; } if (!$exclude_itself) { rmdir($dir) or die "rmdir_recursively: Can't remove directory $dir: $!"; section_time('rmdir'); } 1; } # read a multiline string from a file - may be called from amavisd.conf # sub read_text($;$) { my($filename, $encoding) = @_; my($inp) = IO::File->new; $inp->open($filename,'<') or die "Can't open file $filename for reading: $!"; if (defined($encoding) && $encoding ne '') { binmode($inp, ":encoding($encoding)") or die "Can't set :encoding($encoding) on file $filename: $!"; } my($str) = ''; # must not be undef, work around a Perl utf8 bug my($nbytes,$buff); while (($nbytes=$inp->read($buff,16384)) > 0) { $str .= $buff } defined $nbytes or die "Error reading from $filename: $!"; $inp->close or die "Error closing $filename: $!"; undef $buff; # release storage $str; } # attempt to read all user-visible replies from a l10n dir # This function auto-fills $notify_sender_templ, $notify_virus_sender_templ, # $notify_virus_admin_templ, $notify_virus_recips_templ, # $notify_spam_sender_templ and $notify_spam_admin_templ from files named # template-dsn.txt, template-virus-sender.txt, template-virus-admin.txt, # template-virus-recipient.txt, template-spam-sender.txt, # template-spam-admin.txt. If this is available, it uses the charset # file to do automatic charset conversion. Used by the Debian distribution. # sub read_l10n_templates($;$) { my($dir) = @_; if (@_ > 1) # compatibility with Debian { my($l10nlang, $l10nbase) = @_; $dir = "$l10nbase/$l10nlang" } my($file_chset) = Amavis::Util::read_text("$dir/charset"); local($1,$2); if ($file_chset =~ m{^(?:\#[^\n]*\n)*([^./\n\s]+)(\s*[\#\n].*)?$}s) { $file_chset = untaint($1); } else { die "Invalid charset $file_chset\n"; } $Amavis::Conf::notify_sender_templ = Amavis::Util::read_text("$dir/template-dsn.txt", $file_chset); $Amavis::Conf::notify_virus_sender_templ = Amavis::Util::read_text("$dir/template-virus-sender.txt", $file_chset); $Amavis::Conf::notify_virus_admin_templ = Amavis::Util::read_text("$dir/template-virus-admin.txt", $file_chset); $Amavis::Conf::notify_virus_recips_templ = Amavis::Util::read_text("$dir/template-virus-recipient.txt", $file_chset); $Amavis::Conf::notify_spam_sender_templ = Amavis::Util::read_text("$dir/template-spam-sender.txt", $file_chset); $Amavis::Conf::notify_spam_admin_templ = Amavis::Util::read_text("$dir/template-spam-admin.txt", $file_chset); } #use CDB_File; #sub tie_hash($$) { # my($hashref, $filename) = @_; # CDB_File::create(%$hashref, $filename, "$filename.tmp$$") # or die "Can't create cdb $filename: $!"; # my($cdb) = tie(%$hashref,'CDB_File',$filename) # or die "Tie to $filename failed: $!"; # $hashref; #} # read an associative array (=Perl hash) (as used in lookups) from a file; # may be called from amavisd.conf # # Format: one key per line, anything from '#' to the end of line # is considered a comment, but '#' within correctly quoted RFC 5321 # addresses is not treated as a comment introducer (e.g. a hash sign # within "strange # \"foo\" address"@example.com is part of the string). # Lines may contain a pair: key value, separated by whitespace, # or key only, in which case a value 1 is implied. Trailing whitespace # is discarded (iff $trim_trailing_space_in_lookup_result_fields), # empty lines (containing only whitespace or comment) are ignored. # Addresses (lefthand-side) are converted from RFC 5321 -quoted form # into internal (raw) form and inserted as keys into a given hash. # NOTE: the format is partly compatible with Postfix maps (not aliases): # no continuation lines are honoured, Postfix maps do not allow # RFC 5321 -quoted addresses containing whitespace, Postfix only allows # comments starting at the beginning of a line. # # The $hashref argument is returned for convenience, so that one can do # for example: # $per_recip_whitelist_sender_lookup_tables = { # '.my1.example.com' => read_hash({},'/var/amavis/my1-example-com.wl'), # '.my2.example.com' => read_hash({},'/var/amavis/my2-example-com.wl') } # or even simpler: # $per_recip_whitelist_sender_lookup_tables = { # '.my1.example.com' => read_hash('/var/amavis/my1-example-com.wl'), # '.my2.example.com' => read_hash('/var/amavis/my2-example-com.wl') } # sub read_hash(@) { unshift(@_,{}) if !ref $_[0]; # first argument is optional, defaults to {} my($hashref, $filename, $keep_case) = @_; my($lpcs) = c('localpart_is_case_sensitive'); my($inp) = IO::File->new; $inp->open($filename,'<') or die "Can't open file $filename for reading: $!"; my($ln); for ($! = 0; defined($ln=$inp->getline); $! = 0) { chomp($ln); # carefully handle comments, '#' within "" does not count as a comment my($lhs) = ''; my($rhs) = ''; my($at_rhs) = 0; my($trailing_comment) = 0; for my $t ( $ln =~ /\G ( " (?: \\. | [^"\\] )* " | [^#" \t]+ | [ \t]+ | . )/gsx) { if ($t eq '#') { $trailing_comment = 1; last } if (!$at_rhs && $t =~ /^[ \t]+\z/) { $at_rhs = 1 } else { ($at_rhs ? $rhs : $lhs) .= $t } } $rhs =~ s/[ \t]+\z// if $trailing_comment || $trim_trailing_space_in_lookup_result_fields; next if $lhs eq '' && $rhs eq ''; my($source_route,$localpart,$domain) = Amavis::rfc2821_2822_Tools::parse_quoted_rfc2821($lhs,1); $localpart = lc($localpart) if !$lpcs; my($addr) = $localpart . lc($domain); $hashref->{$addr} = $rhs eq '' ? 1 : $rhs; # do_log(5, 'read_hash: address: <%s>: %s', $addr, $hashref->{$addr}); } defined $ln || $!==0 or # returning EBADF at EOF is a perl bug $!==EBADF ? do_log(0,'Error reading from %s: %s', $filename,$!) : die "Error reading from $filename: $!"; $inp->close or die "Error closing $filename: $!"; $hashref; } sub read_array(@) { unshift(@_,[]) if !ref $_[0]; # first argument is optional, defaults to [] my($arrref, $filename, $keep_case) = @_; my($inp) = IO::File->new; $inp->open($filename,'<') or die "Can't open file $filename for reading: $!"; my($ln); for ($! = 0; defined($ln=$inp->getline); $! = 0) { chomp($ln); my($lhs) = ''; # carefully handle comments, '#' within "" does not count as a comment for my $t ( $ln =~ /\G ( " (?: \\. | [^"\\] )* " | [^#" \t]+ | [ \t]+ | . )/gsx) { last if $t eq '#'; $lhs .= $t; } $lhs =~ s/[ \t]+\z//; # trim trailing whitespace push(@$arrref, Amavis::rfc2821_2822_Tools::unquote_rfc2821_local($lhs)) if $lhs ne ''; } defined $ln || $!==0 or # returning EBADF at EOF is a perl bug $!==EBADF ? do_log(0,'Error reading from %s: %s', $filename,$!) : die "Error reading from $filename: $!"; $inp->close or die "Error closing $filename: $!"; $arrref; } sub dump_hash($) { my($hr) = @_; do_log(0, 'dump_hash: %s => %s', $_, $hr->{$_}) for (sort keys %$hr); } sub dump_array($) { my($ar) = @_; do_log(0, 'dump_array: %s', $_) for @$ar; } # (deprecated, only still used with Amavis::OS_Fingerprint) sub dynamic_destination($$) { my($method,$conn) = @_; if ($method =~ /^(?:[a-z][a-z0-9.+-]*)?:/si) { my(@list); $list[0] = ''; my($j) = 0; for ($method =~ /\G \[ (?: \\. | [^\]\\] )* \] | " (?: \\. | [^"\\] )* " | : | [ \t]+ | [^:"\[ \t]+ | . /gsx) { # real parsing if ($_ eq ':') { $list[++$j] = '' } else { $list[$j] .= $_ } }; if ($list[1] =~ m{^/}) { # presumably the second field is a Unix socket name, keep unchanged } else { my($new_method); my($proto,$relayhost,$relayport) = @list; if ($relayhost eq '*') { my($client_ip); $client_ip = $conn->client_ip if defined $conn; $relayhost = "[$client_ip]" if defined $client_ip && $client_ip ne ''; } if ($relayport eq '*') { my($socket_port); $socket_port = $conn->socket_port if defined $conn; $relayport = $socket_port + 1 if defined $socket_port && $socket_port ne ''; } if ($relayhost eq '*' || $relayport eq '*') { do_log(0,'dynamic destination expected, no client addr/port info: %s', $method); } $list[1] = $relayhost; $list[2] = $relayport; $new_method = join(':',@list); if ($new_method ne $method) { do_log(3, 'dynamic destination: %s -> %s', $method,$new_method); $method = $new_method; } } } $method; } # collect unfinished recipients matching a $filter sub and a delivery # method regexp; assumes all list elements of a delivery_method list # use the same protocol name, hence only the first one is inspected # sub collect_equal_delivery_recips($$$) { my($msginfo, $filter, $deliv_meth_regexp) = @_; my(@per_recip_data_subset, $proto_sockname); my(@per_recip_data) = grep(!$_->recip_done && (!$filter || &$filter($_)) && grep(/$deliv_meth_regexp/, (ref $_->delivery_method ? $_->delivery_method->[0] : $_->delivery_method)), @{$msginfo->per_recip_data}); if (@per_recip_data) { # take the first remaining recipient as a model $proto_sockname = $per_recip_data[0]->delivery_method; defined $proto_sockname or die "undefined recipient's delivery_method"; my($proto_sockname_key) = !ref $proto_sockname ? $proto_sockname : join("\n", @$proto_sockname); # collect recipients with the same delivery method as the first one $per_recip_data_subset[0] = shift(@per_recip_data); # always equals self push(@per_recip_data_subset, grep((ref $_->delivery_method ? join("\n", @{$_->delivery_method}) : $_->delivery_method) eq $proto_sockname_key, @per_recip_data) ); } # return a ref to a filtered list of still-to-be-delivered recipient objects # and a single string or a ref to a list of delivery methods common to # these recipients (\@per_recip_data_subset, $proto_sockname); } 1; # package Amavis::ProcControl; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&exit_status_str &proc_status_ok &kill_proc &cloexec &run_command &run_command_consumer &run_as_subprocess &collect_results &collect_results_structured); import Amavis::Conf qw(:platform c cr ca); import Amavis::Util qw(ll do_log prolong_timer untaint); # freeze thaw import Amavis::Log qw(open_log close_log log_fd); } use subs @EXPORT_OK; use POSIX qw(:sys_wait_h WIFEXITED WIFSIGNALED WIFSTOPPED WEXITSTATUS WTERMSIG WSTOPSIG); use Errno qw(ENOENT EACCES EAGAIN ESRCH); use IO::File (); use Time::HiRes (); # use Fcntl qw(F_GETFD F_SETFD FD_CLOEXEC); # used in cloexec, if enabled # map process termination status number to an informative string, and # append optional mesage (dual-valued errno or a string or a number), # returning the resulting string # sub exit_status_str($;$) { my($stat,$errno) = @_; my($str); if (!defined($stat)) { $str = '(no status)'; } elsif (WIFEXITED($stat)) { $str = sprintf('exit %d', WEXITSTATUS($stat)); } elsif (WIFSTOPPED($stat)) { $str = sprintf('stopped, signal %d', WSTOPSIG($stat)); } else { my($sig) = WTERMSIG($stat); $str = sprintf('%s, signal %d (%04x)', $sig == 1 ? 'HANGUP' : $sig == 2 ? 'INTERRUPTED' : $sig == 6 ? 'ABORTED' : $sig == 9 ? 'KILLED' : $sig == 15 ? 'TERMINATED' : 'DIED', $sig, $stat); } if (defined $errno) { # deal with dual-valued and plain variables $str .= ', '.$errno if (0+$errno) != 0 || ($errno ne '' && $errno ne '0'); } $str; } # check errno to be 0 and a process exit status to be in the list of success # status codes, returning true if both are ok, and false otherwise # sub proc_status_ok($;$@) { my($exit_status,$errno,@success) = @_; my($ok) = 0; if ((!defined $errno || $errno == 0) && WIFEXITED($exit_status)) { my($j) = WEXITSTATUS($exit_status); if (!@success) { $ok = $j==0 } # empty list implies only status 0 is good elsif (grep($_==$j, @success)) { $ok = 1 } } $ok; } # kill a process, typically a runaway external decoder or checker # sub kill_proc($;$$$$) { my($pid,$what,$timeout,$proc_fh,$reason) = @_; $pid >= 0 or die "Shouldn't be killing process groups: [$pid]"; $pid != 1 or die "Shouldn't be killing process 'init': [$pid]"; $what = defined $what ? " running $what" : ''; $reason = defined $reason ? " (reason: $reason)" : ''; # # the following order is a must: SIGTERM first, _then_ close a pipe; # otherwise the following can happen: closing a pipe first (explicitly or # implicitly by undefining $proc_fh) blocks us so we never send SIGTERM # until the external process dies of natural death; on the other hand, # not closing the pipe after SIGTERM does not necessarily let the process # notice SIGTERM, so SIGKILL is always needed to stop it, which is not nice # my($n) = kill(0,$pid); # does the process really exist? if ($n == 0 && $! != ESRCH) { die sprintf("Can't send SIG 0 to process [%s]%s: %s", $pid,$what,$!); } elsif ($n == 0) { do_log(2, 'no need to kill process [%s]%s, already gone', $pid,$what); } else { do_log(-1,'killing process [%s]%s%s', $pid,$what,$reason); kill('TERM',$pid) or $! == ESRCH # be gentle on the first attempt or die sprintf("Can't send SIGTERM to process [%s]%s: %s",$pid,$what,$!); } # close the pipe if still open, ignoring status $proc_fh->close if defined $proc_fh; my($child_stat) = defined $pid && waitpid($pid,WNOHANG) > 0 ? $? : undef; $n = kill(0,$pid); # is the process still there? if ($n > 0 && defined($timeout) && $timeout > 0) { sleep($timeout); $n = kill(0,$pid); # wait a little and recheck } if ($n == 0 && $! != ESRCH) { die sprintf("Can't send SIG 0 to process [%s]%s: %s", $pid,$what,$!); } elsif ($n > 0) { # the process is still there, try a stronger signal do_log(-1,'process [%s]%s is still alive, using a bigger hammer', $pid,$what); kill('KILL',$pid) or $! == ESRCH or die sprintf("Can't send SIGKILL to process [%s]%s: %s",$pid,$what,$!); } } sub cloexec($;$$) { undef } # sub cloexec($;$$) { # supposedly not needed for Perl >= 5.6.0 # my($fh,$newsetting,$name) = @_; my($flags); # $flags = fcntl($fh, F_GETFD, 0) # or die "Can't get close-on-exec flag for file handle $fh $name: $!"; # $flags = 0 + $flags; # turn into numeric, avoid: "0 but true" # if (defined $newsetting) { # change requested? # my($newflags) = $newsetting ? ($flags|FD_CLOEXEC) : ($flags&~FD_CLOEXEC); # if ($flags != $newflags) { # do_log(4,"cloexec: turning %s flag FD_CLOEXEC for file handle %s %s", # $newsetting ? "ON" : "OFF", $fh, $name); # fcntl($fh, F_SETFD, $newflags) # or die "Can't set FD_CLOEXEC for file handle $fh $name: $!"; # } # } # ($flags & FD_CLOEXEC) ? 1 : 0; # returns old setting # } # POSIX::open a file or dup an existing fd (Perl open syntax), with a # requirement that it gets opened on a prescribed file descriptor $fd_target. # Returns a file descriptor number (not a Perl file handle, there is no # associated file handle). Usually called from a forked process prior to exec. # sub open_on_specific_fd($$$$) { my($fd_target,$fname,$flags,$mode) = @_; my($fd_got); # fd directly given as argument, or obtained from POSIX::open my($logging_safe) = 0; if (ll(5)) { # crude attempt to prevent a forked process from writing log records # to its parent process on STDOUT or STDERR my($log_fd) = log_fd(); $logging_safe = 1 if !defined($log_fd) || $log_fd > 2; } local($1); if ($fname =~ /^&=?(\d+)\z/) { $fd_got = $1 } # fd directly specified my($flags_displayed) = $flags == &POSIX::O_RDONLY ? '<' : $flags == &POSIX::O_WRONLY ? '>' : $flags; if (!defined($fd_got) || $fd_got != $fd_target) { # close whatever is on a target descriptor but don't shoot self in the foot # with Net::Server <= 0.90 fd0 was main::stdin, but no longer is in 0.91 do_log(5, "open_on_specific_fd: target fd%s closing, to become %s %s", $fd_target,$flags_displayed,$fname) if $logging_safe; # it pays off to close explicitly, with some luck open will get a target fd POSIX::close($fd_target); # ignore error; we may have just closed a log } if (!defined($fd_got)) { # a file name was given, not a descriptor $fd_got = POSIX::open($fname,$flags,$mode); defined $fd_got or die "Can't open $fname: $!"; $fd_got = 0 + $fd_got; # turn into numeric, avoid: "0 but true" } if ($fd_got != $fd_target) { # dup, ensuring we get a requested descriptor eval { # we may have been left without a log file descriptor, must not die do_log(5, "open_on_specific_fd: target fd%s dup2 from fd%s %s %s", $fd_target,$fd_got,$flags_displayed,$fname) if $logging_safe; }; # POSIX mandates we got the lowest fd available (but some kernels have # bugs), let's be explicit that we require a specified file descriptor defined POSIX::dup2($fd_got,$fd_target) or die "Can't dup2 from $fd_got to $fd_target: $!"; if ($fd_got > 2) { # let's get rid of the original fd, unless 0,1,2 my($err); defined POSIX::close($fd_got) or $err = $!; $err = defined $err ? ": $err" : ''; eval { # we may have been left without a log file descriptor, don't die do_log(5, "open_on_specific_fd: source fd%s closed%s", $fd_got,$err) if $logging_safe; }; } } $fd_got; } sub release_parent_resources() { $Amavis::sql_dataset_conn_lookups->dbh_inactive(1) if $Amavis::sql_dataset_conn_lookups; $Amavis::sql_dataset_conn_storage->dbh_inactive(1) if $Amavis::sql_dataset_conn_storage; # undef $Amavis::sql_dataset_conn_lookups; # undef $Amavis::sql_dataset_conn_storage; # undef $Amavis::snmp_db; # undef $Amavis::db_env; } # Run specified command as a subprocess (like qx operator, but more careful # with error reporting and cancels :utf8 mode). If $stderr_to is undef or # an empty string it is converted to '&1', merging stderr to stdout on fd1. # Return a file handle open for reading from the subprocess. # sub run_command($$@) { my($stdin_from, $stderr_to, $cmd, @args) = @_; my($cmd_text) = join(' ', $cmd, @args); $stdin_from = '/dev/null' if !defined $stdin_from || $stdin_from eq ''; $stderr_to = '&1' if !defined $stderr_to || $stderr_to eq ''; # to stdout my($msg) = join(' ', $cmd, @args, "<$stdin_from", "2>$stderr_to"); # $^F == 2 or do_log(-1,"run_command: SYSTEM_FD_MAX not 2: %d", $^F); my($proc_fh) = IO::File->new; # parent reading side of the pipe my($child_out_fh) = IO::File->new; # child writing side of the pipe pipe($proc_fh,$child_out_fh) or die "run_command: Can't create a pipe: $!"; my($pid); eval { # Avoid using open('-|') which is just too damn smart: possibly waiting # indefinitely when resources are tight, and not catching fork errors as # expected but just bailing out of eval; make a pipe explicitly and fork. # Note that fork(2) returns ENOMEM on lack of swap space, and EAGAIN when # process limit is reached; we want it to fail in both cases and not obey # the EAGAIN and keep retrying, as perl open() does. $pid = fork(); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die "run_command (forking): $eval_stat"; }; defined($pid) or die "run_command: can't fork: $!"; if (!$pid) { # child alarm(0); my($interrupt) = ''; my($h1) = sub { $interrupt = $_[0] }; my($h2) = sub { die "Received signal ".$_[0] }; @SIG{qw(INT HUP TERM TSTP QUIT USR1 USR2)} = ($h1) x 7; eval { # die must be caught, otherwise we end up with two running daemons local(@SIG{qw(INT HUP TERM TSTP QUIT USR1 USR2)}) = ($h2) x 7; if ($interrupt ne '') { my($i) = $interrupt; $interrupt = ''; die $i } # use Devel::Symdump (); # my($dumpobj) = Devel::Symdump->rnew; # for my $k ($dumpobj->ios) { # no strict 'refs'; my($fn) = fileno($k); # if (!defined($fn)) { do_log(2, "not open %s", $k) } # elsif ($fn == 1 || $fn == 2) { do_log(2, "KEEP %s, fileno=%s",$k,$fn) } # else { $! = 0; # close(*{$k}{IO}) and do_log(2, "DID CLOSE %s (fileno=%s)", $k,$fn); # } # } $proc_fh->close or die "Child can't close parent side of a pipe: $!"; release_parent_resources(); # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] my($opt_rdonly) = untaint(&POSIX::O_RDONLY); my($opt_wronly) = untaint(&POSIX::O_WRONLY); open_on_specific_fd(0, $stdin_from, $opt_rdonly, 0); open_on_specific_fd(1, '&='.fileno($child_out_fh), $opt_wronly, 0); open_on_specific_fd(2, $stderr_to, $opt_wronly, 0); # eval { close_log() }; # may have been closed by open_on_specific_fd # BEWARE of Perl older that 5.6.0: sockets and pipes were not FD_CLOEXEC exec {$cmd} ($cmd,@args); die "run_command: failed to exec $cmd_text: $!"; }; my($err) = $@ ne '' ? $@ : "errno=$!"; chomp $err; eval { local(@SIG{qw(INT HUP TERM TSTP QUIT USR1 USR2)}) = ($h2) x 7; if ($interrupt ne '') { my($i) = $interrupt; $interrupt = ''; die $i } open_log(); # oops, exec failed, we will need logging after all... # we're in trouble if stderr was attached to a terminal, but no longer is do_log(-1,"run_command: child process [%s]: %s", $$,$err); }; { no warnings; POSIX::_exit(6); # avoid END and destructor processing kill('KILL',$$); exit 1; # still kicking? die! } } # parent ll(5) && do_log(5,"run_command: [%s] %s", $pid,$msg); $child_out_fh->close or die "Parent failed to close child side of the pipe: $!"; binmode($proc_fh) or die "Can't set pipe to binmode: $!"; # dflt Perl 5.8.1 ($proc_fh, $pid); # return pipe file handle to the subprocess and its PID } # Run a specified command as a subprocess. Return a file handle open for # WRITING to the subprocess, utf8 mode canceled and autoflush turned OFF ! # If $stderr_to is undef or is an empty string it is converted to '&1', # merging stderr to stdout on fd1. # sub run_command_consumer($$@) { my($stdout_to, $stderr_to, $cmd, @args) = @_; my($cmd_text) = join(' ', $cmd, @args); $stdout_to = '/dev/null' if !defined $stdout_to || $stdout_to eq ''; $stderr_to = '&1' if !defined $stderr_to || $stderr_to eq ''; # to stdout my($msg) = join(' ', $cmd, @args, ">$stdout_to", "2>$stderr_to"); # $^F == 2 or do_log(-1,"run_command_consumer: SYSTEM_FD_MAX not 2: %d", $^F); my($proc_fh) = IO::File->new; # parent writing side of the pipe my($child_in_fh) = IO::File->new; # child reading side of the pipe pipe($child_in_fh,$proc_fh) or die "run_command_consumer: Can't create a pipe: $!"; my($pid); eval { # Avoid using open('|-') which is just too damn smart: possibly waiting # indefinitely when resources are tight, and not catching fork errors as # expected but just bailing out of eval; make a pipe explicitly and fork. # Note that fork(2) returns ENOMEM on lack of swap space, and EAGAIN when # process limit is reached; we want it to fail in both cases and not obey # the EAGAIN and keep retrying, as perl open() does. $pid = fork(); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die "run_command_consumer (fork): $eval_stat"; }; defined($pid) or die "run_command_consumer: can't fork: $!"; if (!$pid) { # child alarm(0); my($interrupt) = ''; my($h1) = sub { $interrupt = $_[0] }; my($h2) = sub { die "Received signal ".$_[0] }; @SIG{qw(INT HUP TERM TSTP QUIT USR1 USR2)} = ($h1) x 7; eval { # die must be caught, otherwise we end up with two running daemons local(@SIG{qw(INT HUP TERM TSTP QUIT USR1 USR2)}) = ($h2) x 7; if ($interrupt ne '') { my($i) = $interrupt; $interrupt = ''; die $i } $proc_fh->close or die "Child can't close parent side of a pipe: $!"; release_parent_resources(); # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] my($opt_rdonly) = untaint(&POSIX::O_RDONLY); my($opt_wronly) = untaint(&POSIX::O_WRONLY); open_on_specific_fd(0, '&='.fileno($child_in_fh), $opt_rdonly, 0); open_on_specific_fd(1, $stdout_to, $opt_wronly, 0); open_on_specific_fd(2, $stderr_to, $opt_wronly, 0); # eval { close_log() }; # may have been closed by open_on_specific_fd # BEWARE of Perl older that 5.6.0: sockets and pipes were not FD_CLOEXEC exec {$cmd} ($cmd,@args); die "run_command_consumer: failed to exec $cmd_text: $!"; }; my($err) = $@ ne '' ? $@ : "errno=$!"; chomp $err; eval { local(@SIG{qw(INT HUP TERM TSTP QUIT USR1 USR2)}) = ($h2) x 7; if ($interrupt ne '') { my($i) = $interrupt; $interrupt = ''; die $i } open_log(); # oops, exec failed, we will need logging after all... # we're in trouble if stderr was attached to a terminal, but no longer is do_log(-1,"run_command_consumer: child process [%s]: %s", $$,$err); }; { no warnings; POSIX::_exit(6); # avoid END and destructor processing kill('KILL',$$); exit 1; # still kicking? die! } } # parent ll(5) && do_log(5,"run_command_consumer: [%s] %s", $pid,$msg); $child_in_fh->close or die "Parent failed to close child side of the pipe: $!"; binmode($proc_fh) or die "Can't set pipe to binmode: $!"; # dflt Perl 5.8.1 $proc_fh->autoflush(0); # turn it off here, must call ->flush when needed ($proc_fh, $pid); # return pipe file handle to the subprocess and its PID } # run a specified subroutine with given arguments as a (forked) subprocess, # collecting results (if any) over a pipe from a subprocess and propagating # them back to a caller; (useful to prevent a potential process crash from # bringing down the main process, and allows cleaner timeout aborts) # sub run_as_subprocess($@) { my($code,@args) = @_; alarm(0); # stop the timer my($proc_fh) = IO::File->new; # parent reading side of the pipe my($child_out_fh) = IO::File->new; # child writing side of the pipe pipe($proc_fh,$child_out_fh) or die "run_as_subprocess: Can't create a pipe: $!"; my($pid); eval { # Avoid using open('-|') which is just too damn smart: possibly waiting # indefinitely when resources are tight, and not catching fork errors as # expected but just bailing out of eval; make a pipe explicitly and fork. # Note that fork(2) returns ENOMEM on lack of swap space, and EAGAIN when # process limit is reached; we want it to fail in both cases and not obey # the EAGAIN and keep retrying, as perl open() does. $pid = fork(); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die "run_as_subprocess (forking): $eval_stat"; }; defined($pid) or die "run_as_subprocess: can't fork: $!"; if (!$pid) { # child # timeouts will be also be handled by a parent process my($t0) = Time::HiRes::time; my(@result); my($interrupt) = ''; my($h1) = sub { $interrupt = $_[0] }; my($h2) = sub { die "Received signal ".$_[0] }; @SIG{qw(INT HUP TERM TSTP QUIT USR1 USR2)} = ($h1) x 7; $SIG{PIPE} = 'IGNORE'; # don't signal on a write to a widowed pipe my($myownpid) = $$; # $$ is a syscall $0 = 'sub-' . c('myprogram_name'); # let it show in ps(1) my($eval_stat); eval { # die must be caught, otherwise we end up with two running daemons local(@SIG{qw(INT HUP TERM TSTP QUIT USR1 USR2)}) = ($h2) x 7; if ($interrupt ne '') { my($i) = $interrupt; $interrupt = ''; die $i } prolong_timer("child[$myownpid]"); # restart the timer $proc_fh->close or die "Child can't close parent side of a pipe: $!"; binmode($child_out_fh) or die "Can't set pipe to binmode: $!"; release_parent_resources(); # we don't really need STDOUT here, but just in case the supplied code # happens to write there, let's make STDOUT a dup of a pipe close STDOUT; # ignoring status # prefer dup(2) here instead of fdopen, with some luck this gives us fd1 open(STDOUT, '>&'.fileno($child_out_fh)) or die "Child can't dup pipe to STDOUT: $!"; binmode(STDOUT) or die "Can't set STDOUT to binmode: $!"; ll(5) && do_log(5,"[%s] run_as_subprocess: running as child, ". "stdin=%s, stdout=%s, pipe=%s", $myownpid, fileno(STDIN), fileno(STDOUT), fileno($child_out_fh)); @result = &$code(@args); # invoke the caller-specified subroutine 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; my($dt) = Time::HiRes::time - $t0; eval { # must not use die in forked process, or we end up with two daemons local(@SIG{qw(INT HUP TERM TSTP QUIT USR1 USR2)}) = ($h2) x 7; if ($interrupt ne '') { my($i) = $interrupt; $interrupt = ''; die $i } my($status); my($ll) = 3; if (defined $eval_stat) { # failure chomp $eval_stat; $ll = -2; $status = sprintf("STATUS: FAILURE %s", $eval_stat); } else { # success $status = sprintf("STATUS: SUCCESS, %d results", scalar(@result)); }; my($frozen) = Amavis::Util::freeze([$status,@result]); ll($ll) && do_log($ll, "[%s] run_as_subprocess: child done (%.1f ms), ". "sending results: res_len=%d, %s", $myownpid, $dt*1000, length($frozen), $status); # write results back to a parent process over a pipe as a frozen struct. # writing to broken pipe must return an error, not throw a signal local $SIG{PIPE} = sub { die "Broken pipe\n" }; # locale-independent err $child_out_fh->print($frozen) or die "Can't write result to pipe: $!"; $child_out_fh->close or die "Child can't close its side of a pipe: $!"; close STDOUT or die "Child can't close its STDOUT: $!"; POSIX::_exit(0); # normal completion, avoid END and destructor processing }; my($eval2_stat) = $@ ne '' ? $@ : "errno=$!"; eval { chomp $eval2_stat; if ($interrupt ne '') { my($i) = $interrupt; $interrupt = ''; die $i } # broken pipe is common when parent process is shutting down my($ll) = $eval2_stat =~ /^Broken pipe\b/ ? 1 : -1; do_log($ll,"run_as_subprocess: child process [%s]: %s", $myownpid, $eval2_stat); }; POSIX::_exit(6); # avoid END and destructor processing in a subprocess } # parent ll(5) && do_log(5,"run_as_subprocess: spawned a subprocess [%s]", $pid); $child_out_fh->close or die "Parent failed to close child side of the pipe: $!"; binmode($proc_fh) or die "Can't set pipe to binmode: $!"; # dflt Perl 5.8.1 prolong_timer('run_as_subprocess'); # restart the timer ($proc_fh, $pid); # return pipe file handle to the subprocess and its PID } # read results from a subprocess over a pipe, returns a ref to a results string # and a subprocess exit status; close the pipe and dismiss the subprocess, # by force if necessary; if $success_list_ref is defined, check also the # subprocess exit status against the provided list and log results # sub collect_results($$;$$$) { my($proc_fh,$pid, $what,$results_max_size,$success_list_ref) = @_; # $results_max_size is interpreted as follows: # undef .. no limit, read and return all data; # 0 ... no limit, read and discard all data, returns ref to empty string # >= 1 ... read all data, but truncate results string at limit my($child_stat); my($close_err) = 0; my($pid_orig) = $pid; my($result) = ''; my($result_l) = 0; my($skipping) = 0; my($eval_stat); eval { # read results; could be aborted by a read error or a timeout my($nbytes,$buff); while (($nbytes=$proc_fh->read($buff,16384)) > 0) { if (!defined($results_max_size)) { $result .= $buff } # keep all data elsif ($results_max_size == 0 || $skipping) {} # discard data elsif ($result_l < $results_max_size) { $result .= $buff } else { $skipping = 1; # sanity limit exceeded do_log(-1,'collect_results from [%s] (%s): results size limit '. '(%d bytes) exceeded', $pid_orig,$what,$results_max_size); } $result_l += $nbytes; } defined $nbytes or die "Error reading from a subprocess [$pid_orig]: $!"; ll(5) && do_log(5,'collect_results from [%s] (%s), %d bytes, (limit %s)', $pid_orig,$what,$result_l,$results_max_size); 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; if (defined($results_max_size) && $results_max_size > 0 && length($result) > $results_max_size) { $result = substr($result,0,$results_max_size) . "..."; } if (defined $eval_stat) { # read error or timeout; abort the subprocess chomp $eval_stat; undef $_[0]; # release the caller's copy of $proc_fh kill_proc($pid,$what,1,$proc_fh, "on reading: $eval_stat") if defined $pid; undef $proc_fh; undef $pid; die "collect_results - reading aborted: $eval_stat"; } # normal subprocess exit, close pipe, collect exit status $eval_stat = undef; eval { $proc_fh->close or $close_err = $!; $child_stat = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; undef $_[0]; # release also the caller's copy of $proc_fh 1; } or do { # just in case close itself timed out $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; undef $_[0]; # release the caller's copy of $proc_fh kill_proc($pid,$what,1,$proc_fh, "on closing: $eval_stat") if defined $pid; undef $proc_fh; undef $pid; die "collect_results - closing aborted: $eval_stat"; }; if (defined $success_list_ref) { proc_status_ok($child_stat,$close_err, @$success_list_ref) or do_log(-2, 'collect_results from [%s] (%s): %s %s', $pid_orig,$what, exit_status_str($child_stat,$close_err), $result); } elsif ($close_err != 0) { die "Can't close pipe to subprocess [$pid_orig]: $close_err"; } (\$result,$child_stat); } # read results from a subprocess over a pipe as a frozen data structure; # close the pipe and dismiss the subprocess; returns results as a ref to a list # sub collect_results_structured($$;$$) { my($proc_fh,$pid, $what,$results_max_size) = @_; my($result_ref,$child_stat) = collect_results($proc_fh,$pid, $what,$results_max_size,[0]); $result_ref = Amavis::Util::thaw($$result_ref); my(@result) = !ref($result_ref) ? () : @$result_ref; @result or die "collect_results_structured: no results from subprocess [$pid]"; my($status) = shift(@result); $status =~ /^STATUS: (?:SUCCESS|FAILURE)\b/ or die "collect_results_structured: subprocess [$pid] returned: $status"; (\@result,$child_stat); } 1; # package Amavis::rfc2821_2822_Tools; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT = qw( &rfc2822_timestamp &iso8601_timestamp &iso8601_utc_timestamp &iso8601_week &iso8601_yearweek &iso8601_year_and_week &format_time_interval &make_received_header_field &parse_received &fish_out_ip_from_received &parse_message_id &split_address &split_localpart &replace_addr_fields &make_query_keys "e_rfc2821_local &qquote_rfc2821_local &parse_quoted_rfc2821 &unquote_rfc2821_local &parse_address_list &wrap_string &wrap_smtp_resp &one_response_for_all &EX_OK &EX_NOUSER &EX_UNAVAILABLE &EX_TEMPFAIL &EX_NOPERM); import Amavis::Conf qw(:platform c cr ca $myproduct_name); import Amavis::Util qw(ll do_log unique_ref unique_list); } use subs @EXPORT; use POSIX qw(locale_h strftime); BEGIN { eval { require 'sysexits.ph' }; # try to use the installed version # define the most important constants if undefined do { sub EX_OK() {0} } unless defined(&EX_OK); do { sub EX_NOUSER() {67} } unless defined(&EX_NOUSER); do { sub EX_UNAVAILABLE() {69} } unless defined(&EX_UNAVAILABLE); do { sub EX_TEMPFAIL() {75} } unless defined(&EX_TEMPFAIL); do { sub EX_NOPERM() {77} } unless defined(&EX_NOPERM); } # Given a Unix time, return the local time zone offset at that time # as a string +HHMM or -HHMM, appropriate for the RFC 2822 date format. # Works also for non-full-hour zone offsets, and on systems where strftime # cannot return TZ offset as a number; (c) Mark Martinec, GPL # sub get_zone_offset($) { my($t) = int(shift); my($d) = 0; # local zone offset in seconds for (1..3) { # match the date (with a safety loop limit just in case) my($r) = sprintf("%04d%02d%02d", (localtime($t))[5, 4, 3]) cmp sprintf("%04d%02d%02d", (gmtime($t + $d))[5, 4, 3]); if ($r == 0) { last } else { $d += $r * 24 * 3600 } } my($sl,$su) = (0,0); for ((localtime($t))[2,1,0]) { $sl = $sl * 60 + $_ } for ((gmtime($t + $d))[2,1,0]) { $su = $su * 60 + $_ } $d += $sl - $su; # add HMS difference (in seconds) my($sign) = $d >= 0 ? '+' : '-'; $d = -$d if $d < 0; $d = int(($d + 30) / 60.0); # give minutes, rounded sprintf("%s%02d%02d", $sign, int($d / 60), $d % 60); } # Given a Unix time, provide date-time timestamp as specified in RFC 5322 # (local time), to be used in header fields such as 'Date:' and 'Received:' # See also RFC 3339. # sub rfc2822_timestamp($) { my($t) = @_; my(@lt) = localtime(int($t)); # can't use %z because some systems do not support it (is treated as %Z) # my($old_locale) = POSIX::setlocale(LC_TIME,"C"); # English dates required! my($zone_name) = strftime("%Z",@lt); my($s) = strftime("%a, %e %b %Y %H:%M:%S ", @lt); $s .= get_zone_offset($t); $s .= " (" . $zone_name . ")" if $zone_name !~ /^\s*\z/; # POSIX::setlocale(LC_TIME, $old_locale); # restore the locale $s; } # Given a Unix numeric time (seconds since 1970-01-01T00:00Z), # provide date-time timestamp (local time) as specified in ISO 8601 (EN 28601) # sub iso8601_timestamp($;$$$) { my($t,$suppress_zone,$dtseparator,$with_field_separators) = @_; # can't use %z because some systems do not support it (is treated as %Z) my($fmt) = $with_field_separators ? "%Y-%m-%dT%H:%M:%S" : "%Y%m%dT%H%M%S"; $fmt =~ s/T/$dtseparator/ if defined $dtseparator; my($s) = strftime($fmt,localtime(int($t))); $s .= get_zone_offset($t) unless $suppress_zone; $s; } # Given a Unix numeric time (seconds since 1970-01-01T00:00Z), # provide date-time timestamp (UTC) as specified in ISO 8601 (EN 28601) # sub iso8601_utc_timestamp($;$$$) { my($t,$suppress_zone,$dtseparator,$with_field_separators) = @_; my($fmt) = $with_field_separators ? "%Y-%m-%dT%H:%M:%S" : "%Y%m%dT%H%M%S"; $fmt =~ s/T/$dtseparator/ if defined $dtseparator; my($s) = strftime($fmt,gmtime(int($t))); $s .= 'Z' unless $suppress_zone; $s; } # Does the given year have 53 weeks? Using a formula by Simon Cassidy. # sub iso8601_year_is_long($) { my($y) = @_; my($p) = $y + int($y/4) - int($y/100) + int($y/400); if (($p % 7) == 4) { return 1 } $y--; $p = $y + int($y/4) - int($y/100) + int($y/400); if (($p % 7) == 3) { return 1 } else { return 0 } } # Given a Unix numeric time (seconds since 1970-01-01T00:00Z), # provide a week number 1..53 (local time) as specified in ISO 8601 (EN 28601) # ( equivalent to PostgreSQL extract(week from ...), and MySQL week(date,3) ) # sub iso8601_year_and_week($) { my($unix_time) = @_; my($y,$dowm0,$doy0) = (localtime($unix_time))[5,6,7]; $y += 1900; $dowm0--; $dowm0=6 if $dowm0<0; # normalize, Monday==0 my($dow0101) = ($dowm0 - $doy0 + 53*7) % 7; # dow Jan 1 my($wn) = int(($doy0 + $dow0101) / 7); if ($dow0101 < 4) { $wn++ } if ($wn == 0) { $y--; $wn = iso8601_year_is_long($y) ? 53 : 52 } elsif ($wn == 53 && !iso8601_year_is_long($y)) { $y++; $wn = 1 } ($y,$wn); } sub iso8601_week($) { my($y,$wn) = iso8601_year_and_week(shift); $wn; } sub iso8601_yearweek($) { my($y,$wn) = iso8601_year_and_week(shift); $y*100+$wn; } sub format_time_interval($) { my($t) = @_; return 'undefined' if !defined $t; my($sign) = ''; if ($t < 0) { $sign = '-'; $t = - $t }; my($dd) = int($t / (24*3600)); $t = $t - $dd*(24*3600); my($hh) = int($t / 3600); $t = $t - $hh*3600; my($mm) = int($t / 60); $t = $t - $mm*60; sprintf("%s%d %d:%02d:%02d", $sign,$dd,$hh,$mm,int($t+0.5)); } sub make_received_header_field($$) { my($msginfo, $folded) = @_; my($conn) = $msginfo->conn_obj; my($id) = $msginfo->mail_id; my($smtp_proto, $recips) = ($conn->appl_proto, $msginfo->recips); my($client_ip) = $conn->client_ip; if ($client_ip =~ /:.*:/ && $client_ip !~ /^IPv6:/i) { $client_ip = 'IPv6:' . $client_ip; # RFC 5321 (ex RFC 2821), section 4.1.3 } my($tls) = $msginfo->tls_cipher; my($s) = sprintf("from %s%s%s\n by %s%s (%s, %s)", $conn->smtp_helo eq '' ? 'unknown' : $conn->smtp_helo, $client_ip eq '' ? '' : " ([$client_ip])", !defined $tls ? '' : " (using TLS with cipher $tls)", c('localhost_name'), $conn->socket_ip eq '' ? '' : sprintf(" (%s [%s])", c('myhostname'), $conn->socket_ip), $myproduct_name, $conn->socket_port eq '' ? 'unix socket' : "port ".$conn->socket_port); $s .= "\n with $smtp_proto" if $smtp_proto=~/^(ES|S|L)MTPS?A?\z/i; #RFC 3848 $s .= "\n id $id" if defined $id && $id ne ''; # do not disclose recipients if more than one $s .= "\n for " . qquote_rfc2821_local(@$recips) if @$recips == 1; $s .= ";\n " . rfc2822_timestamp($msginfo->rx_time); $s =~ s/\n//g if !$folded; $s; } # parse Received header field according to RFC 5321, somewhat loosened syntax # Stamp = From-domain By-domain [Via] [With] [ID] [For] datetime # From-domain = "FROM" FWS Extended-Domain CFWS # By-domain = "BY" FWS Extended-Domain CFWS # Via = "VIA" FWS ("TCP" / Atom) CFWS # With = "WITH" FWS ("ESMTP" / "SMTP" / Atom) CFWS # ID = "ID" FWS (Atom / DQUOTE *qcontent DQUOTE / msg-id) CFWS # For = "FOR" FWS 1*( Path / Mailbox ) CFWS # Path = "<" [ A-d-l ":" ] Mailbox ">" # datetime = ";" FWS [ day-of-week "," ] date FWS time [CFWS] # Extended-Domain = # (Domain / Address-literal) [ FWS "(" [ Domain FWS ] Address-literal ")" ] # Avoid regexps like ( \\. | [^"\\] )* which cause recursion trouble / crashes! # sub parse_received($) { local($_) = $_[0]; my(%fld); local($1); tr/\n//d; # unfold, chomp my($comm_lvl) = 0; my($in_option) = ''; my($in_ext_dom) = 0; my($in_tcp_info) = 0; my($in_qcontent) = 0; my($in_literal) = 0; my($in_angle) = 0; my($str_l) = length($_); my($new_pos); for (my $pos=-1; $new_pos=pos($_), $new_pos<$str_l; $pos=$new_pos) { $new_pos > $pos or die "parse_received PANIC1 $new_pos"; # just in case # comment (may be nested: RFC 5322 section 3.2.2) if ($comm_lvl > 0 && /\G( \) )/gcsx) { if ($comm_lvl > 1 || $in_tcp_info) { $fld{$in_option} .= $1 } # nested if ($comm_lvl == 1 && !$in_tcp_info) { $in_option =~ s/-com\z// } $comm_lvl--; next; # pop up one level of comments } if ($in_tcp_info && /\G( \) )/gcsx) # leaving TCP-info { $in_option =~ s/-tcp\z//; $in_tcp_info = 0; $in_ext_dom = 4; next } if (!$in_qcontent && !$in_literal && !$comm_lvl && !$in_tcp_info && $in_ext_dom==1 && /\G( \( )/gcsx) { # entering TCP-info part, only once after 'from' or 'by' $in_option .= '-tcp'; $in_tcp_info = 1; $in_ext_dom = 2; next; } if (!$in_qcontent && !$in_literal && /\G( \( )/gcsx) { $comm_lvl++; # push one level of comments if ($comm_lvl > 1 || $in_tcp_info) { $fld{$in_option} .= $1 } # nested if ($comm_lvl == 1 && !$in_tcp_info) { # comment starts here $in_option .= '-com'; $fld{$in_option} .= ' ' if defined $fld{$in_option}; # looks better } next; } if ($comm_lvl > 0 && /\G( \\. )/gcsx) { $fld{$in_option} .= $1; next } if ($comm_lvl > 0 && /\G( [^()\\]+ )/gcsx) { $fld{$in_option} .= $1; next } # quoted content if ($in_qcontent && /\G( " )/gcsx) # normal exit from qcontent { $in_qcontent = 0; $fld{$in_option} .= $1; next } if ($in_qcontent && /\G( > )/gcsx) # bail out of qcontent { $in_qcontent = 0; $in_angle = 0; $fld{$in_option} .= $1; next } if ($in_qcontent && /\G( \\. )/gcsx) { $fld{$in_option} .= $1; next } if ($in_qcontent && /\G( [^"\\>]+ )/gcsx) { $fld{$in_option} .= $1; next } # address literal if ($in_literal && /\G( \] )/gcsx) { $in_literal = 0; $fld{$in_option} .= $1; next } if ($in_literal && /\G( > )/gcsx) # bail out of address literal { $in_literal = 0; $in_angle = 0; $fld{$in_option} .= $1; next } if (!$comm_lvl && !$in_qcontent && /\G( \[ )/gcsx) { $in_literal = 1; $fld{$in_option} .= $1; next } if ($in_literal && /\G( \\. )/gcsx) { $fld{$in_option} .= $1; next } if ($in_literal && /\G( [^\]\\>]+ )/gcsx) { $fld{$in_option} .= $1; next } if (!$comm_lvl && !$in_qcontent && !$in_literal && !$in_tcp_info) { # top if (!$in_angle && /\G( < )/gcsx) { $in_angle = 1; $fld{$in_option} .= $1; next } if ( $in_angle && /\G( > )/gcsx) { $in_angle = 0; $fld{$in_option} .= $1; next } if (!$in_angle && /\G (from|by) (?:[ \t]+|\z|(?=[\[\(",;<]))/gcsxi) { $in_option = lc($1); $in_ext_dom = 1; next } if (!$in_angle && /\G(via|with|id|for)(?:[ \t]+|\z|(?=[\[\(",;<]))/gcsxi) { $in_option = lc($1); $in_ext_dom = 0; next } if (!$in_angle && /\G( ; )/gcsxi) { $in_option = lc($1); $in_ext_dom = 0; next } if (/\G( [ \t]+ )/gcsx) { $fld{$in_option} .= $1; next } if (/\G( [^ \t,:;\@<>()"\[\]\\]+ )/gcsx) { $fld{$in_option} .= $1; next } } if (/\G( . )/gcsx) { $fld{$in_option} .= $1; next } # other junk die "parse_received PANIC2 $new_pos"; # just in case } for my $f ('from-tcp','by-tcp') { # a tricky part is handling the syntax: # (Domain/Addr-literal) [ FWS "(" [ Domain FWS ] Addr-literal ")" ] CFWS # where absence of Address-literal in TCP-info means that what looked # like a domain in the optional TCP-info, is actually a comment in CFWS local($_) = $fld{$f}; if (!defined($_)) {} elsif (/\[ (\d{1,3} (?: \. \d{1,3}){3}) \] /x) {} elsif (/\[ [^:\]]* : [^\]]* \]/x && # triage, must contain a colon /\[ (?: IPv6: )? [0-9a-f]{0,4} (?: : [0-9a-f]{0,4} | \. [0-9]{1,3} ){2,9} \]/xi) {} # elsif (/ (?: ^ | \D ) ( \d{1,3} (?: \. \d{1,3}){3}) (?! [0-9.] ) /x) {} elsif (/^(?: localhost | ( [a-z0-9_\/+-]{1,63} \. )+ [a-z-]{2,} )\b/xi) {} else { my($fc) = $f; $fc =~ s/-tcp\z/-com/; $fld{$fc} = '' if !defined $fld{$fc}; $fld{$fc} = $_ . (/[ \t]\z/||$fld{$fc}=~/^[ \t]/?'':' ') .$fld{$fc}; delete $fld{$f}; } } for (values %fld) { s/[ \t]+\z//; s/^[ \t]+// } # for my $f (sort {$fld{$a} cmp $fld{$b}} keys %fld) # { do_log(5, "RECVD: %-8s -> /%s/", $f,$fld{$f}) } \%fld; } sub fish_out_ip_from_received($) { my($received) = @_; my($fields_ref) = parse_received($received); my($ip); local($1); for (@$fields_ref{qw(from-tcp from from-com)}) { next if !defined($_); if (/ \[ (\d{1,3} (?: \. \d{1,3}){3}) (?: \. \d{4,5} )? \] /x) { $ip = $1; last; } elsif (/\[ [^:\]]* : [^\]]* \]/x && # triage, must contain a colon /\[ ( (?: IPv6: )? [0-9a-f]{0,4} (?: : [0-9a-f]{0,4} | \. [0-9]{1,3} ){2,9} ) \]/xi) { $ip = $1; last; } elsif (/ (?: ^ | \D ) ( \d{1,3} (?: \. \d{1,3}){3}) (?! [0-9.] ) /x) { $ip = $1; last; } } return undef if !defined $ip; $ip =~ s/^IPv6://i; # discard 'IPv6:' prefix if any do_log(5, "fish_out_ip_from_received: %s", $ip); $ip; } # Splits unquoted fully qualified e-mail address, or an address # with a missing domain part. Returns a pair: (localpart, domain). # The domain part (if nonempty) includes the '@' as the first character. # If the syntax is badly broken, everything ends up as a localpart. # The domain part can be an address literal, as specified by RFC 5322. # Does not handle explicit route paths, use parse_quoted_rfc2821 for that. # sub split_address($) { my($mailbox) = @_; local($1,$2); $mailbox =~ /^ (.*?) ( \@ (?: \[ (?: \\. | [^\]\\] ){0,999} (?: \] | \z) | [^\[\@] )* ) \z/xs ? ($1, $2) : ($mailbox, ''); } # split_localpart() splits localpart of an e-mail address at the first # occurrence of the address extension delimiter character. (based on # equivalent routine in Postfix) # # Reserved addresses are not split: postmaster, mailer-daemon, # double-bounce. Addresses that begin with owner-, or addresses # that end in -request are not split when the owner_request_special # parameter is set. # sub split_localpart($$) { my($localpart, $delimiter) = @_; my($owner_request_special) = 1; # configurable ??? my($extension); local($1,$2); if ($localpart =~ /^(postmaster|mailer-daemon|double-bounce)\z/i) { # do not split these, regardless of what the delimiter is } elsif ($delimiter eq '-' && $owner_request_special && $localpart =~ /^owner-.|.-request\z/si) { # don't split owner-foo or foo-request } elsif ($localpart =~ /^(.+?)(\Q$delimiter\E.*)\z/s) { ($localpart, $extension) = ($1, $2); # extension includes a delimiter # do not split the address if the result would have a null localpart } ($localpart, $extension); } # replace localpart/extension/domain fields of an original email address # with nonempty fields of a replacement # sub replace_addr_fields($$;$) { my($orig_addr, $repl_addr, $delim) = @_; my($localpart_o, $domain_o, $ext_o, $localpart_r, $domain_r, $ext_r); ($localpart_o,$domain_o) = split_address($orig_addr); ($localpart_r,$domain_r) = split_address($repl_addr); $localpart_r = $localpart_o if $localpart_r eq ''; $domain_r = $domain_o if $domain_r eq ''; if (defined $delim && $delim ne '') { ($localpart_o,$ext_o) = split_localpart($localpart_o,$delim); ($localpart_r,$ext_r) = split_localpart($localpart_r,$delim); $ext_r = $ext_o if !defined $ext_r; } $localpart_r . (defined $ext_r ? $ext_r : '') . $domain_r; } # given a (potentially multiline) header field Message-ID, Resent-Message-ID. # In-Reply-To, or References, parse the RFC 5322 (RFC 2822) syntax extracting # all message IDs while ignoring comments, and return them as a list # Note: currently does not handle nested comments. # See also: RFC 2392 - Content-ID and Message-ID Uniform Resource Locators # sub parse_message_id($) { my($str) = @_; $str =~ tr/\n//d; my(@message_id); my($garbage) = 0; $str =~ s/[ \t]+/ /g; # compress whitespace as a bandaid for regexp trouble for my $t ( $str =~ /\G ( [ \t]+ | \( (?: \\. | [^()\\] ){0,999} \) | < (?: " (?: \\. | [^"\\>] ){0,999} " | \[ (?: \\. | [^\]\\>]){0,999} \] | [^"<>\[\]\\]+ )* > | [^<( \t]+ | . )/gsx ) { if ($t =~ /^<.*>\z/) { push(@message_id,$t) } elsif ($t =~ /^[ \t]*\z/) {} # ignore FWS elsif ($t =~ /^\(.*\)\z/) # ignore CFWS { do_log(2, "parse_message_id ignored comment: /%s/ in %s", $t,$str) } else { $garbage = 1 } } if (@message_id > 1) { @message_id = unique_list(\@message_id); # remove possible duplicates } elsif ($garbage && !@message_id) { local($_) = $str; s/^[ \t]+//; s/[ \t\n]+\z//; # trim and sanitize <...> s/^\z//; s/>/_/g; $_ = '<'.$_.'>'; @message_id = ($_); do_log(5, "parse_message_id sanitizing garbage: /%s/ to %s", $str,$_); } @message_id; } # For a given email address (e.g. for User+Foo@sub.exAMPLE.CoM) # prepare and return a list of lookup keys in the following order: # User+Foo@sub.exAMPLE.COM (as-is, no lowercasing) # user+foo@sub.example.com # user@sub.example.com (only if $recipient_delimiter nonempty) # user+foo(@) (only if $include_bare_user) # user(@) (only if $include_bare_user and $recipient_delimiter nonempty) # (@)sub.example.com # (@).sub.example.com # (@).example.com # (@).com # (@). # Note about (@): if $at_with_user is true the user-only keys (without domain) # get an '@' character appended (e.g. 'user+foo@'). Usual for lookup_hash. # If $at_with_user is false the domain-only (without localpart) keys # get a '@' prepended (e.g. '@.example.com'). Usual for SQL and LDAP lookups. # # The domain part is lowercased in all but the first item in the resulting # list; the localpart is lowercased iff $localpart_is_case_sensitive is true. # sub make_query_keys($$$;$) { my($addr,$at_with_user,$include_bare_user,$append_string) = @_; my($localpart,$domain) = split_address($addr); $domain = lc($domain); my($saved_full_localpart) = $localpart; $localpart = lc($localpart) if !c('localpart_is_case_sensitive'); # chop off leading @, and trailing dots local($1); $domain = $1 if $domain =~ /^\@?(.*?)\.*\z/s; my($extension); my($delim) = c('recipient_delimiter'); if ($delim ne '') { ($localpart,$extension) = split_localpart($localpart,$delim); # extension includes a delimiter since amavisd-new-2.5.0! } $extension = '' if !defined $extension; # mute warnings my($append_to_user,$prepend_to_domain) = $at_with_user ? ('@','') : ('','@'); my(@keys); # a list of query keys push(@keys, $addr); # as is push(@keys, $localpart.$extension.'@'.$domain) if $extension ne ''; # user+foo@example.com push(@keys, $localpart.'@'.$domain); # user@example.com if ($include_bare_user) { # typically enabled for local users only push(@keys, $localpart.$extension.$append_to_user) if $extension ne ''; # user+foo(@) push(@keys, $localpart.$append_to_user); # user(@) } push(@keys, $prepend_to_domain.$domain); # (@)sub.example.com if ($domain =~ /\[/) { # don't split address literals push(@keys, $prepend_to_domain.'.'); # (@). } else { my(@dkeys); my($d) = $domain; for (;;) { # (@).sub.example.com (@).example.com (@).com (@). push(@dkeys, $prepend_to_domain.'.'.$d); last if $d eq ''; $d = ($d =~ /^([^.]*)\.(.*)\z/s) ? $2 : ''; } if (@dkeys > 10) { @dkeys = @dkeys[$#dkeys-9 .. $#dkeys] } # sanity limit push(@keys,@dkeys); } if (defined $append_string && $append_string ne '') { $_ .= $append_string for @keys; } my($keys_ref) = unique_ref(\@keys); # remove duplicates ll(5) && do_log(5,"query_keys: %s", join(', ',@$keys_ref)); # the rhs replacement strings are similar to what would be obtained # by lookup_re() given the following regular expression: # /^( ( ( [^\@]*? ) ( \Q$delim\E [^\@]* )? ) (?: \@ (.*) ) )$/xs my($rhs) = [ # a list of right-hand side replacement strings $addr, # $1 = User+Foo@Sub.Example.COM $saved_full_localpart, # $2 = User+Foo $localpart, # $3 = user (lc if localpart_is_case_sensitive) $extension, # $4 = +foo (lc if localpart_is_case_sensitive) $domain, # $5 = sub.example.com (lowercased unconditionally) ]; ($keys_ref, $rhs); } # quote_rfc2821_local() quotes the local part of a mailbox address # (given in internal (unquoted) form), and returns external (quoted) # mailbox address, as per RFC 5321 (ex RFC 2821). # # internal (unquoted) form is used internally by amavisd-new and other mail sw, # external (quoted) form is used in SMTP commands and in message header section # # To re-insert message back via SMTP, the local-part of the address needs # to be quoted again if it contains reserved characters or otherwise # does not obey the dot-atom syntax, as specified in RFC 5321 (ex RFC 2821). # sub quote_rfc2821_local($) { my($mailbox) = @_; # atext: any character except controls, SP, and specials (RFC 5321/RFC 5322) my($atext) = "a-zA-Z0-9!#\$%&'*/=?^_`{|}~+-"; # my($specials) = '()<>\[\]\\\\@:;,."'; my($localpart,$domain) = split_address($mailbox); if ($localpart !~ /^[$atext]+(\.[$atext]+)*\z/so) { # not dot-atom, needs q. local($1); # qcontent = qtext / quoted-pair $localpart =~ s/([\000-\037\177-\377"\\])/\\$1/g; # quote non-qtext $localpart = '"'.$localpart.'"'; # make it a qcontent # Postfix hates ""@domain but is not so harsh on @domain # Late breaking news: don't bother, both forms are rejected by Postfix # when strict_rfc821_envelopes=yes, and both are accepted otherwise } # we used to strip off empty domain (just '@') unconditionally, but this # leads Postfix to interpret an address with a '@' in the quoted local part # e.g. <"h@example.net"@> as (subject to Postfix setting # 'resolve_dequoted_address'), which is not what the sender requested; # we no longer do that if localpart contains an '@': $domain = '' if $domain eq '@' && $localpart =~ /\@/; $localpart . $domain; } # wraps the result of quote_rfc2821_local into angle brackets <...> ; # If given a list, it returns a list (possibly converted to # comma-separated scalar if invoked in scalar context), quoting each element; # sub qquote_rfc2821_local(@) { my(@r) = map($_ eq '' ? '<>' : ('<'.quote_rfc2821_local($_).'>'), @_); wantarray ? @r : join(', ', @r); } sub parse_quoted_rfc2821($$) { my($addr,$unquote) = @_; # the angle-bracket stripping is not really a duty of this subroutine, # as it should have been already done elsewhere, but we allow it here anyway: $addr =~ s/^\s*\s*\z//s; # tolerate unmatched angle brkts local($1,$2); my($source_route,$localpart,$domain) = ('','',''); # RFC 2821: so-called "source route" MUST BE accepted, # SHOULD NOT be generated, and SHOULD be ignored. # Path = "<" [ A-d-l ":" ] Mailbox ">" # A-d-l = At-domain *( "," A-d-l ) # At-domain = "@" domain if (index($addr,':') >= 0 && # triage before more testing for source route $addr =~ m{^ ( [ \t]* \@ (?: [0-9A-Za-z.!\#\$%&*/^{}=_+-]* | \[ (?: \\. | [^\]\\] ){0,999} \] ) [ \t]* (?: , [ \t]* \@ (?: [0-9A-Za-z.!\#\$%&*/^{}=_+-]* | \[ (?: \\. | [^\]\\] ){0,999} \] ) [ \t]* )* : [ \t]* ) (.*) \z }xs) { # NOTE: we are quite liberal on allowing whitespace around , and : here, # and liberal in allowed character set and syntax of domain names, # we mainly avoid stop-characters in the domain names of source route $source_route = $1; $addr = $2; } if ($addr =~ m{^ ( .*? ) ( \@ (?: [^\@\[\]]+ | \[ (?: \\. | [^\]\\] ){0,999} \] | [^\@] )* ) \z}xs) { ($localpart,$domain) = ($1,$2); } else { ($localpart,$domain) = ($addr,''); } $localpart =~ s/ " | \\ (.) | \\ \z /$1/xsg if $unquote; # undo quoted-pairs ($source_route, $localpart, $domain); } # unquote_rfc2821_local() strips away the quoting from the local part # of an external (quoted) mailbox address, and returns internal (unquoted) # mailbox address, as per RFC 5321 (ex RFC 2821). # Internal (unquoted) form is used internally by amavisd-new and other mail sw, # external (quoted) form is used in SMTP commands and in message header section # sub unquote_rfc2821_local($) { my($mailbox) = @_; my($source_route,$localpart,$domain) = parse_quoted_rfc2821($mailbox,1); # make address with '@' in the localpart but no domain (like <"aa@bb.com"> ) # distinguishable from by representing it as aa@bb.com@ in # unquoted form; (it still obeys all regular rules, it is not a dirty trick) $domain = '@' if $domain eq '' && $localpart ne '' && $localpart =~ /\@/; $localpart . $domain; } # Parse an rfc2822.address-list, returning a list of RFC 5322 (quoted) # addresses. Properly deals with group addresses, nested comments, address # literals, qcontent, addresses with source route, discards display # names and comments. The following header fields accept address-list: # To, Cc, Bcc, Reply-To. A header field 'From' accepts a 'mailbox-list' # syntax (which is similar, but does not allow groups); a header field # 'Sender' accepts a 'mailbox' syntax, i.e. only one address and not a group. # use vars qw($s $p @addresses); sub flush_a() { $s =~ s/^[ \t]+//s; $s =~ s/[ \t]\z//s; # trim $p =~ s/^[ \t]+//s; $p =~ s/[ \t]\z//s; if ($p ne '') { $p =~ s/^\z//; push(@addresses,$p) } elsif ($s ne '') { push(@addresses,$s) } $p = ''; $s = ''; } sub parse_address_list($) { local($_) = $_[0]; local($1); s/\n(?=[ \t])//gs; s/\n+\z//s; # unfold, chomp my($str_l) = length($_); $p = ''; $s = ''; @addresses = (); my($comm_lvl) = 0; my($in_qcontent) = 0; my($in_literal) = 0; my($in_group) = 0; my($in_angle) = 0; my($after_at) = 0; my($new_pos); for (my $pos=-1; $new_pos=pos($_), $new_pos<$str_l; $pos=$new_pos) { $new_pos > $pos or die "parse_address_list PANIC1 $new_pos"; # just in case # comment (may be nested: RFC 5322 section 3.2.2) if ($comm_lvl > 0 && /\G( \) )/gcsx) { $comm_lvl--; next } if (!$in_qcontent && !$in_literal && /\G( \( )/gcsx) { $comm_lvl++; next } if ($comm_lvl > 0 && /\G( \\. )/gcsx) { next } if ($comm_lvl > 0 && /\G( [^()\\]+ )/gcsx) { next } # quoted content if ($in_qcontent && /\G( " )/gcsx) # normal exit from qcontent { $in_qcontent = 0; ($in_angle?$p:$s) .= $1; next } if ($in_qcontent && /\G( > )/gcsx) # bail out of qcontent { $in_qcontent = 0; $in_angle = 0; $after_at = 0; ($in_angle?$p:$s) .= $1; next } if (!$comm_lvl && !$in_qcontent && !$in_literal && /\G( " )/gcsx) { $in_qcontent = 1; ($in_angle?$p:$s) .= $1; next } if ($in_qcontent && /\G( \\. )/gcsx) { ($in_angle?$p:$s) .= $1; next } if ($in_qcontent && /\G( [^"\\>]+ )/gcsx) { ($in_angle?$p:$s) .= $1; next } # address literal if ($in_literal && /\G( \] )/gcsx) { $in_literal = 0; ($in_angle?$p:$s) .= $1; next } if ($in_literal && /\G( > )/gcsx) # bail out of address literal { $in_literal = 0; $in_angle = 0; $after_at = 0; ($in_angle?$p:$s) .= $1; next } if (!$comm_lvl && !$in_qcontent && /\G( \[ )/gcsx) { $in_literal = 1 if $after_at; ($in_angle?$p:$s) .= $1; next } if ($in_literal && /\G( \\. )/gcsx) { ($in_angle?$p:$s) .= $1; next } if ($in_literal && /\G( [^\]\\>]+ )/gcsx) { ($in_angle?$p:$s) .= $1; next } # normal content if (!$comm_lvl && !$in_qcontent && !$in_literal) { if (!$in_angle && /\G( < )/gcsx) { $in_angle = 1; $after_at = 0; flush_a() if $p ne ''; $p .= $1; next } if ( $in_angle && /\G( > )/gcsx) { $in_angle = 0; $after_at = 0; $p .= $1; next } if (/\G( , )/gcsx) # top-level addr separator or source route delimiter { !$in_angle ? flush_a() : ($p.=$1); $after_at = 0; next } if (!$in_angle && !$in_group && /\G( : )/gcsx) # group name terminator { $in_group = 1; $s .= $1; $p=$s=''; next } # discard group name if ($after_at && /\G( : )/gcsx) # source route terminator { $after_at = 0; ($in_angle?$p:$s) .= $1; next } if ( $in_group && /\G( ; )/gcsx) # group terminator { $in_group = 0; $after_at = 0; next } if (!$in_group && /\G( ; )/gcsx) # out of place special { ($in_angle?$p:$s) .= $1; $after_at = 0; next } if (/\G( \@ )/gcsx) { $after_at = 1; ($in_angle?$p:$s) .= $1; next } if (/\G( [ \t]+ )/gcsx) { ($in_angle?$p:$s) .= $1; next } if (/\G( [^,:;\@<>()"\[\]\\]+ )/gcsx) { ($in_angle?$p:$s) .= $1; next } } if (/\G( . )/gcsx) { ($in_angle?$p:$s) .= $1; next } # other junk die "parse_address_list PANIC2 $new_pos"; # just in case } flush_a(); @addresses; } # compute a total displayed line size if a string (possibly containing TAB # characters) would be displayed at the given character position (0-based) # sub displayed_length($$) { my($str,$ind) = @_; for my $t ($str =~ /\G ( \t | [^\t]+ )/gsx) { $ind += $t ne "\t" ? length($t) : 8 - $ind % 8 } $ind; } # Wrap a string into a multiline string, inserting \n as appropriate to keep # each line length at $max_len or shorter (not counting \n). A string $prefix # is prepended to each line. Continuation lines get their first space or TAB # character replaced by a string $indent (unless $indent is undefined, which # keeps the leading whitespace character unchanged). Both the $prefix and # $indent are included in line size calculation, and for the purpose of line # size calculations TABs are treated as an appropriate number of spaces. # Parameter $structured indicates where line breaks are permitted: true # indicates that line breaks may only occur where a \n character is already # present in the source line, indicating possible (tentative) line breaks. # If $structured is false, permitted line breaks are chosen within existing # whitespace substrings so that all-whitespace lines are never generated # (even at the expense of producing longer than allowed lines if necessary), # and that each continuation line starts by at least one whitespace character. # Whitespace is neither added nor removed, but simply spliced into trailing # and leading whitespace of subsequent lines. Typically leading whitespace # is a single character, but may include part of the trailing whitespace of # the preceeding line if it would otherwise be too long. This is appropriate # and required for wrapping of mail haeder fields. An exception to preservation # of whitespace is when $indent string is defined but is an empty string, # causing leading and trailing whitespace to be trimmed, producing a classical # plain text wrapping results. Intricate! # sub wrap_string($;$$$$) { my($str,$max_len,$prefix,$indent,$structured) = @_; $max_len = 78 if !defined $max_len; $prefix = '' if !defined $prefix; $structured = 0 if !defined $structured; my(@chunks); # split a string into chunks where each chunk starts with exactly one SP or # TAB character (except possibly the first chunk), followed by an unbreakable # string (consisting typically entirely of non-whitespace characters, at # least one character must be non-whitespace), followed by an all-whitespace # string consisting of only SP or TAB characters. if ($structured) { local($1); # unfold all-whitespace chunks, just in case 1 while $str =~ s/^([ \t]*)\n/$1/; # prefixed? $str =~ s/\n(?=[ \t]*(\n|\z))//g; # within and at end $str =~ s/\n(?![ \t])/\n /g; # insert a space at line folds if missing # unbreakable parts are substrings between newlines, determined by caller @chunks = split(/\n/,$str,-1); } else { $str =~ s/\n(?![ \t])/\n /g; # insert a space at line folds if missing $str =~ s/\n//g; # unfold (knowing a space at folds is not missing) # unbreakable parts are non- all-whitespace substrings @chunks = $str =~ /\G ( (?: ^ .*? | [ \t]) [^ \t]+ [ \t]* ) (?= \z | [ \t] [^ \t] )/gsx; } # do_log(5,"wrap_string chunk: <%s>", $_) for @chunks; my($result) = ''; # wrapped multiline string will accumulate here my($s) = ''; # collects partially assembled single line my($s_displ_ind) = # display size of string in $s, including $prefix displayed_length($prefix,0); my($contin_line) = 0; # are we assembling a continuation line? while (@chunks) { # walk through input substrings and join shorter sections my($chunk) = shift(@chunks); # replace leading space char with $indent if starting a continuation line $chunk =~ s/^[ \t]/$indent/ if defined $indent && $contin_line && $s eq ''; my($s_displ_l) = displayed_length($chunk, $s_displ_ind); if ($s_displ_l <= $max_len # collecting in $s while still fits || (@chunks==0 && $s =~ /^[ \t]*\z/)) { # or we are out of options $s .= $chunk; $s_displ_ind = $s_displ_l; # absorb entire chunk } else { local($1,$2); $chunk =~ /^ ( .* [^ \t] ) ( [ \t]* ) \z/xs # split to head and allwhite or die "Assert 1 failed in wrap: /$result/, /$chunk/"; my($solid,$white_tail) = ($1,$2); my($min_displayed_s_len) = displayed_length($solid, $s_displ_ind); if (@chunks > 0 # not being at the last chunk gives a chance to shove # part of the trailing whitespace off to the next chunk && ($min_displayed_s_len <= $max_len # non-whitespace part fits || $s =~ /^[ \t]*\z/) ) { # or still allwhite even if too long $s .= $solid; $s_displ_ind = $min_displayed_s_len; # take nonwhite if (defined $indent && $indent eq '') { # discard leading whitespace in continuation lines on a plain wrap } else { # preserve all original whitespace while ($white_tail ne '') { # stash-in as much trailing whitespace as it fits to the curr. line my($c) = substr($white_tail,0,1); # one whitespace char. at a time my($dlen) = displayed_length($c, $s_displ_ind); if ($dlen > $max_len) { last } else { $s .= $c; $s_displ_ind = $dlen; # absorb next whitespace char. $white_tail = substr($white_tail,1); # one down, more to go... } } # push remaining trailing whitespace characters back to input $chunks[0] = $white_tail . $chunks[0] if $white_tail ne ''; } } elsif ($s =~ /^[ \t]*\z/) { die "Assert 2 failed in wrap: /$result/, /$chunk/"; } else { # nothing more fits to $s, flush it to $result if ($contin_line) { $result .= "\n" } else { $contin_line = 1 } # trim trailing whitespace when wrapping as a plain text (not headers) $s =~ s/[ \t]+\z// if defined $indent && $indent eq ''; $result .= $prefix.$s; $s = ''; $s_displ_ind = displayed_length($prefix,0); unshift(@chunks,$chunk); # reprocess the chunk } } } if ($s !~ /^[ \t]*\z/) { # flush last chunk if nonempty if ($contin_line) { $result .= "\n" } else { $contin_line = 1 } $s =~ s/[ \t]+\z// if defined $indent && $indent eq ''; # trim plain text $result .= $prefix.$s; $s = ''; } $result; } # wrap an SMTP response at each \n char according to RFC 5321 (ex RFC 2821), # returning resulting lines as a listref # sub wrap_smtp_resp($) { my($resp) = @_; # RFC 5321: The maximum total length of a reply line including the # reply code and the is 512 octets. More information # may be conveyed through multiple-line replies. my($max_len) = 512-2; my(@result_list); local($1,$2,$3,$4); if ($resp !~ /^ ([1-5]\d\d) (\ |-|\z) ([245] \. \d{1,3} \. \d{1,3} (?: \ |\z) )? (.*) \z/xs) { die "wrap_smtp_resp: bad SMTP response code: '$resp'" } my($resp_code,$more,$enhanced,$tail) = ($1,$2,$3,$4); my($lead_len) = length($resp_code) + 1 + length($enhanced); while (length($tail) > $max_len-$lead_len || $tail =~ /\n/) { # RFC 2034: When responses are continued across multiple lines the same # status code must appear at the beginning of the text in each line # of the response. my($head) = substr($tail, 0, $max_len-$lead_len); if ($head =~ /^([^\n]*\n)/s) { $head = $1 } $tail = substr($tail,length($head)); chomp($head); push(@result_list, $resp_code.'-'.$enhanced.$head); } push(@result_list, $resp_code.' '.$enhanced.$tail); \@result_list; } # Prepare a single SMTP response and an exit status as per sysexits.h # from individual per-recipient response codes, taking into account # sendmail milter specifics. Returns a triple: (smtp response, exit status, # an indication whether a non delivery notification (NDN, a form of DSN) # is needed). # sub one_response_for_all($$;$) { my($msginfo, $dsn_per_recip_capable, $suppressed) = @_; my($smtp_resp, $exit_code, $ndn_needed); my($am_id) = $msginfo->log_id; my($sender) = $msginfo->sender; my($per_recip_data) = $msginfo->per_recip_data; my($any_not_done) = scalar(grep(!$_->recip_done, @$per_recip_data)); if (!@$per_recip_data) { # no recipients, nothing to do $smtp_resp = "250 2.5.0 Ok, id=$am_id"; $exit_code = EX_OK; do_log(5, "one_response_for_all <%s>: no recipients, '%s'", $sender, $smtp_resp); } if (!defined $smtp_resp) { for my $r (@$per_recip_data) { # any 4xx code ? if ($r->recip_smtp_response =~ /^4/) # pick the first 4xx code { $smtp_resp = $r->recip_smtp_response; last } } } if (!defined $smtp_resp) { for my $r (@$per_recip_data) { my($fwd_m) = $r->delivery_method; if (!defined $fwd_m) { die "one_response_for_all: delivery_method not defined"; } elsif ($fwd_m ne '' && $any_not_done) { die "Explicit forwarding, but not all recips done"; } } for my $r (@$per_recip_data) { # any invalid code ? if ($r->recip_done && $r->recip_smtp_response !~ /^[245]/) { $smtp_resp = '451 4.5.0 Bad SMTP response code??? "' . $r->recip_smtp_response . '"'; last; # pick the first } } if (defined $smtp_resp) { $exit_code = EX_TEMPFAIL; do_log(5, "one_response_for_all <%s>: 4xx found, '%s'", $sender,$smtp_resp); } } # NOTE: a 2xx SMTP response code is set both by internal Discard # and by a genuine successful delivery. To distinguish between the two # we need to check $r->recip_destiny as well. # if (!defined $smtp_resp) { # if destiny for _all_ recipients is D_DISCARD, give Discard my($notall); for my $r (@$per_recip_data) { if ($r->recip_destiny == D_DISCARD) # pick the first DISCARD code { $smtp_resp = $r->recip_smtp_response if !defined $smtp_resp } else { $notall=1; last } # one is not a discard, nogood } if ($notall) { $smtp_resp = undef } if (defined $smtp_resp) { $exit_code = 99; # helper program will interpret 99 as discard do_log(5, "one_response_for_all <%s>: all DISCARD, '%s'", $sender,$smtp_resp); } } if (!defined $smtp_resp) { # destiny for _all_ recipients is Discard or Reject, give 5xx # (and there is at least one Reject) my($notall, $done_level); my($bounce_cnt) = 0; for my $r (@$per_recip_data) { my($dest, $resp) = ($r->recip_destiny, $r->recip_smtp_response); if ($dest == D_DISCARD) { # ok, this one is a discard, let's see the rest } elsif ($resp =~ /^5/ && $dest != D_BOUNCE) { # prefer to report SMTP response code of genuine rejects # from MTA, over internal rejects by content filters if (!defined $smtp_resp || $r->recip_done > $done_level) { $smtp_resp = $resp; $done_level = $r->recip_done } } else { $notall=1; last; # one is a Pass or Bounce, nogood } } if ($notall) { $smtp_resp = undef } if (defined $smtp_resp) { $exit_code = EX_UNAVAILABLE; do_log(5, "one_response_for_all <%s>: REJECTs, '%s'",$sender,$smtp_resp); } } if (!defined $smtp_resp) { # mixed destiny => 2xx, but generate dsn for bounces and rejects my($rej_cnt) = 0; my($bounce_cnt) = 0; my($drop_cnt) = 0; for my $r (@$per_recip_data) { my($dest, $resp) = ($r->recip_destiny, $r->recip_smtp_response); if ($resp =~ /^2/ && $dest == D_PASS) # genuine successful delivery { $smtp_resp = $resp if !defined $smtp_resp } $drop_cnt++ if $dest == D_DISCARD; if ($resp =~ /^5/) { if ($dest == D_BOUNCE) { $bounce_cnt++ } else { $rej_cnt++ } } } $exit_code = EX_OK; if (!defined $smtp_resp) { # no genuine Pass/2xx # declare success, we'll handle bounce $smtp_resp = "250 2.5.0 Ok, id=$am_id"; if ($any_not_done) { $smtp_resp .= ", continue delivery" } else { $exit_code = 99 } # helper program DISCARD (e.g. milter) } if ($rej_cnt + $bounce_cnt + $drop_cnt > 0) { $smtp_resp .= ", "; $smtp_resp .= "but " if $rej_cnt+$bounce_cnt+$drop_cnt<@$per_recip_data; $smtp_resp .= join ", and ", map { my($cnt, $nm) = @$_; !$cnt ? () : $cnt == @$per_recip_data ? $nm : "$cnt $nm" } ([$rej_cnt, 'REJECT'], [$bounce_cnt, $suppressed ? 'DISCARD(bounce.suppressed)' :'BOUNCE'], [$drop_cnt, 'DISCARD']); } $ndn_needed = ($bounce_cnt > 0 || ($rej_cnt > 0 && !$dsn_per_recip_capable)) ? 1 : 0; ll(5) && do_log(5, "one_response_for_all <%s>: %s, r=%d,b=%d,d=%s, ndn_needed=%s, '%s'", $sender, $rej_cnt + $bounce_cnt + $drop_cnt > 0 ? 'mixed' : 'success', $rej_cnt, $bounce_cnt, $drop_cnt, $ndn_needed, $smtp_resp); } ($smtp_resp, $exit_code, $ndn_needed); } 1; # package Amavis::Lookup::RE; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Util qw(ll do_log fmt_struct); } # Make an object out of the supplied lookup list # to make it distinguishable from simple ACL array sub new($$) { my($class) = shift; bless [@_], $class } # lookup_re() performs a lookup for an e-mail address or other key string # against a list of regular expressions. # # A full unmodified e-mail address is always used, so splitting to localpart # and domain or lowercasing is NOT performed. The regexp is powerful enough # that this can be accomplished by its own mechanisms. The routine is useful # for other RE tests besides the usual e-mail addresses, such as looking for # banned file names. # # Each element of the list can be a ref to a pair, or directly a regexp # ('Regexp' object created by a qr operator, or just a (less efficient) # string containing a regular expression). If it is a pair, the first # element is treated as a regexp, and the second provides a value in case # the regexp matches. If not a pair, the implied result of a match is 1. # # The regular expression is taken as-is, no implicit anchoring or setting # case insensitivity is done, so do use a qr'(?i)^user\@example\.com$', # and not a sloppy qr'user@example.com', which can easily backfire. # Also, if qr is used with a delimiter other than ' (apostrophe), make sure # to quote the @ and $ when they are not introducing a variable name. # # The pattern allows for capturing of parenthesized substrings, which can # then be referenced from the result string using the $1, $2, ... notation, # as with a Perl m// operator. The number after a $ may be a multi-digit # decimal number. To avoid possible ambiguity a ${n} or $(n) form may be used # Substring numbering starts with 1. Nonexistent references evaluate to empty # strings. If any substitution is done, the result inherits the taintedness # of $addr. Keep in mind that $ and @ characters needs to be backslash-quoted # in qq() strings. Example: # $virus_quarantine_to = new_RE( # [ qr'^(.*)\@example\.com$'i => 'virus-${1}@example.com' ], # [ qr'^(.*)(\@[^\@]*)?$'i => 'virus-${1}${2}' ] ); # # Example (equivalent to the example in lookup_acl): # $acl_re = Amavis::Lookup::RE->new( # qr'\@me\.ac\.uk$'i, [qr'[\@.]ac\.uk$'i=>0], qr'\.uk$'i ); # ($r,$k) = $acl_re->lookup_re('user@me.ac.uk'); # or $r = lookup(0, 'user@me.ac.uk', $acl_re); # # 'user@me.ac.uk' matches me.ac.uk, returns true and search stops # 'user@you.ac.uk' matches .ac.uk, returns false (because of =>0) # and search stops # 'user@them.co.uk' matches .uk, returns true and search stops # 'user@some.com' does not match anything, falls through and # returns false (undef) # # As a special allowance, the $addr argument may be a ref to a list of search # keys. At each step in traversing the supplied regexp list, all elements of # @$addr are tried. If any of them matches, the search stops. This is currently # used in banned names lookups, where all attributes of a part are given as a # list @$addr, as a loop on attributes must be an inner loop. # sub lookup_re($$;$%) { my($self, $addr,$get_all,%options) = @_; local($1,$2,$3,$4); my(@matchingkey,@result); $addr .= $options{AppendStr} if defined $options{AppendStr}; for my $e (@$self) { # try each regexp in the list my($key,$r); if (ref($e) eq 'ARRAY') { # a pair: (regexp,result) ($key,$r) = ($e->[0], @$e < 2 ? 1 : $e->[1]); } else { # a single regexp (not a pair), implies result 1 ($key,$r) = ($e, 1); } ""=~/x{0}/; # braindead Perl: serves as explicit deflt for an empty regexp my(@rhs); # match, capturing parenthesized subpatterns into @rhs if (!ref($addr)) { @rhs = $addr =~ /$key/ } else { for (@$addr) { @rhs = /$key/; last if @rhs } } # inner loop if (@rhs) { # regexp matches # do the righthand side replacements if any $n, ${n} or $(n) is specified if (defined($r) && !ref($r) && index($r,'$') >= 0) { # triage my($any) = $r =~ s{ \$ ( (\d+) | \{ (\d+) \} | \( (\d+) \) ) } { my($j)=$2+$3+$4; $j<1 ? '' : $rhs[$j-1] }gxse; # bring taintedness of input to the result $r .= substr($addr,0,0) if $any; } push(@result,$r); push(@matchingkey,$key); last if !$get_all; } } if (!ll(5)) { # don't bother preparing log report which will not be printed } elsif (!@result) { do_log(5, "lookup_re(%s), no matches", fmt_struct($addr)); } else { # pretty logging my(%esc) = (r => "\r", n => "\n", f => "\f", b => "\b", e => "\e", a => "\a", t => "\t"); my(@mk) = @matchingkey; for my $mk (@mk) # undo the \-quoting, will be redone by logging routines { $mk =~ s{ \\(.) }{ exists($esc{$1}) ? $esc{$1} : $1 }egsx } if (!$get_all) { # first match wins do_log(5, 'lookup_re(%s) matches key "%s", result=%s', fmt_struct($addr), $mk[0], fmt_struct($result[0])); } else { # want all matches do_log(5, "lookup_re(%s) matches keys: %s", fmt_struct($addr), join(', ', map {sprintf('"%s"=>%s', $mk[$_],fmt_struct($result[$_]))} (0..$#result))); } } if (!$get_all) { !wantarray ? $result[0] : ($result[0], $matchingkey[0]) } else { !wantarray ? \@result : (\@result, \@matchingkey) } } 1; # package Amavis::Lookup::IP; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION $have_patricia); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&lookup_ip_acl); import Amavis::Util qw(ll do_log); } use subs @EXPORT_OK; BEGIN { eval { require Net::Patricia; Net::Patricia->VERSION(1.015); # need AF_INET6 support import Net::Patricia; $have_patricia = 1; }; } # ip_to_vec() takes IPv6 or IPv4 IP address with optional prefix length # (or IPv4 mask), parses and validates it, and returns it as a 128-bit # vector string that can be used as operand to Perl bitwise string operators. # Syntax and other errors in the argument throw exception (die). # If the second argument $allow_mask is 0, the prefix length or mask # specification is not allowed as part of the IP address. # # The IPv6 syntax parsing and validation adheres to RFC 4291 (ex RFC 3513). # All the following IPv6 address forms are supported: # x:x:x:x:x:x:x:x preferred form # x:x:x:x:x:x:d.d.d.d alternative form # ...::... zero-compressed form # addr/prefix-length prefix length may be specified (defaults to 128) # Optionally an "IPv6:" prefix may be prepended to the IPv6 address # as specified by RFC 5321 (ex RFC 2821). Brackets enclosing the address # are optional, e.g. [::1]/128 . # # The following IPv4 forms are allowed: # d.d.d.d # d.d.d.d/prefix-length CIDR mask length is allowed (defaults to 32) # d.d.d.d/m.m.m.m network mask (gets converted to prefix-length) # If prefix-length or a mask is specified with an IPv4 address, the address # may be shortened to d.d.d/n or d.d/n or d/n. Such truncation is allowed # for compatibility with earlier version, but is deprecated and is not # allowed for IPv6 addresses. # # IPv4 addresses and masks are converted to IPv4-mapped IPv6 addresses # of the form ::FFFF:d.d.d.d, The CIDR mask length (0..32) is converted # to IPv6 prefix-length (96..128). The returned vector strings resulting # from IPv4 and IPv6 forms are indistinguishable. # # NOTE: # d.d.d.d is equivalent to ::FFFF:d.d.d.d (IPv4-mapped IPv6 address) # which is not the same as ::d.d.d.d (IPv4-compatible IPv6 address) # # A triple is returned: # - IP address represented as a 128-bit vector (a string) # - network mask derived from prefix length, a 128-bit vector (string) # - prefix length as an integer (0..128) # sub ip_to_vec($;$) { my($ip,$allow_mask) = @_; my($ip_len); my(@ip_fields); local($1,$2,$3,$4,$5,$6); $ip =~ s/^[ \t]+//; $ip =~ s/[ \t\r\n]+\z//s; # trim my($ipa) = $ip; ($ipa,$ip_len) = ($1,$2) if $allow_mask && $ip =~ m{^([^/]*)/(.*)\z}s; $ipa = $1 if $ipa =~ m{^ \[ (.*) \] \z}xs; # discard optional brackets $ipa =~ s/%[A-Z0-9:._-]+\z//si; # discard interface specification if ($ipa =~ m{^(IPv6:)?(.*:)(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\z}si){ # IPv6 alternative form x:x:x:x:x:x:d.d.d.d my(@d) = ($3,$4,$5,$6); !grep($_ > 255, @d) or die "Invalid decimal field value in IPv6 address: [$ip]\n"; $ipa = $2 . sprintf("%02X%02X:%02X%02X", @d); } elsif ($ipa =~ m{^\d{1,3}(?:\.\d{1,3}){0,3}\z}) { # IPv4 form my(@d) = split(/\./,$ipa,-1); !grep($_ > 255, @d) or die "Invalid field value in IPv4 address: [$ip]\n"; defined($ip_len) || @d==4 or die "IPv4 address [$ip] contains fewer than 4 fields\n"; $ipa = '::FFFF:' . sprintf("%02X%02X:%02X%02X", @d); # IPv4-mapped IPv6 if (!defined($ip_len)) { $ip_len = 32; # no length, defaults to /32 } elsif ($ip_len =~ /^\d{1,9}\z/) { # /n, IPv4 CIDR notation } elsif ($ip_len =~ /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\z/) { !grep($_ > 255, ($1,$2,$3,$4)) or die "Illegal field value in IPv4 mask: [$ip]\n"; my($mask1) = pack('C4',$1,$2,$3,$4); # /m.m.m.m my($len) = unpack("%b*",$mask1); # count ones my($mask2) = pack('B32', '1' x $len); # reconstruct mask from count $mask1 eq $mask2 or die "IPv4 mask not representing a valid CIDR mask: [$ip]\n"; $ip_len = $len; } else { die "Invalid IPv4 network mask or CIDR prefix length: [$ip]\n"; } $ip_len<=32 or die "IPv4 network prefix length greater than 32: [$ip]\n"; $ip_len += 128-32; # convert IPv4 net mask length to IPv6 prefix length } $ipa =~ s/^IPv6://i; # now we presumably have an IPv6 compressed or preferred form x:x:x:x:x:x:x:x if ($ipa !~ /^(.*?)::(.*)\z/s) { # zero-compressing form used? @ip_fields = split(/:/,$ipa,-1); # no, have preferred form } else { # expand zero-compressing form my(@a) = split(/:/,$1,-1); my(@b) = split(/:/,$2,-1); my($missing_cnt) = 8-(@a+@b); $missing_cnt = 1 if $missing_cnt<1; @ip_fields = (@a, ('0') x $missing_cnt, @b); } @ip_fields<8 and die "IPv6 address [$ip] contains fewer than 8 fields\n"; @ip_fields>8 and die "IPv6 address [$ip] contains more than 8 fields\n"; !grep(!/^[0-9a-zA-Z]{1,4}\z/, @ip_fields) # this is quite slow or die "Invalid syntax of IPv6 address: [$ip]\n"; my($vec) = pack("n8", map(hex($_),@ip_fields)); if (!defined($ip_len)) { $ip_len = 128 } elsif ($ip_len !~ /^\d{1,3}\z/) { die "Invalid prefix length syntax in IP address: [$ip]\n" } elsif ($ip_len > 128) { die "IPv6 network prefix length greater than 128: [$ip]\n" } my($mask) = pack('B128', '1' x $ip_len); # do_log(5, "ip_to_vec: %s => %s/%d\n", $ip,unpack("B*",$vec),$ip_len); ($vec,$mask,$ip_len); } # lookup_ip_acl() performs a lookup for an IPv4 or IPv6 address against a list # of lookup tables, each may be a constant, or a ref to an access control # list or a ref to an associative array (hash) of network or host addresses. # # IP address is compared to each member of an access list in turn, # the first match wins (terminates the search), and its value decides # whether the result is true (yes, permit, pass) or false (no, deny, drop). # Falling through without a match produces false (undef). # # The presence of character '!' prepended to a list member decides # whether the result will be true (without a '!') or false (with '!') # in case this list member matches and terminates the search. # # Because search stops at the first match, it only makes sense # to place more specific patterns before the more general ones. # # For IPv4 a network address can be specified in classless notation # n.n.n.n/k, or using a mask n.n.n.n/m.m.m.m . Missing mask implies /32, # i.e. a host address. For IPv6 addresses all RFC 3513 forms are allowed. # See also comments at ip_to_vec(). # # Although not a special case, it is good to remember that '::/0' # always matches any IPv4 or IPv6 address (even syntactically invalid address). # # The '0/0' is equivalent to '::FFFF:0:0/96' and matches any syntactically # valid IPv4 address (including IPv4-mapped IPv6 addresses), but not other # IPv6 addresses! # # Example # given: @acl = qw( !192.168.1.12 172.16.3.3 !172.16.3.0/255.255.255.0 # 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 # !0.0.0.0/8 !:: 127.0.0.0/8 ::1 ); # matches RFC 1918 private address space except host 192.168.1.12 # and net 172.16.3/24 (but host 172.16.3.3 within 172.16.3/24 still matches). # In addition, the 'unspecified' (null, i.e. all zeros) IPv4 and IPv6 # addresses return false, and IPv4 and IPv6 loopback addresses match # and return true. # # If the supplied lookup table is a hash reference, match a canonical # IP address: dot-quad IPv4, or preferred IPv6 form, against hash keys. # For IPv4 addresses a simple classful subnet specification is allowed in # hash keys by truncating trailing bytes from the looked up IPv4 address. # A syntactically invalid IP address cannot match any hash entry. # sub lookup_ip_acl($@) { my($ip, @nets_ref) = @_; my($ip_vec,$ip_mask); my($eval_stat); eval { ($ip_vec,$ip_mask) = ip_to_vec($ip,0); 1 } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; my($label,$fullkey,$result,$lookup_type); my($found) = 0; for my $tb (@nets_ref) { my($t) = ref($tb) eq 'REF' ? $$tb : $tb; # allow one level of indirection if (!ref($t) || ref($t) eq 'SCALAR') { # a scalar always matches my($r) = ref($t) ? $$t : $t; # allow direct or indirect reference $result = $r; $fullkey = "(constant:$r)"; $lookup_type = 'const'; $found=1 if defined $result; } elsif (ref($t) eq 'HASH') { $lookup_type = 'hash'; if (!defined $ip_vec) { # syntactically invalid IP address $fullkey = undef; $result = $t->{$fullkey}; # only matches undef key $found=1 if defined $result; } else { # valid IP address # match a canonical IP address: dot-quad IPv4, or preferred IPv6 form my($ip_c); # IP address in the canonical form: x:x:x:x:x:x:x:x my($ip_dq); # IPv4 in a dotted-quad form if IPv4-mapped, or undef $ip_c = join(':', map(sprintf('%04x',$_), unpack('n8',$ip_vec))); my($ipv4_vec,$ipv4_mask) = ip_to_vec('::FFFF:0:0/96',1); if ( ($ip_vec & $ipv4_mask) eq ($ipv4_vec & $ipv4_mask) ) { # is an IPv4-mapped IPv6 address, format it as a dot-quad form $ip_dq = join('.', unpack('C4',substr($ip_vec,12,4))); # last 32 bits } do_log(5, 'lookup_ip_acl keys: "%s", "%s"', $ip_dq,$ip_c); if (defined $ip_dq) { # try dot-quad, stripping off trailing bytes for (my(@f)=split(/\./,$ip_dq); @f && !$found; $#f--) { $fullkey = join('.',@f); $result = $t->{$fullkey}; $found=1 if defined $result; } } if (!$found) { # try the 'preferred IPv6 form' $fullkey = $ip_c; $result = $t->{$fullkey}; $found=1 if defined $result; } } } elsif (ref($t) eq 'ARRAY') { $lookup_type = 'array'; my($key,$acl_ip_vec,$acl_mask,$acl_mask_len); local($1,$2); for my $net (@$t) { $fullkey = $key = $net; $result = 1; if ($key =~ /^(!+)(.*)\z/s) { # starts with exclamation mark(s) $key = $2; $result = 1 - $result if (length($1) & 1); # negate if odd } ($acl_ip_vec, $acl_mask, $acl_mask_len) = ip_to_vec($key,1); if ($acl_mask_len == 0) { $found=1 } #even an invalid addr matches ::/0 elsif (!defined($ip_vec)) {} # no other matches for invalid address elsif (($ip_vec & $acl_mask) eq ($acl_ip_vec & $acl_mask)) { $found=1 } last if $found; } } elsif ($t->isa('Net::Patricia::AF_INET6')) { # Patricia Trie $lookup_type = 'patricia'; local($1,$2,$3,$4); local($_) = $ip; $_ = $1 if /^ \[ ( [^\]]* ) \] \z/xs; # discard optional brackets s/%[A-Z0-9:._-]+\z//si; # discard interface specification if (m{^ (\d+) \. (\d+) \. (\d+) \. (\d+) \z}x) { $_ = sprintf('::ffff:%d.%d.%d.%d', $1,$2,$3,$4); } else { s/^IPv6://i; # discard optional 'IPv6:' prefix } eval { $result = $t->match_string($_); 1 } or $result=undef; if (defined $result) { $fullkey = $result; if ($fullkey =~ s/^!//) { $result = 0 } else { $result = 1; $found = 1 } } } elsif ($t->isa('Amavis::Lookup::IP')) { # pre-parsed IP lookup array obj $lookup_type = 'arr.obj'; my($acl_ip_vec, $acl_mask, $acl_mask_len); for my $e (@$t) { ($fullkey, $acl_ip_vec, $acl_mask, $acl_mask_len, $result) = @$e; if ($acl_mask_len == 0) { $found=1 } #even an invalid addr matches ::/0 elsif (!defined($ip_vec)) {} # no other matches for invalid address elsif (($ip_vec & $acl_mask) eq ($acl_ip_vec & $acl_mask)) { $found=1 } last if $found; } } elsif ($t->isa('Amavis::Lookup::Label')) { # logging label # just a convenience for logging purposes, not a real lookup method $label = $t->display; # grab the name, and proceed with the next table } else { die "TROUBLE: lookup table is an unknown object: " . ref($t); } last if $found; } $fullkey = $result = undef if !$found; if ($label ne '') { $label = " ($label)" } ll(4) && do_log(4, 'lookup_ip_acl%s %s: key="%s"%s', $label, $lookup_type, $ip, !$found ? ", no match" : " matches \"$fullkey\", result=$result"); if (defined $eval_stat) { chomp $eval_stat; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout $eval_stat = "lookup_ip_acl$label: $eval_stat"; do_log(2, "%s", $eval_stat); } !wantarray ? $result : ($result, $fullkey, $eval_stat); } # create a pre-parsed object from a list of IP networks, # which may be used as an argument to lookup_ip_acl to speed up its searches # sub new($@) { my($class,@nets) = @_; my($build_patricia_trie) = $have_patricia && (@nets > 20); if (!$build_patricia_trie) { # build a traditional pre-parsed search list for a small number of entries my(@list); local($1,$2); for my $net (@nets) { my($key) = $net; my($result) = 1; if ($key =~ /^(!+)(.*)\z/s) { # starts with exclamation mark(s) $key = $2; $result = 1 - $result if (length($1) & 1); # negate if odd } my($ip_vec, $ip_mask, $ip_mask_len) = ip_to_vec($key,1); push(@list, [$net, $ip_vec, $ip_mask, $ip_mask_len, $result]); } return bless(\@list, $class); } else { # build a patricia trie, it offers more efficient searching in large sets my($pt) = Net::Patricia->new(&AF_INET6); do_log(5, "building a patricia trie out of %d nets", scalar(@nets)); for my $net (@nets) { local $_ = $net; local($1,$2,$3,$4); my($masklen); if (s{ / ([0-9.]+) \z }{}x) { $masklen = $1; $masklen =~ /^\d{1,3}\z/ or die "Network mask not supported, use a CIDR syntax: $net"; } s/^!//; # strip a negation from a key, it will be retained in data $_ = $1 if /^ \[ ( [^\]]* ) \] \z/xs; # discard optional brackets s/%[A-Z0-9:._-]+\z//si; # discard interface specification if (/^ \d+ (\. | \z) /x) { # triage for an IPv4 network address if (/^ (\d+) \. (\d+) \. (\d+) \. (\d+) \z/x) { $_ = sprintf('::ffff:%d.%d.%d.%d', $1,$2,$3,$4); $masklen = 32 if !defined $masklen; } elsif (/^ (\d+) \. (\d+) \. (\d+) \.? \z/x) { $_ = sprintf('::ffff:%d.%d.%d.0', $1,$2,$3); $masklen = 24 if !defined $masklen; } elsif (/^ (\d+) \. (\d+) \.? \z/x) { $_ = sprintf('::ffff:%d.%d.0.0', $1,$2); $masklen = 16 if !defined $masklen; } elsif (/^ (\d+) \.? \z/x) { $_ = sprintf('::ffff:%d.0.0.0', $1); $masklen = 8 if !defined $masklen; } $masklen += 96 if defined $masklen; } else { # looks like an IPv6 network s/^IPv6://i; # discard optional 'IPv6:' prefix } $masklen = 128 if !defined $masklen; $_ .= '/' . $masklen; eval { $pt->add_string($_, $net); 1 } or die "Adding a network $net to a patricia trie failed: $@"; } # ll(5) && $pt->climb(sub { do_log(5,"patricia trie, node $_[0]") }); return $pt; # a Net::Patricia::AF_INET6 object } } 1; # package Amavis::Lookup::Opaque; use strict; use re 'taint'; # Make an object out of the supplied argument, pretecting it # from being interpreted as an acl- or a hash- type lookup. # sub new($$) { my($class,$obj) = @_; bless \$obj, $class } sub get($) { ${$_[0]} } 1; # package Amavis::Lookup::OpaqueRef; use strict; use re 'taint'; # Make an object out of the supplied argument, pretecting it # from being interpreted as an acl- or a hash- type lookup. # The argument to new() is expected to be a ref to a variable, # which will be dereferenced by a method get(). # sub new($$) { my($class,$obj) = @_; bless \$obj, $class } sub get($) { ${${$_[0]}} } 1; # package Amavis::Lookup::Label; use strict; use re 'taint'; # Make an object out of the supplied string, to serve as label # in log messages generated by sub lookup # sub new($$) { my($class,$str) = @_; bless \$str, $class } sub display($) { ${$_[0]} } 1; # package Amavis::Lookup::SQLfield; use strict; use re 'taint'; sub new($$$;$$) { my($class, $sql_query, $fieldname, $fieldtype, $implied_args) = @_; my $self = bless { fieldname => $fieldname, fieldtype => $fieldtype }, $class; $self->{sql_query} = $sql_query if defined $sql_query; $self->{args} = ref($implied_args) eq 'ARRAY' ? [@$implied_args] # copy : [$implied_args] if defined $implied_args; $self; } 1; # package Amavis::Lookup::LDAPattr; use strict; use re 'taint'; sub new($$$;$) { my($class, $ldap_query, $attrname, $attrtype) = @_; my $self = bless { attrname => $attrname, attrtype => $attrtype }, $class; $self->{ldap_query} = $ldap_query if defined $ldap_query; $self; } 1; # package Amavis::Lookup; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&lookup &lookup2 &lookup_hash &lookup_acl); import Amavis::Util qw(ll do_log fmt_struct unique_list); import Amavis::Conf qw(:platform c cr ca); import Amavis::Timing qw(section_time); import Amavis::rfc2821_2822_Tools qw(split_address make_query_keys); } use subs @EXPORT_OK; # lookup_hash() performs a lookup for an e-mail address against a hash map. # If a match is found (a hash key exists in the Perl hash) the function returns # whatever the map returns, otherwise undef is returned. First match wins, # aborting further search sequence. # sub lookup_hash($$;$%) { my($addr, $hash_ref,$get_all,%options) = @_; ref($hash_ref) eq 'HASH' or die "lookup_hash: arg2 must be a hash ref: $hash_ref"; local($1,$2,$3,$4); my(@matchingkey,@result); my($append_string); $append_string = $options{AppendStr} if defined $options{AppendStr}; my($keys_ref,$rhs_ref) = make_query_keys($addr,1,1,$append_string); for my $key (@$keys_ref) { # do the search if (exists $$hash_ref{$key}) { # got it push(@result,$$hash_ref{$key}); push(@matchingkey,$key); last if !$get_all; } } # do the right-hand side replacements if any $n, ${n} or $(n) is specified for my $r (@result) { # remember that $r is just an alias to array elements if (defined($r) && !ref($r) && index($r,'$') >= 0) { # plain string with $ my($any) = $r =~ s{ \$ ( (\d+) | \{ (\d+) \} | \( (\d+) \) ) } { my($j)=$2+$3+$4; $j<1 ? '' : $rhs_ref->[$j-1] }gxse; # bring taintedness of input to the result $r .= substr($addr,0,0) if $any; } } if (!ll(5)) { # only bother with logging when needed } elsif (!@result) { do_log(5,"lookup_hash(%s), no matches", $addr); } elsif (!$get_all) { # first match wins do_log(5,'lookup_hash(%s) matches key "%s", result=%s', $addr, $matchingkey[0], !defined($result[0])?'undef':$result[0]); } else { # want all matches do_log(5,"lookup_hash(%s) matches keys: %s", $addr, join(', ', map {sprintf('"%s"=>%s',$matchingkey[$_],$result[$_])} (0..$#result)) ); } if (!$get_all) { !wantarray ? $result[0] : ($result[0], $matchingkey[0]) } else { !wantarray ? \@result : (\@result, \@matchingkey) } } # lookup_acl() performs a lookup for an e-mail address against # access control list. # # The supplied e-mail address is compared with each member of the # lookup list in turn, the first match wins (terminates the search), # and its value decides whether the result is true (yes, permit, pass) # or false (no, deny, drop). Falling through without a match produces # false (undef). Search is always case-insensitive on domain part, # local part matching depends on $localpart_is_case_sensitive setting. # # NOTE: lookup_acl is not aware of address extensions and they are # not handled specially! # # If a list element contains a '@', the full e-mail address is compared, # otherwise if a list element has a leading dot, the domain name part is # matched only, and the domain as well as its subdomains can match. If there # is no leading dot, the domain must match exactly (subdomains do not match). # # The presence of a character '!' prepended to a list element decides # whether the result will be true (without a '!') or false (with '!') # in case where this list element matches and terminates the search. # # Because search stops at the first match, it only makes sense # to place more specific patterns before the more general ones. # # Although not a special case, it is good to remember that '.' always matches, # so a '.' would stop the search and return true, whereas '!.' would stop the # search and return false (0). # # Examples: # # given: @acl = qw( me.ac.uk !.ac.uk .uk ) # 'me.ac.uk' matches me.ac.uk, returns true and search stops # # given: @acl = qw( me.ac.uk !.ac.uk .uk ) # 'you.ac.uk' matches .ac.uk, returns false (because of '!') and search stops # # given: @acl = qw( me.ac.uk !.ac.uk .uk ) # 'them.co.uk' matches .uk, returns true and search stops # # given: @acl = qw( me.ac.uk !.ac.uk .uk ) # 'some.com' does not match anything, falls through and returns false (undef) # # given: @acl = qw( me.ac.uk !.ac.uk .uk !. ) # 'some.com' similar to previous, except it returns 0 instead of undef, # which would only make a difference if this ACL is not the last argument # in a call to lookup(), because a defined result stops further lookups # # given: @acl = qw( me.ac.uk !.ac.uk .uk . ) # 'some.com' matches catchall ".", and returns true. The ".uk" is redundant # # more complex example: @acl = qw( # !The.Boss@dept1.xxx.com .dept1.xxx.com # .dept2.xxx.com .dept3.xxx.com lab.dept4.xxx.com # sub.xxx.com !.sub.xxx.com # me.d.aaa.com him.d.aaa.com !.d.aaa.com .aaa.com # ); # sub lookup_acl($$%) { my($addr, $acl_ref,%options) = @_; ref($acl_ref) eq 'ARRAY' or die "lookup_acl: arg2 must be a list ref: $acl_ref"; return undef if !@$acl_ref; # empty list can't match anything my($lpcs) = c('localpart_is_case_sensitive'); my($localpart,$domain) = split_address($addr); $domain = lc($domain); $localpart = lc($localpart) if !$lpcs; local($1,$2); # chop off leading @ and trailing dots $domain = $1 if $domain =~ /^\@?(.*?)\.*\z/s; $domain .= $options{AppendStr} if defined $options{AppendStr}; my($matchingkey, $result); my($found) = 0; for my $e (@$acl_ref) { $result = 1; $matchingkey = $e; my($key) = $e; if ($key =~ /^(!+)(.*)\z/s) { # starts with an exclamation mark(s) $key = $2; $result = 1-$result if length($1) & 1; # negate if odd } if ($key =~ /^(.*?)\@([^\@]*)\z/s) { # contains '@', check full address $found=1 if $localpart eq ($lpcs?$1:lc($1)) && $domain eq lc($2); } elsif ($key =~ /^\.(.*)\z/s) { # leading dot: domain or subdomain my($key_t) = lc($1); $found=1 if $domain eq $key_t || $domain =~ /(\.|\z)\Q$key_t\E\z/s; } else { # match domain (but not its subdomains) $found=1 if $domain eq lc($key); } last if $found; } $matchingkey = $result = undef if !$found; do_log(5, "lookup_acl(%s)%s", $addr, (!$found ? ", no match" : " matches key \"$matchingkey\", result=$result")); !wantarray ? $result : ($result, $matchingkey); } # Perform a lookup for an e-mail address against any number of supplied maps: # - SQL map, # - LDAP map, # - hash map (associative array), # - (access control) list, # - a list of regular expressions (an Amavis::Lookup::RE object), # - a (defined) scalar always matches, and returns itself as the map value # (useful as a catchall for a final 'pass' or 'fail'); # (see lookup_hash, lookup_acl, lookup_sql and lookup_ldap for details). # # when $get_all is 0 (the common usage): # If a match is found (a defined value), returns whatever the map returns, # otherwise returns undef. FIRST match aborts further search sequence. # when $get_all is true: # Collects a list of results from ALL matching tables, and within each # table from ALL matching key. Returns a ref to a list of results # (and a ref to a list of matching keys if returning a pair). # The first element of both lists is supposed to be what lookup() would # have returned if $get_all were 0. The order of returned elements # corresponds to the order of the search. # # traditional API, deprecated # sub lookup($$@) { my($get_all, $addr, @tables) = @_; lookup2($get_all, $addr, \@tables); } # generalized API # sub lookup2($$$%) { my($get_all, $addr, $tables_ref, %options) = @_; (@_ - 3) % 2 == 0 or die "lookup2: options argument not in pairs (not hash)"; my($label, @result, @matchingkey); for my $tb (!$tables_ref ? () : @$tables_ref) { my($t) = ref($tb) eq 'REF' ? $$tb : $tb; # allow one level of indirection my($reft) = ref($t); if ($reft eq 'CODE') { # lazy evaluation $t = &$t($addr,$get_all,%options); $reft = ref($t); } if (!$reft || $reft eq 'SCALAR') { # a scalar always matches my($r) = $reft ? $$t : $t; # allow direct or indirect reference if (defined $r) { do_log(5,'lookup: (scalar) matches, result="%s"', $r); push(@result,$r); push(@matchingkey,"(constant:$r)"); } } elsif ($reft eq 'HASH') { my($r,$mk); ($r,$mk) = lookup_hash($addr,$t,$get_all,%options) if %$t; if (!defined $r) {} elsif (!$get_all) { push(@result,$r); push(@matchingkey,$mk) } elsif (@$r) { push(@result,@$r); push(@matchingkey,@$mk) } } elsif ($reft eq 'ARRAY') { my($r,$mk); ($r,$mk) = lookup_acl($addr,$t,%options) if @$t; if (defined $r) { push(@result,$r); push(@matchingkey,$mk) } } elsif ($t->isa('Amavis::Lookup::Label')) { # logging label # just a convenience for logging purposes, not a real lookup method $label = $t->display; # grab the name, and proceed with the next table } elsif ($t->isa('Amavis::Lookup::Opaque') || # a structured constant $t->isa('Amavis::Lookup::OpaqueRef')) { # ref to structured const my($r) = $t->get; # behaves like a constant pseudo-lookup if (defined $r) { do_log(5,'lookup: (opaque) matches, result="%s"', $r); push(@result,$r); push(@matchingkey,"(opaque:$r)"); } } elsif ($t->isa('Amavis::Lookup::RE')) { my($r,$mk); ($r,$mk) = $t->lookup_re($addr,$get_all,%options) if @$t; if (!defined $r) {} elsif (!$get_all) { push(@result,$r); push(@matchingkey,$mk) } elsif (@$r) { push(@result,@$r); push(@matchingkey,@$mk) } } elsif ($t->isa('Amavis::Lookup::SQL')) { my($r,$mk) = $t->lookup_sql($addr,$get_all,%options); if (!defined $r) {} elsif (!$get_all) { push(@result,$r); push(@matchingkey,$mk) } elsif (@$r) { push(@result,@$r); push(@matchingkey,@$mk) } } elsif ($t->isa('Amavis::Lookup::SQLfield')) { if ($Amavis::sql_lookups) { # triage my($r,$mk) = $t->lookup_sql_field($addr,$get_all,%options); if (!defined $r) {} elsif (!$get_all) { push(@result,$r); push(@matchingkey,$mk) } elsif (@$r) { push(@result,@$r); push(@matchingkey,@$mk) } } } elsif ($t->isa('Amavis::Lookup::LDAP')) { my($r,$mk) = $t->lookup_ldap($addr,$get_all,%options); if (!defined $r) {} elsif (!$get_all) { push(@result,$r); push(@matchingkey,$mk) } elsif (@$r) { push(@result,@$r); push(@matchingkey,@$mk) } } elsif ($t->isa('Amavis::Lookup::LDAPattr')) { if ($Amavis::ldap_lookups) { # triage my($r,$mk) = $t->lookup_ldap_attr($addr,$get_all,%options); if (!defined $r) {} elsif (!$get_all) { push(@result,$r); push(@matchingkey,$mk) } elsif (@$r) { push(@result,@$r); push(@matchingkey,@$mk) } } } else { die "TROUBLE: lookup table is an unknown object: " . $reft; } last if @result && !$get_all; } # pretty logging if (ll(4)) { # only bother preparing log report which will be printed my($opt_label); $opt_label = $options{Label}; my(@lbl) = grep(defined $_ && $_ ne '', ($opt_label,$label)); $label = ' [' . join(',',unique_list(\@lbl)) . ']' if @lbl; if (!$tables_ref || !@$tables_ref) { do_log(4, "lookup%s => undef, %s, no lookup tables", $label, fmt_struct($addr)); } elsif (!@result) { do_log(4, "lookup%s => undef, %s does not match", $label, fmt_struct($addr)); } elsif (!$get_all) { # first match wins do_log(4, 'lookup%s => %-6s %s matches, result=%s, matching_key="%s"', $label, $result[0] ? 'true,' : 'false,', fmt_struct($addr), fmt_struct($result[0]), $matchingkey[0]); } else { # want all matches do_log(4, 'lookup%s, %d matches for %s, results: %s', $label, scalar(@result), fmt_struct($addr), join(', ', map { sprintf('"%s"=>%s', $matchingkey[$_], fmt_struct($result[$_])) } (0 .. $#result) )); } } if (!$get_all) { !wantarray ? $result[0] : ($result[0], $matchingkey[0]) } else { !wantarray ? \@result : (\@result, \@matchingkey) } } 1; # package Amavis::Expand; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&expand &tokenize); import Amavis::Util qw(ll do_log); } use subs @EXPORT_OK; # Given a string reference and a hashref of predefined (builtin) macros, # expand() performs a macro expansion and returns a ref to a resulting string. # # This is a simple, yet fully fledged macro processor with proper lexical # analysis, call stack, quoting levels, user supplied and builtin macros, # three builtin flow-control macros: selector, regexp selector and iterator, # a macro-defining macro and a macro '#' that eats input to the next newline. # Also recognized are the usual \c and \nnn forms for specifying special # characters, where c can be any of: r, n, f, b, e, a, t. # Details are described in file README.customize, practical examples of use # are in the supplied notification messages; # Author: Mark Martinec , 2002, 2006 use vars qw(%builtins_cached %lexmap %esc); use vars qw($lx_lb $lx_lbS $lx_lbT $lx_lbA $lx_lbC $lx_lbE $lx_lbQQ $lx_rbQQ $lx_rb $lx_sep $lx_h $lx_ph); BEGIN { no warnings 'qw'; # avoid "Possible attempt to put comments in qw()" my(@lx_str) = qw( [ [? [~ [@ [: [= [" "] ] | # %# %0 %1 %2 %3 %4 %5 %6 %7 %8 %9); # lexical elem. # %lexmap maps string to reference in order to protect lexels $lexmap{$_} = \$_ for @lx_str; # maps lexel strings to references ($lx_lb, $lx_lbS, $lx_lbT, $lx_lbA, $lx_lbC, $lx_lbE, $lx_lbQQ, $lx_rbQQ, $lx_rb, $lx_sep, $lx_h, $lx_ph) = map($lexmap{$_}, @lx_str); %esc = (n => \"\n", r => "\r", f => "\f", b => "\b", e => "\e", a => "\a", t => "\t"); # NOTE that \n is specific, it is represented by a ref to a newline and not # by a newline itself; this makes it possible for a macro '#' to skip input # to a true newline from source, making it possible to comment-out entire # lines even if they contain "\n" tokens } # make an object out of the supplied list of tokens sub newmacro { my($class) = shift; bless [@_], $class } # turn a ref to a list of tokens into a single plain string sub tokens_list_to_str($) { join('', map(ref($_) ? $$_ : $_, @{$_[0]})) } sub tokenize($;$) { my($str_ref,$tokens_ref) = @_; local($1); $tokens_ref = [] if !defined $tokens_ref; # parse lexically, replacing lexical element strings with references, # unquoting backslash-quoted characters and %%, and dropping \NL and \_ @$tokens_ref = map { exists $lexmap{$_} ? $lexmap{$_} # replace with ref : $_ eq "\\\n" || $_ eq "\\_" ? '' # drop \NEWLINE and \_ : $_ eq '%%' ? '%' # %% -> % : /^(%#?.)\z/s ? \"$1" # unknown builtins : /^\\([0-7]{1,3})\z/ ? chr(oct($1)) # \nnn : /^\\(.)\z/s ? (exists($esc{$1}) ? $esc{$1} : $1) # \r, \n, \f, ... : /^(_ [A-Z]+ (?: \( [^)]* \) )? _)\z/sx ? \"$1" # SpamAssassin-compatible : $_ } $$str_ref =~ /\G \# | \[ [?~\@:="]? | "\] | \] | \| | % \#? . | \\ [^0-7] | \\ [0-7]{1,3} | _ [A-Z]+ (?: \( [^)]* \) )? _ | [^\[\]\\|%\n#"_]+ | [^\n]+? | \n /gsx; $tokens_ref; } sub evalmacro($$;@) { my($macro_type,$builtins_href,@args) = @_; my(@result); local($1,$2); if ($macro_type == $lx_lbS) { # selector built-in macro my($sel) = tokens_list_to_str(shift(@args)); if ($sel =~ /^\s*\z/) { $sel = 0 } elsif ($sel =~ /^\s*(\d+)\s*\z/) { $sel = 0+$1 } # make numeric else { $sel = 1 } # provide an empty second alternative if we only have one specified if (@args < 2) {} # keep $sel beyond $#args elsif ($sel > $#args) { $sel = $#args } # use last alternative @result = @{$args[$sel]} if $sel >= 0 && $sel <= $#args; } elsif ($macro_type == $lx_lbT) { # regexp built-in macro # args: string, regexp1, then1, regexp2, then2, ... regexpN, thenN[, else] my($str) = tokens_list_to_str(shift(@args)); # collect the first argument my($match,@repl); while (@args >= 2) { # at least a regexp and a 'then' argument still there @repl = (); my($regexp) = tokens_list_to_str(shift(@args)); # collect a regexp arg ""=~/x{0}/; #braindead Perl: serves as explicit deflt for an empty regexp eval { # guard against invalid regular expression local($1,$2,$3,$4,$5,$6,$7,$8,$9); $match = $str=~/$regexp/ ? 1 : 0; @repl = ($1,$2,$3,$4,$5,$6,$7,$8,$9) if $match; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout do_log(2,"invalid macro regexp arg: %s", $eval_stat); $match = 0; @repl = (); }; if ($match) { last } else { shift(@args) } # skip 'then' arg if no match } if (@args > 0) { unshift(@repl,$str); # prepend the whole string as a %0 # formal arg lexels %0, %1, ... %9 are replaced by captured substrings @result = map(!ref || $$_!~/^%([0-9])\z/ ? $_ : $repl[$1], @{$args[0]}); } } elsif ($macro_type == $lx_lb) { # iterator macro my($cvar_r,$sep_r,$body_r); my($cvar); # give meaning to arguments if (@args >= 3) { ($cvar_r,$body_r,$sep_r) = @args } else { ($body_r,$sep_r) = @args; $cvar_r = $body_r } # find the iterator name for (@$cvar_r) { if (ref && $$_ =~ /^%(.)\z/s) { $cvar = $1; last } } my($name) = $cvar; # macro name is usually the same as the iterator name if (@args >= 3 && !defined($name)) { # instead of iterator like %x, the first arg may be a long macro name, # in which case the iterator name becomes a hard-wired 'x' $name = tokens_list_to_str($cvar_r); $name =~ s/^[ \t\n]+//; $name =~ s/[ \t\n]+\z//; # trim whitespace if ($name eq '') { $name = undef } else { $cvar = 'x' } } if (exists($builtins_href->{$name})) { my($s) = $builtins_href->{$name}; if (ref($s) eq 'Amavis::Expand') { # expand a dynamically defined macro my(@margs) = ($name); # no arguments beyond %0 my(@res) = map(!ref || $$_ !~ /^%([0-9])\z/ ? $_ : ref($margs[$1]) ? @{$margs[$1]} : (), @$s); $s = tokens_list_to_str(\@res); } elsif (ref($s) eq 'CODE') { if (exists($builtins_cached{$name})) { $s = $builtins_cached{$name}; } else { while (ref($s) eq 'CODE') { $s = &$s($name) } $builtins_cached{$name} = $s; } } my($ind) = 0; for my $val (ref($s) ? @$s : $s) { # do substitutions in the body push(@result, @$sep_r) if ++$ind > 1 && ref($sep_r); push(@result, map(ref && $$_ eq "%$cvar" ? $val : $_, @$body_r)); } } } elsif ($macro_type == $lx_lbE) { # define a new macro my($name) = tokens_list_to_str(shift(@args)); # first arg is a macro name $name =~ s/^[ \t\n]+//; $name =~ s/[ \t\n]+\z//; # trim whitespace on name delete $builtins_cached{$name}; $builtins_href->{$name} = Amavis::Expand->newmacro(@{$args[0]}); } elsif ($macro_type == $lx_lbA || $macro_type == $lx_lbC || # macro call $$macro_type =~ /^%(\#)?(.)\z/s) { my($name); my($cardinality_only) = 0; if ($macro_type == $lx_lbA || $macro_type == $lx_lbC) { $name = tokens_list_to_str($args[0]); # arg %0 is a macro name $name =~ s/^[ \t\n]+//; $name =~ s/[ \t\n]+\z//; # trim whitespace } else { # simple macro call %x or %#x $name = $2; $cardinality_only = 1 if defined $1; } my($s) = $builtins_href->{$name}; if (!ref($s)) { # macro expands to a plain string if (!$cardinality_only) { @result = $s } else { @result = $s !~ /^\s*\z/ ? 1 : 0 }; # %#x => nonwhite=1, other 0 } elsif (ref($s) eq 'Amavis::Expand') { # dynamically defined macro $args[0] = $name; # replace name with a stringified and trimmed form # expanding a dynamically-defined macro produces a list of tokens; # formal argument lexels %0, %1, ... %9 are replaced by actual arguments @result = map(!ref || $$_ !~ /^%([0-9])\z/ ? $_ : ref($args[$1]) ? @{$args[$1]} : (), @$s); if ($cardinality_only) { # macro call form %#x @result = tokens_list_to_str(\@result) !~ /^\s*\z/ ? 1 : 0; } } else { # subroutine or array ref if (ref($s) eq 'CODE') { if (exists($builtins_cached{$name}) && @args <= 1) { $s = $builtins_cached{$name}; } elsif (@args <= 1) { while (ref($s) eq 'CODE') { $s = &$s($name) } # callback $builtins_cached{$name} = $s; } else { shift(@args); # discard original form of a macro name while (ref($s) eq 'CODE') # subroutine callback { $s = &$s($name, map(tokens_list_to_str($_), @args)) } } } if ($cardinality_only) { # macro call form %#x # for array: number of elements; for scalar: nonwhite=1, other 0 @result = ref($s) ? scalar(@$s) : $s !~ /^\s*\z/ ? 1 : 0; } else { # macro call %x evaluates to the value of macro x @result = ref($s) ? join(', ',@$s) : $s; } } } \@result; } sub expand($$) { my($str_ref) = shift; # a ref to a source string to be macro expanded; my($builtins_href) = shift; # a hashref, mapping builtin macro names # to macro values: strings or array refs my(@tokens); if (ref($str_ref) eq 'ARRAY') { @tokens = @$str_ref } else { tokenize($str_ref,\@tokens) } my($call_level) = 0; my($quote_level) = 0; my(@arg); # stack of arguments lists to nested calls, [0] is top of stack my(@macro_type); # call stack of macro types (leading lexels) of nested calls my(@implied_q); # call stack: is implied quoting currently active? # 0 (not active) or 1 (active); element [0] stack top my(@open_quote); # quoting stack: opening quote lexel for each quoting level %builtins_cached = (); my($whereto); local($1,$2); # preallocate some storage my($output_str) = ''; vec($output_str,2048,8) = 0; $output_str = ''; while (@tokens) { my($t) = shift(@tokens); # do_log(5, "TOKEN: %s", ref($t) ? "<$$t>" : "'$t'"); if (!ref($t)) { # a plain string, no need to check for quoting levels if (defined $whereto) { push(@$whereto,$t) } else { $output_str .= $t } } elsif ($quote_level > 0 && $$t =~ /^\[/) { # go even deeper into quoting $quote_level += ($t == $lx_lbQQ) ? 2 : 1; unshift(@open_quote,$t); if (defined $whereto) { push(@$whereto,$t) } else { $output_str .= $$t } } elsif ($t == $lx_lbQQ) { # just entering a [" ... "] quoting context $quote_level += 2; unshift(@open_quote,$t); # drop a [" , thus stripping one level of quotes } elsif ($$t =~ /^\[/) { # $lx_lb $lx_lbS lx_lbT $lx_lbA $lx_lbC $lx_lbE $call_level++; # open a macro call, start collecting arguments unshift(@arg, [[]]); unshift(@macro_type, $t); unshift(@implied_q, 0); $whereto = $arg[0][0]; if ($t == $lx_lb) { # iterator macro implicitly quotes all arguments $quote_level++; unshift(@open_quote,$t); $implied_q[0] = 1; } } elsif ($quote_level <= 1 && $call_level>0 && $t == $lx_sep) { # next arg unshift(@{$arg[0]}, []); $whereto = $arg[0][0]; if ($macro_type[0]==$lx_lbS && @{$arg[0]} == 2) { # selector macro implicitly quotes arguments beyond first argument $quote_level++; unshift(@open_quote,$macro_type[0]); $implied_q[0] = 1; } } elsif ($quote_level > 1 && ($t == $lx_rb || $t == $lx_rbQQ)) { $quote_level -= ($open_quote[0] == $lx_lbQQ) ? 2 : 1; shift(@open_quote); # pop the quoting stack if ($t == $lx_rb || $quote_level > 0) { # pass-on if still quoted if (defined $whereto) { push(@$whereto,$t) } else { $output_str .= $$t} } } elsif ($call_level > 0 && ($t == $lx_rb || $t == $lx_rbQQ)) { # evaluate $call_level--; my($m_type) = $macro_type[0]; if ($t == $lx_rbQQ) { # fudge for compatibility: treat "] as two chars if (defined $whereto) { push(@$whereto,'"') } else { $output_str.='"' } } if ($implied_q[0] && $quote_level > 0) { $quote_level -= ($open_quote[0] == $lx_lbQQ) ? 2 : 1; shift(@open_quote); # pop the quoting stack } my($result_ref) = evalmacro($m_type, $builtins_href, reverse @{$arg[0]}); shift(@macro_type); shift(@arg); shift(@implied_q); # pop the call stack $whereto = $call_level > 0 ? $arg[0][0] : undef; if ($m_type == $lx_lbC) { # neutral macro call, result implicitly quoted if (defined $whereto) { push(@$whereto, @$result_ref) } else { $output_str .= tokens_list_to_str($result_ref) } } else { # active macro call, push result back to input for reprocessing unshift(@tokens, @$result_ref); } } elsif ($quote_level > 0 ) { # still protect %x and # macro calls if (defined $whereto) { push(@$whereto,$t) } else { $output_str .= $$t } } elsif ($t == $lx_h) { # discard tokens up to and including a newline while (@tokens) { last if shift(@tokens) eq "\n" } } elsif ($$t =~ /^%\#?.\z/s) { # neutral simple macro call %x or %#x my($result_ref) = evalmacro($t, $builtins_href); if (defined $whereto) { push(@$whereto,@$result_ref) } # else { $output_str .= tokens_list_to_str($result_ref) } else { $output_str .= join('', map(ref($_) ? $$_ : $_, @$result_ref)) } } elsif ($$t =~ /^_ ([A-Z]+) (?: \( ( [^)]* ) \) )? _\z/sx) { # neutral simple SA-like macro call, $1 is name, $2 is a single! argument my($result_ref) = evalmacro($lx_lbC, $builtins_href, [$1], !defined($2) ? () : [$2] ); if (defined $whereto) { push(@$whereto, @$result_ref) } else { $output_str .= tokens_list_to_str($result_ref) } } else { # misplaced top-level lexical element if (defined $whereto) { push(@$whereto,$t) } else { $output_str .= $$t } } } %builtins_cached = (); # clear memory \$output_str; } 1; # package Amavis::TempDir; # Handles creation and cleanup of persistent temporary directory, # file 'email.txt' therein, and subdirectory 'parts' use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::Timing qw(section_time); import Amavis::Util qw(ll do_log add_entropy rmdir_recursively); import Amavis::rfc2821_2822_Tools qw(iso8601_timestamp); } use Errno qw(ENOENT EACCES EEXIST); use IO::File qw(O_RDONLY O_WRONLY O_RDWR O_APPEND O_CREAT O_EXCL); use File::Temp (); sub new { my($class) = @_; my($self) = bless {}, $class; $self->{tempdir_path} = undef; undef $self->{tempdir_dev}; undef $self->{tempdir_ino}; undef $self->{fh_pers}; undef $self->{fh_dev}; undef $self->{fh_ino}; $self->{empty} = 1; $self->{preserve} = 0; $self; } sub path { # path to a temporary directory my $self=shift; !@_ ? $self->{tempdir_path} : ($self->{tempdir_path}=shift) } sub fh { # email.txt file handle my $self=shift; !@_ ? $self->{fh_pers} : ($self->{fh_pers}=shift); } sub empty { # whether the directory is empty my $self=shift; !@_ ? $self->{empty} : ($self->{empty}=shift) } sub preserve { # whether to preserve directory when current task is done my $self=shift; !@_ ? $self->{preserve} : ($self->{preserve}=shift); } # Clean up the tempdir on shutdown # sub DESTROY { my $self = shift; local($@,$!,$_); my($myactualpid) = $$; if (defined($my_pid) && $myactualpid != $my_pid) { eval { do_log(5,"TempDir::DESTROY skip, clone [%s] (born as [%s])", $myactualpid, $my_pid) }; } else { eval { do_log(5,"TempDir::DESTROY called") }; eval { # must step out of the directory which is about to be deleted, # otherwise rmdir can fail (e.g. on Solaris) chdir($TEMPBASE) or do_log(-1,"TempDir::DESTROY can't chdir to %s: %s", $TEMPBASE, $!); $self->{fh_pers}->close or do_log(-1,"Error closing temp file: %s",$!) if $self->{fh_pers}; undef $self->{fh_pers}; my($dname) = $self->{tempdir_path}; my($errn) = !defined($dname) || $dname eq '' ? ENOENT : lstat($dname) ? 0 : 0+$!; if (defined($dname) && $errn != ENOENT) { # this will not be included in the TIMING report, # but it only occurs infrequently and doesn't take that long if ($self->{preserve} && !$self->{empty}) { do_log(-1,"TempDir removal: tempdir is to be PRESERVED: %s", $dname); } else { do_log(3, "TempDir removal: %s is being removed: %s%s", $self->{empty} ? 'empty tempdir' : 'tempdir', $dname, $self->{preserve} ? ', nothing to preserve' : ''); rmdir_recursively($dname); } }; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; eval { do_log(1,"TempDir removal: %s",$eval_stat) }; }; } } # Creates a temporary directory, or checks that inode did not change on reuse # sub prepare_dir { my($self) = @_; my(@stat_list); my($errn); my($reuse) = 0; my($dname) = $self->{tempdir_path}; if (defined $dname) { # hope to reuse existing directory @stat_list = lstat($dname); $errn = @stat_list ? 0 : 0+$!; if ($errn != ENOENT) { $reuse = 1; # good, it exists, try reusing it } else { do_log(2,"TempDir::prepare_dir: directory %s no longer exists", $dname); $self->{tempdir_path} = $dname = undef; $self->{empty} = 1; } } if (!defined $dname) { # invent a name of a temporary directory for this child my($dirtemplate) = sprintf("amavis-%s-%05d-XXXXXXXX", iso8601_timestamp(time,1), $my_pid); $dname = File::Temp::tempdir($dirtemplate, DIR => $TEMPBASE); defined $dname && $dname ne '' or die "Can't create a temporary directory $TEMPBASE/$dirtemplate: $!"; do_log(4,"TempDir::prepare_dir: created directory %s", $dname); chmod(0750,$dname) or die "Can't change protection on directory $dname: $!"; @stat_list = lstat($dname); @stat_list or die "Failed to access directory $dname: $!"; $self->{tempdir_path} = $dname; ($self->{tempdir_dev}, $self->{tempdir_ino}) = @stat_list; $self->{empty} = 1; add_entropy($dname, @stat_list); section_time('mkdir tempdir'); } $errn = @stat_list ? 0 : 0+$!; if ($errn != 0) { die "TempDir::prepare_dir: Can't access temporary directory $dname: $!"; } elsif (! -d _) { # exists, but is not a directory !? die "TempDir::prepare_dir: $dname is not a directory!!!"; } elsif ($reuse) { # existing directory my($dev,$ino,$mode,$nlink) = @stat_list; if ($dev != $self->{tempdir_dev} || $ino != $self->{tempdir_ino}) { do_log(-1,"TempDir::prepare_dir: %s is no longer the same directory!", $dname); ($self->{tempdir_dev}, $self->{tempdir_ino}) = ($dev, $ino); } if ($nlink > 3) { # when a directory's link count is > 2, it has "n-2" sub-directories; # this does not apply to file systems like AFS, FAT, ISO-9660, # but it also seems it does not apply to Mac OS 10 (Leopard) do_log(5, "TempDir::prepare_dir: directory %s has %d subdirectories", $dname, $nlink-2); } } } # Prepares the email.txt temporary file for writing (and reading later) # sub prepare_file { my($self) = @_; my($fname) = $self->path . '/email.txt'; my(@stat_list) = lstat($fname); my($errn) = @stat_list ? 0 : 0+$!; if ($errn == ENOENT) { # no file do_log(0,"TempDir::prepare_file: %s no longer exists, can't re-use it", $fname) if $self->{fh_pers}; undef $self->{fh_pers}; } elsif ($errn != 0) { # some other error undef $self->{fh_pers}; die "TempDir::prepare_file: can't access temporary file $fname: $!"; } elsif (! -f _) { # not a regular file !? undef $self->{fh_pers}; die "TempDir::prepare_file: $fname is not a regular file!!!"; } elsif ($self->{fh_pers}) { my($dev,$ino) = @stat_list; if ($dev != $self->{file_dev} || $ino != $self->{file_ino}) { # may happen if some user code has replaced the file, e.g. by altermime undef $self->{fh_pers}; do_log(1,"TempDir::prepare_file: %s is no longer the same file, ". "won't re-use it, deleting", $fname); unlink($fname) or die "Can't remove file $fname: $!"; } } if ($self->{fh_pers} && !$can_truncate) { # just in case clean() retained it undef $self->{fh_pers}; do_log(1,"TempDir::prepare_file: unable to truncate temporary file %s, ". "deleting it", $fname); unlink($fname) or die "Can't remove file $fname: $!"; } if ($self->{fh_pers}) { # rewind and truncate existing file $self->{fh_pers}->flush or die "Can't flush mail file: $!"; $self->{fh_pers}->seek(0,0) or die "Can't rewind mail file: $!"; $self->{fh_pers}->truncate(0) or die "Can't truncate mail file: $!"; } else { do_log(4,"TempDir::prepare_file: creating file %s", $fname); # $^F == 2 # or do_log(-1,"TempDir::prepare_file: SYSTEM_FD_MAX not 2: %d", $^F); my($newfh) = IO::File->new; # this can fail if a previous task of this process just recently stumbled # on some error and preserved its evidence, not deleting a file email.txt $newfh->open($fname, O_CREAT|O_EXCL|O_RDWR, 0640) or die "Can't create file $fname: $!"; binmode($newfh,':bytes') or die "Can't cancel :utf8 mode on $fname: $!"; if (ll(5) && $] >= 5.008001) { # get_layers was added with Perl 5.8.1 my(@layers) = PerlIO::get_layers($newfh); do_log(5,"TempDir::prepare_file: layers: %s", join(',',@layers)); } $self->{fh_pers} = $newfh; @stat_list = lstat($fname); @stat_list or die "Failed to access temporary file $fname: $!"; add_entropy(@stat_list); ($self->{file_dev}, $self->{file_ino}) = @stat_list; section_time('create email.txt'); } } # Cleans the temporary directory for reuse, unless it is set to be preserved # sub clean { my($self) = @_; if ($self->{preserve} && !$self->{empty}) { # keep evidence in case of trouble do_log(-1,"PRESERVING EVIDENCE in %s", $self->{tempdir_path}); if ($self->{fh_pers}) { $self->{fh_pers}->close or die "Error closing mail file: $!" } undef $self->{fh_pers}; $self->{tempdir_path} = undef; $self->{empty} = 1; } # cleanup, but leave directory (and file handle if possible) for reuse if ($self->{fh_pers} && !$can_truncate) { # truncate is not standard across all Unix variants, # it is not Posix, but is XPG4-UNIX. # So if we can't truncate a file and leave it open, # we have to create it anew later, at some cost. # $self->{fh_pers}->close or die "Error closing mail file: $!"; undef $self->{fh_pers}; unlink($self->{tempdir_path}.'/email.txt') or die "Can't delete file ".$self->{tempdir_path}."/email.txt: $!"; section_time('delete email.txt'); } if (defined $self->{tempdir_path}) { # prepare for the next one $self->strip; $self->{empty} = 1; } $self->{preserve} = 0; # reset } # Remove files and subdirectories from the temporary directory, leaving only # the directory itself, file email.txt, and empty subdirectory ./parts . # Leaving directories for reuse can represent an important saving in time, # as directory creation + deletion can be an expensive operation, # requiring atomic file system operation, including flushing buffers # to disk (depending on the file system in use). # sub strip { my $self = shift; my($dname) = $self->{tempdir_path}; do_log(4, "TempDir::strip: %s", $dname); # must step out of the directory which is about to be deleted, # otherwise rmdir can fail (e.g. on Solaris) chdir($TEMPBASE) or die "TempDir::strip: can't chdir to $TEMPBASE: $!"; my(@stat_list) = lstat($dname); my($errn) = @stat_list ? 0 : 0+$!; if ($errn == ENOENT) { do_log(-1,"TempDir::strip: directory %s no longer exists", $dname); $self->{tempdir_path} = $dname = undef; $self->{empty} = 1; } elsif ($errn != 0) { die "TempDir::strip: error accessing directory $dname: $!"; } else { my($dev,$ino) = @stat_list; if ($dev != $self->{tempdir_dev} || $ino != $self->{tempdir_ino}) { do_log(-1,"TempDir::strip: %s is no longer the same directory!", $dname); ($self->{tempdir_dev}, $self->{tempdir_ino}) = ($dev, $ino); } # now deal with the 'parts' subdirectory my($errn) = lstat("$dname/parts") ? 0 : 0+$!; if ($errn == ENOENT) {} # fine, no such directory elsif ($errn!=0) { die "TempDir::strip: error accessing $dname/parts: $!" } elsif ( -l _) { die "TempDir::strip: $dname/parts is a symbolic link" } elsif (!-d _) { die "TempDir::strip: $dname/parts is not a directory" } else { rmdir_recursively("$dname/parts", 1) } $self->check; # check for any remains in the top directory just in case } 1; } # Checks tempdir after being cleaned. # It may only contain subdirectory 'parts' and file email.txt, nothing else. # sub check { my $self = shift; my($eval_stat); my($dname) = $self->{tempdir_path}; local(*DIR); opendir(DIR,$dname) or die "Can't open directory $dname: $!"; eval { # avoid slurping the whole directory contents into memory $! = 0; my($f); while (defined($f = readdir(DIR))) { next if $f eq '.' || $f eq '..'; my($fname) = $dname . '/' . $f; my(@stat_list) = lstat($fname); my($errn) = @stat_list ? 0 : 0+$!; if ($errn) { die "Inaccessible $fname: $!"; } elsif (-f _) { warn "Unexpected file $fname" if $f ne 'email.txt'; } elsif (-l _) { die "Unexpected link $fname"; } elsif (-d _) { my($nlink) = $stat_list[3]; if ($f ne 'parts') { die "Unexpected directory $fname"; } elsif ($nlink > 2) { # number of hard links # when a directory's link count is > 2, it has "n-2" sub-directories; # this does not apply to file systems like AFS, FAT, ISO-9660, # but it also seems it does not apply to Mac OS 10 (Leopard) do_log(5, "TempDir::check: directory %s has %d subdirectories", $dname, $nlink-2); } } else { die "Unexpected non-regular file $fname"; } } # checking status on directory read ops doesn't work as expected, Perl bug # $!==0 or die "Error reading directory $dname: $!"; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; closedir(DIR) or die "Error closing directory $dname: $!"; if (defined $eval_stat) { chomp $eval_stat; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout die "TempDir::check: $eval_stat\n"; } 1; } 1; # package Amavis::IO::FileHandle; # Provides a virtual file (a filehandle tie - a TIEHANDLE) representing # a view to a mail message (accessed on an open file handle) prefixed by # a couple of synthesized mail header fields supplied as an array of lines. use strict; use re 'taint'; use Errno qw(EAGAIN); sub new { shift->TIEHANDLE(@_) } sub TIEHANDLE { my($class) = shift; my($self) = bless { 'fileno' => undef }, $class; if (@_) { $self->OPEN(@_) or return undef } $self; } sub UNTIE { my($self,$count) = @_; $self->CLOSE if !$count && defined $self->FILENO; 1; } sub DESTROY { my $self = $_[0]; local($@,$!,$_); $self->CLOSE if defined $self->FILENO; 1; } sub BINMODE { 1 } sub FILENO { my $self = $_[0]; $self->{'fileno'} } sub CLOSE { my $self = $_[0]; undef $self->{'fileno'}; 1 } sub EOF { my $self = $_[0]; defined $self->{'fileno'} ? $self->{'eof'} : 1 } # creates a view on an already open file, prepended by some text # sub OPEN { my($self, $filehandle,$prefix_lines_ref,$size_limit) = @_; # $filehandle is a fh of an already open file; # $prefix_lines_ref is a ref to an array of lines, to be prepended # to a created view on an existing file; these lines must each # be terminated by a \n, and must not include other \n characters $self->CLOSE if defined $self->FILENO; $self->{'fileno'} = 9999; $self->{'eof'} = 0; $self->{'prefix'} = $prefix_lines_ref; $self->{'prefix_n'} = 0; # number of lines of a prefix $self->{'prefix_l'} = 0; # number of characters of a prefix $self->{'pos'} = 0; $self->{'rec_ind'} = 0; $self->{'size_limit'} = $size_limit; # pretend file ends at the byte limit if (ref $prefix_lines_ref) { my($len) = 0; for (@$prefix_lines_ref) { $len += length($_) } $self->{'prefix_l'} = $len; $self->{'prefix_n'} = @$prefix_lines_ref; } $self->{'handle'} = $filehandle; seek($filehandle, 0,0); # also provides a return value and errno }; sub SEEK { my($self,$offset,$whence) = @_; $whence == 0 or die "Only absolute SEEK is supported on this file"; $offset == 0 or die "Only SEEK(0,0) is supported on this file"; $self->{'eof'} = 0; $self->{'pos'} = 0; $self->{'rec_ind'} = 0; seek($self->{'handle'}, 0,0); # also provides a return value and errno } # sub TELL (not implemented) # Returns the current position in bytes for FILEHANDLE, or -1 on error. # mixing of READ and READLINE is not supported (without rewinding inbetween) # sub READLINE { my($self) = @_; my($size_limit) = $self->{'size_limit'}; my($pos) = $self->{'pos'}; if ($self->{'eof'}) { return undef; } elsif (defined $size_limit && $pos >= $size_limit) { $self->{'eof'} = 1; return undef; } elsif (wantarray) { # return entire file as an array my($rec_ind) = $self->{'rec_ind'}; $self->{'eof'} = 1; my($fh) = $self->{'handle'}; if (!defined $size_limit) { $self->{'rec_ind'} = $self->{'prefix_n'}; # just an estimate $self->{'pos'} = $self->{'prefix_l'}; # just an estimate if ($rec_ind >= $self->{'prefix_n'}) { return readline($fh); } elsif ($rec_ind == 0) { # common case: get the whole thing return ( @{$self->{'prefix'}}, readline($fh) ); } else { return ( @{$self->{'prefix'}}[ $rec_ind .. $#{$self->{'prefix'}} ], readline($fh) ); } } else { # take size limit into account my(@array); if ($rec_ind == 0) { @array = @{$self->{'prefix'}}; } elsif ($rec_ind < $self->{'prefix_n'}) { @array = @{$self->{'prefix'}}[ $rec_ind .. $#{$self->{'prefix'}} ]; } for my $j (0..$#array) { $pos += length($array[$j]); if ($pos >= $size_limit) { # truncate at NL past limit $#array = $j; last; } } my($nread) = 0; if ($pos < $size_limit) { my($inbuf,$carry); my($beyond_limit) = 0; while ( $nread=read($fh,$inbuf,16384) ) { # faster than line-by-line if ($pos+$nread >= $size_limit) { my($k) = index($inbuf, "\n", # find a clean break at next NL $pos >= $size_limit ? 0 : $size_limit-$pos); $inbuf = substr($inbuf, 0, $k >= 0 ? $k+1 : $size_limit-$pos); $beyond_limit = 1; } $pos += $nread; my($k) = $#array + 1; # insertion point push(@array, split(/^/m, $inbuf, -1)); if (defined $carry) { $array[$k] = $carry.$array[$k]; $carry=undef } $carry = pop(@array) if substr($array[-1],-1,1) ne "\n"; last if $beyond_limit; } push(@array,$carry) if defined $carry; } $self->{'rec_ind'} = $rec_ind + @array; $self->{'pos'} = $pos; if (!defined $nread) { undef @array; # errno should still be in $!, caller should be checking it # die "error reading: $!"; } return @array; } } else { # read one line if ($self->{'rec_ind'} < $self->{'prefix_n'}) { my($line) = $self->{'prefix'}->[$self->{'rec_ind'}]; $self->{'rec_ind'}++; $self->{'pos'} += length($line); return $line; } else { my($line) = scalar(readline($self->{'handle'})); if (!defined($line)) { $self->{'eof'} = 1 } # errno in $! else { $self->{'rec_ind'}++; $self->{'pos'} += length($line) } return $line; } } } # mixing of READ and READLINE is not supported (without rewinding inbetween) # sub READ { # SCALAR,LENGTH,OFFSET my $self = shift; my($len) = $_[1]; my($offset) = $_[2]; my($str) = ''; my($nbytes) = 0; my($pos) = $self->{'pos'}; my($beyond_limit) = 0; my($size_limit) = $self->{'size_limit'}; if (defined $size_limit && $pos+$len > $size_limit) { $len = $pos >= $size_limit ? 0 : $size_limit - $pos; $beyond_limit = 1; } if ($len > 0 && $pos < $self->{'prefix_l'}) { # not terribly efficient, but typically only occurs once $str = substr(join('',@{$self->{'prefix'}}), $pos, $len); $nbytes += length($str); $len -= $nbytes; } my($msg); my($buff_directly_accessed) = 0; if ($len > 0) { # avoid shuffling data through multiple buffers for a common case $buff_directly_accessed = $nbytes == 0; my($nb) = $buff_directly_accessed ? read($self->{'handle'}, $_[0], $len, $offset) : read($self->{'handle'}, $str, $len, $nbytes); if (!defined $nb) { $msg = "Error reading: $!"; } elsif ($nb < 1) { # read returns 0 at eof $self->{'eof'} = 1; } else { $nbytes += $nb; $len -= $nb; } } if (defined $msg) { undef $nbytes; # $! already set by a failed sysread } elsif ($beyond_limit && $nbytes == 0) { $self->{'eof'} = 1; } else { if (!$buff_directly_accessed) { ($offset ? substr($_[0],$offset) : $_[0]) = $str; } $pos += $nbytes; $self->{'pos'} = $pos; } $nbytes; # eof: 0; error: undef } sub close { shift->CLOSE(@_) } sub fileno { shift->FILENO(@_) } sub binmode { shift->BINMODE(@_) } sub seek { shift->SEEK(@_) } #sub tell { shift->TELL(@_) } sub read { shift->READ(@_) } sub readline { shift->READLINE(@_) } sub getlines { shift->READLINE(@_) } sub getline { scalar(shift->READLINE(@_)) } 1; # package Amavis::IO::Zlib; # A simple IO::File -compatible wrapper around Compress::Zlib, # much like IO::Zlib but simpler: does only what we need and does it carefully use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); } use Errno qw(EIO); use Compress::Zlib; sub new { my($class) = shift; my($self) = bless {}, $class; if (@_) { $self->open(@_) or return undef } $self; } sub close { my $self = shift; my($status); my($eval_stat); local($1,$2); eval { $status = $self->{fh}->gzclose; 1 } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; delete $self->{fh}; if (defined $eval_stat) { chomp $eval_stat; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout # can't stash arbitrary text into $! die "gzclose error: $eval_stat, $gzerrno"; $! = EIO; return undef; # not reached } elsif ($status != Z_OK) { die "gzclose error: $gzerrno"; # can't stash arbitrary text into $! $! = EIO; return undef; # not reached } 1; } sub DESTROY { my $self = shift; local($@,$!,$_); if (ref $self && $self->{fh}) { eval { $self->close } } } sub open { my($self,$fname,$mode) = @_; if (exists($self->{fh})) { eval { $self->close }; delete $self->{fh} } $self->{fname} = $fname; $self->{mode} = $mode; $self->{pos} = 0; my($gz) = gzopen($fname,$mode); if ($gz) { $self->{fh} = $gz; } else { die "gzopen error: $gzerrno"; # can't stash arbitrary text into $! $! = EIO; undef $gz; # not reached } $gz; } sub seek { my($self,$pos,$whence) = @_; $whence == 0 or die "Only absolute seek is supported on gzipped file"; $pos >= 0 or die "Can't seek to a negative absolute position"; $self->{mode} eq 'rb' or die "Seek to $whence,$pos on gzipped file only supported for 'rb' mode"; if ($pos < $self->{pos}) { $self->close or die "seek: can't close gzipped file: $!"; $self->open($self->{fname},$self->{mode}) or die "seek: can't reopen gzipped file: $!"; } my($skip) = $pos - $self->{pos}; while ($skip > 0) { my($s); my($nbytes) = $self->read($s,$skip); # acceptable for small skips defined $nbytes && $nbytes > 0 or die "seek: error skipping $skip bytes on gzipped file: $!"; $skip -= $nbytes; } 1; # seek is supposed to return 1 upon success, 0 otherwise } sub read { # SCALAR,LENGTH,OFFSET my $self = shift; my($len) = $_[1]; my($offset) = $_[2]; defined $len or die "Amavis::IO::Zlib::read: length argument undefined"; my($nbytes); if (!defined($offset) || $offset == 0) { $nbytes = $self->{fh}->gzread($_[0], $len); } else { my($buff); $nbytes = $self->{fh}->gzread($buff, $len); substr($_[0],$offset) = $buff; } if ($nbytes < 0) { die "gzread error: $gzerrno"; # can't stash arbitrary text into $! $! = EIO; undef $nbytes; # not reached } else { $self->{pos} += $nbytes; } $nbytes; # eof: 0; error: undef } sub getline { my $self = shift; my($nbytes,$line); $nbytes = $self->{fh}->gzreadline($line); if ($nbytes <= 0) { # eof (0) or error (-1) $! = 0; $line = undef; if ($nbytes < 0 && $gzerrno != Z_STREAM_END) { die "gzreadline error: $gzerrno"; # can't stash arbitrary text into $! $! = EIO; # not reached } } else { $self->{pos} += $nbytes; } $line; # eof: undef, $! zero; error: undef, $! nonzero } sub print { my $self = shift; my($buff_ref) = @_ == 1 ? \$_[0] : \join('',@_); my($nbytes); my($len) = length($$buff_ref); if ($len <= 0) { $nbytes = "0 but true"; } else { $nbytes = $self->{fh}->gzwrite($$buff_ref); $self->{pos} += $len; if ($nbytes <= 0) { die "gzwrite error: $gzerrno"; # can't stash arbitrary text into $! $! = EIO; undef $nbytes; # not reached } } $nbytes; } sub printf { shift->print(sprintf(shift,@_)) } 1; # package Amavis::IO::RW; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform); import Amavis::Util qw(ll do_log min max minmax); } use Errno qw(EIO EINTR EAGAIN EPIPE ENOTCONN ECONNRESET); use Time::HiRes (); use IO::Socket; use IO::Socket::UNIX; use IO::Socket::INET; #use IO::Socket::SSL; BEGIN { use vars qw($have_inet6); $have_inet6 = eval { require IO::Socket::INET6 }; } # Connect to one of the specified sockets. The $socket_specs may be a # simple string ([inet-host]:port, [inet6-host]:port, or a unix socket name), # optionally prefixed by a protocol name (scheme) and a colon (the prefix is # ignored here, just avoids a need for parsing by a caller); or it can be # a ref to a list of such socket specifications, which are tried one after # another until a connection is successful. In case of a listref, it leaves # a good socket as the first entry in the list so that it will be tried first # on a next call. # The 'Timeout' argument controls both the connect timeout as well as the # timeout of a select() call in rw_loop() - but may be changed through a # timeout() method. # sub new { my($class, $socket_specs, %arg) = @_; my($self) = bless {}, $class; $self->timeout($arg{Timeout}); $self->{eol_str} = !defined $arg{Eol} ? "\n" : $arg{Eol}; $self->{inp_sane_size} = !$arg{InpSaneSize} ? 500000 : $arg{InpSaneSize}; $self->{last_event_time} = 0; $self->{inp} = ''; $self->{out} = ''; $self->{inpeof} = 0; $self->{ssl_active} = 0; $socket_specs = [ $socket_specs ] if !ref $socket_specs; my($protocol,$socketname,$sock,$eval_stat); my($attempts) = 0; my(@failures); my($n_candidates) = scalar @$socket_specs; $n_candidates > 0 or die "Can't connect, no sockets specified!?"; # sanity for (;;) { if ($n_candidates > 1) { # pick one at random, put it to head of the list my($j) = int(rand($n_candidates)); ll(5) && do_log(5, "picking candidate #%d (of %d) in %s", $j+1, $n_candidates, join(', ',@$socket_specs)); @$socket_specs[0,$j] = @$socket_specs[$j,0] if $j != 0; } $socketname = $socket_specs->[0]; # try the first on the list local($1); $socketname =~ s/^([a-z][a-z0-9.+-]*)?://si; # strip protocol name $protocol = lc($1); # kept for the benefit of a caller $self->{socketname} = undef; $attempts++; eval { $sock = $self->connect_attempt($socketname, %arg); $sock or die "Error connecting to socket $socketname\n"; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; undef $sock; }; if ($sock) { # mission accomplished if (!@failures) { do_log(5, "connected to %s successfully%s", $self->{socketname}); } else { do_log(1, "connected to %s successfully after %d failures on: %s", $self->{socketname}, scalar(@failures), join(', ',@failures)); } last; } else { # failure, prepare for a retry with a next entry if any $n_candidates--; my($ll) = $attempts > 1 || $n_candidates <= 0 ? -1 : 1; ll($ll) && do_log($ll, "connect to %s failed, attempt #%d: %s%s", $socketname, $attempts, $eval_stat, $n_candidates <= 0 ? '' : ', trying next'); push(@failures, $socketname); # circular shift left, move a bad candidate to the end of the list push(@$socket_specs, shift @$socket_specs) if @$socket_specs > 1; last if $n_candidates <= 0; } } $sock or die("All attempts ($attempts) failed connecting to ". join(', ',@$socket_specs) . "\n"); $self->{last_event_time} = Time::HiRes::time; $self->{socket} = $sock; $self->{protocol} = $protocol; $self; } sub connect_attempt { my($self, $socketname, %arg) = @_; my($sock); my($localaddr, $localport) = ($arg{LocalAddr}, $arg{LocalPort}); my($blocking) = 1; # blocking mode defaults to on $blocking = 0 if defined $arg{Blocking} && !$arg{Blocking}; my($timeout) = $self->timeout; my($timeout_displ) = !defined $timeout ? 'undef' : int($timeout) == $timeout ? "$timeout" : sprintf("%.3f",$timeout); my($local_sock_displ) = ''; my($peeraddress,$peerport,$is_inet); local($1,$2); if ($socketname =~ m{^/}) { # simpleminded: unix vs. inet $is_inet = 0; } elsif ($socketname =~ /^(?: \[ ([^\]]*) \] | ([^:]*) ) : ([^:]*)/sx) { # ignore possible further fields after the "proto:addr:port:..." last colon $peeraddress = defined $1 ? $1 : $2; $peerport = $3; $is_inet = 1; } elsif ($socketname =~ /^(?: \[ ([^\]]*) \] | ([0-9a-fA-F.:]+) ) \z/sx) { $peeraddress = defined $1 ? $1 : $2; $is_inet = 1; } else { # probably a syntax error, but let's assume it is a Unix socket $is_inet = 0; } if ($is_inet) { if (defined $localaddr || defined $localport) { $local_sock_displ = ', local ['; $local_sock_displ .= $localaddr if defined $localaddr; $local_sock_displ .= ']'; $local_sock_displ .= ':'.$localport if defined $localport; } if (defined $peeraddress && $peeraddress eq '*') { $peeraddress = $arg{WildcardImpliedHost}; defined $peeraddress or die "Wildcarded host, but client's address not known: $socketname"; } if (!defined $peeraddress || $peeraddress eq '') { die "Empty/unknown host address in socket specification: $socketname"; } $peerport = $arg{Port} if !defined $peerport || $peerport eq ''; if (defined $peerport && $peerport eq '*') { $peerport = $arg{WildcardImpliedPort}; defined $peerport or die "Wildcarded port, but client's port not known: $socketname"; } if (!defined $peerport || $peerport eq '') { die "Empty/unknown port number in socket specification: $socketname"; } elsif ($peerport !~ /^\d{1,5}\z/ || $peerport < 1 || $peerport > 65535) { die "Invalid port number in socket specification: $socketname"; } } my($is_inet4) = $is_inet && $peeraddress =~ /^\d+\.\d+\.\d+\.\d+\z/ ? 1 : 0; if ($is_inet && ($is_inet4 || !$have_inet6)) { # inet socket (IPv4) ll(3) && do_log(3, "new socket by IO::Socket::INET to [%s]:%s, ". "timeout %s, %sblocking%s", $peeraddress, $peerport, $timeout_displ, $blocking ? '' : 'non', $local_sock_displ); $sock = IO::Socket::INET->new( Proto => 'tcp', Blocking => $blocking, # Timeout => $timeout, # produces: Invalid argument PeerAddr => $peeraddress, PeerPort => $peerport, LocalAddr => $localaddr, LocalPort => $localport); $sock or die "Can't connect to INET4 socket $socketname: $!\n"; $self->{last_event} = 'new-inet'; } elsif ($is_inet) { # inet6 socket (IPv6) or unknown IP ll(3) && do_log(3, "new socket by IO::Socket::INET6 to [%s]:%s, ". "timeout %s, %sblocking%s", $peeraddress, $peerport, $timeout_displ, $blocking ? '' : 'non', $local_sock_displ); $sock = IO::Socket::INET6->new( Proto => 'tcp', Blocking => $blocking, # Timeout => $timeout, # produces: Invalid argument PeerAddr => $peeraddress, PeerPort => $peerport, LocalAddr => $localaddr, LocalPort => $localport); $sock or die "Can't connect to INET6 socket $socketname: $!\n"; $self->{last_event} = 'new-inet6'; } else { # unix socket do_log(3,"new socket by IO::Socket::UNIX to %s, timeout %s", $socketname,$timeout_displ); $sock = IO::Socket::UNIX->new(Type => SOCK_STREAM, Timeout => $timeout); $sock or die "Can't create UNIX socket: $!\n"; $sock->connect( pack_sockaddr_un($socketname) ) or die "Can't connect to UNIX socket $socketname: $!\n"; $self->{last_event} = 'new-unix'; } if ($sock) { $self->{socketname} = $is_inet ? "[$peeraddress]:$peerport" : $socketname; } $sock; } sub close { my($self) = @_; my($sock) = $self->{socket}; my($status) = 1; # ok if (!defined($sock)) { # nothing to do } elsif (!defined(fileno($sock))) { # not really open $sock->close; # ignoring errors } else { my($flush_status) = 1; eval { # don't let errors during flush prevent us from closing a socket $flush_status = $self->flush; } or do { undef $flush_status; my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(1, "Amavis::IO::RW: Error flushing on close: %s", $eval_stat); }; $self->{last_event} = 'close'; $self->{last_event_time} = Time::HiRes::time; $! = 0; $status = $sock->close; $status or do_log(1, "Amavis::IO::RW: Error closing socket: %s", !$self->{ssl_active} ? $! : $sock->errstr.", $!" ); $status = $flush_status if $status && !$flush_status; } $status; } sub DESTROY { my $self = $_[0]; local($@,$!,$_); eval { $self->close }; # ignoring errors } sub rw_loop { my($self,$needline,$flushoutput) = @_; # # RFC 2920: Client SMTP implementations MAY elect to operate in a nonblocking # fashion, processing server responses immediately upon receipt, even if # there is still data pending transmission from the client's previous TCP # send operation. If nonblocking operation is not supported, however, client # SMTP implementations MUST also check the TCP window size and make sure that # each group of commands fits entirely within the window. The window size # is usually, but not always, 4K octets. Failure to perform this check can # lead to deadlock conditions. # # We choose to operate in a nonblocking mode. Responses are read as soon as # they become available and stored for later, but not immediately processed # as they come in. This requires some sanity limiting against rogue servers. # my($sock) = $self->{socket}; my($fd_sock) = fileno($sock); my($timeout) = $self->timeout; my($timeout_displ) = !defined $timeout ? 'undef' : int($timeout) == $timeout ? "$timeout" : sprintf("%.3f",$timeout); my($eol_str) = $self->{eol_str}; my($idle_cnt) = 0; my($failed_write_attempts) = 0; local $SIG{PIPE} = 'IGNORE'; # don't signal on a write to a widowed pipe for (;;) { $idle_cnt++; my($rout,$wout,$eout); my($rin,$win,$ein); $rin=$win=$ein=''; my($want_to_write) = $self->{out} ne '' && ($flushoutput || $needline); ll(5) && do_log(5,"rw_loop: needline=%d, flush=%s, wr=%d, timeout=%s", $needline, $flushoutput, $want_to_write, $timeout_displ); if (!defined($fd_sock)) { do_log(3,"rw_loop read: got a closed socket"); $self->{inpeof} = 1; last; } vec($rin,$fd_sock,1) = 1; vec($win,$fd_sock,1) = $want_to_write ? 1 : 0; $ein = $rin | $win; $self->{last_event} = 'select'; $self->{last_event_time} = Time::HiRes::time; my($nfound,$timeleft) = select($rout=$rin, $wout=$win, $eout=$ein, $timeout); $nfound >= 0 or die "Select failed: ". (!$self->{ssl_active} ? $! : $sock->errstr.", $!"); if (vec($rout,$fd_sock,1)) { do_log(5,"rw_loop: receiving"); my($inbuf) = ''; $! = 0; my($nread) = sysread($sock,$inbuf,16384); if (!defined($nread)) { if ($!==EAGAIN || $!==EINTR) { $self->{last_event} = 'read-intr'.(0+$!); $idle_cnt = 0; do_log(2, "rw_loop read interrupted: %s", !$self->{ssl_active} ? $! : $sock->errstr.", $!"); Time::HiRes::sleep(0.1); # slow down, just in case } else { $self->{last_event} = 'read-fail'; $self->{inpeof} = 1; die "Error reading from socket: ". (!$self->{ssl_active} ? $! : $sock->errstr.", $!"); } } elsif ($nread < 1) { # sysread returns 0 at eof $self->{last_event} = 'read-eof'; $self->{inpeof} = 1; do_log(3,"rw_loop read: got eof"); } else { # successful read $self->{last_event} = 'read-ok'; $self->{inpeof} = 0; ll(5) && do_log(5,"rw_loop read %d chars< %s", length($inbuf),$inbuf); $self->{inp} .= $inbuf; $idle_cnt = 0; length($self->{inp}) < $self->{inp_sane_size} or die "rw_loop: Aborting on a runaway server, inp_len=" . length($self->{inp}); } $self->{last_event_time} = Time::HiRes::time; } if (vec($wout,$fd_sock,1)) { do_log(5,"rw_loop: sending"); my($nwrite) = syswrite($sock, $self->{out}); if (!defined($nwrite)) { if ($!==EAGAIN || $!==EINTR) { $self->{last_event} = 'write-intr'.(0+$!); $idle_cnt = 0; $failed_write_attempts++; do_log(2, "rw_loop write interrupted: %s", !$self->{ssl_active} ? $! : $sock->errstr.", $!"); Time::HiRes::sleep(0.1); # slow down, just in case } else { $self->{last_event} = 'write-fail'; die "Error writing to socket: ". (!$self->{ssl_active} ? $! : $sock->errstr.", $!"); } } else { # successful write $self->{last_event} = 'write-ok'; my($out_l) = length($self->{out}); my($ll) = $nwrite != $out_l ? 4 : 5; if (ll($ll)) { my($msg) = $nwrite==$out_l ? sprintf("%d", $nwrite) : sprintf("%d (of %d)", $nwrite,$out_l); my($nlog) = min(200,$nwrite); do_log($ll,"rw_loop sent %s> %s%s", $msg, substr($self->{out},0,$nlog), $nlog<$nwrite?' [...]':''); }; $idle_cnt = 0; if ($nwrite <= 0) { $failed_write_attempts++ } elsif ($nwrite < $out_l) { substr($self->{out},0,$nwrite) = '' } else { $self->{out} = '' } } $self->{last_event_time} = Time::HiRes::time; } if ( ( !$needline || !defined($eol_str) || $eol_str eq '' || index($self->{inp},$eol_str) >= 0 ) && ( !$flushoutput || $self->{out} eq '' ) ) { last; } if ($self->{inpeof}) { if ($self->{out} ne '') { do_log(2, "rw_loop: EOF on input, output buffer not yet empty"); } last; } if ($idle_cnt > 0) { # probably exceeded timeout in select do_log(-1,"rw_loop: leaving rw loop, no progress, ". "last event (%s) %.3f s ago", $self->{last_event}, Time::HiRes::time - $self->{last_event_time}); last; } $failed_write_attempts < 100 or die "rw_loop: Aborting stalled sending"; } } sub socketname { my $self=shift; !@_ ? $self->{socketname} : ($self->{socketname}=shift) } sub protocol { my $self=shift; !@_ ? $self->{protocol} : ($self->{protocol}=shift) } sub timeout { my $self=shift; !@_ ? $self->{timeout} : ($self->{timeout}=shift) } sub ssl_active { my $self=shift; !@_ ? $self->{ssl_active} : ($self->{ssl_active}=shift) } sub eof { my $self=shift; $self->{inpeof} && $self->{inp} eq '' ? 1 : 0 } sub last_io_event_timestamp { my($self,$keyword) = @_; $self->{last_event_time} } sub out_buff_large { my $self=shift; length($self->{out}) > 40000 } sub flush { my $self=shift; $self->rw_loop(0,1) if $self->{out} ne ''; 1 } sub print { my $self = shift; for (@_) { $self->{out} .= $_ } $self->out_buff_large ? $self->flush : 1; } sub at_line_boundary { my $self = $_[0]; my($eol_str) = $self->{eol_str}; my($eol_str_l) = !defined($eol_str) ? 0 : length($eol_str); !$eol_str_l ? 1 : substr($self->{out}, -$eol_str_l, $eol_str_l) eq $eol_str ? 1 : 0; } # returns true if there is any full line (or last incomplete line) # in the buffer waiting to be read, 0 otherwise, undef on eof or error # sub response_line_available { my($self) = @_; my($eol_str) = $self->{eol_str}; if (!defined $eol_str || $eol_str eq '') { return length($self->{inp}); } elsif (index($self->{inp},$eol_str) >= 0) { return 1; } elsif ($self->{inpeof} && $self->{inp} eq '') { return undef; # undef on end-of-file } elsif ($self->{inpeof}) { # partial last line return length($self->{inp}); } } # get one full text line, or last partial line, or undef on eof/error/timeout # sub get_response_line { my($self) = @_; my($ind); my($attempts) = 0; my($eol_str) = $self->{eol_str}; my($eol_str_l) = !defined($eol_str) ? 0 : length($eol_str); for (;;) { if (!$eol_str_l) { my($str) = $self->{inp}; $self->{inp} = ''; return $str; } elsif (($ind=index($self->{inp},$eol_str)) >= 0) { return substr($self->{inp},0,$ind+$eol_str_l,''); } elsif ($self->{inpeof} && $self->{inp} eq '') { $! = 0; return undef; # undef on end-of-file } elsif ($self->{inpeof}) { # return partial last line my($str) = $self->{inp}; $self->{inp} = ''; return $str; } elsif ($attempts > 0) { $! = EIO; return undef; # timeout or error } # try reading some more input, one attempt only $self->rw_loop(1,0); $attempts++; } } # read whatever is available, up to LENGTH bytes # sub read { # SCALAR,LENGTH,OFFSET my $self = shift; my($len) = $_[1]; my($offset) = $_[2]; defined $len or die "Amavis::IO::RW::read: length argument undefined"; $len >= 0 or die "Amavis::IO::RW::read: length argument negative"; $self->rw_loop(0,0); my($nbytes) = length($self->{inp}); $nbytes = $len if $len < $nbytes; if (!defined($offset) || $offset == 0) { $_[0] = substr($self->{inp}, 0, $len, ''); } else { substr($_[0],$offset) = substr($self->{inp}, 0, $len, ''); } $nbytes; # eof: 0; error: undef } use vars qw($ssl_cache); sub ssl_upgrade { my($self,%params) = @_; $self->flush; IO::Socket::SSL->VERSION(1.05); # required minimal version $ssl_cache = IO::Socket::SSL::Session_Cache->new(2) if !defined $ssl_cache; my($sock) = $self->{socket}; IO::Socket::SSL->start_SSL($sock, SSL_session_cache => $ssl_cache, SSL_error_trap => sub { my($sock,$msg)=@_; do_log(-2,"Error on socket: %s",$msg) }, %params, ) or die "Error upgrading socket to SSL: ".IO::Socket::SSL::errstr(); $self->{last_event_time} = Time::HiRes::time; $self->{ssl_active} = 1; ll(3) && do_log(3,"TLS cipher: %s", $sock->get_cipher); ll(5) && do_log(5,"TLS certif: %s", $sock->dump_peer_certificate); 1; } 1; # package Amavis::In::Connection; # Keeps relevant information about how we received the message: # client connection information, SMTP envelope and SMTP parameters use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); } sub new { my($class) = @_; bless {}, $class } sub client_ip # client IP address (immediate SMTP client, i.e. our MTA) { my $self=shift; !@_ ? $self->{client_ip} : ($self->{client_ip}=shift) } sub socket_ip # IP address of our interface that received connection { my $self=shift; !@_ ? $self->{socket_ip} : ($self->{socket_ip}=shift) } sub socket_port # TCP port of our interface that received connection { my $self=shift; !@_ ? $self->{socket_port}: ($self->{socket_port}=shift) } sub socket_proto # TCP/UNIX { my $self=shift; !@_ ? $self->{socket_proto}:($self->{socket_proto}=shift)} sub socket_path # socket path, UNIX sockets only { my $self=shift; !@_ ? $self->{socket_path}: ($self->{socket_path}=shift)} # RFC 3848 sub appl_proto # SMTP/ESMTP(A|S|SA)/LMTP(A|S|SA) / AM.PDP/AM.CL/QMQP/QMQPqq { my $self=shift; !@_ ? $self->{appl_proto} : ($self->{appl_proto}=shift) } sub smtp_helo # (E)SMTP HELO/EHLO parameter { my $self=shift; !@_ ? $self->{smtp_helo} : ($self->{smtp_helo}=shift) } 1; # package Amavis::In::Message::PerRecip; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform); import Amavis::Util qw(setting_by_given_contents_category_all setting_by_given_contents_category cmp_ccat); } sub new # NOTE: this class is a list for historical reasons, not a hash { my($class) = @_; bless [(undef) x 41], $class } # subs to set or access individual elements of a n-tuple by name sub recip_addr # unquoted recipient envelope e-mail address { my $self=shift; !@_ ? $$self[0] : ($$self[0]=shift) } sub recip_addr_smtp # SMTP-encoded recipient envelope e-mail address in <> { my $self=shift; !@_ ? $$self[1] : ($$self[1]=shift) } sub recip_addr_modified # recip. addr. with possible addr. extension inserted { my $self=shift; !@_ ? $$self[2] : ($$self[2]=shift) } sub recip_is_local # recip_addr matches @local_domains_maps { my $self=shift; !@_ ? $$self[3] : ($$self[3]=shift) } sub recip_maddr_id # maddr.id field from SQL corresponding to recip_addr_smtp { my $self=shift; !@_ ? $$self[4] : ($$self[4]=shift) } sub recip_maddr_id_orig # maddr.id field from SQL corresponding to dsn_orcpt { my $self=shift; !@_ ? $$self[5] : ($$self[5]=shift) } sub recip_penpals_age # penpals age in seconds if logging to SQL is enabled { my $self=shift; !@_ ? $$self[6] : ($$self[6]=shift) } sub recip_penpals_score # penpals score (info, also added to spam_level) { my $self=shift; !@_ ? $$self[7] : ($$self[7]=shift) } sub dsn_notify # ESMTP RCPT command NOTIFY option (DSN-RFC 3461, listref) { my $self=shift; !@_ ? $$self[8] : ($$self[8]=shift) } sub dsn_orcpt # ESMTP RCPT command ORCPT option (DSN-RFC 3461, encoded) { my $self=shift; !@_ ? $$self[9] : ($$self[9]=shift) } sub dsn_suppress_reason # if defined disable sending DSN and supply a reason { my $self=shift; !@_ ? $$self[10] : ($$self[10]=shift) } sub recip_destiny # D_REJECT, D_BOUNCE, D_DISCARD, D_PASS { my $self=shift; !@_ ? $$self[11] : ($$self[11]=shift) } sub recip_done # false: not done, true: done (1: faked, 2: truly sent) { my $self=shift; !@_ ? $$self[12] : ($$self[12]=shift) } sub recip_smtp_response # RFC 5321 response (3-digit + enhanced resp + text) { my $self=shift; !@_ ? $$self[13] : ($$self[13]=shift) } sub recip_remote_mta_smtp_response # smtp response as issued by remote MTA { my $self=shift; !@_ ? $$self[14] : ($$self[14]=shift) } sub recip_remote_mta # remote MTA that issued the smtp response { my $self=shift; !@_ ? $$self[15] : ($$self[15]=shift) } sub recip_tagged # message was tagged by address extension or Subject or X-Spam { my $self=shift; !@_ ? $$self[16] : ($$self[16]=shift) } sub recip_mbxname # mailbox name or file when known (local:, bsmtp: or sql:) { my $self=shift; !@_ ? $$self[17] : ($$self[17]=shift) } sub recip_whitelisted_sender # recip considers this sender whitelisted { my $self=shift; !@_ ? $$self[18] : ($$self[18]=shift) } sub recip_blacklisted_sender # recip considers this sender blacklisted { my $self=shift; !@_ ? $$self[19] : ($$self[19]=shift) } sub bypass_virus_checks # boolean: virus checks to be bypassed for this recip { my $self=shift; !@_ ? $$self[20] : ($$self[20]=shift) } sub bypass_banned_checks # bool: ban checks are to be bypassed for this recip { my $self=shift; !@_ ? $$self[21] : ($$self[21]=shift) } sub bypass_spam_checks # boolean: spam checks are to be bypassed for this recip { my $self=shift; !@_ ? $$self[22] : ($$self[22]=shift) } sub banned_parts # banned part descriptions (ref to a list of banned parts) { my $self=shift; !@_ ? $$self[23] : ($$self[23]=shift) } sub banned_parts_as_attr # banned part descriptions - newer syntax (listref) { my $self=shift; !@_ ? $$self[24] : ($$self[24]=shift) } sub banning_rule_key # matching banned rules (lookup table keys) (ref to list) { my $self=shift; !@_ ? $$self[25] : ($$self[25]=shift) } sub banning_rule_comment #comments (or whole expr) from banning_rule_key regexp { my $self=shift; !@_ ? $$self[26] : ($$self[26]=shift) } sub banning_reason_short # just one banned part leaf name with a rule comment { my $self=shift; !@_ ? $$self[27] : ($$self[27]=shift) } sub banning_rule_rhs # a right-hand side of matching rules (a ref to a list) { my $self=shift; !@_ ? $$self[28] : ($$self[28]=shift) } sub mail_body_mangle # mail body is being modified (and how) (e.g. defanged) { my $self=shift; !@_ ? $$self[29] : ($$self[29]=shift) } sub contents_category # sorted listref of "major,minor" strings(category types) { my $self=shift; !@_ ? $$self[30] : ($$self[30]=shift) } sub blocking_ccat # category type most responsible for blocking msg, or undef { my $self=shift; !@_ ? $$self[31] : ($$self[31]=shift) } sub user_id # listref of recipient IDs from a lookup, e.g. SQL field users.id { my $self=shift; !@_ ? $$self[32] : ($$self[32]=shift) } sub user_policy_id # recipient's policy ID, e.g. SQL field users.policy_id { my $self=shift; !@_ ? $$self[33] : ($$self[33]=shift) } sub courier_control_file # path to control file containing this recipient { my $self=shift; !@_ ? $$self[34] : ($$self[34]=shift) } sub courier_recip_index # index of recipient within control file { my $self=shift; !@_ ? $$self[35] : ($$self[35]=shift) } sub delivery_method # delivery method, or empty for implicit delivery (milter) { my $self=shift; !@_ ? $$self[36] : ($$self[36]=shift) } sub spam_level # spam score as returned by spam scanners, ham near 0, spam 5 { my $self=shift; !@_ ? $$self[37] : ($$self[37]=shift) } sub spam_tests # a listref of r/o stringrefs, each: t1=score1,t2=score2,.. { my $self=shift; !@_ ? $$self[38] : ($$self[38]=shift) } # per-recipient spam info - when undefined consult a per-message counterpart sub spam_report # SA terse report of tests hit (for header section reports) { my $self=shift; !@_ ? $$self[39] : ($$self[39]=shift) } sub spam_summary # SA summary of tests hit for standard body reports { my $self=shift; !@_ ? $$self[40] : ($$self[40]=shift) } sub recip_final_addr { # return recip_addr_modified if set, else recip_addr my $self = shift; my($newaddr) = $self->recip_addr_modified; defined $newaddr ? $newaddr : $self->recip_addr; } # The contents_category list is a sorted list of strings, each of the form # "major" or "major,minor", where major and minor are numbers, representing # major and minor category type. Sort order is descending by numeric values, # major first, and subordered by a minor value. When an entry "major,minor" # is added, an entry "major" is added automatically (minor implied to be 0). # A string "major" means the same as "major,0". See CC_* constants for major # category types. Minor category types semantics is specific to each major # category, higher number represent more important finding than a lower number. # add new findings to the contents_category list # sub add_contents_category { my $self = shift; my($major,$minor) = @_; my($aref) = $self->contents_category || []; # major category is always inserted, but "$major,$minor" only if minor>0 if (defined $minor && $minor > 0) { # straight insertion of "$major,$minor" my($el) = sprintf("%d,%d",$major,$minor); my($j)=0; for (@$aref) { if (cmp_ccat($_,$el) <= 0) { last } else { $j++ } }; if ($j > $#{$aref}) { push(@$aref,$el) } # append elsif (cmp_ccat($aref->[$j],$el) != 0) { splice(@$aref,$j,0,$el) } } # straight insertion of "$major" into an ordered array (descending order) my($el) = sprintf("%d",$major); my($j)=0; for (@$aref) { if (cmp_ccat($_,$el) <= 0) { last } else { $j++ } }; if ($j > $#{$aref}) { push(@$aref,$el) } # append elsif (cmp_ccat($aref->[$j],$el) != 0) { splice(@$aref,$j,0,$el) } # insert at index $j $self->contents_category($aref); } # is the "$major,$minor" category in the list? # sub is_in_contents_category { my $self = shift; my($major,$minor) = @_; my($el) = sprintf("%d,%d",$major,$minor); my($aref) = $self->contents_category; !defined($aref) ? undef : scalar(grep(cmp_ccat($_,$el) == 0, @$aref)); } # get a setting corresponding to the most important contents category; # i.e. the highest entry from the category list for which a corresponding entry # in the associative array of settings exists determines returned setting; # sub setting_by_main_contents_category($@) { my $self = shift; my(@settings_href_list) = @_; return undef if !@settings_href_list; my($aref) = $self->contents_category; setting_by_given_contents_category($aref, @settings_href_list); } # get a list of settings corresponding to all relevant contents categories, # sorted from the most important to the least important entry; entries which # have no corresponding setting are not included in the list # sub setting_by_main_contents_category_all($@) { my $self = shift; my(@settings_href_list) = @_; return undef if !@settings_href_list; my($aref) = $self->contents_category; setting_by_given_contents_category_all($aref, @settings_href_list); } sub setting_by_blocking_contents_category($@) { my $self = shift; my(@settings_href_list) = @_; my($blocking_ccat) = $self->blocking_ccat; !defined($blocking_ccat) ? undef : setting_by_given_contents_category($blocking_ccat, @settings_href_list); } sub setting_by_contents_category($@) { my $self = shift; my(@settings_href_list) = @_; my($blocking_ccat) = $self->blocking_ccat; !defined($blocking_ccat) ? $self->setting_by_main_contents_category(@settings_href_list) : setting_by_given_contents_category($blocking_ccat, @settings_href_list); } 1; # package Amavis::In::Message; # this class keeps information about the message being processed use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform); import Amavis::rfc2821_2822_Tools qw(rfc2822_timestamp quote_rfc2821_local qquote_rfc2821_local); import Amavis::Util qw(orcpt_encode); import Amavis::In::Message::PerRecip; } sub new { my($class) = @_; my($self)=bless({},$class); $self->skip_bytes(0); $self } sub conn_obj # ref to a connection object Amavis::In::Connection { my $self=shift; !@_ ? $self->{conn} : ($self->{conn}=shift) } sub rx_time # Unix time (s since epoch) of message reception by amavisd { my $self=shift; !@_ ? $self->{rx_time} : ($self->{rx_time}=shift) } sub partition_tag # SQL partition tag (e.g. an ISO week number 1..53, or 0) { my $self=shift; !@_ ? $self->{partition} : ($self->{partition}=shift) } sub client_proto # orig. client protocol, obtained from XFORWARD or milter { my $self=shift; !@_ ? $self->{cli_proto} : ($self->{cli_proto}=shift) } sub client_addr # original client IP addr, obtained from XFORWARD or milter { my $self=shift; !@_ ? $self->{cli_ip} : ($self->{cli_ip}=shift) } sub client_name # orig. client DNS name, obtained from XFORWARD or milter { my $self=shift; !@_ ? $self->{cli_name} : ($self->{cli_name}=shift) } sub client_port # orig client src port num, obtained from XFORWARD or milter { my $self=shift; !@_ ? $self->{cli_port} : ($self->{cli_port}=shift) } sub client_source # LOCAL/REMOTE/undef, local_header_rewrite_clients/XFORWARD { my $self=shift; !@_ ? $self->{cli_source} : ($self->{cli_source}=shift) } sub client_helo # orig. client EHLO name, obtained from XFORWARD or milter { my $self=shift; !@_ ? $self->{cli_helo} : ($self->{cli_helo}=shift) } sub client_os_fingerprint # SMTP client's OS fingerprint, obtained from p0f { my $self=shift; !@_ ? $self->{cli_p0f} : ($self->{cli_p0f}=shift) } sub originating # originating from our users, copied from c('originating') { my $self=shift; !@_ ? $self->{originating}: ($self->{originating}=shift) } sub queue_id # MTA queue ID of message if known (Courier, milter/AM.PDP, XFORW) { my $self=shift; !@_ ? $self->{queue_id} : ($self->{queue_id}=shift) } sub log_id # task id as shown in the log, also known as am_id { my $self=shift; !@_ ? $self->{log_id} : ($self->{log_id}=shift) } sub mail_id # long-term unique id of the message on this system { my $self=shift; !@_ ? $self->{mail_id} : ($self->{mail_id}=shift) } sub secret_id # secret string to grant access to a message with mail_id { my $self=shift; !@_ ? $self->{secret_id} : ($self->{secret_id}=shift) } sub attachment_password # scrambles a potentially dangerous released mail { my $self=shift; !@_ ? $self->{release_pwd}: ($self->{release_pwd}=shift) } sub msg_size # ESMTP SIZE value, later corrected to actual size,RFC 1870 { my $self=shift; !@_ ? $self->{msg_size} : ($self->{msg_size}=shift) } sub auth_user # ESMTP AUTH username { my $self=shift; !@_ ? $self->{auth_user} : ($self->{auth_user}=shift) } sub auth_pass # ESMTP AUTH password { my $self=shift; !@_ ? $self->{auth_pass} : ($self->{auth_pass}=shift) } sub auth_submitter # ESMTP MAIL command AUTH option value (addr-spec or "<>") { my $self=shift; !@_ ? $self->{auth_subm} : ($self->{auth_subm}=shift) } sub tls_cipher # defined if TLS was on, e.g. contains cipher alg.,RFC 3207 { my $self=shift; !@_ ? $self->{auth_tlscif}: ($self->{auth_tlscif}=shift) } sub dsn_ret # ESMTP MAIL command RET option (DSN-RFC 3461) { my $self=shift; !@_ ? $self->{dsn_ret} : ($self->{dsn_ret}=shift) } sub dsn_envid # ESMTP MAIL command ENVID option (DSN-RFC 3461) xtext enc. { my $self=shift; !@_ ? $self->{dsn_envid} : ($self->{dsn_envid}=shift) } sub dsn_passed_on # obligation to send notification on SUCCESS was relayed { my $self=shift; !@_ ? $self->{dsn_pass_on}: ($self->{dsn_pass_on}=shift) } sub requested_by # Resent-From addr who requested release from a quarantine { my $self=shift; !@_ ? $self->{requested_by}:($self->{requested_by}=shift)} sub body_type # ESMTP BODY param (RFC 1652: 7BIT, 8BITMIME) or BINARYMIME { my $self=shift; !@_ ? $self->{body_type} : ($self->{body_type}=shift) } sub header_8bit # true if header contains characters with code above 255 { my $self=shift; !@_ ? $self->{header_8bit}: ($self->{header_8bit}=shift) } sub body_8bit # true if body contains chars with code above 255 { my $self=shift; !@_ ? $self->{body_8bit}: ($self->{body_8bit}=shift) } sub sender # envelope sender, internal form, e.g.: j doe@example.com { my $self=shift; !@_ ? $self->{sender} : ($self->{sender}=shift) } sub sender_smtp # env sender, SMTP form in <>, e.g.: <"j doe"@example.com> { my $self=shift; !@_ ? $self->{sender_smtp}: ($self->{sender_smtp}=shift) } sub sender_credible # envelope sender is believed to be valid { my $self=shift; !@_ ? $self->{sender_cred}: ($self->{sender_cred}=shift) } sub sender_source # unmangled sender addr. or info from the trace (log/notif) { my $self=shift; !@_ ? $self->{sender_src} : ($self->{sender_src}=shift) } sub sender_maddr_id # maddr.id field from SQL if logging to SQL is enabled { my $self=shift; !@_ ? $self->{maddr_id} : ($self->{maddr_id}=shift) } sub mime_entity # MIME::Parser entity holding the parsed message { my $self=shift; !@_ ? $self->{mime_entity}: ($self->{mime_entity}=shift)} sub parts_root # Amavis::Unpackers::Part root object { my $self=shift; !@_ ? $self->{parts_root} : ($self->{parts_root}=shift)} sub skip_bytes # file offset where mail starts, useful for quar. release { my $self=shift; !@_ ? $self->{file_ofs} : ($self->{file_ofs}=shift) } sub mail_text # RFC 5322 msg: open file handle, or MIME::Entity object { my $self=shift; !@_ ? $self->{mail_text} : ($self->{mail_text}=shift) } sub mail_text_fn # orig. mail filename or undef, e.g. mail_tempdir/email.txt { my $self=shift; !@_ ? $self->{mailtextfn} : ($self->{mailtextfn}=shift) } sub mail_tempdir # work directory, under $TEMPBASE or supplied by client { my $self=shift; !@_ ? $self->{mailtempdir}: ($self->{mailtempdir}=shift)} sub mail_tempdir_obj # Amavis::TempDir obj when non-persistent (quar.release) { my $self=shift; !@_ ? $self->{tempdirobj}: ($self->{tempdirobj}=shift)} sub header_edits # Amavis::Out::EditHeader object or undef { my $self=shift; !@_ ? $self->{hdr_edits} : ($self->{hdr_edits}=shift) } sub rfc2822_from #author addresses list (rfc allows one or more), parsed 'From' { my $self=shift; !@_ ? $self->{hdr_from} : ($self->{hdr_from}=shift) } sub rfc2822_sender # sender address (rfc allows none or one), parsed 'Sender' { my $self=shift; !@_ ? $self->{hdr_sender} : ($self->{hdr_sender}=shift) } sub rfc2822_resent_from # resending author addresses list, parsed 'Resent-From' { my $self=shift; !@_ ? $self->{hdr_rfrom} : ($self->{hdr_rfrom}=shift) } sub rfc2822_resent_sender # resending sender addresses, parsed 'Resent-Sender' { my $self=shift; !@_ ? $self->{hdr_rsender}: ($self->{hdr_rsender}=shift) } sub rfc2822_to # parsed 'To' header field: a list of recipients { my $self=shift; !@_ ? $self->{hdr_to} : ($self->{hdr_to}=shift) } sub rfc2822_cc # parsed 'Cc' header field: a list of Cc recipients { my $self=shift; !@_ ? $self->{hdr_cc} : ($self->{hdr_cc}=shift) } sub orig_header_fields # orig. header fields indices (LAST occurence), hashref { my $self=shift; !@_ ? $self->{orig_hdr_f} : ($self->{orig_hdr_f}=shift) } sub orig_header # orig.h.sect, arrayref of h.fields, with folding & trailing LF { my $self=shift; !@_ ? $self->{orig_header}: ($self->{orig_header}=shift) } sub orig_header_size # size of original header, incl. a separator line,RFC 1870 { my $self=shift; !@_ ? $self->{orig_hdr_s} : ($self->{orig_hdr_s}=shift) } sub orig_body_size # size of original body (in bytes), RFC 1870 { my $self=shift; !@_ ? $self->{orig_bdy_s} : ($self->{orig_bdy_s}=shift) } sub body_digest # message digest of a message body (e.g. MD5, SHA1, SHA256) { my $self=shift; !@_ ? $self->{body_digest}: ($self->{body_digest}=shift) } sub is_mlist # mail is from a mailing list (boolean/string) { my $self=shift; !@_ ? $self->{is_mlist} : ($self->{is_mlist}=shift) } sub is_auto # mail is an auto-response (boolean/string) { my $self=shift; !@_ ? $self->{is_auto} : ($self->{is_auto}=shift) } sub is_bulk # mail from a m.list or bulk or auto-response (bool/string) { my $self=shift; !@_ ? $self->{is_bulk} : ($self->{is_bulk}=shift) } sub dkim_signatures_all # a ref to a list of DKIM signature objects, or undef { my $self=shift; !@_ ? $self->{dkim_sall} : ($self->{dkim_sall}=shift) } sub dkim_signatures_valid # a ref to a list of valid DKIM signature objects { my $self=shift; !@_ ? $self->{dkim_sval} : ($self->{dkim_sval}=shift) } sub dkim_author_sig # author domain signature present and valid (bool/domain) { my $self=shift; !@_ ? $self->{dkim_auth_s}: ($self->{dkim_auth_s}=shift) } sub dkim_thirdparty_sig # third-party signature present and valid (bool/domain) { my $self=shift; !@_ ? $self->{dkim_3rdp_s}: ($self->{dkim_3rdp_s}=shift) } sub dkim_sender_sig # a sender signature is present and is valid (bool/domain) { my $self=shift; !@_ ? $self->{dkim_sndr_s}: ($self->{dkim_sndr_s}=shift) } sub dkim_envsender_sig # boolean: envelope sender signature present and valid { my $self=shift; !@_ ? $self->{dkim_envs_s}: ($self->{dkim_envs_s}=shift) } sub dkim_signatures_new # ref to a list of DKIM signature objects, our signing { my $self=shift; !@_ ? $self->{dkim_snew} : ($self->{dkim_snew}=shift) } sub dkim_signwith_sd # ref to a pair [selector,domain] to force signing with { my $self=shift; !@_ ? $self->{dkim_signsd}: ($self->{dkim_signsd}=shift) } sub quarantined_to # list of quar mailbox names or addresses if quarantined { my $self=shift; !@_ ? $self->{quarantine} : ($self->{quarantine}=shift) } sub quar_type # quarantine type: F/Z/B/Q/M (file/zipfile/bsmtp/sql/mailbox) { my $self=shift; !@_ ? $self->{quar_type} : ($self->{quar_type}=shift) } sub dsn_sent # delivery status notification was sent(1) or suppressed(2) { my $self=shift; !@_ ? $self->{dsn_sent} : ($self->{dsn_sent}=shift) } sub client_delete # don't delete the tempdir, it is a client's reponsibility { my $self=shift; !@_ ? $self->{client_del} :($self->{client_del}=shift)} sub contents_category # sorted arrayref CC_VIRUS/CC_BANNED/CC_SPAM../CC_CLEAN { my $self=shift; !@_ ? $self->{category} : ($self->{category}=shift) } sub blocking_ccat # category type most responsible for blocking msg, or undef { my $self=shift; !@_ ? $self->{bl_ccat} : ($self->{bl_ccat}=shift) } sub checks_performed # a hashref of checks done on a msg (for statistics/log) { my $self=shift; !@_ ? $self->{checks_perf}: ($self->{checks_perf}=shift) } sub actions_performed # listref, summarized actions & SMTP status, for logging { my $self=shift; !@_ ? $self->{act_perf} : ($self->{act_perf}=shift) } sub virusnames # a ref to a list of virus names detected, or undef { my $self=shift; !@_ ? $self->{virusnames} : ($self->{virusnames}=shift) } sub spam_report # SA terse report of tests hit (for header section reports) { my $self=shift; !@_ ? $self->{spam_report} :($self->{spam_report}=shift)} sub spam_summary # SA summary of tests hit for standard body reports { my $self=shift; !@_ ? $self->{spam_summary}:($self->{spam_summary}=shift)} # new style of providing additional information from checkers sub supplementary_info { # holds a hash of tag/value pairs, such as SA get_tag my $self=shift; my $key=shift; !@_ ? $self->{info_tag}{$key} : ($self->{info_tag}{$key}=shift); } { no warnings 'once'; # the following methods apply on a per-message level as well, summarizing # per-recipient information as far as possible *add_contents_category = \&Amavis::In::Message::PerRecip::add_contents_category; *is_in_contents_category = \&Amavis::In::Message::PerRecip::is_in_contents_category; *setting_by_main_contents_category = \&Amavis::In::Message::PerRecip::setting_by_main_contents_category; *setting_by_main_contents_category_all = \&Amavis::In::Message::PerRecip::setting_by_main_contents_category_all; *setting_by_blocking_contents_category = \&Amavis::In::Message::PerRecip::setting_by_blocking_contents_category; *setting_by_contents_category = \&Amavis::In::Message::PerRecip::setting_by_contents_category; } # The order of entries in a per-recipient list is the original order # in which recipient addresses (e.g. obtained via 'MAIL TO:') were received. # Only the entries that were accepted (via SMTP response code 2xx) # are placed in the list. The ORDER MUST BE PRESERVED and no recipients # may be added or removed from the list (without precution)! This is vital # to be able to produce correct per-recipient responses to a LMTP client! # sub per_recip_data { # get or set a listref of envelope recipient objects my $self = shift; # store a copy of the a given listref of recip objects if (@_) { $self->{recips} = [@{$_[0]}] } # caller may modify data if he knows what he is doing $self->{recips}; # return a list of recipient objects } sub recips { # get or set a listref of envelope recipients my $self = shift; if (@_) { # store a copy of a given listref of recipient addresses my($recips_list_ref, $set_dsn_orcpt_too) = @_; $self->per_recip_data([ map { my($per_recip_obj) = Amavis::In::Message::PerRecip->new; $per_recip_obj->recip_addr($_); $per_recip_obj->recip_addr_smtp(qquote_rfc2821_local($_)); $per_recip_obj->dsn_orcpt(orcpt_encode($per_recip_obj->recip_addr_smtp)) if $set_dsn_orcpt_too; $per_recip_obj->recip_destiny(D_PASS); # default is Pass $per_recip_obj } @{$recips_list_ref} ]); } return if !defined wantarray; # don't bother # return listref of recipient addresses [ map($_->recip_addr, @{$self->per_recip_data}) ]; } # for each header field maintain a list of signature indices which covered it; # returns a list of signature indices for a given header field position # sub header_field_signed_by { my($self,$header_field_index) = @_; shift; shift; my($h) = $self->{hdr_sig_ind}; my($hf); if (@_) { $self->{hdr_sig_ind} = $h = [] if !$h; $hf = $h->[$header_field_index]; $h->[$header_field_index] = $hf = [] if !$hf; push(@$hf, @_); # store signature index(es) at a given header position } $hf = $h->[$header_field_index] if $h && !$hf; $hf ? @{$hf} : (); } # return a j-th header field with a given field name, along with its index # into the array of all header fields; if a field name is undef then all # header fields are considered; search proceeds top-down if j >= 0, # or bottom up for negative values (-1=last, -2=next-to-last, ...); # access to the last header field (j=-1) is optimized and avoids a # sequential scan; undefined j is equivalent to -1 # sub get_header_field { my($self,$field_name,$j) = @_; my($field_ind,$field); my($all_fields) = $self->orig_header; $field_name = lc($field_name) if defined $field_name; if (!ref($all_fields)) { # no header section } elsif (defined($field_name) && (!defined($j) || $j == -1)) { # get the last one; this access is commonly used and is quick $field_ind = $self->orig_header_fields->{$field_name}; } elsif ($j >= 0) { # top-down, 0,1,2,... if (!defined($field_name)) { # directly addressed by a field index $field_ind = $j if $j <= $#$all_fields; } else { my($ind) = 0; my($cnt) = 0; local($1); for my $f (@$all_fields) { if ($f =~ /^([^:]*?)[ \t]*:/s && lc($1) eq $field_name) { if ($cnt++ == $j) { $field_ind = $ind; last } } $ind++; } } } else { # bottom-up, -1,-2,-3,... if (!defined($field_name)) { # directly addressed by a field index $j += @$all_fields; # turn into an absolute index $field_ind = $j if $j >= 0; } else { my($cnt) = 0; local($1); $j = -1 - $j; for (my $ind = $#$all_fields; $ind >= 0; $ind--) { my($f) = $all_fields->[$ind]; if ($f =~ /^([^:]*?)[ \t]*:/s && lc($1) eq $field_name) { if ($cnt++ == $j) { $field_ind = $ind; last } } } } } if (defined($field_ind) && wantarray) { $field = $all_fields->[$field_ind]; $field_name = lc($1) if $field =~ /^([^:]*?)[ \t]*:/s; } !wantarray ? $field_ind : ($field_ind, $field_name, $field); } sub get_header_field_body { my($self,$field_name,$j) = @_; my($k); my($f_i,$f_n,$f) = $self->get_header_field($field_name,$j); defined $f && ($k=index($f,':')) >= 0 ? substr($f,$k+1) : $f; } 1; # package Amavis::Out::EditHeader; # Accumulates instructions on what header fields need to be added # to a header section, which deleted, or how to change existing ones. # A call to write_header() then performs these edits on the fly. use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&hdr); import Amavis::Conf qw(:platform c cr ca); import Amavis::Timing qw(section_time); import Amavis::rfc2821_2822_Tools qw(wrap_string); import Amavis::Util qw(ll do_log safe_encode q_encode); } use MIME::Words; use Errno qw(EBADF); sub new { my($class) = @_; bless { prepend=>[], append=>[], addrcvd=>[], edit=>{} }, $class; } sub prepend_header($$$;$) { my($self, $field_name, $field_body, $structured) = @_; unshift(@{$self->{prepend}}, hdr($field_name,$field_body,$structured)); } sub append_header($$$;$) { my($self, $field_name, $field_body, $structured) = @_; push(@{$self->{append}}, hdr($field_name,$field_body,$structured)); } sub append_header_above_received($$$;$) { my($self, $field_name, $field_body, $structured) = @_; push(@{$self->{addrcvd}}, hdr($field_name,$field_body,$structured)); } # now a synonym for append_header_above_received() (old semantics: prepend # or append, depending on setting of $append_header_fields_to_bottom) # sub add_header($$$;$) { my($self, $field_name, $field_body, $structured) = @_; push(@{$self->{addrcvd}}, hdr($field_name,$field_body,$structured)); } # delete all header fields with a $field_name # sub delete_header($$) { my($self, $field_name) = @_; $self->{edit}{lc($field_name)} = [undef]; } # all header fields with $field_name will be edited by a supplied subroutine # sub edit_header($$$;$) { my($self, $field_name, $field_edit_sub, $structured) = @_; # $field_edit_sub will be called with 2 args: a field name and a field body; # It should return a pair consisting of a replacement field body (no field # name and no colon, with or without a trailing NL), and a boolean 'verbatim' # (false in its absence). An undefined replacement field body indicates a # deletion of the entire header field. A value true in the second returned # element indicates that a verbatim replacement is desired (i.e. no other # changes are allowed on a replacement body such as folding or encoding). !defined($field_edit_sub) || ref($field_edit_sub) eq 'CODE' or die "edit_header: arg#3 must be undef or a subroutine ref"; $field_name = lc($field_name); if (!exists($self->{edit}{$field_name})) { $self->{edit}{$field_name} = [$field_edit_sub]; } else { do_log(5, "INFO: multiple header edits: %s", $field_name); push(@{$self->{edit}{$field_name}}, $field_edit_sub); } } # copy all header edits from another header-edits object into this one # sub inherit_header_edits($$) { my($self, $other_edits) = @_; if (defined $other_edits) { for (qw(prepend addrcvd append)) { unshift(@{$self->{$_}}, @{$other_edits->{$_}}) if $other_edits->{$_}; } my $o_edit = $other_edits->{edit}; if ($o_edit) { for my $fn (keys %$o_edit) { if (!exists($self->{edit}{$fn})) { $self->{edit}{$fn} = [ @{$o_edit->{$fn}} ]; # copy list } else { unshift(@{$self->{edit}{$fn}}, @{$o_edit->{$fn}}); } } } } } # Conditioning of a header field to be added. # Insert space after colon if not present, RFC 2047 -encode if field body # contains non-ASCII characters, fold long lines if needed, prepend space # before each NL if missing, append NL if missing. Header lines with only # spaces are not allowed. (RFC 5322: Each line of characters MUST be no more # than 998 characters, and SHOULD be no more than 78 characters, excluding # the CRLF). $structured==0 indicates an unstructured header field, # folding may be inserted at any existing whitespace character position; # $structured==1 indicates that folding is only allowed at positions # indicated by \n in the provided header body, original \n will be removed. # With $structured==2 folding is preserved, wrapping step is skipped. # sub hdr($$$;$) { my($field_name, $field_body, $structured, $wrap_char) = @_; $wrap_char = "\t" if !defined $wrap_char; local($1); if ($field_name =~ /^ (?: Subject\z | Comments\z | X- (?! Envelope- (?:From|To)\z ) )/six && $field_body !~ /^[\t\n\040-\176]*\z/ # not all printable (or TAB or LF) ) { # encode according to RFC 2047 # actually RFC 2047 also allows encoded-words in rfc822 extension # message header fields (now: optional header fields), within comments # in structured header fields, or within 'phrase' (e.g. in From, To, Cc); # we are being sloppy here! $field_body =~ s/\n(?=[ \t])//gs; # unfold chomp($field_body); my($field_body_octets); my($chset) = c('hdr_encoding'); my($qb) = c('hdr_encoding_qb'); $field_body_octets = safe_encode($chset, $field_body); # do_log(5, "hdr - UTF-8 body: %s", $field_body); # do_log(5, "hdr - body octets: %s", $field_body_octets); my($encoder_func) = uc($qb) eq 'Q' ? \&q_encode : \&MIME::Words::encode_mimeword; $field_body = join("\n", map { /^[\001-\011\013\014\016-\177]*\z/ ? $_ : &$encoder_func($_,$qb,$chset) } split(/\n/, $field_body_octets, -1)); } else { # supposed to be in plain ASCII, let's make sure it is $field_body = safe_encode('ascii', $field_body); } $field_name = safe_encode('ascii', $field_name); my($str) = $field_name . ':'; $str .= ' ' if $field_body =~ /^[^ \t]/; $str .= $field_body; if ($structured == 2) { # already folded, keep it that way, sanitize 1 while $str =~ s/^([ \t]*)\n/$1/; # prefixed by whitespace lines? $str =~ s/\n(?=[ \t]*(\n|\z))//g; # whitespace lines within or at end $str =~ s/\n(?![ \t])/\n /g; # insert a space at line folds if missing } else { $wrap_char = "\t" if !defined $wrap_char; $str = wrap_string($str, 78, '', $wrap_char, $structured ) if $structured==1 || length($str) > 78; } if (length($str) > 998) { my(@lines) = split(/\n/,$str); my($trunc) = 0; for (@lines) { if (length($_) > 998) { $_ = substr($_,0,998-3).'...'; $trunc = 1 } } if ($trunc) { do_log(0, "INFO: truncating long header field (len=%d): %s[...]", length($str), substr($str,0,100) ); $str = join("\n",@lines); } } $str =~ s{\n*\z}{\n}s; # ensure a single final NL do_log(5, "header: %s", $str); $str; } # Copy mail header section to the supplied method (line by line) while adding, # removing, or changing certain header fields as required, and append an # empty line (header/body separator). Returns number of original 'Received:' # header fields to make a simple loop detection possible (as required # by RFC 5321 (ex RFC 2821) section 6.3). # # Assumes input file is properly positioned, leaves it positioned # at the beginning of a body. # sub write_header($$$$) { my($self, $msg, $out_fh, $noninitial_submission) = @_; my($fix_whitespace_lines) = 0; my($fix_long_header_lines) = 0; my($fix_bare_cr) = 0; if ($noninitial_submission && c('allow_fixing_improper_header')) { $fix_bare_cr = 1; $fix_long_header_lines = 1 if c('allow_fixing_long_header_lines'); $fix_whitespace_lines = 1 if c('allow_fixing_improper_header_folding'); } my($is_mime) = ref($msg) && $msg->isa('MIME::Entity') ? 1 : 0; do_log(5, "write_header: %s, %s", $is_mime,$out_fh); my(@header); if ($is_mime) { @header = map(/^[ \t]*\n?\z/ ? () # remove empty lines, ensure NL : (/\n\z/ ? $_ : $_ . "\n"), @{$msg->header}); } my($received_cnt) = 0; my($str) = ''; for (@{$self->{prepend}}) { $str .= $_ } for (@{$self->{addrcvd}}) { $str .= $_ } if ($str ne '') { $out_fh->print($str) or die "sending mail header1: $!" } $str = undef; if (!defined($msg)) { # existing header section is empty } else { local($1,$2); my($curr_head,$next_head); my($eof) = 0; my($ill_white_cnt) = 0; my($ill_long_cnt) = 0; my($ill_bare_cr) = 0; for (;;) { if ($eof) { $next_head = "\n"; # fake a missing header/body separator line } elsif ($is_mime) { if (@header) { $next_head = shift @header } else { $eof = 1; $next_head = "\n" } } else { $! = 0; $next_head = $msg->getline; if (!defined($next_head)) { $eof = 1; $next_head = "\n"; $!==0 or # returning EBADF at EOF is a perl bug $!==EBADF ? do_log(0,"Error reading mail header section: $!") : die "Error reading mail header section: $!"; } } if ($next_head =~ /^[ \t]/) { $curr_head .= $next_head; # folded } else { # new header field if (!defined($curr_head)) { # no previous complete header field (we are at the first hdr field) } elsif ($curr_head !~ /^([!-9;-\176]+)[ \t]*:(.*)\z/s) { # parse # invalid header field, but we'll write it anyway } else { # count, edit, or delete # obsolete RFC 822 syntax allowed whitespace before colon my($field_name, $field_body) = ($1, $2); my($field_name_lc) = lc($field_name); $received_cnt++ if $field_name_lc eq 'received'; if (exists($self->{edit}{$field_name_lc})) { chomp($field_body); ### $field_body =~ s/\n(?=[ \t])//gs; # unfold my($edit) = $self->{edit}{$field_name_lc}; # listref of edits for my $e (@$edit) { # possibly multiple (iterative) edits my($new_fbody,$verbatim); ($new_fbody,$verbatim) = &$e($field_name,$field_body) if defined $e; if (!defined($new_fbody)) { do_log(5, "deleted: %s:%s", $field_name, $field_body); $curr_head = undef; last; } $curr_head = $verbatim ? ($field_name . ':' . $new_fbody) : hdr($field_name, $new_fbody, 0); chomp($curr_head); $curr_head .= "\n"; $curr_head =~ /^([!-9;-\176]+)[ \t]*:(.*)\z/s; $field_body = $2; chomp($field_body); # carry to next iteration } } } if (defined $curr_head) { if ($fix_bare_cr) { # sanitize header sect. by removing CR characters $curr_head =~ tr/\r//d and $ill_bare_cr++; } if ($fix_whitespace_lines) { # unfold illegal all-whitespace lines $curr_head =~ s/\n(?=[ \t]*\n)//g and $ill_white_cnt++; } if ($fix_long_header_lines) { # truncate long header lines to 998 ch $curr_head =~ s{^(.{995}).{4,}$}{$1...}mg and $ill_long_cnt++; } $out_fh->print($curr_head) or die "sending mail header2: $!"; } last if $next_head eq "\n"; # header/body separator last if $next_head =~ /^--/; # mime separator (missing h/b separator) $curr_head = $next_head; } } do_log(0, "INFO: unfolded %d illegal all-whitespace ". "continuation lines", $ill_white_cnt) if $ill_white_cnt; do_log(0, "INFO: truncated %d header line(s) longer than 998 characters", $ill_long_cnt) if $ill_long_cnt; do_log(0, "INFO: removed bare CR from %d header line(s)", $ill_bare_cr) if $ill_bare_cr; } $str = ''; for (@{$self->{append}}) { $str .= $_ } $str .= "\n"; # end of header section - a separator line $out_fh->print($str) or die "sending mail header7: $!"; section_time('write-header'); $received_cnt; } 1; # package Amavis::Out; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT = qw(&mail_dispatch); import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::Util qw(ll do_log); } sub mail_dispatch($$$;$) { my($msginfo, $initial_submission, $dsn_per_recip_capable, $filter) = @_; my($tmp_hdr_edits); my($saved_hdr_edits) = $msginfo->header_edits; if (!c('enable_dkim_signing')) { # no signing } elsif ($initial_submission && $initial_submission eq 'Quar') { # do not attempt to sign messages on their way to a quarantine } else { # generate and add DKIM signatures my(@signatures) = Amavis::DKIM::dkim_make_signatures($msginfo,0); if (@signatures) { $msginfo->dkim_signatures_new(\@signatures); if (!defined($tmp_hdr_edits)) { $tmp_hdr_edits = Amavis::Out::EditHeader->new; $tmp_hdr_edits->inherit_header_edits($saved_hdr_edits); } for my $signature (@signatures) { my($s) = $signature->as_string; local($1); $s =~ s{\015\012}{\n}gs; $s =~ s{\n+\z}{}gs; $s =~ s/^((?:DKIM|DomainKey)-Signature)://si; $tmp_hdr_edits->prepend_header($1, $s, 2); } if (c('enable_dkim_verification') && grep($_->recip_is_local, @{$msginfo->per_recip_data})) { # it is too late to split a message now, add the A-R header field # if at least one recipient is local my($allowed_hdrs) = cr('allowed_added_header_fields'); if ($allowed_hdrs && $allowed_hdrs->{lc('Authentication-Results')}) { for my $h (Amavis::DKIM::generate_authentication_results( $msginfo, 0, \@signatures)) { $tmp_hdr_edits->prepend_header('Authentication-Results', $h, 1); } } } } $msginfo->header_edits($tmp_hdr_edits) if defined $tmp_hdr_edits; } my($any_deliveries) = 0; my($per_recip_data) = $msginfo->per_recip_data; my($num_recips_notdone) = scalar(grep(!$_->recip_done && (!$filter || &$filter($_)), @$per_recip_data)); while ($num_recips_notdone > 0) { # a delivery method may be a scalar of a form protocol:socket_specs, or # a listref of such elements; if a list is provided, it is expected that # each entry will be using the same protocol name, otherwise behaviour # is unspecified - so just obtain the protocol name from the first entry # my(%protocols,$any_tempfail); for my $r (@$per_recip_data) { if (!$dsn_per_recip_capable) { my($recip_smtp_response) = $r->recip_smtp_response; # any 4xx code ? if (defined($recip_smtp_response) && $recip_smtp_response =~ /^4/) { $any_tempfail = $recip_smtp_response . ' (' . $r->recip_addr . ')'; } } if (!$r->recip_done && (!$filter || &$filter($r))) { my($proto_sockname) = $r->delivery_method; defined $proto_sockname or die "mail_dispatch: undefined delivery_method"; !ref $proto_sockname || ref $proto_sockname eq 'ARRAY' or die "mail_dispatch: not a scalar or array ref: $proto_sockname"; for (ref $proto_sockname ? @$proto_sockname : $proto_sockname) { local($1); if (/^([a-z][a-z0-9.+-]*):/si) { $protocols{lc($1)} = 1 } else { die "mail_dispatch: no recognized protocol name: $_" } } } } my(@unknown) = grep(!/^(?:smtp|lmtp|pipe|bsmtp|sql|local)\z/i, keys %protocols); !@unknown or die "mail_dispatch: unknown protocol: ".join(', ',@unknown); if (!$dsn_per_recip_capable && defined $any_tempfail) { do_log(0, "temporary failures, giving up further deliveries: %s", $any_tempfail); my($smtp_resp) = "451 4.5.0 Giving up due to previous temporary failures, id=" . $msginfo->log_id; # flag the remaining undelivered recipients as temporary failures for my $r (@$per_recip_data) { next if $r->recip_done; $r->recip_smtp_response($smtp_resp); $r->recip_done(1); } last; } # do one protocol per iteration only, so that we can bail out # as soon as some 4xx temporary failure is detected, avoiding # further deliveries which would lead to duplicate deliveries # if ($protocols{'smtp'} || $protocols{'lmtp'}) { Amavis::Out::SMTP::mail_via_smtp(@_); $any_deliveries = 1; # approximation, will do for the time being } elsif ($protocols{'local'}) { Amavis::Out::Local::mail_to_local_mailbox(@_); $any_deliveries = 1; # approximation, will do for the time being } elsif ($protocols{'pipe'}) { Amavis::Out::Pipe::mail_via_pipe(@_); $any_deliveries = 1; # approximation, will do for the time being } elsif ($protocols{'bsmtp'}) { Amavis::Out::BSMTP::mail_via_bsmtp(@_); $any_deliveries = 1; # approximation, will do for the time being } elsif ($protocols{'sql'}) { $Amavis::extra_code_sql_quar && $Amavis::sql_storage or die "SQL quarantine code not enabled (1)"; Amavis::Out::SQL::Quarantine::mail_via_sql( $Amavis::sql_dataset_conn_storage, @_); $any_deliveries = 1; # approximation, will do for the time being } # are we done yet? my($num_recips_notdone_after) = scalar(grep(!$_->recip_done && (!$filter || &$filter($_)), @$per_recip_data)); if ($num_recips_notdone_after >= $num_recips_notdone) { do_log(-2, "TROUBLE: Number of recipients (%d) not reduced, ". "abandoning effort, proto: %s", $num_recips_notdone_after, join(', ', keys %protocols) ); last; } if ($num_recips_notdone_after > 0) { do_log(3, "Sent to %s recipients, %s still to go", $num_recips_notdone - $num_recips_notdone_after, $num_recips_notdone_after); } $num_recips_notdone = $num_recips_notdone_after; } # restore header edits if modified $msginfo->header_edits($saved_hdr_edits) if defined $tmp_hdr_edits; $any_deliveries; # (estimate) were any successful deliveries actually done? } 1; # package Amavis::UnmangleSender; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&parse_ip_address_from_received &first_received_from); import Amavis::Conf qw(:platform c cr ca); import Amavis::Util qw(ll do_log unique_list); import Amavis::rfc2821_2822_Tools qw( split_address parse_received fish_out_ip_from_received); import Amavis::Lookup qw(lookup lookup2); import Amavis::Lookup::IP qw(lookup_ip_acl); } use subs @EXPORT_OK; # Obtain and parse the first entry (oldest) in the 'Received:' header field # path trace - to be used as the value of a macro %t in customized messages # sub first_received_from($) { my($msginfo) = @_; my($first_received); my($fields_ref) = parse_received($msginfo->get_header_field_body('received')); # last if (exists $fields_ref->{'from'}) { $first_received = join(' ', unique_list(grep(defined($_), @$fields_ref{qw(from from-tcp from-com)}))); do_log(5, "first_received_from: %s", $first_received); } $first_received; } # Try to extract sender's IP address from the Received trace. # When $search_top_down is true: search top-down, use first valid IP address; # otherwise, search bottom-up, use the first *public* IP address from the trace # use vars qw(@nonhostlocalnetworks_maps @publicnetworks_maps); sub parse_ip_address_from_received($;$) { my($msginfo,$search_top_down) = @_; $search_top_down = 0 if !defined $search_top_down; @publicnetworks_maps = ( Amavis::Lookup::Label->new('publicnetworks'), Amavis::Lookup::IP->new(qw( !0.0.0.0/8 !127.0.0.0/8 !169.254.0.0/16 !:: !::1 !FE80::/10 !172.16.0.0/12 !192.168.0.0/16 !10.0.0.0/8 !FEC0::/10 !192.88.99.0/24 !240.0.0.0/4 !224.0.0.0/4 !FF00::/8 ::FFFF:0:0/96 ::/0)) ) if !@publicnetworks_maps; # RFC 5735 (ex RFC 3330), RFC 3513 my($received_from_ip); my(@search_list) = $search_top_down ? (0,1) # the topmost two Received flds : (-1,-2,-3,-4,-5,-6); # bottom-up, first six chronologically for my $j (@search_list) { # walk through a list of Received field indices my($r) = $msginfo->get_header_field_body('received',$j); last if !defined $r; $received_from_ip = fish_out_ip_from_received($r); if ($received_from_ip ne '') { last if $search_top_down; # any valid address would do my($is_public,$fullkey,$err) = lookup_ip_acl($received_from_ip,@publicnetworks_maps); last if (!defined($err) || $err eq '') && $is_public; } } do_log(5, "parse_ip_address_from_received: %s", $received_from_ip); $received_from_ip; } 1; # package Amavis::Unpackers::NewFilename; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&consumed_bytes); import Amavis::Conf qw(c cr ca $MIN_EXPANSION_QUOTA $MIN_EXPANSION_FACTOR $MAX_EXPANSION_QUOTA $MAX_EXPANSION_FACTOR); import Amavis::Util qw(ll do_log min max minmax); } use vars qw($avail_quota); # available bytes quota for unpacked mail use vars qw($rem_quota); # remaining bytes quota for unpacked mail sub new($;$$) { # create a file name generator object my($class, $maxfiles,$mail_size) = @_; # calculate and initialize quota $avail_quota = $rem_quota = # quota in bytes max($MIN_EXPANSION_QUOTA, $mail_size * $MIN_EXPANSION_FACTOR, min($MAX_EXPANSION_QUOTA, $mail_size * $MAX_EXPANSION_FACTOR)); do_log(4,"Original mail size: %d; quota set to: %d bytes", $mail_size,$avail_quota); # create object bless { num_of_issued_names => 0, first_issued_ind => 1, last_issued_ind => 0, maxfiles => $maxfiles, # undef disables limit objlist => [], }, $class; } sub parts_list_reset($) { # clear a list of recently issued names my $self = shift; $self->{num_of_issued_names} = 0; $self->{first_issued_ind} = $self->{last_issued_ind} + 1; $self->{objlist} = []; } sub parts_list($) { # returns a ref to a list of recently issued names my $self = shift; $self->{objlist}; } sub parts_list_add($$) { # add a parts object to the list of parts my($self, $part) = @_; push(@{$self->{objlist}}, $part); } sub generate_new_num($$) { # make-up a new number for a file and return it my($self, $ignore_limit) = @_; if (!$ignore_limit && defined($self->{maxfiles}) && $self->{num_of_issued_names} >= $self->{maxfiles}) { # do not change the text in die without adjusting decompose_part() die "Maximum number of files ($self->{maxfiles}) exceeded"; } $self->{num_of_issued_names}++; $self->{last_issued_ind}++; $self->{last_issued_ind}; } sub consumed_bytes($$;$$) { my($bytes, $bywhom, $tentatively, $exquota) = @_; if (ll(4)) { my($perc) = !$avail_quota ? '' : sprintf(", (%.0f%%)", 100 * ($avail_quota - ($rem_quota - $bytes)) / $avail_quota); do_log(4,"Charging %d bytes to remaining quota %d (out of %d%s) - by %s", $bytes, $rem_quota, $avail_quota, $perc, $bywhom); } if ($bytes > $rem_quota && $rem_quota >= 0) { # Do not modify the following signal text, it gets matched elsewhere! my($msg) = "Exceeded storage quota $avail_quota bytes by $bywhom; ". "last chunk $bytes bytes"; do_log(-1, "%s", $msg); die "$msg\n" if !$exquota; # die, unless allowed to exceed quota } $rem_quota -= $bytes unless $tentatively; $rem_quota; # return remaining quota } 1; # package Amavis::Unpackers::Part; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Util qw(ll do_log); } use vars qw($file_generator_object); sub init($) { $file_generator_object = shift } sub new($;$$$) { # create a part descriptor object my($class, $dir_name,$parent,$ignore_limit) = @_; my($self) = bless {}, $class; if (!defined($dir_name) && !defined($parent)) { # just make an empty object, presumably used as a new root } else { $self->number($file_generator_object->generate_new_num($ignore_limit)); $self->dir_name($dir_name) if defined $dir_name; if (defined $parent) { $self->parent($parent); my($ch_ref) = $parent->children; push(@$ch_ref,$self); $parent->children($ch_ref); } $file_generator_object->parts_list_add($self); # save it ll(4) && do_log(4, "Issued a new %s: %s", defined $dir_name ? "file name" : "pseudo part", $self->base_name); } $self; } sub number { my $self=shift; !@_ ? $self->{number} : ($self->{number}=shift) }; sub dir_name { my $self=shift; !@_ ? $self->{dir_name} : ($self->{dir_name}=shift) }; sub parent { my $self=shift; !@_ ? $self->{parent} : ($self->{parent}=shift) }; sub children { my $self=shift; !@_ ? $self->{children}||[] : ($self->{children}=shift) }; sub mime_placement # part location within a MIME tree, e.g. "1/1/3" { my $self=shift; !@_ ? $self->{place} : ($self->{place}=shift) }; sub type_short # string or a ref to a list of strings, case sensitive { my $self=shift; !@_ ? $self->{ty_short} : ($self->{ty_short}=shift) }; sub type_long { my $self=shift; !@_ ? $self->{ty_long} : ($self->{ty_long}=shift) }; sub type_declared { my $self=shift; !@_ ? $self->{ty_decl} : ($self->{ty_decl}=shift) }; sub name_declared # string or a ref to a list of strings { my $self=shift; !@_ ? $self->{nm_decl} : ($self->{nm_decl}=shift) }; sub report_type # a string, e.g. 'delivery-status', RFC 3462 { my $self=shift; !@_ ? $self->{rep_typ} : ($self->{rep_typ}=shift) }; sub size { my $self=shift; !@_ ? $self->{size} : ($self->{size}=shift) }; sub exists { my $self=shift; !@_ ? $self->{exists} : ($self->{exists}=shift) }; sub attributes # listref of characters representing attributes { my $self=shift; !@_ ? $self->{attr} : ($self->{attr}=shift) }; sub attributes_add { # U=undecodable, C=crypted, D=directory,S=special,L=link my $self = shift; my $a = $self->{attr} || []; for my $arg (@_) { push(@$a,$arg) if $arg ne '' && !grep($_ eq $arg, @$a) } $self->{attr} = $a; }; sub base_name { my $self = shift; sprintf("p%03d",$self->number) } sub full_name { my $self = shift; my $d = $self->dir_name; !defined($d) ? undef : $d.'/'.$self->base_name; } # returns a ref to a list of part ancestors, starting with the root object, # and including the part object itself # sub path { my $self = shift; my(@path); for (my($p)=$self; defined($p); $p=$p->parent) { unshift(@path,$p) } \@path; }; 1; # package Amavis::Unpackers::OurFiler; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter MIME::Parser::Filer); # subclass of MIME::Parser::Filer } # This package will be used by mime_decode(). # # We don't want no heavy MIME::Parser machinery for file name extension # guessing, decoding charsets in filenames (and listening to complaints # about it), checking for evil filenames, checking for filename contention, ... # (which cannot be turned off completely by ignore_filename(1) !!!) # Just enforce our file name! And while at it, collect generated filenames. # sub new($$$) { my($class, $dir, $parent_obj) = @_; $dir =~ s{/+\z}{}; # chop off trailing slashes from directory name bless {parent => $parent_obj, directory => $dir}, $class; } # provide a generated file name # sub output_path($@) { my($self, $head) = @_; my($newpart_obj) = Amavis::Unpackers::Part->new($self->{directory}, $self->{parent}, 1); get_amavisd_part($head, $newpart_obj); # store object into head $newpart_obj->full_name; } sub get_amavisd_part($;$) { my($head) = shift; !@_ ? $head->{amavisd_parts_obj} : ($head->{amavisd_parts_obj} = shift); } 1; # package Amavis::Unpackers::Validity; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&check_header_validity &check_for_banned_names); import Amavis::Util qw(ll do_log min max minmax untaint sanitize_str); import Amavis::Conf qw(:platform %banned_rules c cr ca); import Amavis::Lookup qw(lookup lookup2); } use subs @EXPORT_OK; sub check_header_validity($) { my($msginfo) = @_; local($1,$2,$3); my(@bad); my($minor_badh_category) = 0; my(%field_head_counts); my($allowed_tests) = cr('allowed_header_tests'); my(%t) = !ref($allowed_tests) ? () : %$allowed_tests; # minor category: 2: 8-bit char, 3: NUL/CR control, 4: empty line, 5: long, # 6: syntax, 7: missing, 8: multiple for my $curr_head (@{$msginfo->orig_header}) {#array of hdr fields, not lines my($field_name,$msg1,$msg2,$pre,$mid,$post); # obsolete RFC 822 syntax allowed whitespace before colon $field_name = $1 if $curr_head =~ /^([!-9;-\176]+)[ \t]*:/s; $field_head_counts{lc($field_name)}++ if defined $field_name; if (!defined($field_name) || $field_name =~ /^--/) { if ($t{'syntax'}) { $msg1 = "Invalid header field syntax"; $pre = ''; $mid = ''; $post = $curr_head; $minor_badh_category = max(6, $minor_badh_category); } } elsif ($t{'empty'} && $curr_head =~ /^([ \t]+)(?=\n|\z)/gms) { $mid = $1; $msg1 ="Improper folded header field made up entirely of whitespace"; # note: using //g and pos to avoid deep recursion in regexp $minor_badh_category = max(4, $minor_badh_category); } elsif ($t{'long'} && $curr_head =~ /^([^\n]{999,})(?=\n|\z)/gms) { $mid = $1; $msg1 = "Header line longer than 998 characters"; $minor_badh_category = max(5, $minor_badh_category); } elsif ($t{'control'} && $curr_head =~ /([\000\015])/gs) { $mid = $1; $msg1 = "Improper use of control character"; $minor_badh_category = max(3, $minor_badh_category); } elsif ($t{'8bit'} && $curr_head =~ /([\200-\377])/gs) { $mid = $1; $msg1 = "Non-encoded 8-bit data"; $minor_badh_category = max(2, $minor_badh_category); } elsif ($t{'8bit'} && $curr_head =~ /([^\000-\377])/gs) { $mid = $1; $msg1 = "Non-encoded Unicode character"; # should not happen $minor_badh_category = max(2, $minor_badh_category); } $pre = substr($curr_head,0,pos($curr_head)-length($mid)) if !defined $pre; $post = substr($curr_head,pos($curr_head)) if !defined $post; if (defined $msg1) { chomp($post); if (length($mid) > 20) { $mid = substr($mid, 0,15) . '[...]' } if (length($post) > 20) { $post = substr($post,0,15) . '[...]' } if (length($pre)-length($field_name)-2 > 50-length($post)) { $pre = $field_name . ': ...' . substr($pre, length($pre) - (45-length($post))); } $msg1 .= sprintf(" (char %02X hex)", ord($mid)) if length($mid)==1; $msg2 = sanitize_str($pre . $mid . $post); push(@bad, "$msg1: $msg2"); } last if @bad >= 100; # some sanity limit } # RFC 5322 (ex RFC 2822), RFC 2045, RFC 2183 for (qw(Date From Sender Reply-To To Cc Bcc Subject Message-ID References In-Reply-To MIME-Version Content-Type Content-Transfer-Encoding Content-ID Content-Description Content-Disposition Auto-Submitted)) { my($n) = $field_head_counts{lc($_)}; if ($n < 1 && $t{'missing'} && /^(?:Date|From)\z/i) { push(@bad, "Missing required header field: \"$_\""); $minor_badh_category = max(7, $minor_badh_category); } elsif ($n > 1 && $t{'multiple'}) { if ($n == 2) { push(@bad, "Duplicate header field: \"$_\""); } else { push(@bad, sprintf('Header field occurs more than once: "%s" '. 'occurs %d times', $_, $n)); } $minor_badh_category = max(8, $minor_badh_category); } } if (!@bad) { do_log(5,"check_header: %d, OK", $minor_badh_category) } elsif (ll(2)) { do_log(2,"check_header: %d, %s", $minor_badh_category, $_) for @bad } (\@bad, $minor_badh_category); } sub check_for_banned_names($) { my($msginfo) = @_; do_log(3, "Checking for banned types and filenames"); my($bfnmr) = ca('banned_filename_maps'); # two-level map: recip, partname my(@recip_tables); # a list of records describing banned tables for recips my($any_table_in_recip_tables) = 0; my($any_not_bypassed) = 0; for my $r (@{$msginfo->per_recip_data}) { my($recip) = $r->recip_addr; my(@tables,@tables_m); # list of banned lookup tables for this recipient if (!$r->bypass_banned_checks) { # not bypassed $any_not_bypassed = 1; my($t_ref,$m_ref) = lookup2(1,$recip,$bfnmr); if (defined $t_ref) { for my $ti (0..$#$t_ref) { # collect all relevant tables for each recip my($t) = $t_ref->[$ti]; # an entry may be a ref to a list of lookup tables, or a comma- or # whitespace-separated list of table names (suitable for SQL), # which are mapped to actual lookup tables through %banned_rules if (!defined($t)) { # ignore } elsif (ref($t) eq 'ARRAY') { # a list of actual lookup tables push(@tables, @$t); push(@tables_m, ($m_ref->[$ti]) x @$t); } else { # a list of rules _names_, to be mapped via %banned_rules my(@names); my(@rawnames) = grep(!/^[, ]*\z/, ($t =~ /\G (?: " (?: \\. | [^"\\] ){0,999} " | [^, ] )+ | [, ]+/gsx)); # in principle quoted strings could be used # to construct lookup tables on-the-fly (not implemented) for my $n (@rawnames) { # collect only valid names if (!exists($banned_rules{$n})) { do_log(2,"INFO: unknown banned table name %s, recip=%s", $n,$recip); } elsif (!defined($banned_rules{$n})) { # ignore undef } else { push(@names,$n) } } ll(3) && do_log(3,"collect banned table[%d]: %s, tables: %s", $ti,$recip, join(', ',map($_.'=>'.$banned_rules{$_}, @names))); if (@names) { # any known and valid table names? push(@tables, map($banned_rules{$_}, @names)); push(@tables_m, ($m_ref->[$ti]) x @names); } } } } } push(@recip_tables, { r => $r, recip => $recip, tables => \@tables, tables_m => \@tables_m } ); $any_table_in_recip_tables=1 if @tables; } my($bnpre) = cr('banned_namepath_re'); $bnpre = $$bnpre if ref($bnpre) eq 'REF'; # allow one level of indirection if (!$any_not_bypassed) { do_log(3,"skipping banned check: all recipients bypass banned checks"); } elsif (!$any_table_in_recip_tables && !ref($bnpre)) { do_log(3,"skipping banned check: no applicable lookup tables"); } else { do_log(4,"starting banned checks - traversing message structure tree"); my($parts_root) = $msginfo->parts_root; my($part); for (my(@unvisited)=($parts_root); @unvisited and $part=shift(@unvisited); push(@unvisited,@{$part->children})) { # traverse decomposed parts tree breadth-first my(@path) = @{$part->path}; next if @path <= 1; shift(@path); # ignore place-holder root node next if @{$part->children}; # ignore non-leaf nodes my(@descr_trad); # a part path: list of predecessors of a message part my(@descr); # same, but in form suitable for check on banned_namepath_re for my $p (@path) { my(@k,$n); $n = $p->base_name; if ($n ne '') { $n=~s/[\t\n]/ /g; push(@k,"P=$n") } $n = $p->mime_placement; if ($n ne '') { $n=~s/[\t\n]/ /g; push(@k,"L=$n") } $n = $p->type_declared; $n = [$n] if !ref($n); for (@$n) {if ($_ ne ''){my($m)=$_; $m=~s/[\t\n]/ /g; push(@k,"M=$m")}} $n = $p->type_short; $n = [$n] if !ref($n); for (@$n) {if (defined($_) && $_ ne '') {my($m)=$_; $m=~s/[\t\n]/ /g; push(@k,"T=$m")} } $n = $p->name_declared; $n = [$n] if !ref($n); for (@$n) {if (defined($_) && $_ ne '') {my($m)=$_; $m=~s/[\t\n]/ /g; push(@k,"N=$m")} } $n = $p->attributes; $n = [$n] if !ref($n); for (@$n) {if (defined($_) && $_ ne '') {my($m)=$_; $m=~s/[\t\n]/ /g; push(@k,"A=$m")} } push(@descr, join("\t",@k)); push(@descr_trad, [map { local($1,$2); /^([a-zA-Z0-9])=(.*)\z/s; my($key_what,$key_val) = ($1,$2); $key_what eq 'M' || $key_what eq 'N' ? $key_val : $key_what eq 'T' ? ('.'.$key_val) # prepend a dot (compatibility) : $key_what eq 'A' && $key_val eq 'U' ? 'UNDECIPHERABLE' : ()} @k]); } # we have obtained a description of a part as a list of its predecessors # in a message structure including the part itself at the end of the list my($key_val_str) = join(' | ',@descr); $key_val_str =~ s/\t/,/g; my($key_val_trad_str) = join(' | ', map(join(',',@$_), @descr_trad)); # simplified result to be presented in an SMTP response and DSN my($simple_part_name) = join(',', @{$descr_trad[-1]}); # just leaf node # evaluate current mail component path against each recipients' tables ll(4) && do_log(4, "check_for_banned (%s) %s", join(',', map($_->base_name, @path)), $key_val_trad_str); for my $e (@recip_tables) { @$e{qw(found result matchk part_descr_attr part_descr_trad part_name)} = (0, undef, undef, undef, undef, undef); } my($result,$matchingkey); my($t_ref_old); for my $e (@recip_tables) { # for each recipient and his tables my($found,$recip,$t_ref) = @$e{qw(found recip tables)}; if ($t_ref && @$t_ref) { my($same_as_prev) = $t_ref_old && @$t_ref_old==@$t_ref && !grep($t_ref_old->[$_] ne $t_ref->[$_], (0..$#$t_ref)) ? 1 : 0; if ($same_as_prev) { do_log(4, "skip banned check for %s, same tables as previous, result => %s", $recip,$result); } else { do_log(5,"doing banned check for %s on %s", $recip,$key_val_trad_str); ($result,$matchingkey) = lookup2(0, [map(@$_,@descr_trad)], # check all attribs in one go [map(ref($_) eq 'ARRAY' ? @$_ : $_, @$t_ref)], Label=>"check_bann:$recip"); $t_ref_old = $t_ref; } if (defined $result) { @$e{qw(found result matchk part_descr_attr part_descr_trad part_name)} = (1, $result, $matchingkey, $key_val_str, $key_val_trad_str, $simple_part_name); } } } if (ref $bnpre && grep(!$_->{result}, @recip_tables)) { # any non-true? # try new style: banned_namepath_re; it is global, not per-recipient my $descr_str = join("\n",@descr); if ($] < 5.012003) { # avoid a [perl #62048] bug in lookup_re(): # Unwarranted "Malformed UTF-8 character" on tainted variable $descr_str = untaint($descr_str); } my($result,$matchingkey) = lookup2(0, $descr_str, [$bnpre], Label=>'banned_namepath_re'); if (defined $result) { for my $e (@recip_tables) { if (!$e->{found}) { @$e{qw(found result matchk part_descr_attr part_descr_trad part_name)} = (1, $result, $matchingkey, $key_val_str, $key_val_trad_str, $simple_part_name); } } } } my(%esc) = (r => "\r", n => "\n", f => "\f", b => "\b", e => "\e", a => "\a", t => "\t"); # for pretty-printing my($ll) = grep($_->{result}, @recip_tables) ? 1 : 3; # log level for my $e (@recip_tables) { # log and store results my($r, $recip, $result, $matchingkey, $part_descr_attr, $part_descr_trad, $part_name) = @$e{qw(r recip result matchk part_descr_attr part_descr_trad part_name)}; if (ll($ll)) { # only bother with logging when needed local($1); my($mk) = defined $matchingkey ? $matchingkey : ''; # pretty-print $mk =~ s{ \\(.) }{ exists($esc{$1}) ? $esc{$1} : '\\'.$1 }egsx; do_log($result?1:3, 'p.path%s %s: "%s"%s', !$result?'':" BANNED:$result", $recip, $key_val_str, !defined $result ? '' : ", matching_key=\"$mk\""); } my($a); if ($result) { # the part being tested is banned for this recipient $a = $r->banned_parts || []; push(@$a,$part_descr_trad); $r->banned_parts($a); $a = $r->banned_parts_as_attr || []; push(@$a,$part_descr_attr); $r->banned_parts_as_attr($a); $a = $r->banning_rule_rhs || []; push(@$a,$result); $r->banning_rule_rhs($a); $a = $r->banning_rule_key || []; $matchingkey = "$matchingkey"; # make a plain string out of a qr push(@$a,$matchingkey); $r->banning_rule_key($a); my(@comments) = $matchingkey =~ / \( \? \# \s* (.*?) \s* \) /gsx; $a = $r->banning_rule_comment || []; push(@$a, @comments ? join(' ',@comments) : $matchingkey); $r->banning_rule_comment($a); if (!defined($r->banning_reason_short)) { # just the first my($s) = $part_name; $s =~ s/[ \t]{6,}/ ... /g; # compact whitespace $s = join(' ',@comments) . ':' . $s if @comments; $r->banning_reason_short($s); } } } # last if !grep(!$_->{result}, @recip_tables); # stop if all recips true } # endfor: message tree traversal } # endif: doing parts checking } 1; # package Amavis::Unpackers::MIME; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&mime_decode); import Amavis::Conf qw(:platform c cr ca $MAXFILES); import Amavis::Timing qw(section_time); import Amavis::Util qw(snmp_count untaint ll do_log safe_decode safe_encode); import Amavis::Unpackers::NewFilename qw(consumed_bytes); } use subs @EXPORT_OK; use Errno qw(ENOENT EACCES); use IO::File qw(O_CREAT O_EXCL O_WRONLY); use MIME::Parser; use MIME::Words; # use Scalar::Util qw(tainted); # save MIME preamble and epilogue (if nontrivial) as extra (pseudo)parts # sub mime_decode_pre_epi($$$$$) { my($pe_name, $pe_lines, $tempdir, $parent_obj, $placement) = @_; if (defined $pe_lines && @$pe_lines) { do_log(5, "mime_decode_%s: %d lines", $pe_name, scalar(@$pe_lines)); if (@$pe_lines > 5 || "@$pe_lines" !~ m{^[A-Za-z0-9/\@:;,. \t\n_-]*\z}s) { my($newpart_obj) = Amavis::Unpackers::Part->new("$tempdir/parts",$parent_obj,1); $newpart_obj->mime_placement($placement); $newpart_obj->name_declared($pe_name); my($newpart) = $newpart_obj->full_name; my($outpart) = IO::File->new; # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] $outpart->open($newpart, untaint(O_CREAT|O_EXCL|O_WRONLY), 0640) or die "Can't create $pe_name file $newpart: $!"; binmode($outpart,':bytes') or die "Can't cancel :utf8 mode: $!"; my($len); for (@$pe_lines) { $outpart->print($_) or die "Can't write $pe_name to $newpart: $!"; $len += length($_); } $outpart->close or die "Error closing $pe_name $newpart: $!"; $newpart_obj->size($len); consumed_bytes($len, "mime_decode_$pe_name", 0, 1); } } } # traverse MIME::Entity object depth-first, # extracting preambles and epilogues as extra (pseudo)parts, and # filling-in additional information into Amavis::Unpackers::Part objects # sub mime_traverse($$$$$); # prototype sub mime_traverse($$$$$) { my($entity, $tempdir, $parent_obj, $depth, $placement) = @_; mime_decode_pre_epi('preamble', $entity->preamble, $tempdir, $parent_obj, $placement); my($mt, $et) = ($entity->mime_type, $entity->effective_type); my($part); my($head) = $entity->head; my($body) = $entity->bodyhandle; if (!defined($body)) { # a MIME container only contains parts, no bodypart # create pseudo-part objects for MIME containers (e.g. multipart/* ) $part = Amavis::Unpackers::Part->new(undef,$parent_obj,1); # $part->type_short('no-file'); do_log(2, "%s %s Content-Type: %s", $part->base_name, $placement, $mt); } else { # does have a body part (i.e. not a MIME container) my($fn) = $body->path; my($size); if (!defined($fn)) { $size = length($body->as_string); } else { my($msg); my($errn) = lstat($fn) ? 0 : 0+$!; if ($errn == ENOENT) { $msg = "does not exist" } elsif ($errn) { $msg = "is inaccessible: $!" } elsif (!-r _) { $msg = "is not readable" } elsif (!-f _) { $msg = "is not a regular file" } else { $size = -s _; do_log(4,"mime_traverse: file %s is empty", $fn) if $size==0; } do_log(-1,"WARN: mime_traverse: file %s %s", $fn,$msg) if defined $msg; } consumed_bytes($size, 'mime_decode', 0, 1); # retrieve Amavis::Unpackers::Part object (if any), stashed into head obj $part = Amavis::Unpackers::OurFiler::get_amavisd_part($head); if (defined $part) { $part->size($size); if (defined($size) && $size==0) { $part->type_short('empty'); $part->type_long('empty') } ll(2) && do_log(2, "%s %s Content-Type: %s, size: %d B, name: %s", $part->base_name, $placement, $mt, $size, $entity->head->recommended_filename); my($old_parent_obj) = $part->parent; if ($parent_obj ne $old_parent_obj) { # reparent if necessary ll(5) && do_log(5,"reparenting %s from %s to %s", $part->base_name, $old_parent_obj->base_name, $parent_obj->base_name); my($ch_ref) = $old_parent_obj->children; $old_parent_obj->children([grep($_ ne $part, @$ch_ref)]); $ch_ref = $parent_obj->children; push(@$ch_ref,$part); $parent_obj->children($ch_ref); $part->parent($parent_obj); } } } if (defined $part) { $part->mime_placement($placement); $part->type_declared($mt eq $et ? $mt : [$mt, $et]); $part->attributes_add('U','C') if $mt =~ m{/encrypted}i || $et =~ m{/encrypted}i; my %rn_seen; my @rn; # recommended file names, both raw and RFC 2047 / RFC 2231 decoded for my $attr_name ('content-disposition.filename', 'content-type.name') { my $val_raw = $head->mime_attr($attr_name); next if !defined $val_raw || $val_raw eq ''; my $val_dec = ''; # decoded, represented as native Perl characters eval { my(@chunks) = MIME::Words::decode_mimewords($val_raw); for my $pair (@chunks) { my($data,$encoding) = @$pair; $encoding = 'ISO-8859-1' if !defined $encoding || $encoding eq ''; $encoding =~ s/\*[^*]*\z//; # strip RFC 2231 language suffix $val_dec .= safe_decode($encoding,$data); } }; if ($val_dec ne '' && !$rn_seen{$val_dec}) { push(@rn,$val_dec); $rn_seen{$val_dec} = 1; } if (!$rn_seen{$val_raw}) { push(@rn,$val_raw); $rn_seen{$val_raw} = 1; } } $part->name_declared(@rn==1 ? $rn[0] : \@rn) if @rn; my $val = $head->mime_attr('content-type.report-type'); if (defined $val && $val ne '') { # $val = safe_encode('UTF-8',$val); $part->report_type($val); } } mime_decode_pre_epi('epilogue', $entity->epilogue, $tempdir, $parent_obj, $placement); my($item_num) = 0; for my $e ($entity->parts) { # recursive descent $item_num++; mime_traverse($e,$tempdir,$part,$depth+1,"$placement/$item_num"); } } # Break up mime parts, return a MIME::Entity object # sub mime_decode($$$) { my($fileh, $tempdir, $parent_obj) = @_; # $fileh may be an open file handle, or a file name my($parser) = MIME::Parser->new; $parser->filer(Amavis::Unpackers::OurFiler->new("$tempdir/parts", $parent_obj)); $parser->ignore_errors(1); # also is the default # if bounce killer is enabled, extract_nested_messages must be off, # otherwise we lose headers of attached message/rfc822 messages $parser->extract_nested_messages(0); # $parser->extract_nested_messages("NEST"); # parse embedded message/rfc822 # "NEST" complains with "part did not end with expected boundary" when # the outer message is message/partial and the inner message is chopped $parser->extract_uuencode(1); # to enable or not to enable ??? $parser->max_parts($MAXFILES) if defined $MAXFILES && $MAXFILES > 0 && $parser->UNIVERSAL::can('max_parts'); my($entity); snmp_count('OpsDecByMimeParser'); if (ref($fileh)) { # assume open file handle do_log(4, "Extracting mime components"); $fileh->seek(0,0) or die "Can't rewind mail file: $!"; local($1,$2,$3,$4); # avoid Perl 5.8.0 & 5.8.2 bug, $1 gets tainted ! $entity = $parser->parse($fileh); } else { # assume $fileh is a file name do_log(4, "Extracting mime components from %s", $fileh); local($1,$2,$3,$4); # avoid Perl 5.8.0 & 5.8.2 bug, $1 gets tainted ! $entity = $parser->parse_open("$tempdir/parts/$fileh"); } my($mime_err) = $parser->results->errors; if (defined $mime_err) { $mime_err=~s/\s+\z//; $mime_err=~s/[ \t\r]*\n+/; /g; $mime_err=~s/\s+/ /g; $mime_err = substr($mime_err,0,250) . '[...]' if length($mime_err) > 250; do_log(1, "WARN: MIME::Parser %s", $mime_err) if $mime_err ne ''; } elsif (!defined($entity)) { $mime_err = "Unable to parse, perhaps message contains too many parts"; do_log(1, "WARN: MIME::Parser %s", $mime_err); $entity = ''; } mime_traverse($entity, $tempdir, $parent_obj, 0, '1') if $entity; section_time('mime_decode'); ($entity, $mime_err); } 1; # package Amavis::MIME::Body::OnOpenFh; # A body class that keeps data on an open file handle, read-only, # while allowing to prepend a couple of lines when reading from it. # $skip_bytes bytes at the beginning of a given open file are ignored. use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter MIME::Body); # subclass of MIME::Body import Amavis::Util qw(ll do_log); } sub init { my($self, $fh,$prefix_lines,$skip_bytes) = @_; $self->{MB_Am_fh} = $fh; $self->{MB_Am_prefix} = defined $prefix_lines ? join('',@$prefix_lines) : ''; $self->{MB_Am_prefix_l} = length($self->{MB_Am_prefix}); $self->{MB_Am_skip_bytes} = !defined $skip_bytes ? 0 : $skip_bytes; $self->is_encoded(1); $self; } sub open { my($self,$mode) = @_; $self->close; # ignoring status $mode eq 'r' or die "Only offers read-only access, mode: $mode"; my($fh) = $self->{MB_Am_fh}; my($skip) = $self->{MB_Am_skip_bytes}; $fh->seek($skip,0) or die "Can't rewind mail file: $!"; $self->{MB_Am_pos} = 0; bless { parent => $self }; } sub close { 1 } sub read { # SCALAR,LENGTH,OFFSET my $self = shift; my($len) = $_[1]; my($offset) = $_[2]; my($parent) = $self->{parent}; my($pos) = $parent->{MB_Am_pos}; my($str1) = ''; my($str2) = ''; my($nbytes) = 0; if ($len > 0 && $pos < $parent->{MB_Am_prefix_l}) { $str1 = substr($parent->{MB_Am_prefix}, $pos, $len); $nbytes += length($str1); $len -= $nbytes; } my($msg); if ($len > 0) { my($nb) = $parent->{MB_Am_fh}->read($str2,$len); if (!defined $nb) { $msg = "Error reading: $!"; } elsif ($nb < 1) { # read returns 0 at eof } else { $nbytes += $nb; $len -= $nb; } } if (defined $msg) { undef $nbytes; # $! already set by a failed read } else { ($offset ? substr($_[0],$offset) : $_[0]) = $str1.$str2; $pos += $nbytes; $parent->{MB_Am_pos} = $pos; } $nbytes; # eof: 0; error: undef } 1; # package Amavis::Notify; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&delivery_status_notification &delivery_short_report &build_mime_entity &defanged_mime_entity &msg_from_quarantine &expand_variables); import Amavis::Util qw(ll do_log safe_encode sanitize_str min max minmax untaint make_password orcpt_decode xtext_decode ccat_split ccat_maj); import Amavis::Timing qw(section_time); import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::ProcControl qw(exit_status_str proc_status_ok run_command collect_results); import Amavis::Out::EditHeader qw(hdr); import Amavis::Lookup qw(lookup lookup2); import Amavis::Expand qw(expand); import Amavis::rfc2821_2822_Tools; } use subs @EXPORT_OK; use IO::File qw(O_RDONLY O_WRONLY O_RDWR O_APPEND O_CREAT O_EXCL); use MIME::Entity; use Time::HiRes (); # use Encode; # Perl 5.8 UTF-8 support # replace substring ${myhostname} with a value of a corresponding variable sub expand_variables($) { my($str) = @_; local($1,$2); $str =~ s{ \$ (?: \{ ([^\}]+) \} | ([a-zA-Z](?:[a-zA-Z0-9_]*[a-zA-Z0-9])?\b) ) } { { 'myhostname' => c('myhostname') }->{lc($1.$2)} }egx; $str; } sub wrap_message_into_archive($$) { my($msginfo,$prefix_lines_ref) = @_; # a file with a copy of a mail msg as retrieved from a quarantine: my($attachment_email_name) = c('attachment_email_name'); # 'msg-%m.eml' # an archive file (will contain a retrieved message) to be attached: my($attachment_outer_name) = c('attachment_outer_name'); # 'msg-%m.zip' my($email_fh, $arch_size); my($mail_id) = $msginfo->mail_id; if (!defined $mail_id || $mail_id eq '') { $mail_id = ''; } else { $mail_id =~ /^[A-Za-z0-9_-]*\z/ or die "unsafe mail_id: $mail_id"; $mail_id = untaint($mail_id); } for ($attachment_email_name, $attachment_outer_name) { local $1; s{%(.)}{ $1 eq 'b' ? $msginfo->body_digest : $1 eq 'P' ? $msginfo->partition_tag : $1 eq 'm' ? $mail_id : $1 eq 'n' ? $msginfo->log_id : $1 eq 'i' ? iso8601_timestamp($msginfo->rx_time,1) #,'-') : $1 eq '%' ? '%' : '%'.$1 }egs; $_ = $msginfo->mail_tempdir . '/' . $_; } my($eval_stat); eval { # copy a retrieved message to a file my($inp_fh) = $msginfo->mail_text; $inp_fh->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; $email_fh = IO::File->new; $email_fh->open($attachment_email_name, O_CREAT|O_EXCL|O_RDWR, 0640) or die "Can't create file $attachment_email_name: $!"; binmode($email_fh,':bytes') or die "Can't cancel :utf8 mode: $!"; # copy quarantined mail starting at skip_bytes to $attachment_email_name for (@$prefix_lines_ref) { $email_fh->print($_) or die "Error writing to $attachment_email_name: $!"; } my($nbytes,$buff); while (($nbytes=$inp_fh->read($buff,16384)) > 0) { $email_fh->print($buff) or die "Error writing to $attachment_email_name: $!"; } defined $nbytes or die "Error reading mail file: $!"; $email_fh->close or die "Can't close file $attachment_email_name: $!"; undef $email_fh; undef $buff; # release storage # create a password-protected archive containing the just prepared file; # no need to shell-protect arguments, as this does not invoke a shell my($password) = $msginfo->attachment_password; my(@command) = ( qw(zip -q -j -l), $password eq '' ? () : ('-P', $password), $attachment_outer_name, $attachment_email_name ); # supplying a password on a command line is lame as it shows in ps(1), # but an option -e would require a pseudo terminal, which is really # an overweight cannon unnecessary here: the password is used as a # scrambler only, protecting against accidental opening of a file, # so there is no security issue here $password = 'X' x length($password); # can't hurt to hide it my($proc_fh,$pid) = run_command(undef,undef,@command); my($r,$status) = collect_results($proc_fh,$pid,'zip',16384,[0]); undef $proc_fh; undef $pid; do_log(2,'archiver said: %s',$$r) if ref $r && $$r ne ''; $status == 0 or die "Error creating an archive: $status, $$r"; my($errn) = lstat($attachment_outer_name) ? 0 : 0+$!; if ($errn) { die "Archive $attachment_outer_name is inaccessible: $!" } else { $arch_size = 0 + (-s _) } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; }; if ($eval_stat ne '' || !$arch_size) { # handle failure my($msg) = $eval_stat ne '' ? $eval_stat : sprintf("archive size %d", $arch_size); do_log(-1,'Preparing an archive from a quarantined message failed: %s', $msg); if (defined $email_fh && $email_fh->fileno) { $email_fh->close or do_log(-1,"Can't close %s: %s", $attachment_email_name, $!); } undef $email_fh; if (-e $attachment_email_name) { unlink($attachment_email_name) or do_log(-1,"Can't remove %s: %s", $attachment_email_name, $!); } if (-e $attachment_outer_name) { unlink($attachment_outer_name) or do_log(-1,"Can't remove %s: %s", $attachment_outer_name, $!); } die "Preparing an archive from a quarantined message failed: $msg\n"; } $attachment_outer_name; } # Create a MIME::Entity object. If $mail_as_string_ref points to a string # (multiline mail header with a plain text body) it is added as the first # MIME part. Optionally attach a message header section from original mail, # or attach a complete original message. # sub build_mime_entity($$$$$$$) { my($mail_as_string_ref, $msginfo, $mime_type, $msg_format, $flat, $attach_orig_headers, $attach_orig_message) = @_; $msg_format = '' if !defined $msg_format; if (!defined $mime_type || $mime_type !~ m{^multipart(/|\z)}i) { my($multipart_cnt) = 0; $multipart_cnt++ if $mail_as_string_ref; $multipart_cnt++ if defined $msginfo && ($attach_orig_headers || $attach_orig_message); $mime_type = 'multipart/mixed' if $multipart_cnt > 1; } my($entity,$m_hdr,$m_body); if (!$mail_as_string_ref) { # no plain text part } elsif (substr($$mail_as_string_ref, 0,1) eq "\n") { # empty header section? $m_hdr = ''; $m_body = substr($$mail_as_string_ref,1); } else { # calling index and substr is much faster than an equiv. split into $1,$2 # by a regular expression: /^( (?!\n) .*? (?:\n|\z))? (?: \n (.*) )? \z/sx my($ind) = index($$mail_as_string_ref,"\n\n"); # find hdr/body separator if ($ind < 0) { $m_hdr = $$mail_as_string_ref; $m_body = '' } # no body else { # normal mail structure, nonempty header section and nonempty body $m_hdr = substr($$mail_as_string_ref, 0, $ind+1); $m_body = substr($$mail_as_string_ref, $ind+2); } } $m_body = safe_encode(c('bdy_encoding'), $m_body) if defined $m_body; # make sure _our_ source line number is reported in case of failure my($multipart_cnt) = 0; eval { $entity = MIME::Entity->build( Type => defined $mime_type ? $mime_type : 'multipart/mixed', Encoding => '7bit', 'X-Mailer' => undef); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat; }; if (defined $m_hdr) { # insert header fields into MIME::Head entity # Mail::Header::modify allows all-or-nothing control over automatic header # fields folding by Mail::Header, which is too bad - we would prefer # to have full control on folding of header fields that are explicitly # inserted here, and let Mail::Header handle the rest. Sorry, can't be # done, so let's just disable folding by Mail::Header (which does a poor # job when presented with few break opportunities), and wrap our header # fields ourselves, hoping the remaining automatically generated header # fields won't be too long. local($1,$2); my($head) = $entity->head; $head->modify(0); $m_hdr =~ s/\r?\n(?=[ \t])//gs; # unfold header fields in a template for my $hdr_line (split(/\r?\n/, $m_hdr)) { if ($hdr_line =~ /^([^:]*?)[ \t]*:[ \t]*(.*)\z/s) { my($fhead,$fbody) = ($1,$2); my($str) = hdr($fhead,$fbody,0,' '); # encode, wrap, ... # re-split the result ($fhead,$fbody) = ($1,$2) if $str =~ /^([^:]*):[ \t]*(.*)\z/s; chomp($fbody); do_log(5, "build_mime_entity %s: %s", $fhead,$fbody); eval { # make sure _our_ source line number is reported on failure $head->replace($fhead,$fbody); 1; } or do { $@ = "errno=$!" if $@ eq ''; chomp $@; die $@ if $@ =~ /^timed out\b/; # resignal timeout die sprintf("%s header field '%s: %s'", ($@ eq '' ? "invalid" : "$@, "), $fhead,$fbody); }; } } } my(@prefix_lines); if (defined $m_body) { if ($flat && $attach_orig_message) { my($pos,$j); # split $m_body into lines, retaining each \n for ($pos=0; ($j=index($m_body,"\n",$pos)) >= 0; $pos = $j+1) { push(@prefix_lines, substr($m_body,$pos,$j-$pos+1)) } push(@prefix_lines, substr($m_body,$pos)) if $pos < length($m_body); } else { eval { # make sure _our_ source line number is reported on failure $entity->attach( Type => 'text/plain', Data => $m_body, Encoding => '-SUGGEST', Charset => c('bdy_encoding'), ); $multipart_cnt++; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat; }; } } # prepend a Return-Path to make available the envelope sender address push(@prefix_lines, "\n") if @prefix_lines; # separates text from a message push(@prefix_lines, sprintf("Return-Path: %s\n",$msginfo->sender_smtp)); if (defined $msginfo && $attach_orig_headers && !$attach_orig_message) { # attach a header section only do_log(4, "build_mime_entity: attaching just original header section"); eval { # make sure _our_ source line number is reported on failure $entity->attach( Type => $flat ? 'text/plain' : 'text/rfc822-headers', # RFC 3462 Encoding => $msginfo->header_8bit ? '8bit' : '7bit', Data => [@prefix_lines, @{$msginfo->orig_header}], Disposition => 'inline', Filename => 'header', Description => 'Message header section', ); $multipart_cnt++; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat; }; } elsif (defined $msginfo && $attach_orig_message) { # attach a complete message my($password); if ($msg_format eq 'attach') { # not 'arf' and not 'dsn' $password = $msginfo->attachment_password; # already have it? if (!defined $password) { # make one, and store it for later $password = make_password(c('attachment_password'), $msginfo); $msginfo->attachment_password($password); } } if ($msg_format eq 'attach' && # not 'arf' and not 'dsn' defined $password && $password ne '') { # attach as a ZIP archive $password = 'X' x length($password); # can't hurt to hide it do_log(4, "build_mime_entity: attaching entire original message as zip"); my($archive_fn) = wrap_message_into_archive($msginfo,\@prefix_lines); local($1); $archive_fn =~ m{([^/]*)\z}; my($att_filename) = $1; eval { # make sure _our_ source line number is reported on failure my($att) = $entity->attach( # RFC 2046 Type => 'application/zip', Filename => $att_filename, Path => $archive_fn, Encoding => 'base64', Disposition => 'attachment', Description => 'Original message', ); $multipart_cnt++; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat; }; } else { # attach as a normal message do_log(4, "build_mime_entity: attaching entire original message, plain"); my($orig_mail_as_body) = Amavis::MIME::Body::OnOpenFh->new($msginfo->mail_text, \@prefix_lines, $msginfo->skip_bytes); $orig_mail_as_body or die "Can't create Amavis::MIME::Body object: $!"; eval { # make sure _our_ source line number is reported on failure my($att) = $entity->attach( # RFC 2046 Type => $flat ? 'text/plain' : 'message/rfc822', Encoding => ($msginfo->header_8bit || $msginfo->body_8bit) ? '8bit' : '7bit', Data => [], # Path => $msginfo->mail_text_fn, $flat ? () : (Disposition => 'attachment', Filename => 'message', Description => 'Original message'), ); $att->bodyhandle($orig_mail_as_body); #direct access to tempfile handle $multipart_cnt++; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat; }; } } $entity->make_singlepart if $multipart_cnt < 2; $entity; # return the constructed MIME::Entity } # If $msg_format is 'dsn' generate a delivery status notification according # to RFC 3462 (ex RFC 1892), RFC 3464 (ex RFC 1894) and RFC 3461 (ex RFC 1891). # If $msg_format is 'arf' generate an abuse report according to RFC 5965 # - "An Extensible Format for Email Feedback Reports". If $msg_format is # 'attach', generate a report message and attach the original message. # If $msg_format is 'plain', generate a simple (flat) mail with the only # MIME part being the original message (abuse@yahoo.com can't currently # handle attachments in reports). Returns a message object, or undef if # DSN is requested but not needed. # $request_type: dsn, release, requeue, report # $msg_format: dsn, arf, attach, plain, resend # $feedback_type: abuse, dkim, fraud, miscategorized, not-spam, # opt-out, virus, other # sub delivery_status_notification($$$;$$$$) { # ..._or_report my($msginfo,$dsn_per_recip_capable,$builtins_ref, $notif_recips,$request_type,$feedback_type,$msg_format) = @_; my($notification); my($suppressed) = 0; if (!defined($msg_format)) { $msg_format = $request_type eq 'dsn' ? 'dsn' : $request_type eq 'report' ? c('report_format') : c('release_format'); } my($is_arf) = 0; my($is_dsn) = 0; my($is_attach) = 0; my($is_plain) = 0; if ($msg_format eq 'dsn') { $is_dsn = 1 } elsif ($msg_format eq 'arf') { $is_arf = 1 } elsif ($msg_format eq 'attach') { $is_attach = 1 } else { $is_plain = 1 } # 'plain' my($dsn_time) = $msginfo->rx_time; # time of dsn creation - same as message # use a reception time for consistency and to be resilient to clock jumps $dsn_time = Time::HiRes::time if !$dsn_time; # now my($rfc2822_dsn_time) = rfc2822_timestamp($dsn_time); my($sender) = $msginfo->sender; my($dsn_passed_on) = $msginfo->dsn_passed_on; # NOTIFY=SUCCESS passed to MTA my($per_recip_data) = $msginfo->per_recip_data; my($txt_recip) = ''; # per-recipient part of dsn text according to RFC 3464 my($all_rejected) = 0; if (@$per_recip_data) { $all_rejected = 1; for my $r (@$per_recip_data) { if ($r->recip_destiny != D_REJECT || $r->recip_smtp_response !~ /^5/) { $all_rejected = 0; last } } } my($min_spam_level, $max_spam_level) = minmax(map($_->spam_level, @{$msginfo->per_recip_data})); $min_spam_level = 0 if !defined $min_spam_level; $max_spam_level = 0 if !defined $max_spam_level; my($is_credible) = $msginfo->sender_credible || ''; my($os_fingerprint) = $msginfo->client_os_fingerprint; my($cutoff_byrecip_maps, $cutoff_bysender_maps); my($dsn_cutoff_level_bysender, $dsn_cutoff_level); if ($is_dsn && $sender ne '') { # for null sender it doesn't matter, as DSN will not be sent regardless if ($is_credible) { do_log(3, "DSN: sender is credible (%s), SA: %.3f, <%s>", $is_credible, $max_spam_level, $sender); $cutoff_byrecip_maps = ca('spam_crediblefrom_dsn_cutoff_level_maps'); $cutoff_bysender_maps = ca('spam_crediblefrom_dsn_cutoff_level_bysender_maps'); } else { do_log(5, "DSN: sender NOT credible, SA: %.3f, <%s>", $max_spam_level, $sender); $cutoff_byrecip_maps = ca('spam_dsn_cutoff_level_maps'); $cutoff_bysender_maps = ca('spam_dsn_cutoff_level_bysender_maps'); } $dsn_cutoff_level_bysender = lookup2(0,$sender,$cutoff_bysender_maps); } my($any_succ,$any_fail,$any_delayed) = (0,0,0); local($1); for my $r (!$is_dsn ? () : @$per_recip_data) { # prepare per-recip fields my($recip) = $r->recip_addr; my($smtp_resp) = $r->recip_smtp_response; my($recip_done) = $r->recip_done; # 2=relayed to MTA, 1=faked deliv/quarant my($ccat_name) = $r->setting_by_contents_category(\%ccat_display_names); $ccat_name = "NonBlocking:$ccat_name" if !defined($r->blocking_ccat); my($spam_level) = $r->spam_level; if (!$recip_done) { my($fwd_m) = $r->delivery_method; if (!defined $fwd_m) { do_log(-2,"TROUBLE: recipient not done, undefined delivery_method: ". "<%s> %s", $recip,$smtp_resp); } elsif ($fwd_m eq '') { # e.g. milter # as far as we are concerned all is ok, delivery will be performed # by a helper program or MTA $smtp_resp = "250 2.5.0 Ok, continue delivery"; } else { do_log(-2,"TROUBLE: recipient not done: <%s> %s", $recip,$smtp_resp); } } my($smtp_resp_class) = $smtp_resp =~ /^(\d)/ ? $1 : '0'; my($smtp_resp_code) = $smtp_resp =~ /^(\d+)/ ? $1 : '0'; my($dsn_notify) = $r->dsn_notify; my($notify_on_failure,$notify_on_success,$notify_on_delay,$notify_never) = (0,0,0,0); if (!defined($dsn_notify)) { $notify_on_failure = $notify_on_delay = 1; } else { for (@$dsn_notify) { # validity of the list has already been checked if ($_ eq 'FAILURE') { $notify_on_failure = 1 } elsif ($_ eq 'SUCCESS') { $notify_on_success = 1 } elsif ($_ eq 'DELAY') { $notify_on_delay = 1 } elsif ($_ eq 'NEVER') { $notify_never = 1 } } } if ($notify_never || $sender eq '') { $notify_on_failure = $notify_on_success = $notify_on_delay = 0 } my($dest) = $r->recip_destiny; my($remote_or_local) = $recip_done==2 ? 'from MTA' : $recip_done==1 ? '.' : # this agent 'status-to-be-passed-back'; # warn_sender is an old relict and does not fit well into DSN concepts; # we'll sneak it in, pretending to cause a DELAY notification my($warn_sender) = $notify_on_delay && $smtp_resp_class eq '2' && $recip_done==2 && $r->setting_by_contents_category(cr('warnsender_by_ccat')); ll(5) && do_log(5, "dsn: %s %s %s <%s> -> <%s>: on_succ=%d, on_dly=%d, ". "on_fail=%d, never=%d, warn_sender=%s, DSN_passed_on=%s, ". "destiny=%s, mta_resp: \"%s\"", $remote_or_local, $smtp_resp_code, $ccat_name, $sender, $recip, $notify_on_success, $notify_on_delay, $notify_on_failure, $notify_never, $warn_sender, $dsn_passed_on, $dest, $smtp_resp); # clearly log common cases to facilitate troubleshooting; # first look for some standard reasons for not sending a DSN if ($smtp_resp_class eq '4') { do_log(4, "DSN: TMPFAIL %s %s %s, not to be reported: <%s> -> <%s>", $remote_or_local,$smtp_resp_code,$ccat_name,$sender,$recip); } elsif ($smtp_resp_class eq '5' && $dest==D_REJECT && ($dsn_per_recip_capable || $all_rejected)) { do_log(4, "DSN: FAIL %s %s %s, status propagated back: <%s> -> <%s>", $remote_or_local,$smtp_resp_code,$ccat_name,$sender,$recip); } elsif ($smtp_resp_class eq '5' && !$notify_on_failure) { $suppressed = 1; do_log($recip_done==2 ? 0 : 4, # log level 0 for remotes, RFC 3461 5.2.2d "DSN: FAIL %s %s %s, %s requested to be IGNORED: <%s> -> <%s>", $remote_or_local,$smtp_resp_code,$ccat_name, $notify_never?'explicitly':'implicitly', $sender, $recip); } elsif ($smtp_resp_class eq '2' && !$notify_on_success && !$warn_sender) { my($fmt) = $dest==D_DISCARD ? "SUCC (discarded) %s %s %s, destiny=DISCARD" : "SUCC %s %s %s, no DSN requested"; do_log(5, "DSN: $fmt: <%s> -> <%s>", $remote_or_local,$smtp_resp_code,$ccat_name,$sender,$recip); } elsif ($smtp_resp_class eq '2' && $notify_on_success && $dsn_passed_on && !$warn_sender) { do_log(5, "DSN: SUCC %s %s %s, DSN parameters PASSED-ON: <%s> -> <%s>", $remote_or_local,$smtp_resp_code,$ccat_name,$sender,$recip); } elsif ($notify_never || $sender eq '') { # test sender just in case $suppressed = 1; do_log(5, "DSN: NEVER %s %s, <%s> -> %s", $smtp_resp_code,$ccat_name,$sender,$recip); # next, look for some good _excuses_ for not sending a DSN } elsif ($dest==D_DISCARD) { # requested by final_*_destiny $suppressed = 1; do_log(4, "DSN: FILTER %s %s %s, destiny=DISCARD: <%s> -> <%s>", $remote_or_local,$smtp_resp_code,$ccat_name,$sender,$recip); } elsif (defined $r->dsn_suppress_reason) { $suppressed = 1; do_log(3, "DSN: FILTER %s %s, suppress reason: %s, <%s> -> <%s>", $smtp_resp_code, $ccat_name, $r->dsn_suppress_reason, $sender,$recip); } elsif (defined $dsn_cutoff_level_bysender && $spam_level >= $dsn_cutoff_level_bysender) { $suppressed = 1; do_log(3, "DSN: FILTER %s %s, spam level %.3f exceeds cutoff %s%s, ". "<%s> -> <%s>", $smtp_resp_code, $ccat_name, $spam_level, $dsn_cutoff_level_bysender, !$is_credible ? '' : ", (credible: $is_credible)", $sender, $recip); } elsif (defined($cutoff_byrecip_maps) && ( $dsn_cutoff_level=lookup2(0,$recip,$cutoff_byrecip_maps), defined($dsn_cutoff_level) && ( $spam_level >= $dsn_cutoff_level || ( $r->recip_blacklisted_sender && !$r->recip_whitelisted_sender) ) ) ) { $suppressed = 1; do_log(3, "DSN: FILTER %s %s, spam level %.3f exceeds ". "by-recipient cutoff %s%s, <%s> -> <%s>", $smtp_resp_code, $ccat_name, $spam_level, $dsn_cutoff_level, !$is_credible ? '' : ", (credible: $is_credible)", $sender, $recip); } elsif (defined($msginfo->is_bulk) && ccat_maj($r->contents_category) > CC_CLEAN) { $suppressed = 1; do_log(3, "DSN: FILTER %s %s, suppressed, bulk mail (%s), <%s> -> <%s>", $smtp_resp_code,$ccat_name,$msginfo->is_bulk,$sender,$recip); } elsif ($os_fingerprint =~ /^Windows\b/ && # hard-coded limits! !$msginfo->dkim_envsender_sig && # a hack $spam_level >= ($os_fingerprint=~/^Windows XP(?![^(]*\b2000 SP)/ ? 5 : 8)) { $os_fingerprint =~ /^(\S+\s+\S+)/; do_log(3, "DSN: FILTER %s %s, suppressed for mail from %s ". "at %s, score=%s, <%s> -> <%s>", $smtp_resp_code, $ccat_name, $1, $msginfo->client_addr, $spam_level, $sender,$recip); } else { # RFC 3461, section 5.2.8: "A single DSN may describe attempts to deliver # a message to multiple recipients of that message. If a DSN is issued # for some recipients in an SMTP transaction and not for others according # to the rules above, the DSN SHOULD NOT contain information for # recipients for whom DSNs would not otherwise have been issued." $txt_recip .= "\n"; # empty line between groups of per-recipient fields my($dsn_orcpt) = $r->dsn_orcpt; if (defined $dsn_orcpt) { my($addr_type,$orcpt) = orcpt_decode($dsn_orcpt); $txt_recip .= "Original-Recipient: " . sanitize_str($addr_type.';'.$orcpt) . "\n"; } my($remote_mta) = $r->recip_remote_mta; if (!defined($dsn_orcpt) && $remote_mta ne '' && $r->recip_final_addr ne $recip) { $txt_recip .= "X-NextToLast-Final-Recipient: rfc822;" . quote_rfc2821_local($recip) . "\n"; $txt_recip .= "Final-Recipient: rfc822;" . quote_rfc2821_local($r->recip_final_addr) . "\n"; } else { $txt_recip .= "Final-Recipient: rfc822;" . quote_rfc2821_local($recip) . "\n"; } local($1,$2,$3); my($smtp_resp_code,$smtp_resp_enhcode,$smtp_resp_msg); if ($smtp_resp =~ /^ (\d{3}) [ \t-] [ \t]* ([245] \. \d{1,3} \. \d{1,3})? \s* (.*) \z/xs) { ($smtp_resp_code, $smtp_resp_enhcode, $smtp_resp_msg) = ($1,$2,$3); } else { $smtp_resp_msg = $smtp_resp; } if ($smtp_resp_enhcode eq '' && $smtp_resp_class =~ /^([245])\z/) { $smtp_resp_enhcode = "$1.0.0"; } my($action); # failed / relayed / delivered / expanded if ($recip_done == 2) { # truly forwarded to MTA $action = $smtp_resp_class eq '5' ? 'failed' # remote reject : $smtp_resp_class ne '2' ? undef # shouldn't happen : !$dsn_passed_on ? 'relayed' # relayed to non-conforming MTA : $warn_sender ? 'delayed' # disguised as a DELAY notification : undef; # shouldn't happen } elsif ($recip_done == 1) { # faked delivery to bit bucket or quarantine $action = $smtp_resp_class eq '5' ? 'failed' # local reject : $smtp_resp_class eq '2' ? 'delivered' # discard / bit bucket : undef; # shouldn't happen } elsif (!defined($recip_done) || $recip_done == 0) { $action = $smtp_resp_class eq '2' ? 'relayed' #???? : undef; # shouldn't happen } defined $action or die "Assert failed: $smtp_resp, $smtp_resp_class, ". "$recip_done, $dsn_passed_on"; if ($action eq 'failed') { $any_fail=1 } elsif ($action eq 'delayed') { $any_delayed=1 } else { $any_succ=1 } $txt_recip .= "Action: $action\n"; $txt_recip .= "Status: $smtp_resp_enhcode\n"; my($rem_smtp_resp) = $r->recip_remote_mta_smtp_response; if ($warn_sender && $action eq 'delayed') { $smtp_resp = '250 2.6.0 Bad message, but will be delivered anyway'; } elsif ($remote_mta ne '' && $rem_smtp_resp ne '') { $txt_recip .= "Remote-MTA: dns; $remote_mta\n"; $smtp_resp = $rem_smtp_resp; } elsif ($smtp_resp !~ /\n/ && length($smtp_resp) > 78-23) { # wrap magic # take liberty to wrap our own SMTP responses $smtp_resp = wrap_string("x" x (23-11) . $smtp_resp, 78-11,'','',0); # length(" 554 5.0.0 ") = 11; length("Diagnostic-Code: smtp; ") = 23 # insert and then remove prefix to maintain consistent wrapped size $smtp_resp =~ s/^x{12}//; # wrap response code according to RFC 3461 section 9.2 $smtp_resp = join("\n", @{wrap_smtp_resp($smtp_resp)}); } $smtp_resp =~ s/\n(?![ \t])/\n /gs; $txt_recip .= "Diagnostic-Code: smtp; $smtp_resp\n"; $txt_recip .= "Last-Attempt-Date: $rfc2822_dsn_time\n"; my($final_log_id) = $msginfo->log_id; $final_log_id .= '/' . $msginfo->mail_id if defined $msginfo->mail_id; $txt_recip .= sprintf("Final-Log-ID: %s\n", $final_log_id); do_log(2, "DSN: NOTIFICATION: Action:%s, %s %s %s, spam level %.3f, ". "<%s> -> <%s>", $action, $recip_done==2 && $action ne 'delayed' ? 'RELAYED' : 'LOCAL', $smtp_resp_code, $ccat_name, $spam_level, $sender, $recip); } } # endfor per_recip_data if ( $is_arf || $is_plain || $is_attach || ($is_dsn && ($any_succ || $any_fail || $any_delayed)) ) { my(@hdr_to) = defined $notif_recips ? qquote_rfc2821_local(@$notif_recips) : map($_->recip_addr_smtp, @$per_recip_data); my($hdr_from) = $msginfo->setting_by_contents_category( $is_dsn ? cr('hdrfrom_notify_sender_by_ccat') : $request_type eq 'report' ? cr('hdrfrom_notify_report_by_ccat') : cr('hdrfrom_notify_release_by_ccat') ); $hdr_from = expand_variables($hdr_from); # use the provided template text my(%mybuiltins) = %$builtins_ref; # make a local copy # not really needed, these header fields are overridden later $mybuiltins{'f'} = $hdr_from; $mybuiltins{'T'} = \@hdr_to; $mybuiltins{'d'} = $rfc2822_dsn_time; $mybuiltins{'report_format'} = $msg_format; $mybuiltins{'feedback_type'} = $feedback_type; # RFC 3461 section 6.2: "If a DSN contains no notifications of # delivery failure, the MTA SHOULD return only the header section." my($dsn_ret) = $msginfo->dsn_ret; my($attach_full_msg) = !$is_dsn ? 1 : (defined $dsn_ret && $dsn_ret eq 'FULL' && $any_fail); if ($attach_full_msg && $is_dsn) { # apologize in the log, we should have supplied the full message, yet # RFC 3461 section 6.2 gives us an excuse: "However, if the length of the # message is greater than some implementation-specified length, the MTA # MAY return only the headers even if the RET parameter specified FULL." do_log(1, "DSN RET=%s requested, but we'll only attach a header section", $dsn_ret); $attach_full_msg = 0; # override, just attach a header section } my($template_ref) = $msginfo->setting_by_contents_category( $is_dsn ? cr('notify_sender_templ_by_ccat') : $request_type eq 'report' ? cr('notify_report_templ_by_ccat') : cr('notify_release_templ_by_ccat') ); my($report_str_ref) = expand($template_ref, \%mybuiltins); my($report_entity) = build_mime_entity($report_str_ref, $msginfo, $is_dsn ? 'multipart/report; report-type=delivery-status' : $is_arf ? 'multipart/report; report-type=feedback-report' : 'multipart/mixed', $msg_format, $is_plain, 1, $attach_full_msg); my($head) = $report_entity->head; # RFC 3464: The From field of the message header section of the DSN SHOULD # contain the address of a human who is responsible for maintaining the # mail system at the Reporting MTA site (e.g. Postmaster), so that a reply # to the DSN will reach that person. # Override header fields from the template: eval { $head->replace('From', $hdr_from); 1 } or do { chomp $@; die $@ }; eval { $head->replace('To', join(', ',@hdr_to)); 1 } or do { chomp $@; die $@ }; eval { $head->replace('Date', $rfc2822_dsn_time); 1 } or do { chomp $@; die $@ }; my($dsn_envid) = $msginfo->dsn_envid; # ENVID is encoded as xtext: RFC 3461 $dsn_envid = sanitize_str(xtext_decode($dsn_envid)) if defined $dsn_envid; my($txt_msg) = ''; # per-message part of a report if ($is_arf) { # abuse report format - RFC 5965 # abuse, dkim, fraud, miscategorized, not-spam, opt-out, virus, other $txt_msg .= "Version: 1\n"; # required $txt_msg .= "Feedback-Type: $feedback_type\n"; # required # User-Agent must comply with RFC 2616, section 14.43 my $ua_version = "$myproduct_name/$myversion_id ($myversion_date)"; $txt_msg .= "User-Agent: $ua_version\n"; # required $txt_msg .= "Reporting-MTA: dns; " . c('myhostname') . "\n"; # optional fields: $txt_msg .= "Arrival-Date: ". rfc2822_timestamp($msginfo->rx_time) ."\n"; my($cl_ip) = $msginfo->client_addr; $cl_ip = 'IPv6:'.$cl_ip if $cl_ip =~ /:.*:/ && $cl_ip !~ /^IPv6:/i; $txt_msg .= "Source-IP: $cl_ip\n" if defined $cl_ip; $txt_msg .= "Original-Envelope-Id: $dsn_envid\n" if defined $dsn_envid; $txt_msg .= "Original-Mail-From: " . $msginfo->sender_smtp . "\n"; for my $r (@$per_recip_data) { $txt_msg .= "Original-Rcpt-To: " . $r->recip_addr_smtp . "\n" } my($sigs_ref) = $msginfo->dkim_signatures_valid; if ($sigs_ref) { for my $sig (@$sigs_ref) { my $type = $sig->isa('Mail::DKIM::DkSignature') ? 'DK' : 'DKIM'; $txt_msg .= sprintf("Reported-Domain: %s (valid %s signature by)\n", $sig->domain, $type); } } if (c('enable_dkim_verification')) { for (Amavis::DKIM::generate_authentication_results($msginfo,0)) { my $h = $_; $h =~ tr/\n//d; # remove potential folding points $txt_msg .= "Authentication-Results: $h\n"; } } $txt_msg .= "Incidents: 1\n"; # Reported-URI } elsif ($is_dsn) { # DSN - per-msg part of dsn text according to RFC 3464 my($conn) = $msginfo->conn_obj; my($from_mta) = $conn->smtp_helo; my($client_ip) = $conn->client_ip; $txt_msg .= "Reporting-MTA: dns; " . c('myhostname') . "\n"; $txt_msg .= "Received-From-MTA: smtp; $from_mta ([$client_ip])\n" if $from_mta ne ''; $txt_msg .= "Arrival-Date: ". rfc2822_timestamp($msginfo->rx_time) ."\n"; $txt_msg .= "Original-Envelope-Id: $dsn_envid\n" if defined $dsn_envid; } if ($is_dsn || $is_arf) { # attach a delivery-status or a feedback-report eval { # make sure our source line number is reported in case of failure $report_entity->add_part( MIME::Entity->build(Top => 0, Type => $is_dsn ? 'message/delivery-status' : 'message/feedback-report', Encoding => '7bit', Disposition => 'inline', Filename => $is_arf ? 'arf_status' : 'dsn_status', Description => $is_arf ? "\u$feedback_type report" : $any_fail ? 'Delivery error report' : $any_delayed ? 'Delivery delay report' : 'Delivery report', Data => $txt_msg.$txt_recip), 1); # insert as a second mime part (at offset 1) 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat; }; } my($mailfrom) = $is_dsn ? '' # DSN envelope sender must be empty : unquote_rfc2821_local( (parse_address_list($hdr_from))[0] ); $notification = Amavis::In::Message->new; $notification->rx_time($dsn_time); $notification->log_id($msginfo->log_id); $notification->partition_tag($msginfo->partition_tag); $notification->conn_obj($msginfo->conn_obj); $notification->originating( ($request_type eq 'dsn' || $request_type eq 'report') ? 1 : 0); # $notification->body_type('7BIT'); $notification->mail_text($report_entity); $notification->sender($mailfrom); $notification->sender_smtp(qquote_rfc2821_local($mailfrom)); $notification->auth_submitter('<>'); $notification->auth_user(c('amavis_auth_user')); $notification->auth_pass(c('amavis_auth_pass')); if (defined $hdr_from) { my(@rfc2822_from) = map(unquote_rfc2821_local($_), parse_address_list($hdr_from)); $notification->rfc2822_from($rfc2822_from[0]); } my($bcc); if ($request_type eq 'dsn' || $request_type eq 'report') { $bcc = $msginfo->setting_by_contents_category(cr('dsn_bcc_by_ccat')); } $notification->recips([(defined $notif_recips ? @$notif_recips : map($_->recip_addr, @$per_recip_data)), defined $bcc && $bcc ne '' ? $bcc : () ], 1); my($notif_m) = c('notify_method'); $_->delivery_method($notif_m) for @{$notification->per_recip_data}; } # $suppressed is true if DNS would be needed, but either the sender requested # that DSN is not to be sent, or it is believed the bounce would not reach # the correct sender (faked sender with viruses or spam); # $notification is undef if DSN is not needed ($notification,$suppressed); } # Return a triple of arrayrefs of quoted recipient addresses (the first lists # recipients with successful delivery status, the second lists all the rest), # plus a list of short per-recipient delivery reports for failed deliveries, # that can be used in the first MIME part (the free text format) of delivery # status notifications. # sub delivery_short_report($) { my($msginfo) = @_; my(@succ_recips, @failed_recips, @failed_recips_full); for my $r (@{$msginfo->per_recip_data}) { my($remote_mta) = $r->recip_remote_mta; my($smtp_resp) = $r->recip_smtp_response; my($qrecip_addr) = scalar(qquote_rfc2821_local($r->recip_addr)); if ($r->recip_destiny == D_PASS && ($smtp_resp=~/^2/ || !$r->recip_done)) { push(@succ_recips, $qrecip_addr); } else { push(@failed_recips, $qrecip_addr); push(@failed_recips_full, sprintf("%s:%s\n %s", $qrecip_addr, (!defined($remote_mta)||$remote_mta eq '' ?'' :" [$remote_mta] said:"), $smtp_resp)); } } (\@succ_recips, \@failed_recips, \@failed_recips_full); } # Build a new MIME::Entity object based on the original mail, but hopefully # safer to mail readers: conventional mail header fields are retained, # original mail becomes an attachment of type 'message/rfc822'. # Text in $first_part becomes the first MIME part of type 'text/plain', # $first_part may be a scalar string or a ref to a list of lines # sub defanged_mime_entity($$) { my($msginfo,$first_part) = @_; my($new_entity); $_ = safe_encode(c('bdy_encoding'), $_) for (ref $first_part ? @$first_part : $first_part); eval { # make sure _our_ source line number is reported in case of failure $new_entity = MIME::Entity->build( Type => 'multipart/mixed', 'X-Mailer' => undef); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat; }; # reinserting some of the original header fields to a new header, sanitized my($hdr_edits) = $msginfo->header_edits; if (!$hdr_edits) { $hdr_edits = Amavis::Out::EditHeader->new; $msginfo->header_edits($hdr_edits); } my(%desired_field); for (qw(Received From Sender To Cc Reply-To Date Message-ID Resent-From Resent-Sender Resent-To Resent-Cc Resent-Date Resent-Message-ID In-Reply-To References Subject Comments Keywords Organization Organisation User-Agent X-Mailer DKIM-Signature DomainKey-Signature)) { $desired_field{lc($_)} = 1 }; local($1,$2); for my $curr_head (@{$msginfo->orig_header}) { # array of header fields # obsolete RFC 822 syntax allowed whitespace before colon my($field_name, $field_body) = $curr_head =~ /^([!-9;-\176]+)[ \t]*:(.*)\z/s ? ($1, $2) : (undef, $curr_head); if ($desired_field{lc($field_name)}) { # only desired header fields # protect NUL, CR, and characters with codes above \177 $field_body =~ s{ ( [^\001-\014\016-\177] ) } { sprintf(ord($1)>255 ? '\\x{%04x}' : '\\%03o', ord($1)) }gsxe; # protect NL in illegal all-whitespace continuation lines $field_body =~ s{\n([ \t]*)(?=\n)}{\\012$1}gs; $field_body =~ s{^(.{995}).{4,}$}{$1...}mg; # truncate lines to 998 chomp($field_body); # note that field body is already folded if (lc($field_name) eq 'subject') { # needs to be inserted directly into new header section so that it # can be subjected to header edits, like inserting ***UNCHECKED*** eval { $new_entity->head->add($field_name,$field_body); 1 } or do {chomp $@; die $@}; } else { $hdr_edits->append_header($field_name,$field_body,2); } } } eval { $new_entity->attach( Type => 'text/plain', Encoding => '-SUGGEST', Charset => c('bdy_encoding'), Data => $first_part); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat; }; # prepend a Return-Path to make available the envelope sender address my($rp) = sprintf("Return-Path: %s\n",$msginfo->sender_smtp); my($orig_mail_as_body) = Amavis::MIME::Body::OnOpenFh->new($msginfo->mail_text, [$rp], $msginfo->skip_bytes); $orig_mail_as_body or die "Can't create a MIME::Body object: $!"; eval { my($att) = $new_entity->attach( # RFC 2046 Type => 'message/rfc822; x-spam-type=original', Encoding =>($msginfo->header_8bit || $msginfo->body_8bit) ?'8bit':'7bit', Data => [], # Path => $msginfo->mail_text_fn, Description => 'Original message', Filename => 'message', Disposition => 'attachment', ); $att->bodyhandle($orig_mail_as_body); # direct access to tempfile handle 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die $eval_stat; }; $new_entity; } # Fill-in a message object with information based on a quarantined mail. # Expects $msginfo->mail_text to be a file handle (not a Mime::Entity object), # leaves it positioned at the beginning of a mail body (not to be relied upon). # If given a BSMTP file, expects that it contains a single message only. # sub msg_from_quarantine($$$) { my($msginfo,$request_type,$feedback_type) = @_; my($fh) = $msginfo->mail_text; my($sender_override) = $msginfo->sender; my($recips_data_override) = $msginfo->per_recip_data; my($quarantine_id) = $msginfo->mail_id; $quarantine_id = '' if !defined $quarantine_id; my($reporting) = $request_type eq 'report'; my($release_m); if ($request_type eq 'requeue') { $release_m = c('requeue_method'); $release_m ne '' or die "requeue_method is unspecified"; } else { # 'release' or 'report' $release_m = c('release_method'); $release_m = c('notify_method') if !defined $release_m || $release_m eq ''; $release_m ne '' or die "release_method and notify_method are unspecified"; } $msginfo->originating(0); # let's make it explicit; disables DKIM signing $msginfo->auth_submitter('<>'); $msginfo->auth_user(c('amavis_auth_user')); $msginfo->auth_pass(c('amavis_auth_pass')); $fh->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; my($bsmtp) = 0; # message stored in an RFC 2442 format? my($qid,$sender,@recips_all,@recips_blocked); my($have_recips_blocked) = 0; my($curr_head); my($ln); my($eof) = 0; my($position) = 0; my($offset_bytes) = 0; # file position just past the prefixed header fields # extract envelope information from the quarantine file do_log(4, "msg_from_quarantine: releasing %s", $quarantine_id); for (;;) { if ($eof) { $ln = "\n" } else { $! = 0; $ln = $fh->getline; if (!defined($ln)) { $eof = 1; $ln = "\n"; # fake a missing header/body separator line $!==0 or die "Error reading file ".$msginfo->mail_text_fn.": $!"; } } if ($ln =~ /^[ \t]/) { $curr_head .= $ln } else { my($next_head) = $ln; local($1,$2); local($_) = $curr_head; chomp; s/\n(?=[ \t])//gs; # unfold if (!defined($curr_head)) { # first time } elsif (/^(EHLO|HELO)( |$)/i) { $bsmtp = 1; } elsif (/^MAIL FROM:[ \t]*(<.*>)(.*)$/i) { $bsmtp = 1; $sender = $1; $sender = unquote_rfc2821_local($sender); } elsif ( $bsmtp && /^RCPT TO:[ \t]*(<.*>)(.*)$/i) { push(@recips_all, unquote_rfc2821_local($1)); } elsif ( $bsmtp && /^(DATA|NOOP)$/i) { } elsif ( $bsmtp && /^RSET$/i) { $sender = undef; @recips_all = (); @recips_blocked = (); $qid = undef; } elsif ( $bsmtp && /^QUIT$/i) { last; } elsif (!$bsmtp && /^Return-Path:[ \t]*(.*)$/si) { } elsif (!$bsmtp && /^Delivered-To:[ \t]*(.*)$/si) { } elsif (!$bsmtp && /^X-Envelope-From:[ \t]*(.*)$/si) { if (!defined $sender) { my(@addr_list) = parse_address_list($1); @addr_list >= 1 or die "Address missing in X-Envelope-From"; @addr_list <= 1 or die "More than one address in X-Envelope-From"; $sender = unquote_rfc2821_local($addr_list[0]); } } elsif (!$bsmtp && /^X-Envelope-To:[ \t]*(.*)$/si) { my(@addr_list) = parse_address_list($1); push(@recips_all, map(unquote_rfc2821_local($_), @addr_list)); } elsif (!$bsmtp && /^X-Envelope-To-Blocked:[ \t]*(.*)$/si) { my(@addr_list) = parse_address_list($1); push(@recips_blocked, map(unquote_rfc2821_local($_), @addr_list)); $have_recips_blocked = 1; } elsif (/^X-Quarantine-ID:[ \t]*(.*)$/si) { $qid = $1; $qid = $1 if $qid =~ /^<(.*)>\z/s; } elsif (!$reporting && /^X-Amavis-(?:Hold|Alert|Modified|PenPals| PolicyBank|OS-Fingerprint):/xsi) { # skip } elsif (!$reporting && /^(X-Spam|X-CRM114)-.+:/si) { # skip header fields inserted by us } else { last; # end of known header fields, to be marked as 'skip_bytes' } last if $next_head eq "\n"; # end-of-header-section reached $offset_bytes = $position; # move past last processed header field $curr_head = $next_head; } $position += length($ln); } @recips_blocked = @recips_all if !$have_recips_blocked; # pre-2.6.0 compatib my(@except); if (@recips_blocked < @recips_all) { for my $rec (@recips_all) { push(@except,$rec) if !grep($rec eq $_, @recips_blocked) } } my($sender_smtp) = qquote_rfc2821_local($sender); do_log(0,"Quarantined message %s (%s): %s %s -> %s%s", $request_type, $feedback_type, $quarantine_id, $sender_smtp, join(',', qquote_rfc2821_local(@recips_blocked)), !@except ? '' : (", (excluded: ". join(',', qquote_rfc2821_local(@except)) . " )" )); my(@m); if (!defined($qid)) { push(@m, 'missing X-Quarantine-ID') } elsif ($qid ne $quarantine_id) { push(@m, sprintf("stored quar. ID '%s' does not match requested ID '%s'", $qid,$quarantine_id)); } push(@m, 'missing '.($bsmtp?'MAIL FROM':'X-Envelope-From or Return-Path')) if !defined $sender; push(@m, 'missing '.($bsmtp?'RCPT TO' :'X-Envelope-To')) if !@recips_all; do_log(0, "Quarantine %s %s: %s", $request_type, $quarantine_id, join("; ",@m)) if @m; if ($qid ne $quarantine_id) { die "Stored quarantine ID '$qid' does not match ". "requested ID '$quarantine_id'" } if ($bsmtp) { die "Releasing messages in BSMTP format not yet supported ". "(dot de-stuffing not implemented)" } $msginfo->sender($sender); $msginfo->sender_smtp($sender_smtp); $msginfo->recips(\@recips_all); $_->delivery_method($release_m) for @{$msginfo->per_recip_data}; # mark a file location past prefixed header fields where orig message starts $msginfo->skip_bytes($offset_bytes); my($msg_format) = $request_type eq 'dsn' ? 'dsn' : $request_type eq 'report' ? c('report_format') : c('release_format'); my($hdr_edits) = Amavis::Out::EditHeader->new; $msginfo->header_edits($hdr_edits); if ($msg_format eq 'resend') { if (!defined($recips_data_override)) { $msginfo->recips(\@recips_blocked); # override 'all' by 'blocked' recips } else { # recipients specified in the request override stored info ll(5) && do_log(5, 'overriding recips %s by %s', join(',', qquote_rfc2821_local(@recips_blocked)), join(',', map($_->recip_addr_smtp, @$recips_data_override))); $msginfo->per_recip_data($recips_data_override); } $_->delivery_method($release_m) for @{$msginfo->per_recip_data}; } else { # collect more information from a quarantined message, making it available # to a report generator and to macros during template expansion Amavis::get_body_digest($msginfo,'MD5'); # or 'SHA-1' Amavis::collect_some_info($msginfo); if (defined($recips_data_override) && ll(5)) { do_log(5, 'overriding recips %s by %s', join(',', qquote_rfc2821_local(@recips_blocked)), join(',', map($_->recip_addr_smtp, @$recips_data_override))); } my($notification,$suppressed) = delivery_status_notification( $msginfo, 0, \%Amavis::builtins, !defined($recips_data_override) ? \@recips_blocked : [ map($_->recip_addr, @$recips_data_override) ], $request_type, $feedback_type, undef); # pushes original quarantined message into an attachment of a notification $msginfo = $notification; } if (defined $sender_override) { # sender specified in the request, overrides stored info do_log(5, "overriding sender %s by %s", $sender, $sender_override); $msginfo->sender($sender_override); $msginfo->sender_smtp(qquote_rfc2821_local($sender_override)); } if ($msg_format eq 'resend') { # keep quarantined message at a top MIME level # Resent-* header fields must precede corresponding Received header field # "Resent-From:" and "Resent-Date:" are required fields! my($hdrfrom_recip) = $msginfo->setting_by_contents_category( cr('hdrfrom_notify_recip_by_ccat')); $hdrfrom_recip = expand_variables($hdrfrom_recip); if ($msginfo->requested_by eq '') { $hdr_edits->add_header('Resent-From', $hdrfrom_recip); } else { $hdr_edits->add_header('Resent-From', qquote_rfc2821_local($msginfo->requested_by)); $hdr_edits->add_header('Resent-Sender', $hdrfrom_recip) if $hdrfrom_recip ne ''; } my($prd) = $msginfo->per_recip_data; $hdr_edits->add_header('Resent-To', $prd && @$prd==1 ? $prd->[0]->recip_addr_smtp : 'undisclosed-recipients:;'); $hdr_edits->add_header('Resent-Date', # time of the release rfc2822_timestamp($msginfo->rx_time)); $hdr_edits->add_header('Resent-Message-ID', sprintf('', $msginfo->mail_id||'', c('myhostname')) ); } $hdr_edits->add_header('Received', make_received_header_field($msginfo,1),1); my($bcc) = $msginfo->setting_by_contents_category(cr('always_bcc_by_ccat')); if (defined $bcc && $bcc ne '' && $request_type ne 'report') { my($recip_obj) = Amavis::In::Message::PerRecip->new; # leave recip_addr and recip_addr_smtp undefined! $recip_obj->recip_addr_modified($bcc); $recip_obj->recip_destiny(D_PASS); $recip_obj->dsn_notify(['NEVER']); $recip_obj->add_contents_category(CC_CLEAN,0); $msginfo->per_recip_data([@{$msginfo->per_recip_data}, $recip_obj]); do_log(2,"adding recipient - always_bcc: %s", $bcc); } $msginfo; } 1; # package Amavis::Custom; # MAIL PROCESSING SEQUENCE: # child process initialization # loop for each mail: # - receive mail, parse and make available some basic information # * custom hook: new() - may inspect info, may load policy banks # - mail checking and collecting results # * custom hook: checks() - may inspect or modify checking results # - deciding mail fate (lookup on *_lovers, thresholds, ...) # - quarantining # - sending notifications (to admin and recips) # * custom hook: before_send() - may send other notif, quarantine, modify mail # - forwarding (unless blocked) # * custom hook: after_send() - may suppress DSN, send reports, quarantine # - sending delivery status notification (if needed) # - issue main log entry, manage statistics (timing, counters, nanny) # * custom hook: mail_done() - may inspect results # endloop after $max_requests or earlier sub new { my($class,$conn,$msginfo) = @_; undef } sub checks { my($self,$conn,$msginfo) = @_; undef } sub before_send { my($self,$conn,$msginfo) = @_; undef } sub after_send { my($self,$conn,$msginfo) = @_; undef } sub mail_done { my($self,$conn,$msginfo) = @_; undef } 1; # package Amavis; require 5.005; # need qr operator and \z in regexps use strict; use re 'taint'; BEGIN { use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; import Amavis::Conf qw(:platform :sa :confvars c cr ca); import Amavis::Util qw(untaint min max minmax unique_list unique_ref ll do_log update_current_log_level sanitize_str debug_oneshot am_id safe_decode safe_encode proto_decode add_entropy stir_random generate_mail_id make_password prolong_timer get_deadline waiting_for_client switch_to_my_time switch_to_client_time snmp_counters_init snmp_count dynamic_destination ccat_split ccat_maj cmp_ccat cmp_ccat_maj setting_by_given_contents_category_all setting_by_given_contents_category orcpt_encode); import Amavis::ProcControl qw(exit_status_str proc_status_ok cloexec run_command collect_results); import Amavis::Log qw(open_log close_log collect_log_stats); import Amavis::Timing qw(section_time get_time_so_far); import Amavis::rfc2821_2822_Tools; import Amavis::Lookup qw(lookup lookup2); import Amavis::Lookup::IP qw(lookup_ip_acl); import Amavis::Out; import Amavis::Out::EditHeader; import Amavis::UnmangleSender qw(parse_ip_address_from_received first_received_from); import Amavis::Unpackers::Validity qw( check_header_validity check_for_banned_names); import Amavis::Unpackers::MIME qw(mime_decode); import Amavis::Expand qw(expand tokenize); import Amavis::Notify qw(delivery_status_notification delivery_short_report build_mime_entity defanged_mime_entity expand_variables); import Amavis::In::Connection; import Amavis::In::Message; } use Errno qw(ENOENT EACCES EAGAIN ESRCH EBADF); use POSIX qw(locale_h); use IO::Handle; use IO::File qw(O_RDONLY O_WRONLY O_RDWR O_APPEND O_CREAT O_EXCL); use Time::HiRes (); # body digest, either MD5 or SHA-1 (or perhaps SHA-256) #use Digest::SHA; use Digest::MD5; use Net::Server 0.87; # need Net::Server::PreForkSimple::done use MIME::Base64; use vars qw( $extra_code_db $extra_code_sql_base $extra_code_sql_log $extra_code_sql_quar $extra_code_sql_lookup $extra_code_ldap $extra_code_in_ampdp $extra_code_in_smtp $extra_code_in_courier $extra_code_out_smtp $extra_code_out_pipe $extra_code_out_bsmtp $extra_code_out_local $extra_code_p0f $extra_code_antivirus $extra_code_antispam $extra_code_antispam_extprog $extra_code_antispam_spamc $extra_code_antispam_sa $extra_code_unpackers $extra_code_dkim $extra_code_tools); use vars qw(%modules_basic %got_signals); use vars qw($user_id_sql $user_policy_id_sql $wb_listed_sql); use vars qw($implicit_maps_inserted $maps_have_been_labeled); use vars qw($db_env $snmp_db); use vars qw(%builtins); # macros in customizable notification messages use vars qw($last_task_completed_at); use vars qw($child_invocation_count $child_task_count); use vars qw($child_init_hook_was_called); # $child_invocation_count # counts child re-use from 1 to max_requests # $child_task_count # counts check_mail_begin_task (and check_mail) calls; # this often runs in sync with $child_invocation_count, # but with SMTP or LMTP input there may be more than one # message passed during a single SMTP session use vars qw(@config_files); # configuration files provided by -c or defaulted use vars qw($MSGINFO); use vars qw($av_output @virusname @detecting_scanners $banned_filename_any $banned_filename_all @bad_headers); # Amavis::In::AMPDP, Amavis::In::SMTP and In::Courier objects use vars qw($ampdp_in_obj $smtp_in_obj $courier_in_obj); use vars qw($sql_dataset_conn_lookups); # Amavis::Out::SQL::Connection object use vars qw($sql_dataset_conn_storage); # Amavis::Out::SQL::Connection object use vars qw($sql_storage); # Amavis::Out::SQL::Log object use vars qw($sql_lookups $sql_wblist); # Amavis::Lookup::SQL objects use vars qw($ldap_connection); # Amavis::LDAP::Connection object use vars qw($ldap_lookups); # Amavis::Lookup::LDAP object use vars qw($warm_restart); # 1: warm (reload), 0: cold start (restart) sub new { my($class) = shift; # make Amavis a subclass of Net::Server::whatever @ISA = !$daemonize && $max_servers==1 ? 'Net::Server' # facilitates debugging : defined $min_servers ? 'Net::Server::PreFork' : 'Net::Server::PreForkSimple'; # $class->SUPER::new(@_); # available since Net::Server 0.91 bless { server => $_[0] }, $class; # works with all versions } sub get_rusage() { my($usage); if (Unix::Getrusage->UNIVERSAL::can("getrusage")) { $usage = Unix::Getrusage::getrusage(); # ru_minflt no. of page faults serviced without I/O activity # ru_majflt no. of page faults that required I/O activity # ru_nswap no. of times a process was swapped out # ru_inblock no. of times a file system had to perform input # ru_oublock no. of times a file system had to perform output # ru_msgsnd no. of IPC messages sent # ru_msgrcv no. of IPC messages received # ru_nsignals no. of signals delivered # ru_nvcsw no. of voluntary context switches # ru_nivcsw no. of involuntary context switches # ru_maxrss [kB] maximum resident set size utilized # ru_ixrss [kBtics] integral of mem used by the shared text segment # ru_idrss [kBtics] integral of unshared mem in the data segment # ru_isrss [kBtics] integral of unshared mem in the stack segment # ru_utime [s] time spent executing in user mode # ru_stime [s] time spent in the system on behalf of the process } $usage; } # report process resource usage, data from a system service getrusage(2) # sub report_rusage() { my($usage) = get_rusage(); if ($usage) { my(@order) = qw(minflt majflt nswap inblock oublock msgsnd msgrcv nsignals nvcsw nivcsw maxrss ixrss idrss isrss utime stime); my(@result) = map($_.'='.$usage->{'ru_'.$_}, @order); # known delete $usage->{'ru_'.$_} for @order; push(@result, map($_.'='.$usage->{$_}, keys %$usage)); # any other? do_log(2,"RUSAGE: %s", join(', ',@result)); } } sub macro_rusage { my($msginfo,$recip_index,$name,$arg) = @_; my($usage) = get_rusage(); !$usage || !defined($usage->{$arg}) ? '' : $usage->{$arg}; } # implements macros: T, and SA lookalikes: TESTS, TESTSSCORES # sub macro_tests { my($msginfo,$recip_index,$name,$sep) = @_; my(@s); my($per_recip_data) = $msginfo->per_recip_data; if (defined $recip_index) { # return info on one particular recipient my($r); $r = $per_recip_data->[$recip_index] if $recip_index >= 0; if (defined $r) { my($spam_tests) = $r->spam_tests; @s = split(/,/, join(',',map($$_,@$spam_tests))) if defined $spam_tests; } } else { my(%all_spam_tests); for my $r (@$per_recip_data) { my($spam_tests) = $r->spam_tests; if (defined $spam_tests) { $all_spam_tests{$_} = 1 for split(/,/,join(',',map($$_,@$spam_tests))); } } @s = sort keys %all_spam_tests; } if (@s > 50) { $#s = 50-1; push(@s,"...") } # sanity limit @s = map { my($tn,$ts)=split(/=/); $tn } @s if $name eq 'TESTS'; if ($name eq 'T' || !defined($sep)) { \@s } else { join($sep,@s) } }; # implements macros: c, and SA lookalikes: SCORE(pad), STARS(*) # sub macro_score { my($msginfo,$recip_index,$name,$arg) = @_; my($per_recip_data) = $msginfo->per_recip_data; my($result); my($sl_min,$sl_max); my($w) = ''; if ($name eq 'SCORE' && defined($arg) && $arg=~/^(0+| +)\z/) { $w = length($arg)+4; $w = $arg=~/^0/ ? "0$w" : "$w" } # SA style padding my($fmt) = "%$w.3f"; my($fmts) = "%+$w.3f"; # padding, sign if (defined $recip_index) { # return info on one particular recipient my($r); $r = $per_recip_data->[$recip_index] if $recip_index >= 0; $sl_min = $sl_max = $r->spam_level if defined $r; } else { ($sl_min,$sl_max) = minmax(map($_->spam_level, @$per_recip_data)); } if ($name eq 'STARS') { my($slc) = $arg ne '' ? $arg : c('sa_spam_level_char'); $result = $slc eq '' || !defined $sl_min ? '' : $slc x min(50,$sl_min); } elsif (!defined $sl_min) { $result = '-'; # } elsif ($name eq 'SCORE' || abs($sl_min-$sl_max) < 0.1) { } elsif (abs($sl_min-$sl_max) < 0.1) { # users expect a single value, or not worth reporting a small difference $result = sprintf($fmt,$sl_min); $result =~ s/\.?0*\z//; # trim fraction } else { # format SA score as min..max $sl_min = sprintf($fmt,$sl_min); $sl_min =~ s/\.?0*\z//; $sl_max = sprintf($fmt,$sl_max); $sl_max =~ s/\.?0*\z//; $result = $sl_min . '..' . $sl_max; } $result; }; # implements macro header_field, providing a named header field from a message # sub macro_header_field { my($msginfo,$name,$header_field_name,$limit,$hf_index) = @_; undef $hf_index if $hf_index !~ /^[+-]?\d+\z/; # defaults to last local($_) = $msginfo->get_header_field_body($header_field_name,$hf_index); if (defined $_) { # unfold, trim, protect CR, LF, \000 and \200 chomp; s/\n(?=[ \t])//gs; s/^[ \t]+//; s/[ \t\n]+\z//; if ($header_field_name =~ /^(?:Message-ID|Resent-Message-ID|In-Reply-To|References)\z/i) { $_ = join(' ',parse_message_id($_)) if $_ ne ''; # strip CFWS } s{([\r\n\000\200])}{sprintf("\\%03o",ord($1))}eg; }; !defined($limit) || $limit =~ /^\s+\z/ || $limit < 6 || length($_) <= $limit ? $_ : substr($_,0,$limit-5) . '[...]'; }; sub dkim_test { my($name,$which) = @_; my($w) = lc($which); my($sigs_ref) = $MSGINFO->dkim_signatures_valid; $sigs_ref = [] if !$sigs_ref; $w eq 'any' || $w eq '' ? (!@$sigs_ref ? undef : scalar(@$sigs_ref)) : $w eq 'author' ? $MSGINFO->dkim_author_sig : $w eq 'sender' ? $MSGINFO->dkim_sender_sig : $w eq 'thirdparty'? $MSGINFO->dkim_thirdparty_sig : $w eq 'envsender' ? $MSGINFO->dkim_envsender_sig : $w eq 'identity' ? join(',', map($_->identity, @$sigs_ref)) : $w eq 'selector' ? join(',', map($_->selector, @$sigs_ref)) : $w eq 'domain' ? join(',', map($_->domain, @$sigs_ref)) : $w eq 'sig_sd' ? join(',', unique_list(map($_->selector.':'.$_->domain, @$sigs_ref))) : $w eq 'newsig_sd' ? join(',', unique_list(map($_->selector.':'.$_->domain, @{$MSGINFO->dkim_signatures_new||[]}))) : dkim_acceptable_signing_domain($MSGINFO,$which); } sub dkim_acceptable_signing_domain($@) { my($msginfo,@acceptable_sdid) = @_; my($matches) = 0; my($sigs_ref) = $msginfo->dkim_signatures_valid; if (defined($sigs_ref) && @$sigs_ref) { for my $sig (@$sigs_ref) { my($sdid) = lc($sig->domain); for (@acceptable_sdid) { my($ad) = !defined $_ ? '' : lc($_); local($1); $ad = $1 if $ad =~ /\@([^\@]*)\z/; # compatibility with pre-2.6.5 if ($ad eq '') { # checking for author domain signature $matches = 1 if $msginfo->dkim_author_sig; } elsif ($ad =~ /^\.(.*)\z/s) { # domain itself or its subdomain my($d) = $1; if ($sdid eq $d || $sdid =~ /\.\Q$d\E\z/s) { $matches = 1; last } } else { if ($sdid eq $ad) { $matches = 1; last } } } last if $matches; } } $matches; }; # initialize the %builtins, which is an associative array of built-in macros # to be used in notification message expansion and log templates # sub init_builtin_macros() { # A key (macro name) used to be a single character, but can now be a longer # string, typically a name containing letters, numbers and '_' or '-'. # Upper case letters may (as a mnemonic) suggest the value is an array, # lower case may suggest the value is a scalar string - but this is only # a convention and not enforced. All-uppercase multicharacter names are # intended as SpamAssassin-lookalike macros, although there is nothing # special about them and can be called like other macros. # # A value may be a reference to a subroutine which will be called later at # a time of macro expansion. This way we can provide a method for obtaining # information which is not yet available at the time of initialization, such # as AV scanner results, or provide a lazy evaluation for more expensive # calculations. Subroutine will be called in scalar context, its first # argument is a macro name (a string), remaining arguments (strings, if any) # are arguments of a macro call as specified in the call. The subroutine may # return a scalar string (or undef), or an array reference. # # for SpamAssassin-lookalike macros semantics see Mail::SpamAssassin::Conf %builtins = ( '.' => undef, p => sub {c('policy_bank_path')}, # mail reception timestamp (e.g. start of an SMTP transaction): DATE => sub {rfc2822_timestamp($MSGINFO->rx_time)}, d => sub {rfc2822_timestamp($MSGINFO->rx_time)}, # RFC 5322 local time U => sub {iso8601_utc_timestamp($MSGINFO->rx_time)}, # iso8601 UTC u => sub {sprintf("%010d",int($MSGINFO->rx_time))},# s since Unix epoch,UTC # equivalent, but with more descriptive macro names: date_unix_utc => sub {sprintf("%010d",int($MSGINFO->rx_time))}, date_iso8601_utc => sub {iso8601_utc_timestamp($MSGINFO->rx_time)}, date_iso8601_local => sub {iso8601_timestamp($MSGINFO->rx_time)}, date_rfc2822_local => sub {rfc2822_timestamp($MSGINFO->rx_time)}, week_iso8601 => sub {iso8601_week($MSGINFO->rx_time)}, y => sub {sprintf("%.0f", 1000*get_time_so_far())}, # elapsed time in ms h => sub {c('myhostname')}, # fqdn name of this host HOSTNAME => sub {c('myhostname')}, l => sub {$MSGINFO->originating ? 1 : undef}, # our client (mynets/roaming) s => sub {$MSGINFO->sender_smtp}, # orig. unmodified env. sender addr in <> S => sub {$MSGINFO->sender_smtp}, # kept for compatibility, avoid! o => sub { # best attempt at determining true sender (origin) of the virus, sanitize_str($MSGINFO->sender_source) }, # normally same as %s R => sub {$MSGINFO->recips}, # original message recipients list D => sub {my($y,$n,$f)=delivery_short_report($MSGINFO); $y}, #succ. delivrd O => sub {my($y,$n,$f)=delivery_short_report($MSGINFO); $n}, #failed recips N => sub {my($y,$n,$f)=delivery_short_report($MSGINFO); $f}, #short dsn actions_performed => sub {join(',',@{$MSGINFO->actions_performed||[]})}, Q => sub {$MSGINFO->queue_id}, # MTA queue ID of the message if known m => sub {macro_header_field($MSGINFO,'header','Message-ID')}, r => sub {macro_header_field($MSGINFO,'header','Resent-Message-ID')}, j => sub {macro_header_field($MSGINFO,'header','Subject')}, rfc2822_sender => sub {my($s) = $MSGINFO->rfc2822_sender; !defined($s) ? undef : qquote_rfc2821_local($s) }, rfc2822_from => sub {my($f) = $MSGINFO->rfc2822_from; !defined($f) ? undef : qquote_rfc2821_local(ref $f ? @$f : $f)}, rfc2822_resent_sender => sub {my($rs) = $MSGINFO->rfc2822_resent_sender; !defined($rs) ? undef : qquote_rfc2821_local(grep(defined $_, @$rs))}, rfc2822_resent_from => sub {my($rf) = $MSGINFO->rfc2822_resent_from; !defined($rf) ? undef : qquote_rfc2821_local(grep(defined $_, @$rf))}, header_field => sub {macro_header_field($MSGINFO,@_)}, HEADER => sub {macro_header_field($MSGINFO,@_)}, useragent => # argument: 'name' or 'body', or empty to return entire field sub { my($macro_name,$which_part) = @_; my($head,$body); $body = macro_header_field($MSGINFO,'header', $head='User-Agent'); $body = macro_header_field($MSGINFO,'header', $head='X-Mailer') if !defined $body; !defined($body) ? undef : lc($which_part) eq 'name' ? $head : lc($which_part) eq 'body' ? $body : "$head: $body"; }, ccat => sub { my($name,$attr,$which) = @_; $attr = lc($attr); # name | major | minor | # | is_blocking | is_nonblocking # | is_blocked_by_nonmain $which = lc($which); # main | blocking | auto my($result) = ''; my($blocking_ccat) = $MSGINFO->blocking_ccat; if ($attr eq 'is_blocking') { $result = defined($blocking_ccat) ? 1 : ''; } elsif ($attr eq 'is_nonblocking') { $result = !defined($blocking_ccat) ? 1 : ''; } elsif ($attr eq 'is_blocked_by_nonmain') { if (defined($blocking_ccat)) { my($aref) = $MSGINFO->contents_category; $result = 1 if ref($aref) && @$aref > 0 && $blocking_ccat ne $aref->[0]; } } elsif ($attr eq 'name') { $result = $which eq 'main' ? $MSGINFO->setting_by_main_contents_category(\%ccat_display_names) : $which eq 'blocking' ? $MSGINFO->setting_by_blocking_contents_category( \%ccat_display_names) : $MSGINFO->setting_by_contents_category( \%ccat_display_names); } else { # attr = major, minor, or anything else returns a pair my($maj,$min) = ccat_split( ($which eq 'blocking' || $which ne 'main' && defined $blocking_ccat) ? $blocking_ccat : $MSGINFO->contents_category); $result = $attr eq 'major' ? $maj : $attr eq 'minor' ? sprintf("%d",$min) : sprintf("(%d,%d)",$maj,$min); } $result; }, ccat_maj => # deprecated, use [:ccat|major] sub { my($blocking_ccat) = $MSGINFO->blocking_ccat; (ccat_split(defined $blocking_ccat ? $blocking_ccat : $MSGINFO->contents_category))[0]; }, ccat_min => # deprecated, use [:ccat|minor] sub { my($blocking_ccat) = $MSGINFO->blocking_ccat; (ccat_split(defined $blocking_ccat ? $blocking_ccat : $MSGINFO->contents_category))[1]; }, ccat_name => # deprecated, use [:ccat|name] sub { $MSGINFO->setting_by_contents_category(\%ccat_display_names) }, dsn_notify => sub { return 'NEVER' if $MSGINFO->sender eq ''; my(%merged); for my $r (@{$MSGINFO->per_recip_data}) { my($dn) = $r->dsn_notify; for ($dn ? @$dn : ('FAILURE')) { $merged{uc($_)} = 1 } } uc(join(',', sort keys %merged)); }, attachment_password => sub { my($password) = $MSGINFO->attachment_password; # already have it? if (!defined $password) { # make one, and store it for later $password = make_password(c('attachment_password'), $MSGINFO); $MSGINFO->attachment_password($password); } $password; }, b => sub {$MSGINFO->body_digest}, # original message body digest n => sub {$MSGINFO->log_id}, # amavis internal task id (in log and nanny) i => sub {$MSGINFO->mail_id}, # long-term unique mail id on this system LOGID => sub {$MSGINFO->log_id}, # synonym for %n (no equivalent in SA) MAILID => sub {$MSGINFO->mail_id}, # synonym for %i (no equivalent in SA) P => sub {$MSGINFO->partition_tag}, # SQL partition tag partition_tag => sub {$MSGINFO->partition_tag}, # synonym for %P q => sub {my($q) = $MSGINFO->quarantined_to; $q && [map { my($m)=$_; $m=~s{^\Q$QUARANTINEDIR\E/}{}; $m } @$q]; }, # list of quarantine mailboxes v => sub {[split(/[ \t]*\r?\n/,$av_output)]}, # anti-virus scanner output V => sub {my($vn) = $MSGINFO->virusnames; # unique virus names $vn && unique_ref($vn) }, F => sub { my($b); # first banned part name with a comment from a rule regexp for my $r (@{$MSGINFO->per_recip_data}) { $b = $r->banning_reason_short; last if defined $b; } $b }, banning_rule_key => sub { # regexp of a matching banning rules yielding a true rhs result unique_ref(map { my($v) = $_->banning_rule_key; !defined($v) ? () : @$v } @{$MSGINFO->per_recip_data}); }, banning_rule_comment => sub { # just a comment (or a whole regexp if it contains no comments) # from matching banning regexp rules yielding a true rhs result unique_ref(map { my($v) = $_->banning_rule_comment; !defined($v) ? () : @$v } @{$MSGINFO->per_recip_data}); }, banning_rule_rhs => sub { # right-hand-side of those matching banning rules yielding true # (a r.h.s. of a rule can be a string, is treated as a boolean, # but often it is just an implicit 0 or 1) unique_ref(map { my($v) = $_->banning_rule_rhs; !defined($v) ? () : @$v } @{$MSGINFO->per_recip_data}); }, banned_parts => sub { # list of banned parts with their full paths my($b) = unique_ref(map(@{$_->banned_parts}, grep(defined($_->banned_parts),@{$MSGINFO->per_recip_data}))); my($b_chopped) = @$b > 2; @$b = (@$b[0,1],'...') if $b_chopped; s/[ \t]{6,}/ ... /g for @$b; $b }, banned_parts_as_attr => sub { # list of banned parts with their full paths my($b) = unique_ref(map(@{$_->banned_parts_as_attr}, grep(defined($_->banned_parts_as_attr), @{$MSGINFO->per_recip_data}))); my($b_chopped) = @$b > 2; @$b = (@$b[0,1],'...') if $b_chopped; s/[ \t]{6,}/ ... /g for @$b; $b }, X => sub {\@bad_headers}, W => sub {\@detecting_scanners}, # list of av scanners detecting a virus H => sub {[map(split(/\n/,$_), @{$MSGINFO->orig_header})]}, # arry of lines A => sub {[split(/\r?\n/, $MSGINFO->spam_summary)]}, # SA report text SUMMARY => sub {$MSGINFO->spam_summary}, REPORT => sub {sanitize_str($MSGINFO->spam_report,1)}, #contains any octet TESTSSCORES => sub {macro_tests($MSGINFO,undef,@_)}, # tests with scores TESTS => sub {macro_tests($MSGINFO,undef,@_)}, # tests without scores z => sub {$MSGINFO->msg_size}, #mail size as defined by RFC 1870, or approx t => sub { # first entry in the Received trace sanitize_str(first_received_from($MSGINFO)) }, e => sub { # first valid public IP in the Received trace sanitize_str(parse_ip_address_from_received($MSGINFO)) }, a => sub { $MSGINFO->client_addr }, # original SMTP session client IP addr client_addr => sub { $MSGINFO->client_addr }, # synonym with 'a' client_port => sub { $MSGINFO->client_port }, client_addr_port => sub { # original SMTP session client IP addr & port no. my($a,$p) = ($MSGINFO->client_addr, $MSGINFO->client_port); !defined $a || $a eq '' ? undef : ('[' . $a . ']' . ($p ? ":$p" : '')); }, g => sub { # original SMTP session client DNS name sanitize_str($MSGINFO->client_name) }, client_helo => sub { # original SMTP session EHLO/HELO name sanitize_str($MSGINFO->client_helo) }, remote_mta => sub { unique_ref(map($_->recip_remote_mta, @{$MSGINFO->per_recip_data})) }, smtp_response => sub { unique_ref(map($_->recip_smtp_response, @{$MSGINFO->per_recip_data})) }, remote_mta_smtp_response => sub { unique_ref(map($_->recip_remote_mta_smtp_response, @{$MSGINFO->per_recip_data})) }, REMOTEHOSTADDR => # where the request was sent from sub { my($c) = $MSGINFO->conn_obj; !$c ? '' : $c->client_ip }, REMOTEHOSTNAME => sub { my($c) = $MSGINFO->conn_obj; my($ip) = !$c ? '' : $c->client_ip; $ip ne '' ? "[$ip]" : 'localhost' }, # VERSION => Mail::SpamAssassin->Version, # SA version # SUBVERSION => $Mail::SpamAssassin::SUB_VERSION, # SA sub-version/revision AUTOLEARN => sub {$MSGINFO->supplementary_info('AUTOLEARN')}, ADDEDHEADERHAM => sub {$MSGINFO->supplementary_info('ADDEDHEADERHAM')}, ADDEDHEADERSPAM => sub {$MSGINFO->supplementary_info('ADDEDHEADERSPAM')}, supplementary_info => # additional information from SA and other scanners sub { my($name,$key,$fmt)=@_; my($info) = $MSGINFO->supplementary_info($key); $info eq '' ? '' : $fmt eq '' ? $info : sprintf($fmt,$info); }, rusage => sub { macro_rusage($MSGINFO,undef,@_) }, # resource usage REQD => sub { my($tag2_level); for (@{$MSGINFO->per_recip_data}) { # get minimal tag2_level my($tag2_l) = lookup2(0,$_->recip_addr, ca('spam_tag2_level_maps')); $tag2_level = $tag2_l if defined($tag2_l) && (!defined($tag2_level) || $tag2_l < $tag2_level); } !defined($tag2_level) ? '-' : 0+sprintf("%.3f",$tag2_level); }, '1'=> sub { # above tag level and not bypassed for any recipient? grep($_->is_in_contents_category(CC_CLEAN,1), @{$MSGINFO->per_recip_data}) ? 'Y' : '0' }, '2'=> sub { # above tag2 level and not bypassed for any recipient? grep($_->is_in_contents_category(CC_SPAMMY), @{$MSGINFO->per_recip_data}) ? 'Y' : '0' }, YESNO => sub { my($arg_spam, $arg_ham) = @_; # like %2, but gives: Yes/No grep($_->is_in_contents_category(CC_SPAMMY), @{$MSGINFO->per_recip_data}) ? (defined $arg_spam ? $arg_spam : 'Yes') : (defined $arg_ham ? $arg_ham : 'No') }, YESNOCAPS => sub { my($arg_spam, $arg_ham) = @_; # like %2, but gives: YES/NO grep($_->is_in_contents_category(CC_SPAMMY), @{$MSGINFO->per_recip_data}) ? (defined $arg_spam ? $arg_spam : 'YES') : (defined $arg_ham ? $arg_ham : 'NO') }, 'k'=> sub { # above kill level and not bypassed for any recipient? grep($_->is_in_contents_category(CC_SPAM), @{$MSGINFO->per_recip_data}) ? 'Y' : '0' }, score_boost => 0, # legacy c => sub {macro_score($MSGINFO,undef,@_)}, # info on all recipients SCORE => sub {macro_score($MSGINFO,undef,@_)}, # info on all recipients STARS => sub {macro_score($MSGINFO,undef,@_)}, # info on all recipients dkim => \&dkim_test, tls_in => sub {$MSGINFO->tls_cipher}, # currently only shows ciphers in use report_format => undef, # notification message format, supplied elsewhere feedback_type => undef, # (ARF) feedback type or empty, supplied elsewhere wrap => sub {my($name,$width,$prefix,$indent,$str) = @_; wrap_string($str,$width,$prefix,$indent)}, lc => sub {my($name)=shift; lc(join('',@_))}, # to lowercase uc => sub {my($name)=shift; uc(join('',@_))}, # to uppercase substr => sub {my($name,$s,$ofs,$len) = @_; defined $len ? substr($s,$ofs,$len) : substr($s,$ofs)}, index => sub {my($name,$s,$substr,$pos) = @_; index($s, $substr, defined $pos ? $pos : 0)}, len => sub {my($name,$s) = @_; length($s)}, incr => sub {my($name,$v,@rest) = @_; if (!@rest) { $v++ } else { $v += $_ for @rest }; "$v"}, decr => sub {my($name,$v,@rest) = @_; if (!@rest) { $v-- } else { $v -= $_ for @rest }; "$v"}, min => sub {my($name,@args) = @_; min(map(/^\s*\z/?undef:$_, @args))}, max => sub {my($name,@args) = @_; max(map(/^\s*\z/?undef:$_, @args))}, sprintf=> sub {my($name,$fmt,@args) = @_; sprintf($fmt,@args)}, join => sub {my($name,$sep,@args) = @_; join($sep,@args)}, limit => sub {my($name,$lim,$s) = @_; $lim < 6 || length($s) <= $lim ? $s : substr($s,0,$lim-5).'[...]' }, dquote => sub {my($nm)=shift; join('', map { s{"}{""}g; '"'.$_.'"' } @_)}, uquote => sub {my($nm)=shift; join('', map { s{[ \t]+}{_}g; $_ } @_)}, b64encode => sub {my($nm)=shift; join(' ', map(encode_base64($_,''),@_))}, mime2utf8 => sub { # convert to UTF-8 octets, truncate to $max_len if given my($nm,$str,$max_len,$both_if_diff) = @_; if (!defined $str || $str eq '') { $str = ''; } else { eval { my($chars) = safe_decode('MIME-Header',$str); # logical characters my($octets) = safe_encode('UTF-8',$chars); # bytes, UTF-8 encoded if (defined $max_len && $max_len > 0 && length($octets) > $max_len) { local($1); if ($octets =~ /^(.{0,$max_len})(?=[\x00-\x7F\xC0-\xFF]|\z)/s) { $octets = $1; # cleanly chop a UTF-8 byte sequence, RFC 3629 } } if (!$both_if_diff) { $str = $octets; } else { # only compare the visible part if (defined $max_len && $max_len > 0 && length($str) > $max_len) { $str = substr($str,0,$max_len); } $str = $octets . ' (raw: ' . $str . ')' if $octets ne $str; } 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(5, "macro mime2utf8: malformed string, keeping raw bytes: %s", $eval_stat); if (defined $max_len && $max_len > 0 && length($str) > $max_len) { $str = substr($str,0,$max_len); } }; } $str; }, # macros f, T, C, B will be defined for each notification as appropriate # (representing From:, To:, Cc:, and Bcc: respectively) # remaining free letters: wxEGIJKLMYZ ); } # initialize %local_delivery_aliases # sub init_local_delivery_aliases() { # The %local_delivery_aliases maps local virtual 'localpart' to a mailbox # (e.g. to a quarantine filename or a directory). Used by method 'local:', # i.e. in mail_to_local_mailbox(), for direct local quarantining. # The hash value may be a ref to a pair of fixed strings, or a subroutine ref # (which must return a pair of strings (a list, not a list ref)) which makes # possible lazy evaluation when some part of the pair is not known before # the final delivery time. The first string in a pair must be either: # - empty or undef, which will disable saving the message, # - a filename, indicating a Unix-style mailbox, # - a directory name, indicating a maildir-style mailbox, # in which case the second string may provide a suggested file name. # %Amavis::Conf::local_delivery_aliases = ( 'virus-quarantine' => sub { ($QUARANTINEDIR, undef) }, 'banned-quarantine' => sub { ($QUARANTINEDIR, undef) }, 'unchecked-quarantine' => sub { ($QUARANTINEDIR, undef) }, 'spam-quarantine' => sub { ($QUARANTINEDIR, undef) }, 'bad-header-quarantine' => sub { ($QUARANTINEDIR, undef) }, 'clean-quarantine' => sub { ($QUARANTINEDIR, undef) }, 'other-quarantine' => sub { ($QUARANTINEDIR, undef) }, 'archive-quarantine' => sub { ($QUARANTINEDIR, undef) }, # some more examples: 'archive-files' => sub { ("$QUARANTINEDIR", undef) }, 'archive-mbox' => sub { ("$QUARANTINEDIR/archive.mbox", undef) }, 'recip-quarantine' => sub { ("$QUARANTINEDIR/recip-archive.mbox",undef) }, 'sender-quarantine' => sub { my($s) = $MSGINFO->sender; $s = substr($s,0,100)."..." if length($s) > 100+3; $s =~ tr/a-zA-Z0-9@._+-/=/c; $s =~ s/\@/_at_/g; $s = untaint($s) if $s =~ /^(?:[a-zA-Z0-9%=._+-]+)\z/; # untaint ($QUARANTINEDIR, "sender-$s-%m.gz"); # suggested file name }, # 'recip-quarantine2' => sub { # my(@fnames); # my($myfield) = # Amavis::Lookup::SQLfield->new($sql_lookups,'some_field_name','S'); # for my $r (@{$MSGINFO->recips}) { # my($field_value) = lookup(0,$r,$myfield); # my($fname) = $field_value; # or perhaps: my($fname) = $r; # local($1); $fname =~ s/[^a-zA-Z0-9._\@]/=/g; $fname =~ s/\@/%/g; # $fname = untaint($fname) if $fname =~ /^([a-zA-Z0-9._=%]+)\z/; # $fname =~ s/%/%%/g; # protect % # do_log(3, "Recipient: %s, field: %s, fname: %s", # $r, $field_value, $fname); # push(@fnames, $fname); # } # # ???what file name to choose if there is more than one recipient??? # ( $QUARANTINEDIR, "sender-$fnames[0]-%i-%n.gz" ); # suggested file name # }, ); } # tokenize templates (input to macro expansion), after dropping privileges # sub init_tokenize_templates() { my(@templ_names) = qw(log_templ log_recip_templ notify_sender_templ notify_virus_recips_templ notify_virus_sender_templ notify_virus_admin_templ notify_spam_sender_templ notify_spam_admin_templ notify_release_templ notify_report_templ notify_autoresp_templ); for my $bank_name (keys %policy_bank) { for my $n (@templ_names) { # tokenize templates to speed up macro expansion my($s) = $policy_bank{$bank_name}{$n}; $s = $$s if ref($s) eq 'SCALAR'; $policy_bank{$bank_name}{$n} = tokenize(\$s) if defined $s; } } } # pre-parse IP lookup tables to speed up lookups, after dropping privileges # sub init_preparse_ip_lookups() { for my $bank_name (keys %policy_bank) { my($r) = $policy_bank{$bank_name}{'inet_acl'}; if (ref($r) eq 'ARRAY') # should be a ref to a single IP lookup table { $policy_bank{$bank_name}{'inet_acl'} = Amavis::Lookup::IP->new(@$r) } $r = $policy_bank{$bank_name}{'client_ipaddr_policy'}; # listref of pairs if (ref($r) eq 'ARRAY') { # should be an array, test just to make sure my($odd) = 1; for my $table (@$r) { # replace plain lists with pre-parsed objects $table = Amavis::Lookup::IP->new(@$table) if $odd && ref($table) eq 'ARRAY'; $odd = !$odd; } } } } # initialize some remaining global variables in a master process; # invoked after chroot and after privileges have been dropped, before forking # sub after_chroot_init() { $child_invocation_count = $child_task_count = 0; %modules_basic = %INC; # helps to track missing modules in chroot do_log(5,"after_chroot_init: EUID: %s (%s); EGID: %s (%s)", $>,$<, $),$( ); my(@msg); my($euid) = $>; # effective UID $> = 0; # try to become root POSIX::setuid(0) if $> != 0; # and try some more if ($> == 0 || $euid == 0) { # succeded? panic! @msg = ("It is possible to change EUID from $euid to root, ABORTING!", "Please use a recent version of Net::Server", "or start as non-root, e.g. by su(1) or using option -u user"); } elsif ($daemon_chroot_dir eq '') { # A quick check on vulnerability/protection of a config file # (non-exhaustive: doesn't test for symlink tricks and higher directories). # The config file has already been executed by now, so it may be # too late to feel sorry now, but better late then never. my(@actual_c_f) = Amavis::Conf::get_config_files_read(); do_log(2,"config files read: %s", join(", ",@actual_c_f)); for my $config_file (@actual_c_f) { local($1); # IO::Handle::_open_mode_string can taint $1 if mode is '+<' my($fh) = IO::File->new; my($errn) = stat($config_file) ? 0 : 0+$!; if ($errn) { # not accessible, don't bother to test further } elsif ($fh->open($config_file,O_RDWR)) { push(@msg, "Config file \"$config_file\" is writable, ". "UID $<, EUID $>, EGID $)" ); $fh->close; # close, ignoring status } elsif (rename($config_file, $config_file.'.moved')) { my($m) = 'appears writable (unconfirmed)'; my($errn_cf_orig) = stat($config_file) ? 0 : 0+$!; my($errn_cf_movd) = stat($config_file.'.moved') ? 0 : 0+$!; if ($errn_cf_orig==ENOENT && $errn_cf_movd!=ENOENT) { # try to rename back, ignoring status rename($config_file.'.moved', $config_file); $m = 'is writable (confirmed)'; } push(@msg, "Directory of a config file \"$config_file\" $m, ". "UID $<, EUID $>, EGID $)" ); } last if @msg; } } if (@msg) { do_log(-3,"FATAL: %s",$_) for @msg; print STDERR (map("$_\n", @msg)); die "SECURITY PROBLEM, ABORTING"; exit 1; # just in case } init_tokenize_templates(); init_preparse_ip_lookups(); # report versions of some (more interesting) modules for my $m ('Amavis::Conf', sort map { s/\.pm\z//; s[/][::]g; $_ } grep(/\.pm\z/, keys %INC)) { next if !grep($_ eq $m, qw(Amavis::Conf File::Temp Archive::Tar Archive::Zip Compress::Zlib Convert::TNEF Convert::UUlib MIME::Entity MIME::Parser MIME::Tools Mail::Header Mail::Internet Digest::MD5 Digest::SHA Digest::SHA1 Crypt::OpenSSL::RSA Authen::SASL Authen::SASL::XS Authen::SASL::Cyrus Authen::SASL::Perl Socket6 IO::Socket::INET6 IO::Socket::SSL Net::SSLeay Net::Server Mail::ClamAV Mail::SpamAssassin Mail::DKIM::Verifier Mail::DKIM::Signer Mail::SPF Mail::SPF::Query NetAddr::IP URI Razor2::Client::Version Net::LDAP DBI DBD::mysql DBD::Pg DBD::SQLite BerkeleyDB DB_File Net::DNS Unix::Syslog Time::HiRes Net::Patricia SAVI Anomy::Sanitizer Unix::Getrusage)); do_log(0, "Module %-19s %s", $m, eval{$m->VERSION} || '?'); } do_log(0,"Amavis::DB code %s loaded", $extra_code_db ?'':" NOT"); do_log(0,"SQL base code %s loaded", $extra_code_sql_base ?'':" NOT"); do_log(0,"SQL::Log code %s loaded", $extra_code_sql_log ?'':" NOT"); do_log(0,"SQL::Quarantine %s loaded", $extra_code_sql_quar ?'':" NOT"); do_log(0,"Lookup::SQL code %s loaded", $extra_code_sql_lookup ?'':" NOT"); do_log(0,"Lookup::LDAP code %s loaded", $extra_code_ldap ?'':" NOT"); do_log(0,"AM.PDP-in proto code%s loaded", $extra_code_in_ampdp ?'':" NOT"); do_log(0,"SMTP-in proto code %s loaded", $extra_code_in_smtp ?'':" NOT"); do_log(0,"Courier proto code %s loaded", $extra_code_in_courier ?'':" NOT"); do_log(0,"SMTP-out proto code %s loaded", $extra_code_out_smtp ?'':" NOT"); do_log(0,"Pipe-out proto code %s loaded", $extra_code_out_pipe ?'':" NOT"); do_log(0,"BSMTP-out proto code%s loaded", $extra_code_out_bsmtp ?'':" NOT"); do_log(0,"Local-out proto code%s loaded", $extra_code_out_local ?'':" NOT"); do_log(0,"OS_Fingerprint code %s loaded", $extra_code_p0f ?'':" NOT"); do_log(0,"ANTI-VIRUS code %s loaded", $extra_code_antivirus ?'':" NOT"); do_log(0,"ANTI-SPAM code %s loaded", $extra_code_antispam ?'':" NOT"); do_log(0,"ANTI-SPAM-EXT code %s loaded", $extra_code_antispam_extprog ?'':" NOT"); do_log(0,"ANTI-SPAM-C code %s loaded", $extra_code_antispam_spamc ?'':" NOT"); do_log(0,"ANTI-SPAM-SA code %s loaded", $extra_code_antispam_sa?'':" NOT"); do_log(0,"Unpackers code %s loaded", $extra_code_unpackers ?'':" NOT"); do_log(0,"DKIM code %s loaded", $extra_code_dkim ?'':" NOT"); do_log(0,"Tools code %s loaded", $extra_code_tools ?'':" NOT"); # store policy names into 'policy_bank_name' fields, if not explicitly set for my $name (keys %policy_bank) { if (ref($policy_bank{$name}) eq 'HASH' && !exists($policy_bank{$name}{'policy_bank_name'})) { $policy_bank{$name}{'policy_bank_name'} = $name; $policy_bank{$name}{'policy_bank_path'} = $name; } } }; # overlay the current policy bank by settings from the # $policy_bank{$policy_bank_name}, or load the default policy bank (empty name) # sub load_policy_bank($;$) { my($policy_bank_name,$msginfo) = @_; if (!exists $policy_bank{$policy_bank_name}) { do_log(-1,'policy bank "%s" does not exist, ignored', $policy_bank_name); } elsif ($policy_bank_name eq '') { %current_policy_bank = %{$policy_bank{$policy_bank_name}}; # copy base update_current_log_level(); do_log(4,'loaded base policy bank'); } else { my($cpbp) = c('policy_bank_path'); # currently loaded bank my($new_bank_ref) = $policy_bank{$policy_bank_name}; my($do_log5) = ll(5); for my $k (keys %$new_bank_ref) { if ($k eq 'ACTION') { if (ref $new_bank_ref->{$k} eq 'CODE') { do_log(5,'invoking user ACTION on loading a policy bank %s', $policy_bank_name); eval { # $msginfo may be undef when a p. b. load takes place early &{$new_bank_ref->{$k}}($msginfo,$policy_bank_name); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,'failed ACTION on loading a policy bank %s: %s', $policy_bank_name, $eval_stat); }; } } elsif (!exists $current_policy_bank{$k}) { do_log(-1,'loading policy bank "%s": unknown field "%s"', $policy_bank_name,$k); } elsif (ref($new_bank_ref->{$k}) ne 'HASH' || ref($current_policy_bank{$k}) ne 'HASH') { $current_policy_bank{$k} = $new_bank_ref->{$k}; } else { # new hash to be merged into or replacing an existing hash if ($new_bank_ref->{$k}{REPLACE}) { # replace the entire hash $current_policy_bank{$k} = { %{$new_bank_ref->{$k}} }; # copy of new do_log(5,"loading policy bank %s, curr{%s} hash replaced", $policy_bank_name, $k) if $do_log5; } else { # merge field-by-field, old fields missing in new are retained $current_policy_bank{$k} = { %{$current_policy_bank{$k}} }; # copy while (my($key,$val) = each %{$new_bank_ref->{$k}}) { do_log(5,"loading policy bank %s, curr{%s}{%s} = %s, %s", $policy_bank_name, $k, $key, $val, !exists($current_policy_bank{$k}{$key}) ? 'new' : 'replaces '.$current_policy_bank{$k}{$key} ) if $do_log5; $current_policy_bank{$k}{$key} = $val; } } delete $current_policy_bank{$k}{REPLACE}; } } $current_policy_bank{'policy_bank_path'} = ($cpbp eq '' ? '' : $cpbp.'/') . $policy_bank_name; update_current_log_level(); do_log(2,'loaded policy bank "%s"%s', $policy_bank_name, $cpbp eq '' ? '' : " over \"$cpbp\""); } } ### Net::Server hook ### Occurs in the parent (master) process after (possibly) opening a log file, ### creating pid file, reopening STDIN/STDOUT to /dev/null and daemonizing; ### but before binding to sockets # sub post_configure_hook { # umask(0007); # affect protection of Unix sockets created by Net::Server } sub set_sockets_access() { if (defined $unix_socket_mode && $unix_socket_mode ne '') { for my $s (@listen_sockets) { local($1); if ($s =~ m{^(/.+)\|unix\z}si) { my($path) = $1; chmod($unix_socket_mode,$path) or do_log(-1, "Error setting mode 0%o on a socket %s: %s", $unix_socket_mode, $path, $!); } } } } ### Net::Server hook ### Occurs in the parent (master) process after binding to sockets, ### but before chrooting and dropping privileges # sub post_bind_hook { umask(0027); # restore our preferred umask set_sockets_access() if defined $warm_restart && !$warm_restart; } ### Net::Server hook ### This hook occurs in the parent (master) process after chroot, ### after change of user, and change of group has occured. ### It allows for preparation before forking and looping begins. # sub pre_loop_hook { my($self) = @_; local $SIG{CHLD} = 'DEFAULT'; # do_log(5, "entered pre_loop_hook"); eval { after_chroot_init(); # the rest of the top-level initialization # this needs to be done only after chroot, otherwise paths will be wrong find_external_programs([split(/:/,$path,-1)]); # path, decoders, scanners # do some sanity checking my($name) = $TEMPBASE; $name = "$daemon_chroot_dir $name" if $daemon_chroot_dir ne ''; my($errn) = stat($TEMPBASE) ? 0 : 0+$!; if ($errn==ENOENT) { die "No TEMPBASE directory: $name" } elsif ($errn) { die "TEMPBASE directory inaccessible, $!: $name" } elsif (!-d _) { die "TEMPBASE is not a directory: $name" } elsif (!-w _) { die "TEMPBASE directory is not writable: $name" } if ($enable_db && $extra_code_db) { my($name) = $db_home; $name = "$daemon_chroot_dir $name" if $daemon_chroot_dir ne ''; $errn = stat($db_home) ? 0 : 0+$!; if ($errn == ENOENT) { die "Please create an empty directory $name to hold a database". " (config variable \$db_home)\n" } elsif ($errn) { die "db_home $name inaccessible: $!" } elsif (!-d _) { die "db_home $name is not a directory" } elsif (!-w _) { die "db_home $name directory is not writable" } Amavis::DB::init(1, !$warm_restart); } if (!defined($sql_quarantine_chunksize_max)) { die "Variable \$sql_quarantine_chunksize_max is undefined\n"; } elsif ($sql_quarantine_chunksize_max < 1024) { die "Setting of \$sql_quarantine_chunksize_max is too small: ". "$sql_quarantine_chunksize_max bytes, it would be inefficient\n"; } elsif ($sql_quarantine_chunksize_max > 1024*1024) { do_log(-1, "Setting of %s is quite large: %d KiB, it unnecessarily ". "wastes memory", '$sql_quarantine_chunksize_max', $sql_quarantine_chunksize_max/1024); } if ($QUARANTINEDIR ne '') { my($name) = $QUARANTINEDIR; $name = "$daemon_chroot_dir $name" if $daemon_chroot_dir ne ''; $errn = stat($QUARANTINEDIR) ? 0 : 0+$!; if ($errn == ENOENT) { } # ok elsif ($errn) { die "QUARANTINEDIR $name inaccessible: $!" } elsif (-d _ && !-w _){ die "QUARANTINEDIR directory $name not writable"} } $spamcontrol_obj->init_pre_fork if $spamcontrol_obj; my(@modules_extra) = grep(!exists $modules_basic{$_}, keys %INC); if (@modules_extra) { do_log(1, "extra modules loaded after daemonizing/chrooting: %s", join(", ", sort @modules_extra)); %modules_basic = %INC; } do_log(0, "DKIM signature verification disabled, corresponding features ". "not available. If not intentional, consider enabling it by setting: ". "\$enable_dkim_verification to 1, or explicitly disable it by setting ". "it to 0 to mute this warning." ) if !c('enable_dkim_verification') && !defined(c('enable_dkim_verification')); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; my($msg) = "TROUBLE in pre_loop_hook: $eval_stat"; do_log(-2,"%s",$msg); die("Suicide (" . am_id() . ") " . $msg . "\n"); }; 1; } # (!)_DIE: Unable to create sub named "" at /usr/local/sbin/amavisd line 9947. # line 9947 was in sub write_to_log_hook: local $SIG{CHLD} = 'DEFAULT'; # perl #60360: local $SIG{FOO} = sub {...}; sets signal handler to SIG_DFL # # http://www.perlmonks.org/?node_id=721692 # # non-atomic, clears to SIG_DFL, then sets: local $SIG{ALRM} = sub {...}; # use Sub::ScopeFinalizer qw( scope_finalizer ); # my $sentry = local_sassign $SIG{ALRM}, \&alarm_handler; # sub local_sassign { # my $r = \($_[0]); # my $sentry = scope_finalizer { $$r = $_[0] } { args => [ $$r ] }; # $$r = $_[1]; return $sentry; # } # or use: # use POSIX qw(:signal_h) ; # my $sigset = POSIX::SigSet->new ; # my $blockset = POSIX::SigSet->new( SIGALRM ) ; # sigprocmask(SIG_BLOCK, $blockset, $sigset ); # local $SIG{ALRM} = sub .... ; # sigprocmask(SIG_SETMASK, $sigset ); ### log routine Net::Server hook ### (Sys::Syslog MUST NOT be specified as a value of 'log_file'!) # # Redirect Net::Server logging to use Amavis' do_log(). # The main reason is that Net::Server uses Sys::Syslog # (and has two bugs in doing it, at least the Net-Server-0.82), # and Amavis users are acustomed to Unix::Syslog. # sub write_to_log_hook { my($self,$level,$msg) = @_; my($prop) = $self->{server}; local $SIG{CHLD} = 'DEFAULT'; $level = 0 if $level < 0; $level = 4 if $level > 4; # my($ll) = (-2,-1,0,1,3)[$level]; # 0=err, 1=warn, 2=notice, 3=info, 4=debug my($ll) = (-1, 0,1,3,4)[$level]; # 0=err, 1=warn, 2=notice, 3=info, 4=debug chomp($msg); # just call Amavis' traditional logging ll($ll) && do_log($ll, "Net::Server: %s", $msg); 1; } ### user customizable Net::Server hook (Net::Server 0.88 or later), ### hook occurs in the master process !!! # sub run_n_children_hook { # do_log(5, "entered run_n_children_hook"); Amavis::AV::sophos_savi_reload() if $extra_code_antivirus && Amavis::AV::sophos_savi_stale(); add_entropy(Time::HiRes::gettimeofday); } ### compatibility with patched Net::Server by SAVI patch (Net::Server <= 0.87) # sub parent_fork_hook { my $self = $_[0]; $self->run_n_children_hook } ### user customizable Net::Server hook, ### run by every child process during its startup # sub child_init_hook { my($self) = @_; local $SIG{CHLD} = 'DEFAULT'; $child_init_hook_was_called = 1; do_log(5, "entered child_init_hook"); $my_pid = $$; $0 = c('myprogram_name') . ' (virgin child)'; stir_random(); # reset log counters inherited from a master process collect_log_stats(); # my(@signames) = qw(HUP INT QUIT ILL TRAP ABRT EMT FPE KILL BUS SEGV # SYS PIPE ALRM TERM URG TSTP CONT TTIN TTOU IO # XCPU XFSZ VTALRM PROF WINCH INFO USR1 USR2); # my($h) = sub { my($s) = $_[0]; $got_signals{$s}++; # local($SIG{$s})='IGNORE'; kill($my_pid,$s) }; # @SIG{@signames} = ($h) x @signames; my($inherited_entropy); eval { # if ($> == 0 || $< == 0) { # last resort, in case Net::Server didn't do it # do_log(2, "child_init_hook: dropping privileges, user=%s, group=%s", # $daemon_user,$daemon_group); # drop_priv($daemon_user,$daemon_group); # } undef $db_env; undef $snmp_db; # just in case Amavis::Timing::init(); snmp_counters_init(); close_log(); open_log(); # reopen syslog or log file to get per-process fd if ($extra_code_db) { # Berkeley DB handles should not be shared across process forks, # each forked child should acquire its own Berkeley DB handles $db_env = Amavis::DB->new; # get access to a bdb environment $snmp_db = Amavis::DB::SNMP->new($db_env); $snmp_db->register_proc(0,1,'') if defined $snmp_db; # alive and idle my($var_ref) = $snmp_db->read_snmp_variables('entropy'); $inherited_entropy = $var_ref->[0] if $var_ref && @$var_ref; } # if ($extra_code_db) { # is it worth reporting the timing? (probably not) # section_time('bdb-open'); # do_log(2, "%s", Amavis::Timing::report()); # report elapsed times # } # Prepare permanent SQL dataset connection objects, does not connect yet! # $sql_dataset_conn_lookups and $sql_dataset_conn_storage may be the # same dataset (one connection used), or they may be separate objects, # which will make separate connections to (same or distinct) datasets, # possibly using different SQL engine types or servers if ($extra_code_sql_lookup && @lookup_sql_dsn) { $sql_dataset_conn_lookups = Amavis::Out::SQL::Connection->new(@lookup_sql_dsn); } if ($extra_code_sql_log && @storage_sql_dsn) { if (!$sql_dataset_conn_lookups || @storage_sql_dsn != @lookup_sql_dsn || grep($storage_sql_dsn[$_] ne $lookup_sql_dsn[$_], (0..$#storage_sql_dsn)) ) { # DSN differs or no SQL lookups, storage needs its own connection $sql_dataset_conn_storage = Amavis::Out::SQL::Connection->new(@storage_sql_dsn); if ($sql_dataset_conn_lookups) { do_log(2,"storage and lookups will use separate connections to SQL"); } else { do_log(5,"only storage connections to SQL, no lookups"); } } else { # same dataset, use the same database connection object $sql_dataset_conn_storage = $sql_dataset_conn_lookups; do_log(2,"storage and lookups will use the same connection to SQL"); } } # create storage/lookup objs to hold DBI handles and 'prepared' statements $sql_storage = Amavis::Out::SQL::Log->new($sql_dataset_conn_storage) if $sql_dataset_conn_storage; $sql_lookups = Amavis::Lookup::SQL->new($sql_dataset_conn_lookups, 'sel_policy') if $sql_dataset_conn_lookups; $sql_wblist = Amavis::Lookup::SQL->new($sql_dataset_conn_lookups, 'sel_wblist') if $sql_dataset_conn_lookups; $spamcontrol_obj->init_child if $spamcontrol_obj; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-2, "TROUBLE in child_init_hook: %s", $eval_stat); die "Suicide in child_init_hook: $eval_stat\n"; }; add_entropy($inherited_entropy, Time::HiRes::gettimeofday, rand()); Amavis::Timing::go_idle('vir'); } ### user customizable Net::Server hook # sub post_accept_hook { my($self) = @_; local $SIG{CHLD} = 'DEFAULT'; # do_log(5, "entered post_accept_hook"); if (!$child_init_hook_was_called) { # this can happen with base Net::Server (not PreFork nor PreForkSiple) do_log(5, "post_accept_hook: invoking child_init_hook which was skipped"); $self->child_init_hook; } $child_invocation_count++; $0 = sprintf("%s (ch%d-accept)", c('myprogram_name'), $child_invocation_count); Amavis::Util::am_id(''); Amavis::Timing::go_busy('hi '); # establish initial time right after 'accept' Amavis::Timing::init(); snmp_counters_init(); $snmp_db->register_proc(1,1,'A') if defined $snmp_db; # enter 'accept' state load_policy_bank(''); # start with a builtin baseline policy bank } ### user customizable Net::Server hook, load a by-interface policy bank; ### if this hook returns 1 the request is processed ### if this hook returns 0 the request is denied # sub allow_deny_hook { my($self) = @_; local($1,$2,$3,$4); # Perl bug: $1 and $2 come tainted from Net::Server ! local $SIG{CHLD} = 'DEFAULT'; # do_log(5, "entered allow_deny_hook"); my($prop, $sock, $bank_name, $is_ux); $prop = $self->{server}; $sock = $prop->{client}; $is_ux = $sock && $sock->UNIVERSAL::can('NS_proto') && $sock->NS_proto eq 'UNIX'; if ($is_ux) { my($path) = $sock->NS_unix_path; if (defined $interface_policy{$path}) { $bank_name = $interface_policy{$path}; } elsif (defined $interface_policy{"SOCK"}) { $bank_name = $interface_policy{"SOCK"}; } } else { my($myif,$myport) = ($prop->{sockaddr}, $prop->{sockport}); if (defined $interface_policy{"$myif:$myport"}) { $bank_name = $interface_policy{"$myif:$myport"}; } elsif (defined $interface_policy{$myport}) { $bank_name = $interface_policy{$myport}; } } load_policy_bank($bank_name) if defined $bank_name && $bank_name ne c('policy_bank_name'); # note that the new policy bank may have replaced the inet_acl access table if ($is_ux) { # always permit access - unix sockets are immune to this check } else { my($permit,$fullkey,$err) = lookup_ip_acl($prop->{peeraddr}, Amavis::Lookup::Label->new('inet_acl'), ca('inet_acl')); if (defined($err) && $err ne '') { do_log(-1, "DENIED ACCESS due to INVALID PEER IP ADDRESS %s: %s", $prop->{peeraddr}, $err); return 0; } elsif (!$permit) { do_log(-1, "DENIED ACCESS from IP %s, policy bank '%s'%s", $prop->{peeraddr}, c('policy_bank_name'), !defined $fullkey ? '' : ", blocked by rule $fullkey"); return 0; } } 1; } ### The heart of the program ### user customizable Net::Server hook # sub process_request { my $self = shift; local $SIG{CHLD} = 'DEFAULT'; # do_log(5, "entered process_request"); local($1,$2,$3,$4); # Perl bug: $1 and $2 come tainted from Net::Server ! my($prop) = $self->{server}; my($sock) = $prop->{client}; ll(3) && do_log(3, "process_request: fileno sock=%s, STDIN=%s, STDOUT=%s", fileno($sock), fileno(STDIN), fileno(STDOUT)); # Net::Server 0.91 dups a socket to STDIN and STDOUT, which we do not want; # it also forgets to close STDIN & STDOUT afterwards, so session remains # open (smtp QUIT does not work), fixed in 0.92; # Net::Server 0.92 introduced option no_client_stdout, but it # breaks Net::Server::get_client_info by setting it, so we can't use it; # On NetBSD closing fh STDIN (on fd0) somehow leaves fd0 still assigned to # a socket (Net::Server 0.91) and cannot be closed even by a POSIX::close # Let's just leave STDIN and STDOUT as they are, which works for versions # of Net::Server 0.90 and older, is wasteful with 0.91 and 0.92, and is # fine with 0.93. if (ref($sock) !~ /^(?:IO::Socket::SSL|Net::Server::Proto::SSL)\z/) { # binmode not implemented in IO::Socket::SSL and returns false binmode($sock) or die "Can't set socket $sock to binmode: $!"; } local $SIG{ALRM} = sub { die "timed out\n" }; # do not modify the sig text! my($eval_stat); eval { # if ($] < 5.006) # Perl older than 5.6.0 did not set FD_CLOEXEC on sockets # { cloexec($_,1,$_) for @{$prop->{sock}} } switch_to_my_time('new request'); # timer init if ($extra_code_ldap && !defined $ldap_lookups) { # make LDAP lookup object $ldap_connection = Amavis::LDAP::Connection->new($default_ldap); $ldap_lookups = Amavis::Lookup::LDAP->new($default_ldap,$ldap_connection) if $ldap_connection; } if (defined $ldap_lookups && $lookup_maps_imply_sql_and_ldap && !$implicit_maps_inserted) { # make LDAP field lookup objects with incorporated field names # fieldtype: B=boolean, N=numeric, S=string, L=list # B-, N-, S-, L- returns undef if field does not exist # B0: boolean, nonexistent field treated as false, # B1: boolean, nonexistent field treated as true my $lf = sub{Amavis::Lookup::LDAPattr->new($ldap_lookups,@_)}; unshift(@Amavis::Conf::local_domains_maps, $lf->('amavisLocal', 'B1')); unshift(@Amavis::Conf::virus_lovers_maps, $lf->('amavisVirusLover', 'B-')); unshift(@Amavis::Conf::spam_lovers_maps, $lf->('amavisSpamLover', 'B-')); unshift(@Amavis::Conf::unchecked_lovers_maps, $lf->('amavisUncheckedLover', 'B-')); unshift(@Amavis::Conf::banned_files_lovers_maps, $lf->('amavisBannedFilesLover', 'B-')); unshift(@Amavis::Conf::bad_header_lovers_maps, $lf->('amavisBadHeaderLover', 'B-')); unshift(@Amavis::Conf::bypass_virus_checks_maps, $lf->('amavisBypassVirusChecks', 'B-')); unshift(@Amavis::Conf::bypass_spam_checks_maps, $lf->('amavisBypassSpamChecks', 'B-')); unshift(@Amavis::Conf::bypass_banned_checks_maps,$lf->('amavisBypassBannedChecks', 'B-')); unshift(@Amavis::Conf::bypass_header_checks_maps,$lf->('amavisBypassHeaderChecks', 'B-')); unshift(@Amavis::Conf::spam_tag_level_maps, $lf->('amavisSpamTagLevel', 'N-')); unshift(@Amavis::Conf::spam_tag2_level_maps, $lf->('amavisSpamTag2Level', 'N-')); unshift(@Amavis::Conf::spam_tag3_level_maps, $lf->('amavisSpamTag3Level', 'N-')); unshift(@Amavis::Conf::spam_kill_level_maps, $lf->('amavisSpamKillLevel', 'N-')); unshift(@Amavis::Conf::spam_dsn_cutoff_level_maps,$lf->('amavisSpamDsnCutoffLevel','N-')); unshift(@Amavis::Conf::spam_quarantine_cutoff_level_maps,$lf->('amavisSpamQuarantineCutoffLevel','N-')); unshift(@Amavis::Conf::spam_subject_tag_maps, $lf->('amavisSpamSubjectTag', 'S-')); unshift(@Amavis::Conf::spam_subject_tag2_maps, $lf->('amavisSpamSubjectTag2', 'S-')); unshift(@Amavis::Conf::spam_subject_tag3_maps, $lf->('amavisSpamSubjectTag3', 'S-')); unshift(@Amavis::Conf::virus_quarantine_to_maps, $lf->('amavisVirusQuarantineTo', 'S-')); unshift(@Amavis::Conf::spam_quarantine_to_maps, $lf->('amavisSpamQuarantineTo', 'S-')); unshift(@Amavis::Conf::banned_quarantine_to_maps, $lf->('amavisBannedQuarantineTo','S-')); unshift(@Amavis::Conf::unchecked_quarantine_to_maps, $lf->('amavisUncheckedQuarantineTo','S-')); unshift(@Amavis::Conf::bad_header_quarantine_to_maps, $lf->('amavisBadHeaderQuarantineTo', 'S-')); unshift(@Amavis::Conf::clean_quarantine_to_maps, $lf->('amavisCleanQuarantineTo', 'S-')); unshift(@Amavis::Conf::archive_quarantine_to_maps, $lf->('amavisArchiveQuarantineTo', 'S-')); unshift(@Amavis::Conf::message_size_limit_maps, $lf->('amavisMessageSizeLimit', 'N-')); unshift(@Amavis::Conf::addr_extension_virus_maps, $lf->('amavisAddrExtensionVirus', 'S-')); unshift(@Amavis::Conf::addr_extension_spam_maps, $lf->('amavisAddrExtensionSpam', 'S-')); unshift(@Amavis::Conf::addr_extension_banned_maps, $lf->('amavisAddrExtensionBanned','S-')); unshift(@Amavis::Conf::addr_extension_bad_header_maps, $lf->('amavisAddrExtensionBadHeader','S-')); unshift(@Amavis::Conf::warnvirusrecip_maps, $lf->('amavisWarnVirusRecip', 'B-')); unshift(@Amavis::Conf::warnbannedrecip_maps, $lf->('amavisWarnBannedRecip', 'B-')); unshift(@Amavis::Conf::warnbadhrecip_maps, $lf->('amavisWarnBadHeaderRecip', 'B-')); unshift(@Amavis::Conf::newvirus_admin_maps, $lf->('amavisNewVirusAdmin', 'S-')); unshift(@Amavis::Conf::virus_admin_maps, $lf->('amavisVirusAdmin', 'S-')); unshift(@Amavis::Conf::spam_admin_maps, $lf->('amavisSpamAdmin', 'S-')); unshift(@Amavis::Conf::banned_admin_maps, $lf->('amavisBannedAdmin', 'S-')); unshift(@Amavis::Conf::bad_header_admin_maps, $lf->('amavisBadHeaderAdmin', 'S-')); unshift(@Amavis::Conf::banned_filename_maps, $lf->('amavisBannedRuleNames', 'S-')); unshift(@Amavis::Conf::disclaimer_options_bysender_maps, $lf->('amavisDisclaimerOptions', 'S-')); unshift(@Amavis::Conf::forward_method_maps, $lf->('amavisForwardMethod', 'S-')); unshift(@Amavis::Conf::sa_userconf_maps, $lf->('amavisSaUserConf', 'S-')); unshift(@Amavis::Conf::sa_username_maps, $lf->('amavisSaUserName', 'S-')); section_time('ldap-prepare'); } if (defined $sql_lookups && $lookup_maps_imply_sql_and_ldap && !$implicit_maps_inserted) { # make SQL field lookup objects with incorporated field names # fieldtype: B=boolean, N=numeric, S=string, # B-, N-, S- returns undef if field does not exist # B0: boolean, nonexistent field treated as false, # B1: boolean, nonexistent field treated as true my $nf = sub{Amavis::Lookup::SQLfield->new($sql_lookups,@_)}; # shorthand $user_id_sql = $nf->('id', 'S-'); $user_policy_id_sql = $nf->('policy_id', 'S-'); unshift(@Amavis::Conf::local_domains_maps, $nf->('local', 'B1')); unshift(@Amavis::Conf::virus_lovers_maps, $nf->('virus_lover', 'B-')); unshift(@Amavis::Conf::spam_lovers_maps, $nf->('spam_lover', 'B-')); unshift(@Amavis::Conf::unchecked_lovers_maps, $nf->('unchecked_lover', 'B-')); unshift(@Amavis::Conf::banned_files_lovers_maps, $nf->('banned_files_lover', 'B-')); unshift(@Amavis::Conf::bad_header_lovers_maps, $nf->('bad_header_lover', 'B-')); unshift(@Amavis::Conf::bypass_virus_checks_maps, $nf->('bypass_virus_checks', 'B-')); unshift(@Amavis::Conf::bypass_spam_checks_maps, $nf->('bypass_spam_checks', 'B-')); unshift(@Amavis::Conf::bypass_banned_checks_maps, $nf->('bypass_banned_checks', 'B-')); unshift(@Amavis::Conf::bypass_header_checks_maps, $nf->('bypass_header_checks', 'B-')); unshift(@Amavis::Conf::spam_tag_level_maps, $nf->('spam_tag_level', 'N-')); unshift(@Amavis::Conf::spam_tag2_level_maps, $nf->('spam_tag2_level', 'N-')); unshift(@Amavis::Conf::spam_tag3_level_maps, $nf->('spam_tag3_level', 'N-')); unshift(@Amavis::Conf::spam_kill_level_maps, $nf->('spam_kill_level', 'N-')); unshift(@Amavis::Conf::spam_dsn_cutoff_level_maps,$nf->('spam_dsn_cutoff_level','N-')); unshift(@Amavis::Conf::spam_quarantine_cutoff_level_maps,$nf->('spam_quarantine_cutoff_level','N-')); unshift(@Amavis::Conf::spam_subject_tag_maps, $nf->('spam_subject_tag', 'S-')); unshift(@Amavis::Conf::spam_subject_tag2_maps, $nf->('spam_subject_tag2', 'S-')); unshift(@Amavis::Conf::spam_subject_tag3_maps, $nf->('spam_subject_tag3', 'S-')); unshift(@Amavis::Conf::virus_quarantine_to_maps, $nf->('virus_quarantine_to', 'S-')); unshift(@Amavis::Conf::spam_quarantine_to_maps, $nf->('spam_quarantine_to', 'S-')); unshift(@Amavis::Conf::banned_quarantine_to_maps, $nf->('banned_quarantine_to', 'S-')); unshift(@Amavis::Conf::unchecked_quarantine_to_maps, $nf->('unchecked_quarantine_to', 'S-')); unshift(@Amavis::Conf::bad_header_quarantine_to_maps, $nf->('bad_header_quarantine_to','S-')); unshift(@Amavis::Conf::clean_quarantine_to_maps, $nf->('clean_quarantine_to', 'S-')); unshift(@Amavis::Conf::archive_quarantine_to_maps,$nf->('archive_quarantine_to','S-')); unshift(@Amavis::Conf::message_size_limit_maps, $nf->('message_size_limit', 'N-')); unshift(@Amavis::Conf::addr_extension_virus_maps, $nf->('addr_extension_virus', 'S-')); unshift(@Amavis::Conf::addr_extension_spam_maps, $nf->('addr_extension_spam', 'S-')); unshift(@Amavis::Conf::addr_extension_banned_maps,$nf->('addr_extension_banned','S-')); unshift(@Amavis::Conf::addr_extension_bad_header_maps,$nf->('addr_extension_bad_header','S-')); unshift(@Amavis::Conf::warnvirusrecip_maps, $nf->('warnvirusrecip', 'B-')); unshift(@Amavis::Conf::warnbannedrecip_maps, $nf->('warnbannedrecip', 'B-')); unshift(@Amavis::Conf::warnbadhrecip_maps, $nf->('warnbadhrecip', 'B-')); unshift(@Amavis::Conf::newvirus_admin_maps, $nf->('newvirus_admin', 'S-')); unshift(@Amavis::Conf::virus_admin_maps, $nf->('virus_admin', 'S-')); unshift(@Amavis::Conf::spam_admin_maps, $nf->('spam_admin', 'S-')); unshift(@Amavis::Conf::banned_admin_maps, $nf->('banned_admin', 'S-')); unshift(@Amavis::Conf::bad_header_admin_maps, $nf->('bad_header_admin', 'S-')); unshift(@Amavis::Conf::banned_filename_maps, $nf->('banned_rulenames', 'S-')); unshift(@Amavis::Conf::disclaimer_options_bysender_maps, $nf->('disclaimer_options', 'S-')); unshift(@Amavis::Conf::forward_method_maps, $nf->('forward_method', 'S-')); unshift(@Amavis::Conf::sa_userconf_maps, $nf->('sa_userconf', 'S-')); unshift(@Amavis::Conf::sa_username_maps, $nf->('sa_username', 'S-')); section_time('sql-prepare'); } $implicit_maps_inserted = 1; if (!$maps_have_been_labeled) { Amavis::Conf::label_default_maps(); $maps_have_been_labeled = 1 } my($ns_proto) = $sock->NS_proto; # Net::Server::Proto submodules my($conn) = Amavis::In::Connection->new; # keeps info about connection $conn->socket_proto($ns_proto); my($suggested_protocol) = c('protocol'); # suggested by the policy bank $suggested_protocol = '' if !defined $suggested_protocol; do_log(5,"process_request: suggested_protocol=\"%s\" on a %s socket", $suggested_protocol, $ns_proto); # $snmp_db->register_proc(2,0,'b') if defined $snmp_db; # begin protocol if ($ns_proto eq 'UNIX') { $conn->socket_path($sock->NS_unix_path); # how to test: $ socat stdio unix-connect:/var/amavis/amavisd.sock,crnl } else { # TCP, UDP, SSLEAY, SSL (Net::Server::Proto submodules) $conn->socket_ip($prop->{sockaddr}); $conn->socket_port($prop->{sockport}); $conn->client_ip($prop->{peeraddr}); } if ($suggested_protocol eq 'SMTP' || $suggested_protocol eq 'LMTP' || ($suggested_protocol eq '' && $ns_proto =~ /^(?:TCP|SSLEAY|SSL)\z/)) { if (!$extra_code_in_smtp) { die "incoming TCP connection, but dynamic SMTP/LMTP code not loaded"; } $smtp_in_obj = Amavis::In::SMTP->new if !$smtp_in_obj; $smtp_in_obj->process_smtp_request( $sock, ($suggested_protocol eq 'LMTP'?1:0), $conn, \&check_mail); } elsif ($suggested_protocol eq 'AM.PDP') { # amavis policy delegation protocol (e.g. new milter or amavisd-release) $ampdp_in_obj = Amavis::In::AMPDP->new if !$ampdp_in_obj; $ampdp_in_obj->process_policy_request($sock, $conn, \&check_mail, 0); } elsif ($suggested_protocol eq 'COURIER') { die "unavailable support for protocol: $suggested_protocol"; } elsif ($suggested_protocol eq 'QMQPqq') { die "unavailable support for protocol: $suggested_protocol"; } elsif ($suggested_protocol eq 'TCP-LOOKUP') { #postfix maps, experimental process_tcp_lookup_request($sock, $conn); do_log(2, "%s", Amavis::Timing::report()); # report elapsed times # } elsif ($suggested_protocol eq 'AM.CL') { # # defaults to old amavis helper program protocol # $ampdp_in_obj = Amavis::In::AMPDP->new if !$ampdp_in_obj; # $ampdp_in_obj->process_policy_request($sock, $conn, \&check_mail, 1); } elsif ($suggested_protocol eq '') { die "protocol not specified, $ns_proto"; } else { die "unsupported protocol: $suggested_protocol, $ns_proto"; } Amavis::Out::SMTP::Session::rundown_stale_sessions(0) if $extra_code_out_smtp; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; alarm(0); # stop the timer if (defined $eval_stat) { chomp $eval_stat; my($timed_out) = $eval_stat =~ /^timed out\b/; if ($timed_out) { my($msg) = "Requesting process rundown, task exceeded allowed time"; $msg .= " during waiting for input from client" if waiting_for_client(); do_log(-1, $msg); } else { do_log(-2, "TROUBLE in process_request: %s", $eval_stat); $smtp_in_obj->preserve_evidence(1) if $smtp_in_obj; do_log(-1, "Requesting process rundown after fatal error"); } undef $smtp_in_obj; undef $ampdp_in_obj; undef $courier_in_obj; $self->done(1); } elsif ($max_requests > 0 && $child_task_count >= $max_requests) { # in case of multiple-transaction protocols (e.g. SMTP, LMTP) # we do not like to keep running indefinitely at the mercy of MTA my($have_sawampersand)=Devel::SawAmpersand->UNIVERSAL::can("sawampersand"); do_log(2, "Requesting process rundown after %d tasks (and %s sessions)%s", $child_task_count, $child_invocation_count, !$have_sawampersand ? '' : Devel::SawAmpersand::sawampersand() ? ", SawAmpersand is TRUE!" : ", SawAmpersand is false"); undef $smtp_in_obj; undef $ampdp_in_obj; undef $courier_in_obj; $self->done(1); } elsif ($extra_code_antivirus && Amavis::AV::sophos_savi_stale() ) { do_log(0, "Requesting process rundown due to stale Sophos virus data"); undef $smtp_in_obj; undef $ampdp_in_obj; undef $courier_in_obj; $self->done(1); } my(@modules_extra) = grep(!exists $modules_basic{$_}, keys %INC); # do_log(2, "modules loaded: %s", join(", ", sort keys %modules_basic)); if (@modules_extra) { do_log(1, "extra modules loaded: %s", join(", ", sort @modules_extra)); %modules_basic = %INC; } do_log(5, "exiting process_request"); } ### After processing of a request, but before client connection has been closed ### user customizable Net::Server hook # sub post_process_request_hook { my($self) = @_; my($prop) = $self->{server}; my($sock) = $prop->{client}; local $SIG{CHLD} = 'DEFAULT'; # do_log(5, "entered post_process_request_hook"); debug_oneshot(0); $0 = sprintf("%s (ch%d-avail)", c('myprogram_name'), $child_invocation_count); alarm(0); # stop the timer $snmp_db->register_proc(1,0,'') if defined $snmp_db; # alive and idle again Amavis::Timing::go_idle('bye'); if (ll(3)) { my($load_report) = Amavis::Timing::report_load(); do_log(3,$load_report) if defined $load_report; } # workaround: Net::Server 0.91 forgets to disconnect session if (Net::Server->VERSION == 0.91) { close STDIN; close STDOUT } } ### Child is about to be terminated ### user customizable Net::Server hook # sub child_finish_hook { my($self) = @_; local $SIG{CHLD} = 'DEFAULT'; # do_log(5, "entered child_finish_hook"); # for my $m (sort map { s/\.pm\z//; s[/][::]g; $_ } grep(/\.pm\z/, keys %INC)){ # do_log(0, "Module %-19s %s", $m, $m->VERSION || '?') # if grep($m=~/^$_/, qw(Mail::ClamAV Mail::SpamAssassin Razor2 Net::DNS)); # } Amavis::Out::SMTP::Session::rundown_stale_sessions(1) if $extra_code_out_smtp; $spamcontrol_obj->rundown_child if $spamcontrol_obj; report_rusage(); $0 = sprintf("%s (ch%d-finish)", c('myprogram_name'), $child_invocation_count); do_log(5,"child_finish_hook: invoking DESTROY methods"); undef $smtp_in_obj; undef $ampdp_in_obj; undef $courier_in_obj; undef $sql_storage; undef $sql_wblist; undef $sql_lookups; undef $sql_dataset_conn_lookups; undef $sql_dataset_conn_storage; undef $ldap_lookups; undef $ldap_connection; eval { $snmp_db->register_proc(0,0,undef) } if defined $snmp_db; # unregister undef $snmp_db; undef $db_env; } sub END { # runs before exiting the module # do_log(5,"at the END handler: invoking DESTROY methods"); undef $smtp_in_obj; undef $ampdp_in_obj; undef $courier_in_obj; undef $sql_storage; undef $sql_wblist; undef $sql_lookups; undef $sql_dataset_conn_lookups; undef $sql_dataset_conn_storage; undef $ldap_lookups; undef $ldap_connection; eval { $snmp_db->register_proc(0,0,undef) } if defined $snmp_db; # unregister undef $snmp_db; undef $db_env; } # implements Postfix TCP lookup server, see tcp_table(5) man page; experimental # sub process_tcp_lookup_request($$) { my($sock, $conn) = @_; local($/) = "\012"; # set line terminator to LF (regardless of platform) my($req_cnt); my($ln); for ($! = 0; defined($ln=$sock->getline); $! = 0) { $req_cnt++; my($level) = 0; local($1); my($resp_code, $resp_msg) = (400, 'INTERNAL ERROR'); if ($ln =~ /^get (.*?)\015?\012\z/si) { my($key) = proto_decode($1); my($sl); $sl = lookup2(0,$key, ca('spam_lovers_maps')); $resp_code = 200; $level = 2; $resp_msg = $sl ? "OK Recipient <$key> IS spam lover" : "DUNNO Recipient <$key> is NOT spam lover"; } elsif ($ln =~ /^put ([^ ]*) (.*?)\015?\012\z/si) { $resp_code = 500; $resp_msg = 'request not implemented: ' . $ln; } else { $resp_code = 500; $resp_msg = 'illegal request: ' . $ln; } do_log($level, "tcp_lookup(%s): %s %s", $req_cnt,$resp_code,$resp_msg); $sock->printf("%03d %s\012", $resp_code, tcp_lookup_encode($resp_msg)) or die "Can't write to tcp_lookup socket: $!"; } defined $ln || $!==0 or die "Error reading from socket: $!"; do_log(0, "tcp_lookup: RUNDOWN after %d requests", $req_cnt); } sub tcp_lookup_encode($) { my($str) = @_; local($1); $str =~ s/([^\041-\044\046-\176])/sprintf("%%%02x",ord($1))/egs; $str; } sub check_mail_begin_task() { # The check_mail_begin_task (and check_mail) may be called several times # per child lifetime and/or per-SMTP session. The variable $child_task_count # is mainly used by AV-scanner interfaces, e.g. to initialize when invoked # for the first time during child process lifetime $child_task_count++; do_log(4, "check_mail_begin_task: task_count=%d", $child_task_count); # comment out to retain SQL/LDAP cache entries for the whole child lifetime: $sql_wblist->clear_cache if defined $sql_wblist; $sql_lookups->clear_cache if defined $sql_lookups; $ldap_lookups->clear_cache if defined $ldap_lookups; # reset certain global variables for each task undef $av_output; @detecting_scanners = (); @virusname = (); @bad_headers = (); $banned_filename_any = $banned_filename_all = 0; undef $MSGINFO; # just in case } # create a mail_id unique to a database and save preliminary info to SQL; # if SQL is not enabled, just call a plain generate_mail_id() once # sub generate_unique_mail_id($) { my($msginfo) = @_; for (my($attempt)=5;;) { # sanity limit on retries my($mail_id,$secret_id) = generate_mail_id(); $msginfo->secret_id($secret_id); $secret_id = 'X' x length($secret_id); # can't hurt to be conservative $msginfo->mail_id($mail_id); # assign a long-term unique id to the msg if (!$sql_storage) { last; # no need to store and no way to check for uniqueness } else { # attempt to save a message placeholder to SQL, ensuring it is unique $sql_storage->save_info_preliminary($msginfo) and last; if (--$attempt <= 0) { do_log(-2,"ERROR sql_storage: too many retries ". "on storing preliminary, info not saved"); last; } else { snmp_count('GenMailIdRetries'); do_log(2,"sql_storage: retrying preliminary, %d attempts remain", $attempt); sleep(int(1+rand(3))); add_entropy(Time::HiRes::gettimeofday, $attempt); } } }; } # Collects some information derived from the envelope and the message, # do some common lookups, storing the information into a $msginfo object # to make commonly used information quickly and readily avaliable to the # rest of the program, e.g. avoiding a need for repeated lookups or parsing # of the same attribute # sub collect_some_info($) { my($msginfo) = @_; my($partition_tag) = c('partition_tag'); $partition_tag = &$partition_tag($msginfo) if ref $partition_tag eq 'CODE'; $partition_tag = 0 if !defined $partition_tag; $msginfo->partition_tag($partition_tag); my($sender) = $msginfo->sender; $msginfo->sender_source($sender); # obtain RFC 5322 From and Sender from the mail header section, parsed/clean my($rfc2822_sender) = $msginfo->get_header_field_body('sender'); my($rfc2822_from_field) = $msginfo->get_header_field_body('from'); my(@rfc2822_from); # RFC 5322 (ex RFC 2822) allows multiple author's addr if (defined $rfc2822_sender) { my(@sender_parsed) = map(unquote_rfc2821_local($_), parse_address_list($rfc2822_sender)); $rfc2822_sender = !@sender_parsed ? '' : $sender_parsed[0]; # none or one $msginfo->rfc2822_sender($rfc2822_sender); } if (defined $rfc2822_from_field) { @rfc2822_from = map(unquote_rfc2821_local($_), parse_address_list($rfc2822_from_field)); # rfc2822_from is a ref to a list when there are multiple author addresses! $msginfo->rfc2822_from(@rfc2822_from < 1 ? undef : @rfc2822_from < 2 ? $rfc2822_from[0] : \@rfc2822_from); } if (defined $msginfo->get_header_field('to')) { my($rfc2822_to) = $msginfo->get_header_field_body('to'); my(@to_parsed) = map(unquote_rfc2821_local($_), parse_address_list($rfc2822_to)); $msginfo->rfc2822_to(@to_parsed<2 ? $to_parsed[0] : \@to_parsed); } if (defined $msginfo->get_header_field('cc')) { my($rfc2822_cc) = $msginfo->get_header_field_body('cc'); my(@cc_parsed) = map(unquote_rfc2821_local($_), parse_address_list($rfc2822_cc)); $msginfo->rfc2822_cc(@cc_parsed<2 ? $cc_parsed[0] : \@cc_parsed); } my(@rfc2822_resent_from, @rfc2822_resent_sender); if (defined $msginfo->get_header_field('resent-from') || defined $msginfo->get_header_field('resent-sender')) { # triage # Each Resent block should have exactly one Resent-From, and none or one # Resent-Sender address. A HACK: undef in each list is used to separate # addresses obtained from different resent blocks, for the benefit of # those interested in traversing them block by block (e.g. when choosing # a DKIM signing key). The RFC 5322 section 3.6.6 says: All of the resent # fields corresponding to a particular resending of the message SHOULD be # grouped together. my(@r_from, @r_sender); local($1); for (my $j = 0; ; $j++) { # traverse header section by fields, top-down my($f_i,$f_n,$f) = $msginfo->get_header_field(undef,$j); if ( @r_from && ( !defined($f) || # end of a header section $f !~ /^Resent-/si || # presumably end of a resent block $f =~ /^Resent-From\s*:/si || # another Resent-From encountered $f =~ /^Resent-Sender\s*:/si && @r_sender # another Resent-Sender ) ) { # ends of a current resent block # a hack: undef in a list is used to separate addresses # from different resent blocks push(@rfc2822_resent_from, undef, @r_from); @r_from = (); push(@rfc2822_resent_sender, undef, @r_sender); @r_sender = (); } last if !defined $f; if ($f =~ /^Resent-From\s*:(.*)\z/si) { push(@r_from, map(unquote_rfc2821_local($_), parse_address_list($1))); } elsif ($f =~ /^Resent-Sender\s*:(.*)\z/si) { # multiple Resent-Sender in a block are illegal, store them all anyway push(@r_sender,map(unquote_rfc2821_local($_), parse_address_list($1))); } } if (@r_from || @r_sender) { # any leftovers not forming a resent block? push(@rfc2822_resent_from, undef, @r_from); push(@rfc2822_resent_sender, undef, @r_sender); } shift(@rfc2822_resent_from) if @rfc2822_resent_from; # remove undef shift(@rfc2822_resent_sender) if @rfc2822_resent_sender; # remove undef # rfc2822_resent_from and rfc2822_resent_sender are listrefs (or undef) $msginfo->rfc2822_resent_from(\@rfc2822_resent_from) if @rfc2822_resent_from; $msginfo->rfc2822_resent_sender(\@rfc2822_resent_sender) if @rfc2822_resent_sender; } my($mail_size) = $msginfo->msg_size; # use corrected ESMTP size if avail. if (!defined($mail_size) || $mail_size <= 0) { # not yet known? $mail_size = $msginfo->orig_header_size + $msginfo->orig_body_size; $msginfo->msg_size($mail_size); # store back do_log(4,"mail size unknown, size set to %d", $mail_size); } # check for mailing lists, bulk mail and auto-responses my($is_mlist); # mail from a mailing list my($is_auto); # bounce, auto-response, challenge-reesponse, ... my($is_bulk); # bulk mail or $is_mlist or $is_auto if (defined $msginfo->get_header_field('list-id')) { # RFC 2919 $is_mlist = $msginfo->get_header_field_body('list-id'); } elsif (defined $msginfo->get_header_field('list-post')) { $is_mlist = $msginfo->get_header_field_body('list-post'); } elsif (defined $msginfo->get_header_field('list-unsubscribe')) { $is_mlist = $msginfo->get_header_field_body('list-unsubscribe'); } elsif (defined $msginfo->get_header_field('mailing-list')) { $is_mlist = $msginfo->get_header_field_body('mailing-list'); # non-std. } elsif ($sender =~ /^ (?: [^\@]+ -(?:request|bounces|owner|admin) | owner- [^\@]+ ) (?: \@ | \z )/xsi) { $is_mlist = 'sender=' . $sender; } elsif ($rfc2822_from[0] =~ /^ (?: [^\@]+ -(?:request|bounces|owner) | owner- [^\@]+ ) (?: \@ | \z )/xsi) { $is_mlist = 'From:' . $rfc2822_from[0]; } if (defined $is_mlist) { # sanitize a bit local($1); $is_mlist = $1 if $is_mlist =~ / < (.*) > [^>]* \z/xs; $is_mlist =~ s/\s+/ /g; $is_mlist =~ s/^ //; $is_mlist =~ s/ \z//; $is_mlist =~ s/^mailto://i; $is_mlist = 'ml:' . $is_mlist; } if (defined $msginfo->get_header_field('precedence')) { my($prec) = $msginfo->get_header_field_body('precedence'); $prec =~ s/^[ \t]+//; local($1); $is_mlist = $1 if !defined($is_mlist) && $prec =~ /^(list)/si; $is_auto = $1 if $prec =~ /^(auto.?reply)\b/si; $is_bulk = $1 if $prec =~ /^(bulk|junk)\b/si; } if (defined $is_auto) { # already set } elsif (defined $msginfo->get_header_field('auto-submitted')) { my($auto) = $msginfo->get_header_field_body('auto-submitted'); $auto =~ s/ \( [^)]* \) //gx; $auto =~ s/^[ \t]+//; $auto =~ s/[ \t]+\z//; $is_auto = 'Auto-Submitted:' . $auto if lc($auto) ne 'no'; } elsif ($sender eq '') { $is_auto = 'sender=<>'; } elsif ($sender =~ /^ (?: mailer-daemon|double-bounce|mailer|autoreply ) (?: \@ | \z )/xsi) { # 'postmaster' is also common, but a bit risky $is_auto = 'sender=' . $sender; } elsif ($rfc2822_from[0] =~ # just checks the first author, good enough /^ (?: mailer-daemon|double-bounce|mailer|autoreply ) (?: \@ | \z )/xsi) { $is_auto = 'From:' . $rfc2822_from[0]; } if (defined $is_mlist) { $is_bulk = $is_mlist; } elsif (defined $is_auto) { $is_bulk = $is_auto; } elsif (defined $is_bulk) { # already set } elsif ($rfc2822_from[0] =~ # just checks the first author, good enough /^ (?: [^\@]+ -relay | postmaster | uucp ) (?: \@ | \z )/xsi) { $is_bulk = 'From:' . $rfc2822_from[0]; } $is_mlist = 1 if defined $is_mlist && !$is_mlist; # make sure it is true $is_auto = 1 if defined $is_auto && !$is_auto; # make sure it is true $is_bulk = 1 if defined $is_bulk && !$is_bulk; # make sure it is true $msginfo->is_mlist($is_mlist) if defined $is_mlist; $msginfo->is_auto($is_auto) if defined $is_auto; $msginfo->is_bulk($is_bulk) if defined $is_bulk; # now that we have a parsed From, check if we have a valid # author domain signature and do other DKIM pre-processing if (c('enable_dkim_verification')) { Amavis::DKIM::collect_some_dkim_info($msginfo); } if ($sender ne '') { # provide some initial default for sender_credible my(@cred) = ( $msginfo->originating ? 'orig' : (), $msginfo->dkim_envsender_sig ? 'dkim' : () ); $msginfo->sender_credible(join(',',@cred)) if @cred; } } # Checks the message stored on a file. File must already # be open on file handle $msginfo->mail_text; it need not be positioned # properly, check_mail must not close the file handle. # sub check_mail($$) { my($msginfo, $dsn_per_recip_capable) = @_; my($which_section) = 'check_init'; my(%elapsed,$t0_sect); $elapsed{'TimeElapsedReceiving'} = Time::HiRes::time - $msginfo->rx_time; my($point_of_no_return) = 0; # past the point where mail or DSN was sent my($am_id) = $msginfo->log_id; my($conn) = $msginfo->conn_obj; if (!defined($am_id)) { $am_id = am_id(); $msginfo->log_id($am_id) } $snmp_db->register_proc(1,0,'=',$am_id) if defined $snmp_db; # check begins my($smtp_resp, $exit_code, $preserve_evidence); my($mail_id, $custom_object); my($hold); # set to some string causes the message to be placed on hold # (frozen) by MTA. This can be used in cases when we stumble # across some permanent problem making us unable to decide # if the message is to be really delivered. # is any mail component password protected or otherwise non-decodable? my($any_undecipherable) = 0; my($mime_err); # undef, or MIME parsing error string as given by MIME::Parser if (defined $last_task_completed_at) { my($dt) = $msginfo->rx_time - $last_task_completed_at; do_log(3,"smtp connection cache, dt: %.1f, state: %d", $dt, $smtp_connection_cache_enable); if (!$smtp_connection_cache_on_demand) {} elsif (!$smtp_connection_cache_enable && $dt < 5) { do_log(3,"smtp connection cache, dt: %.1f -> enabling", $dt); $smtp_connection_cache_enable = 1; } elsif ($smtp_connection_cache_enable && $dt >= 15) { do_log(3,"smtp connection cache, dt: %.1f -> disabling", $dt); $smtp_connection_cache_enable = 0; } } # ugly - save in a global to make it accessible to %builtins $MSGINFO = $msginfo; eval { $msginfo->checks_performed({}) if !$msginfo->checks_performed; $msginfo->add_contents_category(CC_CLEAN,0); # CC_CLEAN is always present $_->add_contents_category(CC_CLEAN,0) for @{$msginfo->per_recip_data}; $msginfo->header_edits(Amavis::Out::EditHeader->new); add_entropy(Time::HiRes::gettimeofday, $child_task_count, $am_id, $msginfo->queue_id, $msginfo->mail_text_fn, $msginfo->sender); section_time($which_section); $which_section = 'check_init2'; { my($cwd) = $msginfo->mail_tempdir; if (!defined $cwd || $cwd eq '') { $cwd = $TEMPBASE } chdir($cwd) or die "Can't chdir to $cwd: $!"; } # compute body digest, measure mail size, check for 8-bit data, add entropy get_body_digest($msginfo,'MD5'); # or 'SHA-1' $which_section = 'check_init3'; collect_some_info($msginfo); my($mail_size) = $msginfo->msg_size; # use corrected ESMTP size if (!defined($msginfo->client_addr)) { # fetch missing address from header my($cl_ip) = parse_ip_address_from_received($msginfo,1); if (defined $cl_ip && $cl_ip ne '') { do_log(3,"client IP address unknown, fetched from Received: %s", $cl_ip); $cl_ip =~ s/^IPv6://i; $msginfo->client_addr(lc $cl_ip); } } $which_section = 'check_init4'; my($file_generator_object) = # maxfiles 0 disables the $MAXFILES limit Amavis::Unpackers::NewFilename->new($MAXFILES?$MAXFILES:undef,$mail_size); Amavis::Unpackers::Part::init($file_generator_object); # fudge: keep in var my($parts_root) = Amavis::Unpackers::Part->new; $msginfo->parts_root($parts_root); # section_time($which_section); if (!defined $mail_id && ($sql_store_info_for_all_msgs || !$sql_storage)) { $which_section = 'gen_mail_id'; $snmp_db->register_proc(2,0,'G',$am_id) if defined $snmp_db; # create a mail_id unique to a database and save preliminary info to SQL generate_unique_mail_id($msginfo); $mail_id = $msginfo->mail_id; section_time($which_section) if $sql_storage; } $which_section = "custom-new"; eval { my($old_orig) = c('originating'); # may load policy banks $custom_object = Amavis::Custom->new($conn,$msginfo); my($new_orig) = c('originating'); # may have changed by a p.b.load $msginfo->originating($new_orig) if ($old_orig?1:0) != ($new_orig?1:0); update_current_log_level(); 1; } or do { undef $custom_object; my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"custom new err: %s", $eval_stat); }; if (ref $custom_object) { do_log(5,"Custom hooks enabled"); section_time($which_section); } my($cl_ip) = $msginfo->client_addr; my($os_fingerprint_obj,$os_fingerprint); my($os_fingerprint_method) = c('os_fingerprint_method'); if (!defined($os_fingerprint_method) || $os_fingerprint_method eq '') { # no fingerprinting service configured } elsif ($cl_ip eq '' || $cl_ip eq '0.0.0.0' || $cl_ip eq '::') { # original client IP address not available, can't query p0f } else { # launch a query $which_section = "os_fingerprint"; my($dst) = c('os_fingerprint_dst_ip_and_port'); my($dst_ip,$dst_port); local($1,$2,$3); ($dst_ip,$dst_port) = ($1.$2, $3) if defined($dst) && $dst =~ m{^(?: \[ ([^\]]*) \] | ([^:]*) ) : ([^:]*) }six; $os_fingerprint_obj = Amavis::OS_Fingerprint->new( dynamic_destination($os_fingerprint_method,$conn), 0.050, $cl_ip, $msginfo->client_port, $dst_ip, $dst_port, $mail_id); } my($sender) = $msginfo->sender; my(@recips) = map($_->recip_addr, @{$msginfo->per_recip_data}); my($rfc2822_sender) = $msginfo->rfc2822_sender; my($fm) = $msginfo->rfc2822_from; my(@rfc2822_from) = !defined($fm) ? () : ref $fm ? @$fm : $fm; $mail_size = $msginfo->msg_size; # refresh after custom hook, just in case add_entropy("$cl_ip $mail_size $sender", \@recips); if (ll(1)) { my($pbn) = c('policy_bank_path'); ll(1) && do_log(1,"Checking: %s %s%s%s -> %s", $mail_id||'', $pbn eq '' ? '' : "$pbn ", $cl_ip eq '' ? '' : "[$cl_ip] ", qquote_rfc2821_local($sender), join(',', qquote_rfc2821_local(@recips)) ); } if (ll(3)) { my($envsender) = qquote_rfc2821_local($sender); my($hdrsender) = qquote_rfc2821_local($rfc2822_sender), my($hdrfrom) = qquote_rfc2821_local(@rfc2822_from); do_log(3,"2822.From: %s%s%s", $hdrfrom, !defined($rfc2822_sender) ? '' : ", 2822.Sender: $hdrsender", defined $rfc2822_sender && $envsender eq $hdrsender ? '' : $envsender eq $hdrfrom ? '' : ", 2821.Mail_From: $envsender"); } my($cnt_local) = 0; my($cnt_remote) = 0; for my $r (@{$msginfo->per_recip_data}) { my($recip) = $r->recip_addr; my($is_local) = lookup2(0,$recip, ca('local_domains_maps')); $is_local ? $cnt_local++ : $cnt_remote++; $r->recip_is_local($is_local ? 1 : 0); # canonical boolean, untainted if (!defined($r->bypass_virus_checks)) { my($bypassed_v) = lookup2(0,$recip, ca('bypass_virus_checks_maps')); $r->bypass_virus_checks($bypassed_v); } if (!defined($r->bypass_banned_checks)) { my($bypassed_b) = lookup2(0,$recip, ca('bypass_banned_checks_maps')); $r->bypass_banned_checks($bypassed_b); } if (!defined($r->bypass_spam_checks)) { my($bypassed_s) = lookup2(0,$recip, ca('bypass_spam_checks_maps')); $r->bypass_spam_checks($bypassed_s); } if (defined $user_id_sql) { my($user_id_ref,$mk_ref) = # list of all id's that match lookup2(1, $recip, [$user_id_sql], Label=>"users.id"); $r->user_id($user_id_ref) if ref $user_id_ref; # listref or undef } if (defined $user_policy_id_sql) { my($user_policy_id) = lookup2(0, $recip, [$user_policy_id_sql], Label=>"users.policy_id"); $r->user_policy_id($user_policy_id); # just the first match } } # update message count and mesage size snmp counters # orig local # 0 0 InMsgsOpenRelay # 0 1 InMsgsInbound # 0 x (non-originating: inbound or open relay) # 1 0 InMsgsOutbound # 1 1 InMsgsInternal # 1 x InMsgsOriginating (outbound or internal) # x 0 (departing: outbound or open relay) # x 1 (local: inbound or internal) # x x InMsgs snmp_count('InMsgs'); snmp_count('InMsgsBounceNullRPath') if $sender eq ''; snmp_count( ['InMsgsRecips', $cnt_local+$cnt_remote]); # recipients count snmp_count( ['InMsgsSize', $mail_size, 'C64'] ); if ($msginfo->originating) { snmp_count('InMsgsOriginating'); snmp_count( ['InMsgsRecipsOriginating', $cnt_local+$cnt_remote]); snmp_count( ['InMsgsSizeOriginating', $mail_size, 'C64'] ); } if ($cnt_local > 0) { my($d) = $msginfo->originating ? 'Internal' : 'Inbound'; snmp_count('InMsgs'.$d); snmp_count( ['InMsgsRecips'.$d, $cnt_local]); snmp_count( ['InMsgsRecipsLocal', $cnt_local]); snmp_count( ['InMsgsSize'.$d, $mail_size, 'C64'] ); } if ($cnt_remote > 0) { my($d) = $msginfo->originating ? 'Outbound' : 'OpenRelay'; snmp_count('InMsgs'.$d); snmp_count( ['InMsgsRecips'.$d, $cnt_remote]); snmp_count( ['InMsgsSize'.$d, $mail_size, 'C64'] ); if (!$msginfo->originating) { do_log(1,'Open relay? Nonlocal recips but not originating: %s', join(', ', map($_->recip_addr, grep(!$_->recip_is_local, @{$msginfo->per_recip_data})))); } } # mkdir can be a costly operation (must be atomic, flushes buffers). # If we can re-use directory 'parts' from the previous invocation it saves # us precious time. Together with matching rmdir this can amount to 10-15 % # of total elapsed time on some classical file systems! (no spam checking) $which_section = "creating_partsdir"; { my($tempdir) = $msginfo->mail_tempdir; my($errn) = lstat("$tempdir/parts") ? 0 : 0+$!; if ($errn == ENOENT) { # needs to be created mkdir("$tempdir/parts", 0750) or die "Can't create directory $tempdir/parts: $!"; section_time('mkdir parts'); } elsif ($errn != 0) { die "$tempdir/parts is not accessible: $!" } elsif (!-d _) { die "$tempdir/parts is not a directory" } else {} # fine, directory already exists and is accessible } # FIRST: what kind of e-mail did we get? call content scanners my($virus_presence_checked,$spam_presence_checked); my($virus_dejavu) = 0; my($will_do_virus_scanning, $all_bypass_virus_checks); if ($extra_code_antivirus) { $all_bypass_virus_checks = !grep(!$_->bypass_virus_checks, @{$msginfo->per_recip_data}); $will_do_virus_scanning = !$virus_presence_checked && !$all_bypass_virus_checks; } my($will_do_banned_checking) = # banned name checking will be needed? @{ca('banned_filename_maps')} || cr('banned_namepath_re'); my($bounce_header_fields_ref,$bounce_msgid,$bounce_type); if (c('bypass_decode_parts')) { do_log(5, "decoding bypassed"); } elsif (!$will_do_virus_scanning && !$will_do_banned_checking && c('bounce_killer_score') <= 0) { do_log(5, "decoding not needed"); } else { # decoding parts can take a lot of time $which_section = "mime_decode-1"; $snmp_db->register_proc(2,0,'D',$am_id) if defined $snmp_db; # decoding $t0_sect = Time::HiRes::time; $mime_err = ensure_mime_entity($msginfo) if !defined($msginfo->mime_entity); prolong_timer($which_section); if (c('bounce_killer_score') > 0) { $which_section = "dsn_parse"; # analyze a bounce after MIME decoding but before further archive # decoding (which often replaces original MIME parts by decoded files) eval { # just in case ($bounce_header_fields_ref,$bounce_type) = inspect_a_bounce_message($msginfo); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1, "inspect_a_bounce_message failed: %s", $eval_stat); }; if ($bounce_header_fields_ref && exists $bounce_header_fields_ref->{'message-id'}) { $bounce_msgid = $bounce_header_fields_ref->{'message-id'}; } prolong_timer($which_section); } $which_section = "parts_decode_ext"; snmp_count('OpsDec'); ($hold,$any_undecipherable) = Amavis::Unpackers::decompose_mail($msginfo->mail_tempdir, $file_generator_object); if ($hold ne '' || $any_undecipherable) { $msginfo->add_contents_category(CC_UNCHECKED,0); for my $r (@{$msginfo->per_recip_data}) { $r->add_contents_category(CC_UNCHECKED,0) if !$r->bypass_virus_checks; } } $elapsed{'TimeElapsedDecoding'} = Time::HiRes::time - $t0_sect; } my($bphcm) = ca('bypass_header_checks_maps'); if (grep(!lookup2(0,$_->recip_addr,$bphcm), @{$msginfo->per_recip_data})) { $which_section = "check_header"; my($allowed_tests) = cr('allowed_header_tests'); my($allowed_mime_test) = $allowed_tests && $allowed_tests->{'mime'}; # check for bad headers and for bad MIME subheaders / bad MIME structure if ($allowed_mime_test && defined $mime_err && $mime_err ne '') { push(@bad_headers, "MIME error: ".$mime_err); $msginfo->add_contents_category(CC_BADH,1); } my($badh_ref,$minor_badh_cc) = check_header_validity($msginfo); $msginfo->checks_performed->{H} = 1; if (@$badh_ref) { push(@bad_headers, @$badh_ref); $msginfo->add_contents_category(CC_BADH,$minor_badh_cc); } for my $r (@{$msginfo->per_recip_data}) { my($bypassed) = lookup2(0,$r->recip_addr,$bphcm); if (!$bypassed && $allowed_mime_test && defined $mime_err && $mime_err ne '') { $r->add_contents_category(CC_BADH,1) } # CC_BADH min: 1=broken mime if (!$bypassed && @$badh_ref) { $r->add_contents_category(CC_BADH,$minor_badh_cc) } } section_time($which_section); } if ($will_do_banned_checking) { # check for banned file contents $which_section = "check-banned"; check_for_banned_names($msginfo); # saves results in $msginfo $msginfo->checks_performed->{B} = 1; $banned_filename_any = 0; $banned_filename_all = 1; for my $r (@{$msginfo->per_recip_data}) { next if $r->bypass_banned_checks; my($a) = $r->banned_parts; if (!defined $a || !@$a) { $banned_filename_all = 0; } else { my($rhs) = $r->banning_rule_rhs; if (defined $rhs) { for my $j (0..$#{$a}) { $r->dsn_suppress_reason(sprintf("BANNED:%s suggested by rule", $rhs->[$j])) if $rhs->[$j] =~ /^DISCARD/; } } $banned_filename_any = 1; $r->add_contents_category(CC_BANNED,0); } } $msginfo->add_contents_category(CC_BANNED,0) if $banned_filename_any; ll(4) && do_log(4,"banned check: any=%d, all=%s (%d)", $banned_filename_any, $banned_filename_all?'Y':'N', scalar(@{$msginfo->per_recip_data})); } my($virus_checking_failed) = 0; if (!$extra_code_antivirus) { do_log(5, "no anti-virus code loaded, skipping virus_scan"); } elsif ($all_bypass_virus_checks) { do_log(5, "bypassing of virus checks requested"); } elsif (defined $hold && $hold ne '') { # protect virus scanner from bombs do_log(0, "NOTICE: Virus scanning skipped: %s", $hold); $will_do_virus_scanning = 0; } else { if (!$will_do_virus_scanning) { do_log(-1, "NOTICE: will_do_virus_scanning is false???") } $mime_err = ensure_mime_entity($msginfo) if !defined($msginfo->mime_entity) && !c('bypass_decode_parts'); # special case to make available a complete mail file for inspection if ((defined($mime_err) && $mime_err ne '') || !defined($msginfo->mime_entity) || lookup2(0,'MAIL',\@keep_decoded_original_maps) || $any_undecipherable && lookup2(0,'MAIL-UNDECIPHERABLE', \@keep_decoded_original_maps)) { # keep the email.txt by making a hard link to it in ./parts/ $which_section = "linking-to-MAIL"; my($tempdir) = $msginfo->mail_tempdir; my($newpart_obj) = Amavis::Unpackers::Part->new("$tempdir/parts",$parts_root,1); my($newpart) = $newpart_obj->full_name; do_log(3, "presenting full original message to scanners as %s%s%s", $newpart, !$any_undecipherable ? '' :", $any_undecipherable undecipherable", $mime_err eq '' ? '' : ", MIME error: $mime_err"); link($msginfo->mail_text_fn, $newpart) or die sprintf("Can't create hard link %s to %s: %s", $newpart, $msginfo->mail_text_fn, $!); $newpart_obj->type_short('MAIL'); # case sensitive $newpart_obj->type_declared('message/rfc822'); } $which_section = "virus_scan"; $snmp_db->register_proc(2,0,'V',$am_id) if defined $snmp_db; # virus scan my($av_ret); $t0_sect = Time::HiRes::time; $virus_checking_failed = 1; eval { my($vn, $ds); ($av_ret, $av_output, $vn, $ds) = Amavis::AV::virus_scan($msginfo, $child_task_count==1); @virusname = @$vn; @detecting_scanners = @$ds; # copy if (defined $av_ret) { $virus_presence_checked = 1; $virus_checking_failed = 0; $msginfo->checks_performed->{V} = 1; } 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-2, "AV: %s", $eval_stat); $virus_checking_failed = $eval_stat; $virus_checking_failed = 1 if !$virus_checking_failed; }; $elapsed{'TimeElapsedVirusCheck'} = Time::HiRes::time - $t0_sect; snmp_count('OpsVirusCheck'); if ($virus_presence_checked && @virusname && defined $snmp_db) { $which_section = "read_snmp_variables"; # true if none with counter zero or undef $virus_dejavu = 1 if !grep(!defined($_) || $_ == 0, @{$snmp_db->read_snmp_variables( map("virus.byname.$_", @virusname))}); section_time($which_section); } } if ($virus_checking_failed) { $msginfo->add_contents_category(CC_UNCHECKED,0); for my $r (@{$msginfo->per_recip_data}) { $r->add_contents_category(CC_UNCHECKED,0) if !$r->bypass_virus_checks; } if (c('virus_scanners_failure_is_fatal')) { $hold = 'AV: ' . $virus_checking_failed; die "$hold\n"; # TEMPFAIL } } $which_section = "post_virus_scan"; if (@virusname) { my($virus_suppress_reason); my($ccat_maj,$ccat_min) = (CC_VIRUS,0); my($vtfsm) = ca('viruses_that_fake_sender_maps'); if (@$vtfsm) { for my $vn (@virusname) { my($result,$matchingkey) = lookup2(0,$vn,$vtfsm); if ($result) { # is a virus known to fake a sender address do_log(3,"Virus %s matches %s, sender addr ignored", $vn,$matchingkey); # try to get some info on sender source from his IP address my($first_rcvd_from_ip) = parse_ip_address_from_received($msginfo); if (defined $first_rcvd_from_ip && $first_rcvd_from_ip ne '') { $msginfo->sender_source(sprintf('?@[%s]', $first_rcvd_from_ip)); } else { $msginfo->sender_source(undef); } $virus_suppress_reason = 'INFECTED'; # $ccat_min = 1; last; } } } $msginfo->add_contents_category($ccat_maj,$ccat_min); for my $r (@{$msginfo->per_recip_data}) { $r->add_contents_category( $ccat_maj,$ccat_min) if !$r->bypass_virus_checks; if (defined $virus_suppress_reason) { $r->dsn_suppress_reason($virus_suppress_reason . (!defined $_ ? '' : ", $_")) for $r->dsn_suppress_reason; } } $msginfo->virusnames([@virusname]); # copy names to object my($vntpbm) = ca('virus_name_to_policy_bank_maps'); if (@$vntpbm) { my(@bank_names, %bank_names); for my $vn (@virusname) { my($result,$matchingkey) = lookup2(0,$vn,$vntpbm); if ($result) { if ($result eq '1') { # a handy usability trick to supply a hardwired policy bank # name when acl-style lookup table is used, which can only # return a boolean (undef, 0, or 1) $result = 'VIRUS'; } # $result is a list of policy banks as a comma-separated string my(@pbn); # collect list of newly encountered policy bank names for (map { s/^[ \t]+//; s/[ \t]+\z//; $_ } split(/,/,$result)) { next if $_ eq '' || $bank_names{$_}; push(@pbn,$_); $bank_names{$_} = 1; } if (@pbn) { push(@bank_names,@pbn); ll(2) && do_log(2, "virus %s loads policy bank(s) %s, match: %s", $vn, join(',',@pbn), $matchingkey); } } } if (@bank_names) { # ignore nonexisting bank names @bank_names = grep(defined $policy_bank{$_}, unique_list(\@bank_names)); if (@bank_names) { load_policy_bank($_,$msginfo) for @bank_names; $msginfo->originating(c('originating')); # may have changed } } } } if (defined($os_fingerprint_obj)) { $which_section = "fingerprint_collect"; $os_fingerprint = $os_fingerprint_obj->collect_response; if (defined $os_fingerprint && $os_fingerprint ne '') { $msginfo->checks_performed->{F} = 1; if ($msginfo->originating) { $os_fingerprint = 'MYNETWORKS' } # blank-out our smtp clients info $msginfo->client_os_fingerprint($os_fingerprint); # store info } } my($bypass_spam_checks_by_bounce_killer); if (!$bounce_header_fields_ref) { # not a bounce } elsif ($msginfo->originating) { # will be rescued from bounce killing by the originating flag } elsif (defined($bounce_msgid) && $bounce_msgid =~ /(\@[^\@>() \t][^\@>]*?)[ \t]*>?\z/ && lookup2(0,$1, ca('local_domains_maps'))) { # will be rescued from bounce killing by a local domain # in referenced Message-ID } elsif (!defined($sql_storage) || !$sql_store_info_for_all_msgs || c('penpals_bonus_score') <= 0 || c('penpals_halflife') <= 0) { # will be rescued from bounce killing by pen pals disabled } elsif (c('bounce_killer_score') > 20) { # is a bounce and is eligible to bounce killing, no need for spam scan $bypass_spam_checks_by_bounce_killer = 1; } # consider doing spam scanning if (!$extra_code_antispam) { do_log(5, "no anti-spam code loaded, skipping spam_scan"); } elsif ($bypass_spam_checks_by_bounce_killer) { do_log(5, "bypassing of spam checks by a bounce killer"); } elsif (!grep(!$_->bypass_spam_checks, @{$msginfo->per_recip_data})) { do_log(5, "bypassing of spam checks requested for all recips"); } else { # preliminary test - would a message be allowed to pass for any recipient # based on evidence collected so far (virus, banned) my($any_pass) = 0; my($prelim_blocking_ccat); for my $r (@{$msginfo->per_recip_data}) { my($final_destiny) = D_PASS; my(@fd_tuples) = $r->setting_by_main_contents_category_all( cr('final_destiny_by_ccat'), cr('lovers_maps_by_ccat')); for my $tuple (@fd_tuples) { my($cc, $fd, $lovers_map_ref) = @$tuple; if (!defined($fd) || $fd == D_PASS) { } elsif (defined($lovers_map_ref) && lookup2(0, $r->recip_addr, $lovers_map_ref, Label=>'Lovers1')) { } else { $prelim_blocking_ccat = $cc; $final_destiny = $fd; last; } } $any_pass = 1 if $final_destiny == D_PASS; } if (!$any_pass) { do_log(5, "bypassing of spam checks, message will be blocked anyway ". "due to %s", $prelim_blocking_ccat); } else { $which_section = "spam-wb-list"; my($any_wbl, $all_wbl) = Amavis::SpamControl::white_black_list( $msginfo, $sql_wblist, $user_id_sql, $ldap_lookups); section_time($which_section); if ($all_wbl) { do_log(5, "sender white/blacklisted, skipping spam_scan"); } elsif (!$spamcontrol_obj) { do_log(5, "spam scanning disabled, no spamcontrol_obj"); } else { $which_section = "spam_scan"; $snmp_db->register_proc(2,0,'S',$am_id) if defined $snmp_db; $t0_sect = Time::HiRes::time; # sets $msginfo->spam_level, spam_status, # spam_report, spam_summary, supplementary_info $spamcontrol_obj->spam_scan($msginfo); $msginfo->checks_performed->{S} = 1; prolong_timer($which_section); $elapsed{'TimeElapsedSpamCheck'} = Time::HiRes::time - $t0_sect; snmp_count('OpsSpamCheck'); $spam_presence_checked = 1; } } } if (ref $custom_object) { $which_section = "custom-checks"; eval { $custom_object->checks($conn,$msginfo); update_current_log_level(); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"custom checks error: %s", $eval_stat); }; section_time($which_section); } snmp_count("virus.byname.$_") for @virusname; my(@sa_tests,%sa_tests); { my($tests) = $msginfo->supplementary_info('TESTS'); if (defined($tests) && $tests ne 'none') { @sa_tests = $tests =~ /([^=,;]+)(?==)/g; %sa_tests = map(($_,1), @sa_tests); } } # SECOND: now that we know what we got, decide what to do with it $which_section = 'after_scanning'; Amavis::DKIM::adjust_score_by_signer_reputation($msginfo) if $msginfo->dkim_signatures_valid; my($min_spam_level, $max_spam_level) = minmax(map($_->spam_level, @{$msginfo->per_recip_data})); $min_spam_level = 0 if !defined $min_spam_level; $max_spam_level = 0 if !defined $max_spam_level; $which_section = "penpals_check"; my($pp_age); if (!defined $sql_storage || !$sql_store_info_for_all_msgs) { # pen pals disabled - SQL data not available or incomplete } elsif ($msginfo->is_in_contents_category(CC_VIRUS)) { # pen pals disabled, not needed for infected messages } else { my($pp_bonus) = c('penpals_bonus_score'); # score points my($pp_halflife) = c('penpals_halflife'); # seconds if ($pp_bonus <= 0 || $pp_halflife <= 0) { # penpals disabled } elsif (defined($penpals_threshold_low) && !defined($bounce_msgid) && $max_spam_level < $penpals_threshold_low) { # low score for all recipients, no need for aid do_log(5,"penpals: low score, no need for penpals aid"); } elsif (defined($penpals_threshold_high) && !defined($bounce_msgid) && $min_spam_level - $pp_bonus > $penpals_threshold_high) { # spam, can't get below threshold_high even under best circumstances do_log(5,"penpals: high score, penpals won't help"); } elsif ($sender ne '' && !$msginfo->originating && lookup2(0,$sender, ca('local_domains_maps'))) { # no bonus to unauthent. senders from outside claiming a local domain do_log(5,"penpals: local sender from outside, ignored: %s", $sender); } else { $t0_sect = Time::HiRes::time; $snmp_db->register_proc(2,0,'P',$am_id) if defined $snmp_db; # penpals my($sid) = $msginfo->sender_maddr_id; for my $r (@{$msginfo->per_recip_data}) { next if $r->recip_done; # already dealt with my($recip) = $r->recip_addr; my($rid) = $r->recip_maddr_id; if (defined($rid) && $sid ne $rid && $r->recip_is_local) { # inbound or internal_to_internal, except self_to_self my($refs_str) = $msginfo->get_header_field_body('in-reply-to') . $msginfo->get_header_field_body('references'); my(@refs) = $refs_str eq '' ? () : parse_message_id($refs_str); push(@refs,$bounce_msgid) if defined $bounce_msgid && $bounce_msgid ne ''; do_log(4,"penpals: references: %s", join(", ",@refs)) if @refs; # NOTE: swap $rid and $sid as args here, as we are now checking # for a potential reply mail - whether the current recipient has # recently sent any mail to the sender of the current mail: my($pp_mail_id,$pp_subj); ($pp_age,$pp_mail_id,$pp_subj) = $sql_storage->penpals_find($rid,$sid,\@refs,$msginfo->rx_time); $msginfo->checks_performed->{P} = 1; if (defined $pp_age) { # found info about previous correspondence $r->recip_penpals_age($pp_age); # save the information my($weight) = exp(-($pp_age/$pp_halflife) * log(2)); # weight is a factor between 1 and 0, representing # exponential decay: weight(t) = 1 / 2^(t/halflife) # i.e. factors 1, 1/2, 1/4, 1/8... at age 0, hl, 2*hl, 3*hl... my($adj) = - $weight * $pp_bonus; $r->recip_penpals_score($adj); $r->spam_level( ($r->spam_level || 0) + $adj); { my($spam_tests) = 'AM.PENPAL=' . (0+sprintf("%.3f",$adj)); if (!defined($r->spam_tests)) { $r->spam_tests([ \$spam_tests ]); } else { unshift(@{$r->spam_tests}, \$spam_tests); } } if (ll(2)) { do_log(2,"penpals: bonus %.3f, age %s (%d), ". "SA score %.3f, <%s> replying to <%s>, ref mail_id: %s", -$adj, format_time_interval($pp_age), $pp_age, $r->spam_level, $sender, $recip, $pp_mail_id); my($this_subj) = $msginfo->get_header_field_body('subject'); $this_subj = $1 if $this_subj =~ /^\s*(.*?)\s*$/; do_log(2,"penpals: prev Subject: %s", $pp_subj); do_log(2,"penpals: this Subject: %s", $this_subj); } } } } section_time($which_section); $elapsed{'TimeElapsedPenPals'} = Time::HiRes::time - $t0_sect; } } $which_section = "bounce_killer"; if ($bounce_header_fields_ref) { # message looks like a DSN (= bounce) snmp_count('InMsgsBounce'); my($bounce_rescued); if (defined $pp_age && $pp_age < 8*24*3600) { # less than 8 days ago # found by pen pals by a Message-ID in attachment and recip. address; # is a bounce, refers to our previous outgoing message, treat it kindly snmp_count('InMsgsBounceRescuedByPenPals'); $bounce_rescued = 'by penpals'; } elsif ($msginfo->originating) { snmp_count('InMsgsBounceRescuedByOriginating'); $bounce_rescued = 'by originating'; } elsif (defined($bounce_msgid) && $bounce_msgid =~ /(\@[^\@>() \t][^\@>]*?)[ \t]*>?\z/ && lookup2(0,$1, ca('local_domains_maps'))) { # not in pen pals, but domain in Message-ID is a local domain; # it is only useful until spammers figure out the trick, # then it should be disabled snmp_count('InMsgsBounceRescuedByDomain'); $bounce_rescued = 'by domain'; } elsif (!defined($sql_storage) || c('penpals_bonus_score') <= 0 || c('penpals_halflife') <= 0) { $bounce_rescued = 'by: pen pals disabled'; } ll(2) && do_log(2, "bounce %s (%s), %s -> %s, %s", defined $bounce_rescued ?'rescued '.$bounce_rescued :'killed', $bounce_type, qquote_rfc2821_local($sender), join(',', qquote_rfc2821_local(@recips)), join(', ', map { $_ . ': ' . $bounce_header_fields_ref->{$_} } sort( grep(/^(?:From|Return-Path|Message-ID|Date)\z/i, keys %$bounce_header_fields_ref) )) ); if (!$bounce_rescued) { snmp_count('InMsgsBounceKilled'); my($bounce_killer_score) = c('bounce_killer_score'); for my $r (@{$msginfo->per_recip_data}) { $r->spam_level( ($r->spam_level || 0) + $bounce_killer_score); my($spam_tests) = 'AM.BOUNCE=' . $bounce_killer_score; if (!defined($r->spam_tests)) { $r->spam_tests([ \$spam_tests ]); } else { unshift(@{$r->spam_tests}, \$spam_tests); } } } # else: not a recognizable bounce } elsif ($msginfo->is_auto || $sender =~ /^postmaster(?:\@|\z)/si || $rfc2822_from[0] =~ /^postmaster(?:\@|\z)/si || $sa_tests{'ANY_BOUNCE_MESSAGE'} ) { # message could be some kind of a non-standard bounce or autoresponse, # but lacks recognizable structure and a header section from orig. mail ll(2) && do_log(2, "bounce unverifiable%s, %s -> %s", !$msginfo->originating ? '' : ', originating', qquote_rfc2821_local($sender), join(',', qquote_rfc2821_local(@recips))); snmp_count('InMsgsBounce'); snmp_count('InMsgsBounceUnverifiable'); } $which_section = "decide_mail_destiny"; $snmp_db->register_proc(2,0,'r',$am_id) if defined $snmp_db; # results... my($considered_oversize_by_some_recips); my($mslm) = ca('message_size_limit_maps'); for my $r (@{$msginfo->per_recip_data}) { next if $r->recip_done; # already dealt with my($recip) = $r->recip_addr; my($spam_level) = $r->spam_level; # consider adding CC_SPAM or CC_SPAMMY to the contents_category list; # spaminess is an individual matter, we must compare spam level # with each recipient setting, there is no single global criterium my($tag_level,$tag2_level,$tag3_level,$kill_level); my($bypassed) = $r->bypass_spam_checks; if (!$bypassed) { $tag_level = lookup2(0,$recip, ca('spam_tag_level_maps')); $tag2_level = lookup2(0,$recip, ca('spam_tag2_level_maps')); $tag3_level = lookup2(0,$recip, ca('spam_tag3_level_maps')); $kill_level = lookup2(0,$recip, ca('spam_kill_level_maps')); } my($blacklisted) = $r->recip_blacklisted_sender; my($whitelisted) = $r->recip_whitelisted_sender; # penpals_score is already accounted for in spam_level, # it is provided here separately for informational/logging purposes my($penpals_score) = $r->recip_penpals_score; # is zero or negative! my($do_tag) = !$bypassed && ( $blacklisted || !defined $tag_level || $tag_level eq '' || ($spam_level + ($whitelisted?-10:0) >= $tag_level)); my($do_tag2,$do_tag3,$do_kill) = map { !$bypassed && !$whitelisted && ($blacklisted || (defined($_) && $spam_level >= $_) ) } ($tag2_level,$tag3_level,$kill_level); $do_tag2 = $do_tag2 || $do_tag3; # tag3 implies tag2, just in case if ($do_tag) { # spaminess is at or above tag level $msginfo->add_contents_category(CC_CLEAN,1); $r->add_contents_category(CC_CLEAN,1) if !$bypassed; } if ($do_tag2) { # spaminess is at or above tag2 level $msginfo->add_contents_category(CC_SPAMMY); $r->add_contents_category(CC_SPAMMY) if !$bypassed; } if ($do_tag3) { # spaminess is at or above tag3 level $msginfo->add_contents_category(CC_SPAMMY,1); $r->add_contents_category(CC_SPAMMY,1) if !$bypassed; } if ($do_kill) { # spaminess is at or above kill level $msginfo->add_contents_category(CC_SPAM,0); $r->add_contents_category(CC_SPAM,0) if !$bypassed; } # consider adding CC_OVERSIZED to the contents_category list; if (@$mslm) { # checking of mail size is needed? my($size_limit) = lookup2(0,$r->recip_addr,$mslm); if ($enforce_smtpd_message_size_limit_64kb_min && $size_limit && $size_limit < 65536) { $size_limit = 65536 } # RFC 5321 requires at least 64k if ($size_limit && $mail_size > $size_limit) { do_log(1,"OVERSIZED from %s to %s: size %s B, limit %s B", $msginfo->sender_smtp, $r->recip_addr_smtp, $mail_size, $size_limit) if !$considered_oversize_by_some_recips; $considered_oversize_by_some_recips = 1; $r->add_contents_category(CC_OVERSIZED,0); $msginfo->add_contents_category(CC_OVERSIZED,0); } } # determine true reason for blocking,considering lovers and final_destiny my($blocking_ccat); my($final_destiny) = D_PASS; my($to_be_mangled); my(@fd_tuples) = $r->setting_by_main_contents_category_all( cr('final_destiny_by_ccat'), cr('lovers_maps_by_ccat'), cr('defang_maps_by_ccat') ); for my $tuple (@fd_tuples) { my($cc, $fd, $lovers_map_ref, $mangle_map_ref) = @$tuple; if (!defined($fd) || $fd == D_PASS) { do_log(5, "final_destiny (ccat=%s) is PASS, recip %s", $cc,$recip); } elsif (defined($lovers_map_ref) && lookup2(0,$recip,$lovers_map_ref, Label=>'Lovers2')) { do_log(5, "contents lover (ccat=%s) %s", $cc,$recip); } elsif ($fd == D_BOUNCE && ($sender eq '' || defined($msginfo->is_bulk)) && ccat_maj($cc) == CC_BADH) { # have mercy on bad header section in mail from mailing lists and # in DSN: since a bounce for such mail will be suppressed, it is # probably better to just let a mail with a bad header section pass, # it is rather innocent my($is_bulk) = $msginfo->is_bulk; do_log(1, "allow bad header section from %s<%s> -> <%s>: %s", !defined($is_bulk) ? '' : "($is_bulk) ", $sender, $recip, $bad_headers[0]); } else { $blocking_ccat = $cc; $final_destiny = $fd; my($cc_main) = $r->contents_category; $cc_main = $cc_main->[0] if $cc_main; if ($blocking_ccat eq $cc_main) { do_log(3, "blocking contents category is (%s) for %s", $blocking_ccat,$recip); } else { do_log(3, "blocking ccat (%s) differs from ccat_maj=%s, %s", $blocking_ccat,$cc_main,$recip); } last; # first blocking wins, also skips turning on mangling } # topmost mangling reason wins if (!defined($to_be_mangled) && defined($mangle_map_ref)) { my($mangle_type) = !ref($mangle_map_ref) ? $mangle_map_ref # compatibility : lookup2(0,$recip,$mangle_map_ref, Label=>'Mangling1'); $to_be_mangled = $mangle_type if $mangle_type ne ''; } } $r->recip_destiny($final_destiny); if (defined $blocking_ccat) { # save a blocking contents category $r->blocking_ccat($blocking_ccat); # summarize per-recipient blocking_ccat to a message level my($msg_bl_ccat) = $msginfo->blocking_ccat; if (!defined($msg_bl_ccat) || cmp_ccat($blocking_ccat,$msg_bl_ccat)>0) { $msginfo->blocking_ccat($blocking_ccat) } } else { # defanging/mangling only has effect on passed mail # defang_all serves mostly for testing purposes and compatibility $to_be_mangled = 1 if !$to_be_mangled && c('defang_all'); if ($to_be_mangled) { my($orig_to_be_mangled) = $to_be_mangled; if ($to_be_mangled =~ /^(?:disclaimer|nulldisclaimer)\z/i) { # disclaimers can only go to mail originating from internal # networks - the 'allow_disclaimers' should (only) be enabled # by an appropriate policy bank, e.g. MYNETS and/or ORIGINATING if (!c('allow_disclaimers')) { $to_be_mangled = 0; # not for remote or unauthorized clients do_log(5,"will not add disclaimer, allow_disclaimers is false"); } else { my($rf) = $msginfo->rfc2822_resent_from; my($rs) = $msginfo->rfc2822_resent_sender; # disclaimers should only go to mail with 2822.From or # 2822.Sender or 2822.Resent-From or 2822.Resent-Sender # or 2821.mail_from address matching local domains if (!grep(defined($_) && $_ ne '' && lookup2(0,$_, ca('local_domains_maps')), unique_list( (!$rf ? () : @$rf), (!$rs ? () : @$rs), @rfc2822_from, $rfc2822_sender, $sender))) { $to_be_mangled = 0; # not for foreign 'Sender:' or 'From:' do_log(5,"will not add disclaimer, sender not local"); } } } else { # defanging (not disclaiming) # defanging and other mail mangling/munging only applies to # incoming mail, i.e. for recipients matching local_domains_maps $to_be_mangled = 0 if !$r->recip_is_local; } # store a boolean or a mangling name (defang, disclaimer, ...) $r->mail_body_mangle($to_be_mangled) if $to_be_mangled; ll(2) && do_log(2, "mangling %s: %s (was: %s), ". "discl_allowed=%d, <%s> -> <%s>", $to_be_mangled ? 'YES' : 'NO', $to_be_mangled, $orig_to_be_mangled, c('allow_disclaimers'), $sender, $recip); } } if ($penpals_score < 0) { # only for logging and statistics purposes my($do_tag2_nopp,$do_tag3_nopp,$do_kill_nopp) = map { !$whitelisted && ($blacklisted || (defined($_) && $spam_level-$penpals_score >= $_) ) } ($tag2_level,$tag3_level,$kill_level); $do_tag2_nopp = $do_tag2_nopp || $do_tag3_nopp; my($which) = $do_kill_nopp && !$do_kill ? 'kill' : $do_tag3_nopp && !$do_tag3 ? 'tag3' : $do_tag2_nopp && !$do_tag2 ? 'tag2' : ''; if ($which ne '') { snmp_count("PenPalsSavedFrom\u$which") if $final_destiny==D_PASS; do_log(2, "PenPalsSavedFrom%s %.3f%.3f%s, <%s> -> <%s>", "\u$which", $spam_level-$penpals_score, $penpals_score, ($final_destiny==D_PASS ? '' : ', but mail still blocked'), $sender, $recip); } } if ($final_destiny == D_PASS) { # recipient wants this message, malicious or not do_log(5, "final_destiny PASS, recip %s", $recip); } else { # recipient does not want this content do_log(5, "final_destiny %s, recip %s", $final_destiny, $recip); # supply RFC 3463 enhanced status codes my($status) = setting_by_given_contents_category( $blocking_ccat, { CC_VIRUS, "554 5.7.0", CC_BANNED, "554 5.7.0", CC_UNCHECKED, "554 5.7.0", CC_SPAM, "554 5.7.0", CC_SPAMMY, "554 5.7.0", CC_BADH.",2", "554 5.6.3", # nonencoded 8-bit character CC_BADH, "554 5.6.0", CC_OVERSIZED, "552 5.3.4", CC_MTA, "550 5.3.5", CC_CATCHALL, "554 5.7.0", }); my($statoverride,$softfailed); $softfailed = ''; if ($status =~ /^[24]/) { # just in case # keep unchanged } elsif ($final_destiny == D_TEMPFAIL) { $statoverride = '450'; # 5xx -> 450 } elsif (c('soft_bounce')) { $statoverride = '450'; # 5xx -> 450 $softfailed = ' (soft_bounce)'; ll(5) && do_log(5, "soft_bounce: %s %s -> %s", $final_destiny == D_DISCARD ? 'discard' : 'bounce', $status, $statoverride); } elsif ($final_destiny == D_DISCARD) { $statoverride = '250'; # 5xx -> 250 } if (defined $statoverride) { my $code = substr($statoverride,0,1); local($1,$2); $status =~ s{^\d(\d\d) \d(\.\d\.\d)}{$statoverride $code$2}; } # get the custom smtp response reason text my($smtp_reason) = setting_by_given_contents_category( $blocking_ccat, cr('smtp_reason_by_ccat')); $smtp_reason = '' if !defined $smtp_reason; if ($smtp_reason ne '') { my(%mybuiltins) = %builtins; # make a local copy $smtp_reason = expand(\$smtp_reason, \%mybuiltins); $smtp_reason = !ref($smtp_reason) ? '' : $$smtp_reason; chomp($smtp_reason); $smtp_reason = sanitize_str($smtp_reason,1); $smtp_reason = substr($smtp_reason,0,100) . "..." if length($smtp_reason) > 100+3; } my($response) = sprintf("%s %s%s%s", $status, ($final_destiny == D_PASS ? "Ok" : $final_destiny == D_DISCARD ? "Ok, discarded" : $final_destiny == D_REJECT ? "Reject" : $final_destiny == D_BOUNCE ? "Bounce" : $final_destiny == D_TEMPFAIL ? "Temporary failure" : "Not ok ($final_destiny)" ), $softfailed, $smtp_reason eq '' ? '' : ', '.$smtp_reason); ll(4) && do_log(4, "blocking ccat=%s, SMTP response: %s", $blocking_ccat,$response); $r->recip_smtp_response($response); $r->recip_done(1); # fake a delivery (confirm delivery to a bit bucket) # note that 5xx status rejects may later be converted to bounces } } section_time($which_section); $which_section = "quar+notif"; $t0_sect = Time::HiRes::time; $snmp_db->register_proc(2,0,'Q',$am_id) if defined $snmp_db; # notify, quar do_notify_and_quarantine($msginfo, $virus_dejavu); # $which_section = "aux_quarantine"; # do_quarantine($msginfo, undef, ['archive-files'], 'local:archive/%m'); # do_quarantine($msginfo, undef, ['archive@localhost'], 'local:all-%m'); # do_quarantine($msginfo, undef, ['sender-quarantine'], 'local:user-%m' # ) if lookup(0,$sender, ['user1@domain','user2@domain']); # section_time($which_section); $elapsed{'TimeElapsedQuarantineAndNotify'} = Time::HiRes::time - $t0_sect; if (defined $hold && $hold ne '') { do_log(-1, "NOTICE: HOLD reason: %s", $hold) } # THIRD: now that we know what to do with it, do it! (deliver or bounce) { # update Content*Msgs* counters my($ccat_name) = $msginfo->setting_by_contents_category(\%ccat_display_names_major); my($counter_name) = 'Content'.$ccat_name.'Msgs'; snmp_count($counter_name); if ($msginfo->originating) { snmp_count($counter_name.'Originating'); } if ($cnt_local > 0) { my($d) = $msginfo->originating ? 'Internal' : 'Inbound'; snmp_count($counter_name.$d); } if ($cnt_remote > 0) { my($d) = $msginfo->originating ? 'Outbound' : 'OpenRelay'; snmp_count($counter_name.$d); } } # set $r->delivery_method according to forward_method_maps_by_ccat lookup # or defaults for my $r (@{$msginfo->per_recip_data}) { next if defined($r->delivery_method); my($fwd_map) = $r->setting_by_contents_category( cr('forward_method_maps_by_ccat')); my($fwd_m); $fwd_m = lookup2(0, $r->recip_addr, $fwd_map, Label=>"forward_method") if ref $fwd_map; $fwd_m = '' if !defined $fwd_m; $r->delivery_method($fwd_m); } # a custom hook may change $r->delivery_method if (ref $custom_object) { $which_section = "custom-before_send"; eval { $custom_object->before_send($conn,$msginfo); update_current_log_level(); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"custom before_send error: %s", $eval_stat); }; section_time($which_section); } if (ll(3)) { # log delivery method by recipients my(%fwd_m_displ_log); for my $r (@{$msginfo->per_recip_data}) { my($fwd_m) = $r->delivery_method; my($fwd_m_displ) = !defined $fwd_m ? "undefined, mail will not be forwarded" : map(ref eq 'ARRAY' ? '('.join(', ',@$_).')' : $_, $fwd_m); if (!$fwd_m_displ_log{$fwd_m_displ}) { $fwd_m_displ_log{$fwd_m_displ} = [ $r ]; } else { push(@{$fwd_m_displ_log{$fwd_m_displ}}, $r); } } for my $log_msg (sort keys %fwd_m_displ_log) { do_log(3, "delivery method is %s, recips: %s", $log_msg, join(', ', map($_->recip_addr, @{$fwd_m_displ_log{$log_msg}}))); } } my($bcc)= $msginfo->setting_by_contents_category(cr('always_bcc_by_ccat')); if (defined $bcc && $bcc ne '') { my($recip_obj) = Amavis::In::Message::PerRecip->new; # leave recip_addr and recip_addr_smtp undefined! $recip_obj->recip_addr_modified($bcc); $recip_obj->recip_destiny(D_PASS); $recip_obj->dsn_notify(['NEVER']); $recip_obj->contents_category($msginfo->contents_category); # $recip_obj->add_contents_category(CC_CLEAN,0); $msginfo->per_recip_data([@{$msginfo->per_recip_data}, $recip_obj]); do_log(2,"adding recipient - always_bcc: %s", $bcc); } my($hdr_edits) = $msginfo->header_edits; # to be delivered explicitly (not by an AM.PDP client) if (grep(!$_->recip_done && $_->delivery_method ne '', @{$msginfo->per_recip_data})) { # forwarding is needed $which_section = "forwarding"; $t0_sect = Time::HiRes::time; $snmp_db->register_proc(2,0,'F',$am_id) if defined $snmp_db; $hdr_edits = add_forwarding_header_edits_common( $msginfo, $hdr_edits, $hold, $any_undecipherable, $virus_presence_checked, $spam_presence_checked); for (;;) { # do the delivery, in batches if necessary my($r_hdr_edits) = Amavis::Out::EditHeader->new; # per-recip edits set $r_hdr_edits->inherit_header_edits($hdr_edits); my($done_all); my($recip_cl); # ref to a list of recip objects needing same mail edits # prepare header section edits, clusterize ($r_hdr_edits, $recip_cl, $done_all) = add_forwarding_header_edits_per_recip( $msginfo, $r_hdr_edits, $hold, $any_undecipherable, $virus_presence_checked, $spam_presence_checked, undef); last if !@$recip_cl; $msginfo->header_edits($r_hdr_edits); # store edits for this batch # preserve information that may be changed by prepare_modified_mail() my($m_t,$m_tfn,$m_ofs) = ($msginfo->mail_text, $msginfo->mail_text_fn, $msginfo->skip_bytes); my(@m_dm) = map($_->delivery_method, @{$msginfo->per_recip_data}); # mail body mangling/defanging/sanitizing my($body_modified) = prepare_modified_mail($msginfo,$hold,$any_undecipherable,$recip_cl); # defanged_mime_entity have modifed header edits, refetch just in case $r_hdr_edits = $msginfo->header_edits; if ($body_modified) { my($resend_m) = c('resend_method'); do_log(3, "mail body mangling in effect, %s", $resend_m); if ($resend_m ne '') { $_->delivery_method($resend_m) for @{$msginfo->per_recip_data}; } } if (mail_dispatch($msginfo, 0, $dsn_per_recip_capable, sub { my($r) = @_; grep($_ eq $r, @$recip_cl) })) { $point_of_no_return = 1; # now past the point where mail was sent } # close and delete replacement file, if any my($tmp_fh) = $msginfo->mail_text; # replacement file, to be removed if ($tmp_fh && !$tmp_fh->isa('MIME::Entity') && $tmp_fh ne $m_t) { $tmp_fh->close or do_log(-1,"Can't close replacement: %s", $!); if (debug_oneshot()) { do_log(5, "defanging+debug, preserving %s",$msginfo->mail_text_fn); } else { unlink($msginfo->mail_text_fn) or do_log(-1,"Can't remove %s: %s", $msginfo->mail_text_fn, $!); } } # restore temporarily modified settings $msginfo->mail_text($m_t); $msginfo->mail_text_fn($m_tfn); $msginfo->skip_bytes($m_ofs); $_->delivery_method(shift @m_dm) for @{$msginfo->per_recip_data}; last if $done_all; } # turn on CC_MTA in case of MTA trouble (e.g, rejected by MTA on fwding) for my $r (@{$msginfo->per_recip_data}) { my($smtp_resp) = $r->recip_smtp_response; # skip successful deliveries and non- MTA-generated status codes next if $smtp_resp =~ /^2/ || $r->recip_done != 2; my($min_ccat) = $smtp_resp =~ /^5/ ? 2 : $smtp_resp =~ /^4/ ? 1 : 0; $r->add_contents_category(CC_MTA,$min_ccat); $msginfo->add_contents_category(CC_MTA,$min_ccat); my($blocking_ccat) = sprintf("%d,%d", CC_MTA,$min_ccat); $r->blocking_ccat($blocking_ccat); $msginfo->blocking_ccat($blocking_ccat) if !defined($msginfo->blocking_ccat); my($final_destiny) = $r->setting_by_contents_category(cr('final_destiny_by_ccat')); if ($final_destiny == D_PASS) { $final_destiny = D_REJECT; # impossible to pass, change to reject } $r->recip_destiny($final_destiny); local($1,$2); if ($smtp_resp !~ /^5/) { # keep unchanged } elsif ($final_destiny == D_DISCARD) { $smtp_resp =~ s{^\d(\d\d) \d(\.\d\.\d)}{250 2$2}; # 5xx -> 250 } elsif (c('soft_bounce')) { do_log(5, "soft_bounce: (mta) %s -> 450", $smtp_resp); $smtp_resp =~ s{^\d(\d\d) \d(\.\d\.\d)}{450 4$2}; # 5xx -> 450 } my($smtp_reason) = # get the custom smtp response reason text $r->setting_by_contents_category(cr('smtp_reason_by_ccat')); $smtp_reason = '' if !defined $smtp_reason; if ($smtp_reason ne '') { my(%mybuiltins) = %builtins; # make a local copy $smtp_reason = expand(\$smtp_reason, \%mybuiltins); $smtp_reason = !ref($smtp_reason) ? '' : $$smtp_reason; chomp($smtp_reason); $smtp_reason = sanitize_str($smtp_reason,1); $smtp_reason = substr($smtp_reason,0,100) . "..." if length($smtp_reason) > 100+3; } $smtp_resp =~ /^(\d\d\d(?: \d\.\d\.\d)?)\s*(.*)\z/s; my($dis) = $final_destiny == D_DISCARD ? ' Discarded' : ''; $r->recip_smtp_response("$1$dis $smtp_reason, $2"); $r->recip_done(1); # fake a delivery (confirm delivery to a bit bucket) # note that 5xx status rejects may later be converted to bounces } $msginfo->header_edits($hdr_edits); # restore original edits just in case $elapsed{'TimeElapsedForwarding'} = Time::HiRes::time - $t0_sect; } # AM.PDP or AM.CL (milter) if (grep(!$_->recip_done && $_->delivery_method eq '', @{$msginfo->per_recip_data})) { $which_section = "AM.PDP headers"; $hdr_edits = add_forwarding_header_edits_common( $msginfo, $hdr_edits, $hold, $any_undecipherable, $virus_presence_checked, $spam_presence_checked); my($done_all); my($recip_cl); # ref to a list of similar recip objects ($hdr_edits, $recip_cl, $done_all) = add_forwarding_header_edits_per_recip( $msginfo, $hdr_edits, $hold, $any_undecipherable, $virus_presence_checked, $spam_presence_checked, undef); if (c('enable_dkim_signing')) { # add DKIM signatures my(@signatures) = Amavis::DKIM::dkim_make_signatures($msginfo,0); $msginfo->dkim_signatures_new(\@signatures) if @signatures; for my $signature (@signatures) { my($s) = $signature->as_string; local($1); $s =~ s{\015\012}{\n}gs; $s =~ s{\n+\z}{}gs; $s =~ s/^((?:DKIM|DomainKey)-Signature):[ \t]*//si; $hdr_edits->prepend_header($1, $s, 2); } } $msginfo->header_edits($hdr_edits); # store edits (redundant) if (@$recip_cl && !$done_all) { do_log(-1, "AM.PDP: RECIPIENTS REQUIRE DIFFERENT HEADERS"); }; } prolong_timer($which_section); if (ref $custom_object) { $which_section = "custom-after_send"; eval { $custom_object->after_send($conn,$msginfo); update_current_log_level(); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"custom after_send error: %s", $eval_stat); }; section_time($which_section); } $which_section = "delivery-notification"; $t0_sect = Time::HiRes::time; # generate a delivery status notification according to RFC 3462 & RFC 3464 my($notification,$suppressed) = delivery_status_notification( $msginfo, $dsn_per_recip_capable, \%builtins, [$sender], 'dsn', undef, undef); my($ndn_needed); ($smtp_resp, $exit_code, $ndn_needed) = one_response_for_all($msginfo, $dsn_per_recip_capable, $suppressed && !defined($notification) ); do_log(4, "notif=%s, suppressed=%d, ndn_needed=%s, exit=%s, %s", defined $notification ? 'Y' : 'N', $suppressed, $ndn_needed, $exit_code, $smtp_resp); section_time('prepare-dsn'); if ($suppressed && !defined($notification)) { $msginfo->dsn_sent(2); # would-be-bounced, but bounce was suppressed } elsif (defined $notification) { # dsn needed, send delivery notification mail_dispatch($notification, 'Dsn', 0); my($n_smtp_resp, $n_exit_code, $n_dsn_needed) = one_response_for_all($notification, 0); # check status if ($n_smtp_resp =~ /^2/ && !$n_dsn_needed) { # dsn successful? $msginfo->dsn_sent(1); # mark the message as bounced $point_of_no_return = 2; # now past the point where DSN was sent } elsif ($n_smtp_resp =~ /^4/) { die sprintf("temporarily unable to send DSN to <%s>: %s", $msginfo->sender, $n_smtp_resp); } else { do_log(-1,"NOTICE: UNABLE TO SEND DSN to <%s>: %s", $sender, $n_smtp_resp); # # if dsn cannot be sent, try to send it to postmaster # $notification->recips(['postmaster']); # # attempt double bounce # mail_dispatch($notification, 'Notif', 0); } # $notification->purge; } { # increment appropriate InMsgsStatus* SNMP counters and do some sanity # checking along the way; also sets $msginfo->actions_performed # my($err, %which_counts); my $orig = $msginfo->originating; my $dsn_sent = $msginfo->dsn_sent; # 1=bounced, 2=suppressed for my $r (@{$msginfo->per_recip_data}) { my $which; my $done = $r->recip_done; # 2=relayed to MTA, 1=faked deliv/quarant my $dest = $r->recip_destiny; my $resp_code = $smtp_resp; # per-msg status (one_response_for_all) $resp_code = $r->recip_smtp_response if $dsn_per_recip_capable; my $resp_class = substr($resp_code||'0', 0, 1); if (!$done) { $which = 'Accepted'; my $fwd_m = $r->delivery_method; # double-checking our sanity if (defined $fwd_m && $fwd_m ne '') { $err = "Recip not done, nonempty delivery method: $fwd_m"; } } elsif ($resp_class !~ /^[245]\z/) { $err = "Bad response code: $resp_code"; } elsif ($resp_class eq '4') { $which = 'TempFailed'; } elsif ($resp_class eq '5' && $dest == D_REJECT) { $which = 'Rejected'; } else { # $resp_class eq '2' || $resp_class eq '5' && $dest!=D_REJECT # a 2xx SMTP response code is set both by internal Discard and # by a genuine successful delivery. To distinguish between the two # we need to check $r->recip_destiny if ($done == 2) { # successful genuine forwarding $which = $r->recip_tagged ? 'RelayedTagged' : 'RelayedUntagged'; $err = "Forwarded, but destiny not D_PASS? ($dest)" if $dest != D_PASS; $err = "Forwarded, but status not 2xx? ($resp_code)" if $resp_class ne '2'; } elsif ($dest == D_DISCARD) { # forwarded to a bit bucket $which = 'Discarded'; } elsif ( $dest == D_BOUNCE || ($dest == D_REJECT && $resp_class eq '2') ) { if ($dsn_sent && $dsn_sent == 1) { $which = 'Bounced'; # genuine bounce (DSN) sent } elsif ($dsn_sent) { $which = 'NoBounce'; # bounce suppressed } else { # sanity check $err = "To be bounced, but DSN was neither sent nor suppressed?"; } } elsif ($dest == D_REJECT) { $which = 'Rejected'; $err = "Rejected, but status not 5xx? ($resp_code)" if $resp_class ne '5'; } else { # sanity check $err = "Recip forwarding suppressed but not DISCARD?"; } } $which = 'Unknown' if !defined $which; $which_counts{$which}++; # counts status without a direction $which_counts{'Relayed'}++ if $which eq 'RelayedTagged' || $which eq 'RelayedUntagged'; my $islocal = $r->recip_is_local; if ($orig) { if ($islocal) { $which_counts{$which.'Internal'}++ } else { $which_counts{$which.'Outbound'}++ } $which_counts{$which.'Originating'}++; } else { if ($islocal) { $which_counts{$which.'Inbound'}++ } else { $which_counts{$which.'OpenRelay'}++ } } do_log(0, "unexpected status/result, please verify: %s, %s", $err, $r->recip_addr_smtp) if defined $err; } my @which_list = sort keys %which_counts; # prefer this status in the list first, before a 'Quarantined' entry; # ignore a plain status name without mail direction to reduce clutter; # ignore Originating, as it is always paired with Internal or Outbound $msginfo->actions_performed([]) if !$msginfo->actions_performed; unshift(@{$msginfo->actions_performed}, map(/^RelayedUntagged(.*)/ ? "Relayed$1" : $_, # short log name grep(/(?:Inbound|Internal|Outbound|OpenRelay)\z/, @which_list))); snmp_count('InMsgsStatus'.$_) for @which_list; ll(3) && do_log(3, 'status counters: InMsgsStatus{%s}', join(',', @which_list)); } prolong_timer($which_section); $elapsed{'TimeElapsedDSN'} = Time::HiRes::time - $t0_sect; # generate customized log report at log level 0 - this is usually the # only log entry interesting to administrators during normal operation $which_section = 'main_log_entry'; my(%mybuiltins) = %builtins; # make a local copy { # do a per-message log entry # macro %T has overloaded semantics, ugly $mybuiltins{'T'} = $mybuiltins{'TESTSSCORES'}; my($y,$n,$f) = delivery_short_report($msginfo); @mybuiltins{'D','O','N'} = ($y,$n,$f); if (ll(0)) { my($strr) = expand(cr('log_templ'), \%mybuiltins); for my $logline (split(/[ \t]*\n/, $$strr)) { do_log(0, "%s", $logline) if $logline ne ''; } } } if (c('log_recip_templ') ne '') { # do per-recipient log entries # redefine some macros with a by-recipient semantics my($j) = 0; for my $r (@{$msginfo->per_recip_data}) { # recipient counter in macro %. may indicate to the template # that a per-recipient expansion semantics is expected $j++; $mybuiltins{'.'} = sprintf("%d",$j); my($recip) = $r->recip_addr; my($qrecip_addr) = scalar(qquote_rfc2821_local($recip)); my($remote_mta) = $r->recip_remote_mta; my($smtp_resp) = $r->recip_smtp_response; $mybuiltins{'remote_mta'} = $remote_mta; $mybuiltins{'smtp_response'} = $smtp_resp; $mybuiltins{'remote_mta_smtp_response'} = $r->recip_remote_mta_smtp_response; $mybuiltins{'D'} = $mybuiltins{'O'} = $mybuiltins{'N'} = undef; if ($r->recip_destiny==D_PASS &&($smtp_resp=~/^2/ || !$r->recip_done)){ $mybuiltins{'D'} = $qrecip_addr; } else { $mybuiltins{'O'} = $qrecip_addr; $mybuiltins{'N'} = sprintf("%s:%s\n %s", $qrecip_addr, ($remote_mta eq '' ?'' :" [$remote_mta] said:"), $smtp_resp); } my(@b); @b = @{$r->banned_parts} if defined $r->banned_parts; my($b_chopped) = @b > 2; @b = (@b[0,1],'...') if $b_chopped; s/[ \t]{6,}/ ... /g for @b; $mybuiltins{'banned_parts'} = \@b; # list of banned parts $mybuiltins{'F'} = $r->banning_reason_short; # just one name & comment $mybuiltins{'banning_rule_comment'} = !defined($r->banning_rule_comment) ? undef : unique_ref($r->banning_rule_comment); $mybuiltins{'banning_rule_rhs'} = !defined($r->banning_rule_rhs) ? undef : unique_ref($r->banning_rule_rhs); my($dn) = $r->dsn_notify; $mybuiltins{'dsn_notify'} = uc(join(',', $sender eq '' ? 'NEVER' : !$dn ? 'FAILURE' : @$dn)); my($tag_level,$tag2_level,$kill_level); if (!$r->bypass_spam_checks) { $tag_level = lookup2(0,$recip, ca('spam_tag_level_maps')); $tag2_level = lookup2(0,$recip, ca('spam_tag2_level_maps')); $kill_level = lookup2(0,$recip, ca('spam_kill_level_maps')); } my($is_local) = $r->recip_is_local; my($do_tag) = $r->is_in_contents_category(CC_CLEAN,1); my($do_tag2) = $r->is_in_contents_category(CC_SPAMMY); my($do_kill) = $r->is_in_contents_category(CC_SPAM); for ($do_tag,$do_tag2,$do_kill) { $_ = $_ ? 'Y' : '0' } # normalize for ($is_local) { $_ = $_ ? 'L' : '0' } # normalize for ($tag_level,$tag2_level,$kill_level) { $_ = 'x' if !defined($_) } $mybuiltins{'R'} = $recip; $mybuiltins{'c'} = $mybuiltins{'SCORE'} = $mybuiltins{'STARS'} = sub { macro_score($msginfo, $j-1, @_) }; # info on one recipient $mybuiltins{'T'} = $mybuiltins{'TESTSSCORES'} = $mybuiltins{'TESTS'} = sub { macro_tests($msginfo, $j-1, @_)}; # info on one recipient $mybuiltins{'tag_level'} = # replacement for deprecated %3 !defined($tag_level) ? '-' : 0+sprintf("%.3f",$tag_level); $mybuiltins{'tag2_level'} = $mybuiltins{'REQD'} = # replacement for %4 !defined($tag2_level) ? '-' : 0+sprintf("%.3f",$tag2_level); $mybuiltins{'kill_level'} = # replacement for deprecated %5 !defined($kill_level) ? '-' : 0+sprintf("%.3f",$kill_level); @mybuiltins{('0','1','2','k')} = ($is_local,$do_tag,$do_tag2,$do_kill); # macros %3, %4, %5 are deprecated, replaced by tag/tag2/kill_level @mybuiltins{('3','4','5')} = ($tag_level,$tag2_level,$kill_level); $mybuiltins{'ccat'} = sub { my($name,$attr,$which) = @_; $attr = lc($attr); # name | major | minor | # | is_blocking | is_nonblocking # | is_blocked_by_nonmain $which = lc($which); # main | blocking | auto my($result) = ''; my($blocking_ccat) = $r->blocking_ccat; if ($attr eq 'is_blocking') { $result = defined($blocking_ccat) ? 1 : ''; } elsif ($attr eq 'is_nonblocking') { $result = !defined($blocking_ccat) ? 1 : ''; } elsif ($attr eq 'is_blocked_by_nonmain') { if (defined($blocking_ccat)) { my($aref) = $r->contents_category; $result = 1 if ref($aref) && @$aref > 0 && $blocking_ccat ne $aref->[0]; } } elsif ($attr eq 'name') { $result = $which eq 'main' ? $r->setting_by_main_contents_category(\%ccat_display_names) : $which eq 'blocking' ? $r->setting_by_blocking_contents_category( \%ccat_display_names) : $r->setting_by_contents_category( \%ccat_display_names); } else { # attr = major, minor, or anything else returns a pair my($maj,$min) = ccat_split( ($which eq 'blocking' || $which ne 'main' && defined $blocking_ccat) ? $blocking_ccat : $r->contents_category); $result = $attr eq 'major' ? $maj : $attr eq 'minor' ? sprintf("%d",$min) : sprintf("(%d,%d)",$maj,$min); } $result; }; my($strr) = expand(cr('log_recip_templ'), \%mybuiltins); for my $logline (split(/[ \t]*\n/, $$strr)) { do_log(0, "%s", $logline) if $logline ne ''; } } } section_time($which_section); prolong_timer($which_section); if (defined $os_fingerprint && $os_fingerprint ne '') { # log and collect statistics on contents type vs. OS my($spam_ham_thd) = 2.0; # reasonable threshold guesstimate local($1); my($os_short); # extract operating system name when avail. $os_short = $1 if $os_fingerprint =~ /^([^,([]*)/; $os_short = $1 if $os_short =~ /^[ \t,-]*(.*?)[ \t,-]*\z/; my($snmp_counter_name); if ($os_short ne '') { $os_short = $1 if $os_short =~ /^(Windows [^ ]+|[^ ]+)/; # drop vers. $os_short =~ s{[^0-9A-Za-z:./_+-]}{-}g; $os_short =~ s{\.}{,}g; $snmp_counter_name = $msginfo->setting_by_contents_category( { CC_VIRUS,'virus', CC_BANNED,'banned', CC_SPAM,'spam', CC_SPAMMY,'spammy', CC_CATCHALL,'clean' }); if ($snmp_counter_name eq 'clean') { $snmp_counter_name = $max_spam_level <= $spam_ham_thd ?'ham' : undef; } if (defined $snmp_counter_name) { snmp_count("$snmp_counter_name.byOS.$os_short"); if ($snmp_counter_name eq 'ham' && $os_fingerprint =~ /^Windows XP(?![^(]*\b2000 SP)/) { do_log(3, 'Ham from Windows XP? Most weird! %s [%s] score=%.3f', $mail_id||'', $cl_ip, $max_spam_level); } } } do_log(2, "OS_fingerprint: %s %s %s.%s - %s", $msginfo->client_addr, $max_spam_level, defined $snmp_counter_name ? $snmp_counter_name : 'x', $os_short, $os_fingerprint); } if ($sql_storage && defined $msginfo->mail_id) { # save final information to SQL (if enabled) $which_section = 'sql-update'; my($ds) = $msginfo->dsn_sent; $ds = !$ds ? 'N' : $ds==1 ? 'Y' : $ds==2 ? 'q' : '?'; for (my($attempt)=5; $attempt>0; ) { # sanity limit on retries if ($sql_storage->save_info_final($msginfo,$ds)) { last; } elsif (--$attempt <= 0) { do_log(-2,"ERROR sql_storage: too many retries ". "on storing final, info not saved"); } else { do_log(2,"sql_storage: retrying on final, %d attempts remain", $attempt); sleep(int(1+rand(3))); # can't mix Time::HiRes::sleep with alarm } }; section_time($which_section); } if (ll(2)) { # log SpamAssassin timing report if available my($sa_tim) = $msginfo->supplementary_info('TIMING'); do_log(2, "TIMING-SA %s", $sa_tim) if defined($sa_tim) && $sa_tim ne ''; } if (defined $snmp_db) { $which_section = 'update_snmp'; my($log_lines, $log_entries_by_level_ref, $log_retries, $log_status_counts_ref) = collect_log_stats(); snmp_count( ['LogLines', $log_lines, 'C64'] ); my($log_entries_all_cnt) = 0; for my $level_str (keys %$log_entries_by_level_ref) { my($level) = 0+$level_str; my($cnt) = $log_entries_by_level_ref->{$level_str}; $log_entries_all_cnt += $cnt; # snmp_count( ['LogEntriesEmerg', $cnt, 'C64'] ); # not in use # snmp_count( ['LogEntriesAlert', $cnt, 'C64'] ); # not in use snmp_count( ['LogEntriesCrit', $cnt, 'C64'] ) if $level <= -3; snmp_count( ['LogEntriesErr', $cnt, 'C64'] ) if $level <= -2; snmp_count( ['LogEntriesWarning', $cnt, 'C64'] ) if $level <= -1; snmp_count( ['LogEntriesNotice', $cnt, 'C64'] ) if $level <= 0; snmp_count( ['LogEntriesInfo', $cnt, 'C64'] ) if $level <= 1; snmp_count( ['LogEntriesDebug', $cnt, 'C64'] ); if ($level < 0) { $level_str = "0" } elsif ($level > 5) { $level_str = "5" } snmp_count( ['LogEntriesLevel'.$level_str, $cnt, 'C64'] ); } snmp_count( ['LogEntries', $log_entries_all_cnt, 'C64'] ); if ($log_retries > 0) { snmp_count( ['LogRetries', $log_retries] ); do_log(3,"Syslog retries: %d x %s", $log_status_counts_ref->{$_}, $_) for (keys %$log_status_counts_ref); } $elapsed{'TimeElapsedSending'} += # merge similar timing entries delete $elapsed{$_} for ('TimeElapsedQuarantineAndNotify', 'TimeElapsedForwarding', 'TimeElapsedDSN'); snmp_count( ['entropy',0,'STR'] ); $elapsed{'TimeElapsedTotal'} = Time::HiRes::time - $msginfo->rx_time; # Will end up as SNMPv2-TC TimeInterval (INTEGER), units of 0.01 seconds, # but we keep it in milliseconds in the bdb database! # Note also the use of C32 instead of INT, we want cumulative time. snmp_count([$_, int(1000*$elapsed{$_}+0.5), 'C32']) for (keys %elapsed); $snmp_db->update_snmp_variables; section_time($which_section); } if (ref $custom_object) { $which_section = "custom-mail_done"; eval { $custom_object->mail_done($conn,$msginfo); update_current_log_level(); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"custom mail_done error: %s", $eval_stat); }; section_time($which_section); } $which_section = 'finishing'; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; $preserve_evidence = 1 if $allow_preserving_evidence; my($msg) = "$which_section FAILED: $eval_stat"; if ($point_of_no_return) { do_log(-2, "TROUBLE in check_mail, but must continue (%s): %s", $point_of_no_return,$msg); } else { do_log(-2, "TROUBLE in check_mail: %s", $msg); $smtp_resp = "451 4.5.0 Error in processing, id=$am_id, $msg"; $exit_code = EX_TEMPFAIL; for my $r (@{$msginfo->per_recip_data}) { $r->recip_smtp_response($smtp_resp); $r->recip_done(1) } } }; # if ($hold ne '') { # do_log(-1, "NOTICE: Evidence is to be preserved: %s", $hold); # $preserve_evidence = 1 if $allow_preserving_evidence; # } if (!$preserve_evidence && debug_oneshot()) { do_log(0, "DEBUG_ONESHOT CAUSES EVIDENCE TO BE PRESERVED"); $preserve_evidence = 1; # regardless of $allow_preserving_evidence } $snmp_db->register_proc(1,0,'.') if defined $snmp_db; # content checking done do_log(-1, "signal: %s", join(', ',keys %got_signals)) if %got_signals; undef $MSGINFO; # release global reference ($smtp_resp, $exit_code, $preserve_evidence); } # Ensure we have $msginfo->$entity defined when we expect we'll need it, # sub ensure_mime_entity($) { my($msginfo) = @_; my($ent,$mime_err); if (!defined($msginfo->mime_entity)) { ($ent,$mime_err) = mime_decode($msginfo->mail_text, $msginfo->mail_tempdir, $msginfo->parts_root); $msginfo->mime_entity($ent); prolong_timer('mime_decode'); } $mime_err; } # Check if a message is a bounce, and if it is, try to obtain essential # information from a header section of an attached original message, # primarily the Message-ID. # sub inspect_a_bounce_message($) { my($msginfo) = @_; my(%header_field,$bounce_type); my($is_true_bounce) = 0; my($parts_root) = $msginfo->parts_root; if (!defined($parts_root)) { do_log(5, 'inspect_dsn: no parts root'); } else { my($sender) = $msginfo->sender; my($structure_type) = '?'; my($top_main); my($top) = $parts_root->children; for my $e (!defined $top ? () : @$top) { # take a main message component, ignoring preamble/epilogue MIME parts # and pseudo components such as a fabricated 'MAIL' (i.e. a copy of # entire message for the benefit of some virus scanners) my($name) = $e->name_declared; next if !defined($e->type_declared) && defined($name) && ($name eq 'preamble' || $name eq 'epilogue'); next if $e->type_short eq 'MAIL' && lc($e->type_declared) eq 'message/rfc822'; $top_main = $e; last; } my(@parts); my($fname_ind); my($plaintext) = 0; if (defined $top_main) { # one level only my($ch) = $top_main->children; @parts = ($top_main, !defined $ch ? () : @$ch); } my(@t) = map { my($t)=$_->type_declared; lc(ref $t ? $t->[0] : $t) } @parts; ll(5) && do_log(5, "inspect_dsn: parts: %s", join(", ",@t)); my($fm) = $msginfo->rfc2822_from; my(@rfc2822_from) = !defined $fm ? () : ref $fm ? @$fm : $fm; my($p0_report_type); $p0_report_type = $parts[0]->report_type if @parts; $p0_report_type = lc $p0_report_type if defined $p0_report_type; if ( @parts >= 2 && @parts <= 4 && $t[0] eq 'multipart/report' && ( $t[2] eq 'message/delivery-status' || # RFC 3464 $t[2] eq 'message/global-delivery-status' || # RFC 5337 $t[2] eq 'message/disposition-notification' || # RFC 3798 $t[2] eq 'message/global-disposition-notification' || # RFC 5337 $t[2] eq 'message/feedback-report' # RFC 5965 ) && defined $p0_report_type && $t[2] eq 'message/'.$p0_report_type && ( $t[3] eq 'text/rfc822-headers' || $t[3] eq 'message/rfc822' || $t[3] eq 'message/rfc822-headers' || # nonstandard $t[3] eq 'message/partial' ) # nonstandard ) { # standard DSN or MDN or feedback-report $bounce_type = $t[2] eq 'message/disposition-notification' ? 'MDN' : $t[2] eq 'message/global-disposition-notification' ? 'MDN' : $t[2] eq 'message/feedback-report' ? 'ARF' : 'DSN'; $structure_type = 'standard ' . $bounce_type; $fname_ind = $#parts; $is_true_bounce = 1; } elsif ( @parts == 5 && $t[0] eq 'multipart/report' && $t[-2] eq 'message/delivery-status' && defined $p0_report_type && $t[-2] eq 'message/'.$p0_report_type && ( $t[-1] eq 'text/rfc822-headers' || $t[-1] eq 'message/rfc822' ) ) { # almost standard DSN, has two leading plain text parts $bounce_type = 'DSN'; # BorderWare Security Platform $structure_type = 'standard ' . $bounce_type; $fname_ind = $#parts; $is_true_bounce = 1; } elsif ( @parts >= 2 && @parts <= 4 && $t[0] eq 'multipart/report' && $t[2] eq 'message/delivery-status' && defined $p0_report_type && $t[2] eq 'message/'.$p0_report_type && $t[3] eq 'text/plain' ) { # nonstandard DSN, missing header, unless it is stashed in text/plain $fname_ind = 3; $structure_type = 'nostandard DSN-plain'; $plaintext = 1; $bounce_type = 'DSN'; } elsif (@parts >= 3 && @parts <= 4 && # a root with 2 or 3 leaves $t[0] eq 'multipart/report' && defined $p0_report_type && $p0_report_type eq 'delivery-status' && ( $t[-1] eq 'text/rfc822-headers' || $t[-1] eq 'message/rfc822' )) { # not quite std. DSN (missing message/delivery-status), but recognizable $fname_ind = -1; $is_true_bounce = 1; $bounce_type = 'DSN'; $structure_type = 'DSN, missing delivery-status part'; } elsif (@parts >= 3 && @parts <= 5 && $t[0] eq 'multipart/mixed' && ( $t[-1] eq 'text/rfc822-headers' || $t[-1] eq 'message/rfc822' || $t[-1] eq 'message/rfc822-headers') && # nonstandard - Gordano M.S. ( $rfc2822_from[0] =~ /^MAILER-DAEMON(?:\@|\z)/si || $msginfo->get_header_field_body('subject') =~ /\b(?:Delivery Failure Notification|failure notice)\b/ ) ) { # qmail, msn?, mailman, C/R $fname_ind = -1; $structure_type = 'multipart/mixed(' . $msginfo->is_bulk . ')'; } elsif ( $msginfo->is_auto && $sender eq '' && # notify@yahoogroups.com notify@yahoogroupes.fr $rfc2822_from[0] =~ /^notify\@yahoo/si && @parts >= 3 && @parts <= 5 && $t[0] eq 'multipart/mixed' && ( $t[-1] eq 'text/rfc822-headers' || $t[-1] eq 'message/rfc822' ) ) { $fname_ind = -1; $structure_type = 'multipart/mixed(yahoogroups)'; } elsif ( $msginfo->is_auto && $sender eq '' && @parts == 1 && $t[0] ne 'multipart/report' && $rfc2822_from[0] =~ /^(?:MAILER-DAEMON|postmaster)(?:\@|\z)/si ) { # nonstructured, possibly a non-standard bounce (qmail, gmail.com, ...) $fname_ind = 0; $plaintext = 1; $structure_type = 'nonstructured(' . $msginfo->is_auto . ')'; # } elsif ( $msginfo->is_auto && $sender eq '' && # ( grep($_->recip_addr eq 'xxx@example.com', # victim # @{$msginfo->per_recip_data}) ) ) { # # nonstructured, possibly a non-standard bounce # $fname_ind = 0; $plaintext = 1; $is_true_bounce = 1; # $structure_type = 'nonstructured, unknown'; # $bounce_type = 'INFO'; # } elsif (@parts == 3 && # $t[0] eq 'multipart/mixed' && # $t[-1] eq 'application/octet-stream' && # $parts[-1]->name_declared =~ /\.eml\z/) { # # MDaemon; too permissive! test for postmaster or mailer-daemon ? # $fname_ind = -1; # $structure_type = 'multipart/mixed with binary .eml'; # } elsif ( $msginfo->is_auto && @parts == 2 && # $t[0] eq 'multipart/mixed' && $t[1] eq 'text/plain' ) { # # nonstructured, possibly a broken bounce # $fname_ind = 1; $plaintext = 1; # $structure_type = $t[0] .' with '. $t[1] .'(' . $msginfo->is_auto .')'; # } elsif ( $msginfo->is_auto && @parts == 3 && # $t[0] eq 'multipart/alternative' && # $t[1] eq 'text/plain' && $t[2] eq 'text/html' ) { # # text/plain+text/html, possibly a challenge CR message # $fname_ind = 1; $plaintext = 1; # $structure_type = $t[0] .' with '. $t[1] .'(' . $msginfo->is_auto .')'; } if (defined $fname_ind && defined $parts[$fname_ind]) { # we probably have a header section from original mail, scan it $fname_ind = $#parts if $fname_ind == -1; my($fname) = $parts[$fname_ind]->full_name; ll(5) && do_log(5,'inspect_dsn: struct: "%s", basenm(%s): %s, fname: %s', $structure_type, $fname_ind, $parts[$fname_ind]->base_name, $fname); if (defined $fname) { my(%collectable_header_fields); $collectable_header_fields{lc($_)} = 1 for qw(From To Return-Path Message-ID Date Received Subject MIME-Version Content-Type); my($fh) = IO::File->new; $fh->open($fname,'<') or die "Can't open file $fname: $!"; binmode($fh,':bytes') or die "Can't cancel :utf8 mode: $!"; my($have_header_fields_cnt) = 0; my($nonheader_cnt) = 0; my($curr_head,$ln); my($nr) = 0; my($eof) = 0; local($1,$2); my($line_limit) = $plaintext ? 200 : 1000; for (;;) { if ($eof) { $ln = "\n"; # fake a missing header/body separator line } else { $! = 0; $ln = $fh->getline; if (!defined($ln)) { $eof = 1; $ln = "\n"; $!==0 or # returning EBADF at EOF is a perl bug $!==EBADF ? do_log(1,"Error reading mail header section: $!") : die "Error reading mail header section: $!"; } } last if ++$nr > $line_limit; # safety measure if ($ln =~ /^[ \t]/) { # folded $curr_head .= $ln if length($curr_head) < 2000; # safety measure } else { # a new header field, process previous if any if (defined $curr_head) { $curr_head =~ s/^[> ]+// if $plaintext; # be more conservative on accepted h.f.name than RFC 5322 allows # the '_' and '.' are quite rare, digits even rarer; # the longest non-X h.f.name is content-transfer-encoding (25) # the longest h.f.names in the wild are 59 chars, largest ever 77 if ($curr_head !~ /^([a-zA-Z0-9._-]{1,60})[ \t]*:(.*)\z/s) { $nonheader_cnt++; } else { my($hfname) = lc($1); if ($collectable_header_fields{$hfname}) { $have_header_fields_cnt++ if !exists $header_field{$hfname}; $header_field{$hfname} = $2; } } } $curr_head = $ln; if (!$plaintext) { last if $ln eq "\n" || $ln =~ /^--/; } elsif ($ln =~ /^\s*$/ || $ln =~ /^--/) { if (exists $header_field{'from'} && $have_header_fields_cnt >= 4 && $nonheader_cnt <= 1) { last; } else { # reset, hope for the next paragraph to be a header $have_header_fields_cnt = 0; $nonheader_cnt = 0; %header_field = (); $curr_head = undef; } } } } defined $ln || $!==0 or # returning EBADF at EOF is a perl bug $!==EBADF ? do_log(1,"Error reading from %s: %s", $fname,$!) : die "Error reading from $fname: $!"; $fh->close or die "Error closing $fname: $!"; my($thd) = exists $header_field{'message-id'} ? 3 : 5; $is_true_bounce = 1 if exists $header_field{'from'} && $have_header_fields_cnt >= $thd; if ($is_true_bounce) { ll(5) && do_log(5, "inspect_dsn: plain=%s, got %d: %s", $plaintext?"Y":"N", scalar(keys %header_field), join(", ", sort keys %header_field)); for (@header_field{keys %header_field}) { s/\n(?=[ \t])//gs; s/^[ \t]+//; s/[ \t\n]+\z// } if (!defined($header_field{'message-id'}) && $have_header_fields_cnt >= 5 && $nonheader_cnt <= 1) { $header_field{'message-id'} = ''; # fake: defined but empty do_log(5, "inspect_dsn: a header section with no Message-ID"); } elsif (defined($header_field{'message-id'})) { $header_field{'message-id'} = (parse_message_id($header_field{'message-id'}))[0] if defined $header_field{'message-id'}; } } section_time("inspect_dsn"); } } $bounce_type = 'bounce' if !defined $bounce_type; if ($is_true_bounce) { do_log(3, 'inspect_dsn: is a %s, struct: "%s", part(%s/%d), <%s>', $bounce_type, $structure_type, !defined($fname_ind) ? '-' : $fname_ind, scalar(@parts), $sender) if ll(3); } elsif ($msginfo->is_auto) { # bounce likely, but contents unrecognizable do_log(3, 'inspect_dsn: possibly a %s, unrecognizable, '. 'struct: "%s", parts(%s/%d): %s', $bounce_type, $structure_type, !defined($fname_ind) ? '-' : $fname_ind, scalar(@parts), join(", ",@t)) if ll(3); } else { # not a bounce do_log(3, 'inspect_dsn: not a bounce'); } } $bounce_type = undef if !$is_true_bounce; !$is_true_bounce ? () : (\%header_field,$bounce_type); } # obtain authserv-id from an Authentication-Results header field # sub parse_authentication_results($) { local($_) = $_[0]; tr/\n//d; local($1); my($comm_lvl) = 0; my($authservid); while (!/\G \z/gcsx) { if ( /\G \( /gcsx) { $comm_lvl++ } elsif ($comm_lvl > 0 && /\G \) /gcsx) { $comm_lvl-- } elsif ($comm_lvl > 0 && /\G(?: \\. | [^()\\]+ )/gcsx) {} elsif (!$comm_lvl && /\G [ \t]+ /gcsx) {} elsif (!$comm_lvl && /\G ([^\000-\040\177-\377:;,"()<>\[\]\@\\]+)/gcsx) { $authservid = $1; last } else { last }; # syntax error } $authservid; } sub add_forwarding_header_edits_common($$$$$$) { my($msginfo, $hdr_edits, $hold, $any_undecipherable, $virus_presence_checked, $spam_presence_checked) = @_; my($use_our_hdrs) = cr('prefer_our_added_header_fields'); my($allowed_hdrs) = cr('allowed_added_header_fields'); if ($allowed_hdrs && $allowed_hdrs->{lc('X-Amavis-Hold')}) { # discard existing X-Amavis-Hold header field, only allow our own $hdr_edits->delete_header('X-Amavis-Hold'); if ($hold ne '') { $hdr_edits->add_header('X-Amavis-Hold', $hold); do_log(-1, "Inserting header field: X-Amavis-Hold: %s", $hold); } } # RFC 5451: For security reasons, any MTA conforming to this specification # MUST delete any discovered instance of this header field that claims to # have been added within its trust boundary and that did not come from # another trusted MTA. [...] For simplicity and maximum security, a border # MTA MAY remove all instances of this header field on mail crossing into # its trust boundary. [...] (Hmmm...!?) However, an MTA MUST remove such # a header if the [SMTP] connection relaying the message is not from a # trusted internal MTA. my($authservid) = c('myauthservid'); $authservid = c('myhostname') if !defined $authservid || $authservid eq ''; # delete header field if its authserv-id matches ours or is unparseable $hdr_edits->edit_header('Authentication-Results', sub { my($h,$b) = @_; my($aid) = parse_authentication_results($b); if (defined $aid) { $aid =~ s{/.*}{}; $authservid =~ s{/.*}{} }; !defined $aid || lc($aid) eq lc($authservid) ? (undef,0) : ($b,1); } ); # [...] Border MTA MAY elect simply to remove all instances of this # header field on mail crossing into its trust boundary # $hdr_edits->delete_header('Authentication-Results'); # example on how to remove subject tag inserted by some other MTA: # $hdr_edits->edit_header('Subject', # sub { my($h,$s)=@_; $s=~s/^\s*\*\*\* Spam \*\*\*(.*)/$1/si; $s }); if ($extra_code_antivirus) { # $hdr_edits->delete_header('X-Amavis-Alert'); # it does not hurt to keep it my($am_hdr_fld_head) = c('X_HEADER_TAG'); my($am_hdr_fld_body) = c('X_HEADER_LINE'); $hdr_edits->delete_header($am_hdr_fld_head) if c('remove_existing_x_scanned_headers') && defined $am_hdr_fld_body && $am_hdr_fld_body ne '' && defined $am_hdr_fld_head && $am_hdr_fld_head =~ /^[!-9;-\176]+\z/; } for ('X-Spam-Checker-Version') { if ($extra_code_antispam_sa && $allowed_hdrs && $allowed_hdrs->{lc $_} && $use_our_hdrs && $use_our_hdrs->{lc $_}) { no warnings 'once'; $hdr_edits->add_header($_, sprintf("SpamAssassin %s (%s) on %s", Mail::SpamAssassin::Version(), $Mail::SpamAssassin::SUB_VERSION, c('myhostname'))); } } $hdr_edits; } # Prepare header edits for the first not-yet-done recipient. # Inspect remaining recipients, returning the list of recipient objects # that are receiving the same set of header edits (so the message may be # delivered to them in one SMTP transaction). # sub add_forwarding_header_edits_per_recip($$$$$$$) { my($msginfo, $hdr_edits, $hold, $any_undecipherable, $virus_presence_checked, $spam_presence_checked, $filter) = @_; my(@recip_cluster); my(@per_recip_data) = grep(!$_->recip_done && (!$filter || &$filter($_)), @{$msginfo->per_recip_data}); my($per_recip_data_len) = scalar(@per_recip_data); my($first) = 1; my($cluster_key); my($cluster_full_spam_status); my($use_our_hdrs) = cr('prefer_our_added_header_fields'); my($allowed_hdrs) = cr('allowed_added_header_fields'); my($x_header_tag) = c('X_HEADER_TAG'); my($adding_x_header_tag) = $x_header_tag =~ /^[!-9;-\176]+\z/ && c('X_HEADER_LINE') ne '' && $allowed_hdrs && $allowed_hdrs->{lc($x_header_tag)}; my($mail_id) = $msginfo->mail_id; my($os_fp) = $msginfo->client_os_fingerprint; if (defined($os_fp) && $os_fp ne '' && $msginfo->client_addr ne '') { $os_fp .= ', ['. $msginfo->client_addr . ']:' . $msginfo->client_port } my(@headers_to_be_removed); # header fields that may need to be removed if ($extra_code_antispam) { @headers_to_be_removed = qw( X-Spam-Status X-Spam-Level X-Spam-Flag X-Spam-Score X-Spam-Report X-Spam-Checker-Version X-Spam-Tests); @headers_to_be_removed = grep(defined $msginfo->get_header_field($_), @headers_to_be_removed); } my($header_tagged) = 0; for my $r (@per_recip_data) { my($spam_level) = $r->spam_level; my($recip) = $r->recip_addr; my($is_local) = $r->recip_is_local; my($blacklisted) = $r->recip_blacklisted_sender; my($whitelisted) = $r->recip_whitelisted_sender; my($bypassed) = $r->bypass_spam_checks; my($do_tag) = $r->is_in_contents_category(CC_CLEAN,1); my($do_tag2) = $r->is_in_contents_category(CC_SPAMMY); my($do_kill) = $r->is_in_contents_category(CC_SPAM); my($do_tag_badh) = $r->is_in_contents_category(CC_BADH); my($do_tag_banned)= $r->is_in_contents_category(CC_BANNED); my($do_tag_virus) = $r->is_in_contents_category(CC_VIRUS); my($mail_mangle) = $r->mail_body_mangle; my($do_tag_virus_checked) = $adding_x_header_tag && !$r->bypass_virus_checks; my($do_rem_hdr)= @headers_to_be_removed && lookup2(0,$recip,ca('remove_existing_spam_headers_maps')); my($do_p0f) = $is_local && defined($os_fp) && $os_fp ne '' && $allowed_hdrs && $allowed_hdrs->{lc('X-Amavis-OS-Fingerprint')}; my($pp_age); if ($allowed_hdrs && $allowed_hdrs->{lc('X-Amavis-PenPals')}) { $pp_age = $r->recip_penpals_age; $pp_age = format_time_interval($pp_age) if defined $pp_age; } my($tag_level,$tag2_level,$subject_tag); if ($extra_code_antispam && !$bypassed) { $tag_level = lookup2(0,$recip, ca('spam_tag_level_maps')); $tag2_level = lookup2(0,$recip, ca('spam_tag2_level_maps')); } if ($is_local) { # || c('warn_offsite') my(@subj_maps_pairs) = $r->setting_by_main_contents_category_all( cr('subject_tag_maps_by_ccat')); for my $pair (@subj_maps_pairs) { my($cc,$map_ref) = @$pair; next if !ref($map_ref); $subject_tag = lookup2(0,$recip,$map_ref); # take the first nonempty string last if defined $subject_tag && $subject_tag ne ''; } } $subject_tag = '' if !defined $subject_tag; if ($subject_tag ne '') { # expand subject template # just implement a small subset of macro-lookalikes, not true macro calls $subject_tag =~ s{_(SCORE|REQD|YESNO|YESNOCAPS|HOSTNAME|DATE|U|LOGID|MAILID)_} { $1 eq 'SCORE' ? (0+sprintf("%.3f",$spam_level)) : $1 eq 'REQD' ? (!defined($tag2_level) ? '-' : 0+sprintf("%.3f",$tag2_level)) : $1 eq 'YESNO' ? ($do_tag2 ? 'Yes' : 'No') : $1 eq 'YESNOCAPS' ? ($do_tag2 ? 'YES' : 'NO') : $1 eq 'HOSTNAME' ? c('myhostname') : $1 eq 'DATE' ? rfc2822_timestamp($msginfo->rx_time) : $1 eq 'U' ? iso8601_utc_timestamp($msginfo->rx_time) : $1 eq 'LOGID' ? $msginfo->log_id : $1 eq 'MAILID' ? $mail_id||'' : '_'.$1.'_' }egsx; } # normalize $_ = $_?1:0 for ($do_tag_virus_checked, $do_tag_virus, $do_tag_banned, $do_tag_badh, $do_tag, $do_tag2, $do_p0f, $do_rem_hdr, $is_local); my($spam_level_bar, $full_spam_status); if ($is_local && ($do_tag || $do_tag2)) { # prepare status and level bar # spam-related header fields should _not_ be inserted for: # - nonlocal recipients (outgoing mail), as a matter of courtesy # to our users; # - recipients matching bypass_spam_checks: even though spam checking # may have been done for other reasons, these recipients do not expect # such header fields, so let's pretend the check has not been done # and not insert spam-related header fields for them; # - everyone when the spam level is below the tag level # or the sender was whitelisted and tag level is below -10 # (undefined tag level is treated as lower than any spam score). my($autolearn_status) = $msginfo->supplementary_info('AUTOLEARN'); my($slc) = c('sa_spam_level_char'); $spam_level_bar = $slc x min(64, $bypassed || $whitelisted ? 0 : $blacklisted ? 64 : 0+$spam_level) if $slc ne ''; my($spam_tests) = $r->spam_tests; $spam_tests = !defined $spam_tests ?'' : join(',',map($$_,@$spam_tests)); # allow header field wrapping at any comma my($s) = $spam_tests; $s =~ s/,/,\n /g; $full_spam_status = sprintf( "%s,\n score=%s\n %s%s%stests=[%s]\n autolearn=%s", $do_tag2 ? 'Yes' : 'No', !defined $spam_level ? 'x' : 0+sprintf("%.3f",$spam_level), !defined $tag_level || $tag_level eq '' ? '' : sprintf("tagged_above=%s\n ",$tag_level), !defined $tag2_level ? '' : sprintf("required=%s\n ", $tag2_level), join('', $blacklisted ? "BLACKLISTED\n " : (), $whitelisted ? "WHITELISTED\n " : ()), $s, $autolearn_status||'unavailable'); } my($key) = join("\000", map {defined $_ ? $_ : ''} ( $do_tag_virus_checked, $do_tag_virus, $do_tag_banned, $do_tag_badh, $do_tag && $is_local, $do_tag2 && $is_local, $subject_tag, $do_rem_hdr, $spam_level_bar, $full_spam_status, $mail_mangle, $do_p0f, $pp_age) ); if ($first) { if (ll(4)) { my($sl) = !defined($spam_level) ? 'x' : 0+sprintf("%.3f",$spam_level); # trim fraction do_log(4, "headers CLUSTERING: NEW CLUSTER <%s>: score=%s, ". "tag=%s, tag2=%s, local=%s, bl=%s, s=%s, mangle=%s", $recip, $sl, $do_tag, $do_tag2, $is_local, $blacklisted, $subject_tag, $mail_mangle); } $cluster_key = $key; $cluster_full_spam_status = $full_spam_status; } elsif ($key eq $cluster_key) { do_log(5,"headers CLUSTERING: <%s> joining cluster", $recip); } else { do_log(5,"headers CLUSTERING: skipping <%s> (t=%s, t2=%s, r=%s, l=%s)", $recip,$do_tag,$do_tag2,$do_rem_hdr,$is_local); next; # this recipient will be handled in some later pass } if ($first) { # insert header fields required for the new cluster my(%header_field_provided); # mainly applies to spam header fields if ($do_rem_hdr) { $hdr_edits->delete_header($_) for @headers_to_be_removed; } if ($is_local && defined $msginfo->quarantined_to && defined $mail_id) { $hdr_edits->add_header('X-Quarantine-ID', '<'.$mail_id.'>') if $allowed_hdrs && $allowed_hdrs->{lc('X-Quarantine-ID')}; } if ($mail_mangle) { # mail body modified, invalidates DKIM signatures if ($allowed_hdrs && $allowed_hdrs->{lc('X-Amavis-Modified')}) { $hdr_edits->add_header('X-Amavis-Modified', sprintf("Mail body modified (%s) - %s", length($mail_mangle) > 1 ? "using $mail_mangle" : "defanged", c('myhostname') )); } } if ($do_tag_virus_checked) { $hdr_edits->add_header(c('X_HEADER_TAG'), c('X_HEADER_LINE')); } if ($allowed_hdrs && $allowed_hdrs->{lc('X-Amavis-Alert')}) { if ($do_tag_virus) { my($virusname_list) = $msginfo->virusnames; $hdr_edits->add_header('X-Amavis-Alert', "INFECTED, message contains virus: " . (!defined($virusname_list) ? '' : join(", ",@$virusname_list)) ); $header_tagged = 1; } if ($do_tag_banned) { $hdr_edits->add_header('X-Amavis-Alert', 'BANNED, message contains ' . $r->banning_reason_short); $header_tagged = 1; } if ($do_tag_badh) { $hdr_edits->add_header('X-Amavis-Alert', 'BAD HEADER SECTION, ' . $bad_headers[0]); # $header_tagged = 1; # not this one, it is mostly harmless } } if ($is_local && $allowed_hdrs && $use_our_hdrs) { for ('X-Spam-Checker-Version') { if ($extra_code_antispam_sa && $allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { # a hack instead of making %header_field_provided global: # just mark it as already provided, this header field was # already inserted by add_forwarding_header_edits_common() $header_field_provided{lc $_} = 1; } } for ('X-Spam-Flag') { if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { $hdr_edits->add_header($_, $do_tag2 ? 'YES' : 'NO') if $do_tag; $header_field_provided{lc $_} = 1; $header_tagged = 1 if $do_tag2; # SPAMMY } } for ('X-Spam-Score') { if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { if ($do_tag) { my($score) = 0+$spam_level; $score = max(64,$score) if $blacklisted; # not below 64 if bl $score = min( 0,$score) if $whitelisted; # not above 0 if wl $hdr_edits->add_header($_, 0+sprintf("%.3f",$score)); } $header_field_provided{lc $_} = 1; } } for ('X-Spam-Level') { if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { if ($do_tag && defined $spam_level_bar) { $hdr_edits->add_header($_, $spam_level_bar); } $header_field_provided{lc $_} = 1; } } for ('X-Spam-Status') { if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { $hdr_edits->add_header($_, $full_spam_status, 1) if $do_tag; $header_field_provided{lc $_} = 1; } } for ('X-Spam-Report') { # SA reports may contain any octet, i.e. 8-bit data from a mail # that is reported by a matching rule; no charset is associated, so # it doesn't make sense to RFC 2047 -encode it, so just sanitize it if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { if ($do_tag2) { my($report) = $r->spam_report; $report = $msginfo->spam_report if !defined $report; if (defined $report && $report ne '') { $hdr_edits->add_header($_, "\n".sanitize_str($report,1), 2); } } $header_field_provided{lc $_} = 1; } } } if ($is_local && $allowed_hdrs) { # add remaining header fields as provided by spam scanners my($sa_header) = $msginfo->supplementary_info( $do_tag2 ? 'ADDEDHEADERSPAM' : 'ADDEDHEADERHAM'); if (defined $sa_header && $sa_header ne '') { for my $hf (split(/^(?![ \t])/m, $sa_header, -1)) { local($1,$2); if ($hf =~ /^([!-9;-\176]+)[ \t]*:(.*)\z/s) { my($hf_name,$hf_body) = ($1,$2); my($hf_name_lc) = lc($hf_name); chomp($hf_body); if ($header_field_provided{$hf_name_lc}) { do_log(5,'fwd: scanner provided %s, but we preferred our own', $hf_name); } elsif (!$allowed_hdrs->{$hf_name_lc}) { do_log(5,'fwd: scanner provided %s, inhibited '. 'by %%allowed_added_header_fields', $hf_name); } else { do_log(5,'fwd: scanner provided %s, inserting', $hf_name); $hdr_edits->add_header($hf_name, $hf_body, 2); } } } } for my $pair ( ['DSPAMRESULT', 'X-DSPAM-Result'], ['DSPAMSIGNATURE', 'X-DSPAM-Signature'], ['CRM114STATUS', 'X-CRM114-Status'], ['CRM114CACHEID', 'X-CRM114-CacheID'] ) { my($suppl_attr_name, $hf_name) = @$pair; my($suppl_attr_val) = $msginfo->supplementary_info($suppl_attr_name); if (defined $suppl_attr_val && $suppl_attr_val ne '') { if (!$allowed_hdrs->{lc $hf_name}) { do_log(5,'fwd: scanner provided %s, '. 'inhibited by %%allowed_added_header_fields', $hf_name); } else { do_log(5,'fwd: scanner provided %s, inserting', $hf_name); $hdr_edits->add_header($hf_name, sanitize_str($suppl_attr_val), 2); } } } } $hdr_edits->add_header('X-Amavis-OS-Fingerprint', sanitize_str($os_fp)) if $do_p0f; $hdr_edits->add_header('X-Amavis-PenPals', 'age '.$pp_age) if defined $pp_age; if ($is_local && c('enable_dkim_verification') && $allowed_hdrs && $allowed_hdrs->{lc('Authentication-Results')}) { for my $h (Amavis::DKIM::generate_authentication_results($msginfo,0)) { $hdr_edits->add_header('Authentication-Results', $h, 1); } } if ($subject_tag ne '') { if (defined $msginfo->get_header_field('subject')) { $hdr_edits->edit_header('Subject', sub { local($1,$2); $_[1] =~ /^([ \t]?)(.*)\z/s; my($subj) = $2; if (length($subject_tag) >= 3) # precaution { $subj =~ s/\Q$subject_tag\E//sg } ' ' . $subject_tag . $subj }); } else { # no Subject header field present, insert one $subject_tag =~ s/[ \t]+\z//; # trim $hdr_edits->add_header('Subject', $subject_tag); do_log(0,"INFO: no existing header field 'Subject', inserting it"); } $header_tagged = 1; } if ($allowed_hdrs && $allowed_hdrs->{lc('Received')} && grep($_->delivery_method ne '', @{$msginfo->per_recip_data})) { $hdr_edits->add_header('Received', make_received_header_field($msginfo,1), 1); } } # if $first push(@recip_cluster,$r); $first = 0; $r->recip_tagged(1) if $header_tagged; my($delim) = c('recipient_delimiter'); if ($is_local) { # rewrite/replace recipient addresses, possibly with multiple recipients my($rewrite_map) = $r->setting_by_contents_category( cr('addr_rewrite_maps_by_ccat')); my($rewrite) = !ref $rewrite_map ? undef :lookup2(0,$recip,$rewrite_map); if ($rewrite ne '') { my(@replacements) = grep($_ ne '', map { /^ [ \t]* (.*?) [ \t]* \z/sx; $1 } split(/,/, $rewrite, -1)); if (@replacements) { my($repl_addr) = shift @replacements; my($modif_addr) = replace_addr_fields($recip,$repl_addr,$delim); ll(5) && do_log(5,"addr_rewrite_maps: replacing <%s> by <%s>", $recip,$modif_addr); $r->recip_addr_modified($modif_addr); for my $bcc (@replacements) { # remaining addresses are extra Bcc my($new_addr) = replace_addr_fields($recip,$bcc,$delim); ll(5) && do_log(5,"addr_rewrite_maps: recip <%s>, adding <%s>", $recip,$new_addr); # my($clone) = $r->clone; # $clone->recip_addr_modified($new_addr); } } $r->dsn_orcpt(orcpt_encode($r->recip_addr_smtp)) if !defined($r->dsn_orcpt); } } if ($is_local && defined $delim && $delim ne '') { # append address extensions to mailbox names if desired my($ext_map) = $r->setting_by_contents_category( cr('addr_extension_maps_by_ccat')); my($ext) = !ref($ext_map) ? undef : lookup2(0,$recip,$ext_map); if ($ext ne '') { $ext = $delim . $ext; my($orig_extension); my($localpart,$domain) = split_address($recip); ($localpart,$orig_extension) = split_localpart($localpart,$delim) if c('replace_existing_extension'); # strip existing extension my($new_addr) = $localpart.$ext.$domain; if (ll(5)) { if (!defined($orig_extension)) { do_log(5, "appending addr ext '%s', giving '%s'", $ext,$new_addr); } else { do_log(5, "replacing addr ext '%s' by '%s', giving '%s'", $orig_extension,$ext,$new_addr); } } # RFC 3461: If no ORCPT parameter was present in the RCPT command when # the message was received, an ORCPT parameter MAY be added to the # RCPT command when the message is relayed. If an ORCPT parameter is # added by the relaying MTA, it MUST contain the recipient address # from the RCPT command used when the message was received by that MTA. $r->dsn_orcpt(orcpt_encode($r->recip_addr_smtp)) if !defined($r->dsn_orcpt); $r->recip_addr_modified($new_addr); $r->recip_tagged(1); } } } my($done_all); if (@recip_cluster == $per_recip_data_len) { do_log(5,"headers CLUSTERING: done all %d recips in one go", $per_recip_data_len); $done_all = 1; } else { ll(4) && do_log(4, "headers CLUSTERING: got %d recips out of %d: %s", scalar(@recip_cluster), $per_recip_data_len, join(', ', map($_->recip_addr_smtp, @recip_cluster))); } if (ll(2) && defined($cluster_full_spam_status) && @recip_cluster) { my($s) = $cluster_full_spam_status; $s =~ s/\n[ \t]/ /g; do_log(2, "spam-tag, %s -> %s, %s", $msginfo->sender_smtp, join(',', map($_->recip_addr_smtp, @recip_cluster)), $s); } ($hdr_edits, \@recip_cluster, $done_all); } # Mail body mangling (defanging, sanitizing or adding disclaimers); # Prepare mail body replacement for the first recipient # in the @$per_recip_data list (which contains a subset of recipients # with the same mail edits, to be dispatched next as one message) # sub prepare_modified_mail($$$$) { my($msginfo, $hold, $any_undecipherable, $per_recip_data) = @_; my($body_modified) = 0; for my $r (@$per_recip_data) { # a subset of recipients! my($recip) = $r->recip_addr; my($mail_mangle) = $r->mail_body_mangle; my($actual_mail_mangle); if (!$mail_mangle) { # skip } elsif ($mail_mangle =~ /^(?:null|nulldisclaimer)\z/i) { # for testing $body_modified = 1; # pretend mail was modified while actually it was not section_time('mangle-'.$mail_mangle); } elsif (( lc($mail_mangle) ne 'attach' && ($enable_anomy_sanitizer || $altermime ne '') ) || $mail_mangle =~ /^(?:anomy|altermime|disclaimer)\z/i) { do_log(2,"mangling by: %s, <%s>", $mail_mangle,$recip); my($orig_fn) = $msginfo->mail_text_fn; my($repl_fn) = $msginfo->mail_tempdir . '/email-repl.txt'; my($inp_fh) = $msginfo->mail_text; my($out_fh); my($repl_size); eval { $inp_fh->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; $out_fh = IO::File->new; $out_fh->open($repl_fn, O_CREAT|O_EXCL|O_WRONLY, 0640) or die "Can't create file $repl_fn: $!"; binmode($out_fh,':bytes') or die "Can't cancel :utf8 mode: $!"; if (lc $mail_mangle eq 'anomy' && !$enable_anomy_sanitizer) { die 'Anomy requested, but $enable_anomy_sanitizer is false'; } elsif ($enable_anomy_sanitizer && $mail_mangle !~ /^(?:altermime|disclaimer)\z/) { $actual_mail_mangle = 'anomy'; $enable_anomy_sanitizer or die "Anomy not available: $mail_mangle"; my(@scanner_conf); my($e); my($engine) = Anomy::Sanitizer->new; if ($e = $engine->error) { die $e } $engine->configure(@scanner_conf, @{ca('anomy_sanitizer_args')}); if ($e = $engine->error) { die $e } my($ret) = $engine->sanitize($msginfo->mail_text, $out_fh); if ($e = $engine->error) { die $e } # close flushes buffers, makes it possible to check file size below $out_fh->close or die "Can't close file $repl_fn: $!"; # re-open as read-only $out_fh = IO::File->new; $out_fh->open($repl_fn,'<') or die "Can't open file $repl_fn: $!"; binmode($out_fh,':bytes') or die "Can't cancel :utf8 mode: $!"; } else { # use altermime for adding disclaimers or defanging $actual_mail_mangle = 'altermime'; $altermime ne '' or die "altermime not available: $mail_mangle"; # prepare arguments to altermime my(@altermime_args); my($disclaimer_options); if (lc($mail_mangle) ne 'disclaimer') { # defang: no by-sender opts. @altermime_args = @{ca('altermime_args_defang')}; } else { # disclaimer @altermime_args = @{ca('altermime_args_disclaimer')}; my($opt_maps) = ca('disclaimer_options_bysender_maps'); if (defined($opt_maps) && @$opt_maps && # by sender options? grep(/_OPTION_/,@altermime_args)) { # determine whose by-sender options to use my($fm) = $msginfo->rfc2822_from; my($rf) = $msginfo->rfc2822_resent_from; my($rs) = $msginfo->rfc2822_resent_sender; my(@rfc2822_from) = !defined($fm) ? () : ref $fm ? @$fm : $fm; my(@rfc2822_resent_from, @rfc2822_resent_sender); @rfc2822_resent_from = @$rf if defined $rf; @rfc2822_resent_sender = @$rs if defined $rs; # see comments in dkim_make_signatures my(@search_list); # collects candidate originator addresses # author addresses go first push(@search_list, map([$_,'2822.From'], @rfc2822_from)); # merge Resent-From and Resent-Sender addresses by resent blocks while (@rfc2822_resent_from || @rfc2822_resent_sender) { while (@rfc2822_resent_from) { my($addr) = shift(@rfc2822_resent_from); last if !defined $addr; # undef delimits resent blocks push(@search_list, [$addr, '2822.Resent-From']); } while (@rfc2822_resent_sender) { my($addr) = shift(@rfc2822_resent_sender); last if !defined $addr; # undef delimits resent blocks push(@search_list, [$addr, '2822.Resent-Sender']); } } push(@search_list, [$msginfo->rfc2822_sender, '2822.Sender']); push(@search_list, [$msginfo->sender, '2821.mail_from']); # # find disclaimer options pertaining to the # most appropriate originator address my(%addr_seen); for my $pair (@search_list) { my($addr,$addr_src) = @$pair; next if !defined($addr) || $addr eq ''; next if $addr_seen{$addr}++; do_log(5,"disclaimer options lookup (%s) %s",$addr_src,$addr); next if !lookup2(0,$addr, ca('local_domains_maps')); my($opt,$matchingkey) = lookup2(0,$addr,$opt_maps); if (defined $opt) { $disclaimer_options = $opt; do_log(3,"disclaimer options pertaining to (%s) %s: %s", $addr_src, $addr, $disclaimer_options); last; } } s/_OPTION_/$disclaimer_options/gs for @altermime_args; } } ### copy original mail to $repl_fn, altermime can't handle stdin well my($nbytes,$buff); while (($nbytes=$inp_fh->read($buff,16384)) > 0) { $out_fh->print($buff) or die "Error writing to $repl_fn: $!" } defined $nbytes or die "Error reading mail file: $!"; $out_fh->close or die "Can't close file $repl_fn: $!"; undef $out_fh; undef $buff; # release storage my($proc_fh,$pid) = run_command(undef, '&1', $altermime, "--input=$repl_fn", @altermime_args); my($r,$status) = collect_results($proc_fh,$pid,$altermime,16384,[0]); undef $proc_fh; undef $pid; do_log(2,"program %s said: %s", $altermime, $$r) if ref $r && $$r ne ''; $status == 0 or die "Program $altermime failed: $status, $$r"; $out_fh = IO::File->new; $out_fh->open($repl_fn,'<') or die "Can't open file $repl_fn: $!"; binmode($out_fh,':bytes') or die "Can't cancel :utf8 mode: $!"; } my($errn) = lstat($repl_fn) ? 0 : 0+$!; if ($errn) { die "Replacement $repl_fn inaccessible: $!" } else { $repl_size = 0 + (-s _) } 1; } or do { $@ = "errno=$!" if $@ eq '' }; if ($@ ne '' || $repl_size <= 0) { # handle failure my($msg) = $@ ne '' ? $@ : sprintf("replacement size %d",$repl_size); chomp($msg); do_log(-1,"mangling by %s failed: %s, mail will pass unmodified", $actual_mail_mangle, $msg); if (defined $out_fh) { $out_fh->close or do_log(-1,"Can't close %s: %s", $repl_fn,$!); undef $out_fh; } unlink($repl_fn) or do_log(-1,"Can't remove %s: %s", $repl_fn,$!); if ($actual_mail_mangle eq 'altermime') { # check for leftover files my($repl_tmp_fn) = $repl_fn . '.tmp'; # altermime's temporary file my($errn) = lstat($repl_tmp_fn) ? 0 : 0+$!; if ($errn == ENOENT) {} # fine, does not exist elsif ($errn) { do_log(-1,"Temporary file %s is inaccessible: %s",$repl_tmp_fn,$!); } else { # cleanup after failing altermime unlink($repl_tmp_fn) or do_log(-1,"Can't remove %s: %s",$repl_tmp_fn,$!); } } } else { do_log(1,"mangling by %s (%s) done, new size: %d, orig %d bytes", $actual_mail_mangle, $mail_mangle, $repl_size, $msginfo->msg_size); # don't close or delete the original file, we'll still need it $msginfo->mail_text($out_fh); $msginfo->mail_text_fn($repl_fn); $msginfo->skip_bytes(0); $body_modified = 1; } section_time('mangle-'.$actual_mail_mangle); } else { # 'attach' (default) - poor-man's defanging of dangerous contents do_log(2,"mangling by built-in defanger: %s, <%s>", $mail_mangle,$recip); $actual_mail_mangle = 'attach'; my(@explanation); my($spam_summary_inserted) = 0; my(@df_pairs) = $r->setting_by_main_contents_category_all(cr('defang_maps_by_ccat')); for my $pair (@df_pairs) { # collect all defanging reasons that apply my($cc,$mangle_map_ref) = @$pair; my($df) = !defined($mangle_map_ref) ? undef : !ref($mangle_map_ref) ? $mangle_map_ref # compatibility : lookup2(0,$recip,$mangle_map_ref, Label=>'Mangling2'); # the $r->mail_body_mangle happens to be the first noteworthy $df do_log(4,'defang? ccat "%s": %s', $cc,$df); next if !$df; my($ccm) = ccat_maj($cc); if ($ccm==CC_VIRUS) { my($virusname_list) = $msginfo->virusnames; push(@explanation, 'WARNING: contains virus ' . (!defined($virusname_list) ? '' : join(", ",@$virusname_list))); } if ($ccm==CC_BANNED) { push(@explanation, "WARNING: banning rules detected suspect part(s),\n". "do not open unless you know what you are doing"); } if ($ccm==CC_UNCHECKED) { if ($hold ne '') { push(@explanation, "WARNING: NOT CHECKED FOR VIRUSES (mail bomb?):\n $hold"); } elsif ($any_undecipherable) { push(@explanation, "WARNING: contains undecipherable part"); } } if ($ccm==CC_BADH) { my($bad) = join(' ',@bad_headers); if (length($bad) > 1000) { $bad = substr($bad,0,1000) . "..." } push(@explanation, split(/\n/, wrap_string('WARNING: bad headers - '.$bad, 78,'',' ') )); } push(@explanation, 'WARNING: oversized') if $ccm==CC_OVERSIZED; if (!$spam_summary_inserted && # can be both CC_SPAMMY and CC_SPAM ($ccm==CC_SPAM || $ccm==CC_SPAMMY)) { push(@explanation, split(/\n/, $msginfo->spam_summary)); $spam_summary_inserted = 1; } } my($s) = join(' ',@explanation); do_log(1, "DEFANGING MAIL: %s", length($s) <= 150 ? $s : substr($s,0,150-3).'[...]'); for (@explanation) { if (length($_) > 100) { $_ = substr($_,0,100-3) . '...' } } $_ .= "\n" for (@explanation); # append newlines my($d) = defanged_mime_entity($msginfo,\@explanation); $msginfo->mail_text($d); # substitute mail with a rewritten version $msginfo->mail_text_fn(undef); # remove filename information $msginfo->skip_bytes(0); $body_modified = 1; section_time('defang'); } # actually the 'for' loop is bogus and runs only once, all recipients # listed in the argument are known to be using the same setting for # $r->mail_body_mangle, ensured by add_forwarding_header_edits_per_recip; # just exit the loop last; } $body_modified; } sub do_quarantine($$$$;@) { shift(@_) if $_[0]->isa('Amavis::In::Connection'); # for compatibility my($msginfo, $hdr_edits_inherited, $recips_ref, $quarantine_method, @snmp_id) = @_; if ($quarantine_method eq '') { do_log(5, "quarantine disabled"); } else { local($1); my($quar_m_protocol) = !ref $quarantine_method ? $quarantine_method : $quarantine_method->[0]; $quar_m_protocol = lc $1 if $quar_m_protocol =~ /^([a-z][a-z0-9.+-]*):/si; my($quar_msg) = Amavis::In::Message->new; $quar_msg->rx_time($msginfo->rx_time); # copy the reception time $quar_msg->log_id($msginfo->log_id); # use the same log_id $quar_msg->partition_tag($msginfo->partition_tag); # same partition_tag $quar_msg->conn_obj($msginfo->conn_obj); $quar_msg->mail_id($msginfo->mail_id); # use the same mail_id $quar_msg->body_type($msginfo->body_type); # use the same BODY= type $quar_msg->header_8bit($msginfo->header_8bit); $quar_msg->body_8bit($msginfo->body_8bit); $quar_msg->msg_size($msginfo->msg_size); $quar_msg->body_digest($msginfo->body_digest); # copy original digest $quar_msg->dsn_ret($msginfo->dsn_ret); $quar_msg->dsn_envid($msginfo->dsn_envid); $quar_msg->auth_submitter($msginfo->sender_smtp); $quar_msg->auth_user(c('amavis_auth_user')); $quar_msg->auth_pass(c('amavis_auth_pass')); $quar_msg->originating(0); # disables DKIM signing my($orig_env_sender_retained, $orig_env_recips_retained); my($mftq) = c('mailfrom_to_quarantine'); if (!defined $mftq || $quar_m_protocol =~ /^(?:bsmtp|sql)\z/) { # we keep the original envelope sender address if replacement sender # is not provided, or with quarantine methods which store to fixed # locations which do not depend on envelope $quar_msg->sender($msginfo->sender); # original sender $quar_msg->sender_smtp($msginfo->sender_smtp); $orig_env_sender_retained = 1; } elsif (defined $mftq) { # have a replacement and smtp, lmtp, pipe, local $quar_msg->sender($mftq); $mftq = qquote_rfc2821_local($mftq); $quar_msg->sender_smtp($mftq); $quar_msg->auth_submitter($mftq); } my(@recips); if (!$recips_ref || $quar_m_protocol =~ /^(?:bsmtp|sql)\z/) { # we keep the original envelope recipients if replacement recipients # are not provided, or with quarantine methods which store to fixed # locations which do not depend on envelope information for my $r (@{$msginfo->per_recip_data}) { my($recip_obj) = Amavis::In::Message::PerRecip->new; # copy original recipient addresses and DSN info $recip_obj->recip_addr($r->recip_addr); $recip_obj->recip_addr_smtp($r->recip_addr_smtp); $recip_obj->dsn_orcpt($r->dsn_orcpt); $recip_obj->recip_destiny(D_PASS); $recip_obj->dsn_notify(['NEVER']) if $orig_env_sender_retained; $recip_obj->delivery_method($quarantine_method); push(@recips,$recip_obj); } $orig_env_recips_retained = 1; } else { # have a replacement and smtp, lmtp, pipe, local # with these quarantine methods the envelope information is used to # determine where and how to store a quarantined message, and may not # reflect original envelope sender and recipients addresses for my $rec (@$recips_ref) { # use recipients provided by a caller my($recip_obj) = Amavis::In::Message::PerRecip->new; $recip_obj->recip_addr($rec); $recip_obj->recip_addr_smtp(qquote_rfc2821_local($rec)); $recip_obj->recip_destiny(D_PASS); $recip_obj->dsn_notify(['NEVER']) if $orig_env_sender_retained; $recip_obj->delivery_method($quarantine_method); push(@recips,$recip_obj); } } $quar_msg->per_recip_data(\@recips); my($hdr_edits) = Amavis::Out::EditHeader->new; $hdr_edits->inherit_header_edits($hdr_edits_inherited); if (defined $msginfo->mail_id) { $hdr_edits->prepend_header('X-Quarantine-ID', '<'.$msginfo->mail_id.'>'); } if ($quar_m_protocol ne 'bsmtp') { # NOTE: RFC 2821 mentions possible header flds X-SMTP-MAIL & X-SMTP-RCPT # Exim uses: Envelope-To, Sendmail uses X-Envelope-To; # No need with bsmtp, which preserves the envelope. my(@blocked_recips) = map($_->recip_addr_smtp, grep($_->recip_done, @{$msginfo->per_recip_data})); $hdr_edits->prepend_header('X-Envelope-To-Blocked', join(",\n ", @blocked_recips), 1); $hdr_edits->prepend_header('X-Envelope-To', join(",\n ", map($_->recip_addr_smtp, @{$msginfo->per_recip_data})),1); } if (!$orig_env_sender_retained) { # unless X-Envelope-* would be redundant $hdr_edits->prepend_header('X-Envelope-From', $msginfo->sender_smtp); } $hdr_edits->add_header('Received', make_received_header_field($msginfo,1), 1); $quar_msg->header_edits($hdr_edits); $quar_msg->mail_text($msginfo->mail_text); # use the same mail contents $quar_msg->skip_bytes($msginfo->skip_bytes); if (ll(5)) { my($quar_m_displ) = !ref $quarantine_method ? $quarantine_method : '('.join(', ',@$quarantine_method).')'; do_log(5,"DO_QUARANTINE, %s, %s -> %s", $quar_m_displ, $quar_msg->sender_smtp, join(', ', map($_->recip_addr_smtp, @{$quar_msg->per_recip_data})) ); } snmp_count('QuarMsgs'); snmp_count( ['QuarMsgsSize', $quar_msg->msg_size, 'C64'] ); mail_dispatch($quar_msg, 'Quar', 0); my($n_smtp_resp, $n_exit_code, $n_dsn_needed) = one_response_for_all($quar_msg, 0); # check status if ($n_smtp_resp =~ /^2/ && !$n_dsn_needed) { # ok @snmp_id = ('Other') if !@snmp_id; for (unique_list(@snmp_id)) { snmp_count('QuarMsgs'.$_); snmp_count( ['QuarMsgsSize'.$_, $quar_msg->msg_size, 'C64'] ); } my $any_arch = grep($_ eq 'Arch', @snmp_id); my $any_nonarch = grep($_ ne 'Arch', @snmp_id); my $act_perf = $msginfo->actions_performed; $msginfo->actions_performed($act_perf=[]) if !$act_perf; if ($any_nonarch && !grep($_ eq 'Quarantined', @$act_perf)) { push(@$act_perf, 'Quarantined'); } if ($any_arch && !grep($_ eq 'Archived', @$act_perf)) { push(@$act_perf, 'Archived'); } } elsif ($n_smtp_resp =~ /^4/) { snmp_count('QuarAttemptTempFails'); die "temporarily unable to quarantine: $n_smtp_resp"; } else { # abort if quarantining not successful snmp_count('QuarAttemptFails'); die "Can't quarantine: $n_smtp_resp"; } my($quar_type); my(@qa); my(%seen); # collect unique quarantine mailboxes or addresses my($existing_qa) = $msginfo->quarantined_to; if (ref $existing_qa) { @qa = @$existing_qa; $seen{$_}++ for @qa } for my $r (@{$quar_msg->per_recip_data}) { my($mbxname) = $r->recip_mbxname; if ($mbxname ne '' && !$seen{$mbxname}++) { unshift(@qa,$mbxname); local($_) = $quar_m_protocol; $quar_type = $_ eq 'smtp' ? 'M' : $_ eq 'lmtp' ? 'L' : $_ eq 'bsmtp' ? 'B' : $_ eq 'sql' ? 'Q' : $_ eq 'local' ? ($mbxname=~/\@/ ? 'M' : $mbxname=~/\.gz\z/ ? 'Z' : 'F') : '?'; } } # remember only the first quarantine method $msginfo->quar_type($quar_type) if !defined $msginfo->quar_type; $msginfo->quarantined_to(!@qa ? undef : \@qa); # remember quar. location do_log(5, "DO_QUARANTINE done"); } } # prepare header edits for the quarantined message # sub prepare_header_edits_for_quarantine($) { my($msginfo) = @_; my($blacklisted_any,$whitelisted_any) = (0,0); my($do_tag_any,$do_tag2_any,$do_kill_any) = (0,0,0); my($tag_level_min,$tag2_level_min,$kill_level_min); my(%all_spam_tests); my($min_spam_level, $max_spam_level) = minmax(map($_->spam_level, @{$msginfo->per_recip_data})); for my $r (@{$msginfo->per_recip_data}) { my($rec) = $r->recip_addr; my($spam_level) = $r->spam_level; if (ll(2)) { my($blocking_ccat) = $r->blocking_ccat; my($rec_ccat_maj,$rec_ccat_min) = ccat_split( defined $blocking_ccat ? $blocking_ccat : $r->contents_category); my($ccat,$ccat_min) = ccat_split($msginfo->contents_category); do_log(2,"header_edits_for_quar: rec_bl_ccat=(%d,%d), ccat=(%d,%d) %s", $rec_ccat_maj, $rec_ccat_min, $ccat, $ccat_min, $rec) if $rec_ccat_maj != $ccat || $rec_ccat_min != $ccat_min; } my($tag_level,$tag2_level,$kill_level,$do_tag,$do_tag2,$do_kill); $do_tag = $r->is_in_contents_category(CC_CLEAN,1); $do_tag2 = $r->is_in_contents_category(CC_SPAMMY); $do_kill = $r->is_in_contents_category(CC_SPAM); if (!$r->bypass_spam_checks && ($do_tag || $do_tag2 || $do_kill)) { # do the more expensive lookups only when needed $tag_level = lookup2(0,$rec, ca('spam_tag_level_maps')); $tag2_level = lookup2(0,$rec, ca('spam_tag2_level_maps')); $kill_level = lookup2(0,$rec, ca('spam_kill_level_maps')); } # summarize $blacklisted_any = 1 if $r->recip_blacklisted_sender; $whitelisted_any = 1 if $r->recip_whitelisted_sender; $tag_level_min = $tag_level if defined($tag_level) && $tag_level ne '' && (!defined($tag_level_min) || $tag_level < $tag_level_min); $tag2_level_min = $tag2_level if defined($tag2_level) && (!defined($tag2_level_min) || $tag2_level < $tag2_level_min); $kill_level_min = $kill_level if defined($kill_level) && (!defined($kill_level_min) || $kill_level < $kill_level_min); $do_tag_any = 1 if $do_tag; $do_tag2_any = 1 if $do_tag2; $do_kill_any = 1 if $do_kill; my($spam_tests) = $r->spam_tests; if (defined $spam_tests) { $all_spam_tests{$_} = 1 for split(/,/, join(',',map($$_,@$spam_tests))); } } my(%header_field_provided); # mainly applies to spam header fields my($use_our_hdrs) = cr('prefer_our_added_header_fields'); my($allowed_hdrs) = cr('allowed_added_header_fields'); my($hdr_edits) = Amavis::Out::EditHeader->new; if ($allowed_hdrs && $allowed_hdrs->{lc('X-Amavis-Alert')}) { if ($msginfo->is_in_contents_category(CC_VIRUS)) { my($virusname_list) = $msginfo->virusnames; $hdr_edits->add_header('X-Amavis-Alert', "INFECTED, message contains virus: " . (!defined($virusname_list) ? '' : join(", ",@$virusname_list)) ); } if ($msginfo->is_in_contents_category(CC_BANNED)) { for my $r (@{$msginfo->per_recip_data}) { if (defined($r->banning_reason_short)) { $hdr_edits->add_header('X-Amavis-Alert', 'BANNED, message contains ' . $r->banning_reason_short); last; # fudge: only the first recipient's banned hit will be shown } } } if ($msginfo->is_in_contents_category(CC_BADH)) { $hdr_edits->add_header('X-Amavis-Alert', 'BAD HEADER SECTION '.$bad_headers[0]); } } if ($allowed_hdrs) { for ('X-Amavis-OS-Fingerprint') { my($p0f) = $msginfo->client_os_fingerprint; if (defined($p0f) && $p0f ne '' && $allowed_hdrs->{lc $_}) { $hdr_edits->add_header($_, sanitize_str($p0f)); } } } if ($allowed_hdrs && $use_our_hdrs) { my($spam_level_bar); my($slc) = c('sa_spam_level_char'); $spam_level_bar = $slc x min(64, $whitelisted_any ? 0 : $blacklisted_any ? 64 : 0+$max_spam_level) if $slc ne ''; # allow header field wrapping at any comma my($s) = join(",\n ", sort keys %all_spam_tests); my($sl) = 'x'; if (defined $min_spam_level) { my($minsl) = 0+sprintf("%.3f",$min_spam_level); my($maxsl) = 0+sprintf("%.3f",$max_spam_level); $sl = $minsl eq $maxsl ? $minsl : "$minsl..$maxsl"; } my($autolearn_status) = $msginfo->supplementary_info('AUTOLEARN'); my($full_spam_status) = sprintf( "%s,\n score=%s\n tag=%s\n tag2=%s\n kill=%s\n ". "%stests=[%s]\n autolearn=%s", $do_tag2_any||$do_kill_any ? 'Yes' : 'No', $sl, (map { !defined $_ ? 'x' : 0+sprintf("%.3f",$_) } ($tag_level_min, $tag2_level_min, $kill_level_min)), join('', $blacklisted_any ? "BLACKLISTED\n " : (), $whitelisted_any ? "WHITELISTED\n " : ()), $s, $autolearn_status||'unavailable'); if (ll(2)) { # log entry semi-compatible with older log parsers my($s) = $full_spam_status; $s =~ s/\n[ \t]/ /g; do_log(2,"header_edits_for_quar: %s -> %s, %s", $msginfo->sender_smtp, join(',', qquote_rfc2821_local(@{$msginfo->recips})), $s); } for ('X-Spam-Flag') { if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { $hdr_edits->add_header($_, $do_tag2_any ? 'YES' : 'NO'); $header_field_provided{lc $_} = 1; } } for ('X-Spam-Score') { if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { my($score) = 0+$max_spam_level; $score = max(64,$score) if $blacklisted_any; # not below 64 if bl $score = min( 0,$score) if $whitelisted_any; # not above 0 if wl $hdr_edits->add_header($_, 0+sprintf("%.3f",$score)); $header_field_provided{lc $_} = 1; } } for ('X-Spam-Level') { if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { $hdr_edits->add_header($_, $spam_level_bar) if defined $spam_level_bar; $header_field_provided{lc $_} = 1; } } for ('X-Spam-Status') { if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { $hdr_edits->add_header($_, $full_spam_status, 1); $header_field_provided{lc $_} = 1; } } for ('X-Spam-Report') { if ($allowed_hdrs->{lc $_} && $use_our_hdrs->{lc $_}) { my($report) = $msginfo->spam_report; if (defined $report && $report ne '') { $hdr_edits->add_header($_, "\n".sanitize_str($report,1), 2); } $header_field_provided{lc $_} = 1; } } } if ($allowed_hdrs) { # add remaining header fields as provided by spam scanners my($sa_header) = $msginfo->supplementary_info( $do_tag2_any ? 'ADDEDHEADERSPAM' : 'ADDEDHEADERHAM'); if (defined $sa_header && $sa_header ne '') { for my $hf (split(/^(?![ \t])/m, $sa_header, -1)) { local($1,$2); if ($hf =~ /^([!-9;-\176]+)[ \t]*:(.*)\z/s) { my($hf_name,$hf_body) = ($1,$2); my($hf_name_lc) = lc($hf_name); chomp($hf_body); if ($header_field_provided{$hf_name_lc}) { do_log(5,'quar: scanner provided %s, but we preferred our own', $hf_name); } elsif (!$allowed_hdrs->{$hf_name_lc}) { do_log(5,'quar: scanner provided %s, '. 'inhibited by %%allowed_added_header_fields', $hf_name); } else { do_log(5,'quar: scanner provided %s, inserting', $hf_name); $hdr_edits->add_header($hf_name, $hf_body, 2); } } } } for my $pair ( ['DSPAMRESULT', 'X-DSPAM-Result'], ['DSPAMSIGNATURE', 'X-DSPAM-Signature'], ['CRM114STATUS', 'X-CRM114-Status'], ['CRM114CACHEID', 'X-CRM114-CacheID'] ) { my($suppl_attr_name, $hf_name) = @$pair; my($suppl_attr_val) = $msginfo->supplementary_info($suppl_attr_name); if (defined $suppl_attr_val && $suppl_attr_val ne '') { if (!$allowed_hdrs->{lc $hf_name}) { do_log(5,'quar: scanner provided %s, '. 'inhibited by %%allowed_added_header_fields', $hf_name); } else { do_log(5,'quar: scanner provided %s, inserting', $hf_name); $hdr_edits->add_header($hf_name, sanitize_str($suppl_attr_val), 2); } } } } if (c('enable_dkim_verification') && $allowed_hdrs && $allowed_hdrs->{lc('Authentication-Results')}) { for my $h (Amavis::DKIM::generate_authentication_results($msginfo,0)) { $hdr_edits->add_header('Authentication-Results', $h, 1) } } section_time('quar-hdrs'); $hdr_edits; } # Quarantine according to contents and send admin & recip notif. as needed # (this subroutine replaces the former subroutines do_virus and do_spam) # sub do_notify_and_quarantine($$) { my($msginfo, $virus_dejavu) = @_; my($mailfrom_admin, $hdrfrom_admin, $notify_admin_templ_ref) = map { scalar($msginfo->setting_by_contents_category(cr($_))) } qw(mailfrom_notify_admin_by_ccat hdrfrom_notify_admin_by_ccat notify_admin_templ_by_ccat); my($qar_method) = c('archive_quarantine_method'); my(@ccat_names_pairs) = $msginfo->setting_by_main_contents_category_all(\%ccat_display_names); my($ccat,$ccat_min) = ccat_split($msginfo->contents_category); if (ll(3)) { my($ccat_name) = ref $ccat_names_pairs[0] ? $ccat_names_pairs[0][1] :undef; do_log(3,"do_notify_and_quar: ccat=%s (%d,%d) (%s) ccat_block=(%s)". ", qar_mth=%s", $ccat_name, $ccat, $ccat_min, join(', ', map(sprintf('"%s":%s', $_->[0], $_->[1]), @ccat_names_pairs)), $msginfo->blocking_ccat, $qar_method); } my($virusname_list) = $msginfo->virusnames; my($newvirus_admin_maps_ref) = defined($virusname_list) && @$virusname_list && !$virus_dejavu ? ca('newvirus_admin_maps') : undef; my $archive_any = 0; my $archive_transparent = 1; if (defined $qar_method && $qar_method ne '') { # archiving quarantine # test if @archive_quarantine_to_maps for all recipients yields # a magic placeholder '%a', indicating we want transparent archiving # which retains unmodified envelope recipient addresses my $aqtm = ca('archive_quarantine_to_maps'); for my $r (@{$msginfo->per_recip_data}) { my($q) = lookup2(0, $r->recip_addr, $aqtm); $archive_any = 1 if defined $q && $q ne ''; $archive_transparent = 0 if !defined $q || $q ne '%a'; last if $archive_any && !$archive_transparent; } } my(@q_tuples,@a_addr); # per-recip quarantine address(es) and admins for my $r (@{$msginfo->per_recip_data}) { my($rec) = $r->recip_addr; my($blacklisted) = $r->recip_blacklisted_sender; my($whitelisted) = $r->recip_whitelisted_sender; my($spam_level) = $r->spam_level; # an alternative approach to determining which quarantine and notif. to take # my(@qmqta_tuples) = $r->setting_by_main_contents_category_all( # cr('quarantine_method_by_ccat'), cr('quarantine_to_maps_by_ccat'), # cr('admin_maps_by_ccat') ); # my($qq); # quarantine (pseudo) address associated with the recipient # my($quarantining_reason_ccat); # for my $tuple (@qmqta_tuples) { # my($cc, $q_method, $quarantine_to_maps_ref, $admin_maps_ref) = @$tuple; # if (defined($q_method) && $q_method ne '' && $quarantine_to_maps_ref) { # my($q) = lookup2(0,$rec,$quarantine_to_maps_ref); # if (defined $q && $q ne '') # { $qq = $q; $quarantining_reason_ccat = $cc; last } # } # } # my($aa); # administrator's e-mail address # my($admin_notif_reason_ccat); # for my $tuple (@qmqta_tuples) { # my($cc, $q_method, $quarantine_to_maps_ref, $admin_maps_ref) = @$tuple; # if ($admin_maps_ref) { # my($a) = lookup2(0,$rec,$admin_maps_ref); # if (defined $a && $a ne '') # { $aa = $a; $admin_notif_reason_ccat = $cc; last } # } # } # ($rec_ccat_maj,$rec_ccat_min) = ccat_split($quarantining_reason_ccat); my($blocking_ccat) = $r->blocking_ccat; my($rec_ccat_maj,$rec_ccat_min) = ccat_split( defined $blocking_ccat ? $blocking_ccat : $r->contents_category); my($q_method) = $r->setting_by_contents_category(cr('quarantine_method_by_ccat')); my($quarantine_to_maps_ref) = $r->setting_by_contents_category(cr('quarantine_to_maps_by_ccat')); # get per-recipient quarantine address(es) and admins if (!defined($q_method) || $q_method eq '') { do_log(5,"do_notify_and_quarantine: not quarantining, q_method off"); } elsif (!$quarantine_to_maps_ref) { do_log(5,"do_notify_and_quarantine: not quarantining, null q_to maps"); } else { my($q); # quarantine (pseudo) address associated with the recipient ($q) = lookup2(0,$rec,$quarantine_to_maps_ref); if (defined $q && $q ne '' && ($rec_ccat_maj==CC_SPAM || $rec_ccat_maj==CC_SPAMMY)) { # consider suppressing spam quarantine my($cutoff) = lookup2(0,$rec, ca('spam_quarantine_cutoff_level_maps')); if (!defined $cutoff || $cutoff eq '') { # no cutoff, quarantining all } elsif ($blacklisted && !$whitelisted) { do_log(2,"do_notify_and_quarantine: cutoff, blacklisted"); $q = ''; # disable quarantine on behalf of this recipient } elsif (($spam_level||0) >= $cutoff) { do_log(2,"do_notify_and_quarantine: spam level exceeds ". "quarantine cutoff level %s", $cutoff); $q = ''; # disable quarantine on behalf of this recipient } } # keep original recipient when q_to is '%a' or with BSMTP; some day # we may end up doing %k, %a, %l, %u, %e, %d placeholder replacements $q = $rec if defined $q && $q ne '' && ($q eq '%a' || $q_method =~ /^bsmtp:/i); if (!defined($q) || $q eq '') { do_log(5,"do_notify_and_quarantine: not quarantining, q_to off"); } else { my($ccat_name_major) = $r->setting_by_contents_category(\%ccat_display_names_major); push(@q_tuples, [$q_method, $q, $ccat_name_major]); } } my($admin_maps_ref) = $r->setting_by_contents_category(cr('admin_maps_by_ccat')); my($a); # administrator's e-mail address ($a) = lookup2(0,$rec,$admin_maps_ref) if $admin_maps_ref; if (defined $a && $a ne '' && ($rec_ccat_maj==CC_SPAM || $rec_ccat_maj==CC_SPAMMY)) { # consider suppressing spam admin notifications my($cutoff) = lookup2(0,$rec, ca('spam_notifyadmin_cutoff_level_maps')); if (!defined $cutoff || $cutoff eq '') { # no cutoff, sending administrator notifications } elsif ($blacklisted && !$whitelisted) { do_log(2,"do_notify_and_quarantine: spam admin cutoff, blacklisted"); $a = ''; # disable admin notification on behalf of this recipient } elsif (($spam_level||0) >= $cutoff) { do_log(2,"do_notify_and_quarantine: spam level exceeds ". "spam admin cutoff level %s", $cutoff); $a = ''; # disable admin notification on behalf of this recipient } } push(@a_addr, $a) if defined $a && $a ne '' && !grep($_ eq $a, @a_addr); if (ccat_maj($r->contents_category)==CC_VIRUS && $newvirus_admin_maps_ref){ ($a) = lookup2(0,$rec,$newvirus_admin_maps_ref); push(@a_addr, $a) if defined $a && $a ne '' && !grep($_ eq $a, @a_addr); } if ($archive_any && !$archive_transparent) { # archiving quarantine my($q) = lookup2(0,$rec, ca('archive_quarantine_to_maps')); # keep original recipient when q_to is '%a' or with BSMTP $q = $rec if defined $q && $q ne '' && ($q eq '%a' || $qar_method =~ /^bsmtp:/i); push(@q_tuples, [$qar_method, $q, 'Arch']) if defined $q && $q ne ''; } } # endfor per_recip_data if ($ccat == CC_SPAM) { my($sqbsm) = ca('spam_quarantine_bysender_to_maps'); if (@$sqbsm) { # by-sender spam quarantine (hardly useful, rarely used) my($q) = lookup2(0,$msginfo->sender, $sqbsm); if (defined $q && $q ne '') { my($msg_q_method) = $msginfo->setting_by_contents_category( cr('quarantine_method_by_ccat')); push(@q_tuples, [$msg_q_method, $q, 'Spam']) if defined $msg_q_method && $msg_q_method ne ''; } } } section_time('notif-quar'); if (@q_tuples || $archive_any) { if (!defined($msginfo->mail_id) && grep($_->[2] ne 'Arch', @q_tuples)) { # delayed mail_id generation - now we really need it $snmp_db->register_proc(2,0,'G',$msginfo->log_id) if defined $snmp_db; # create a mail_id unique to a database and save preliminary info to SQL generate_unique_mail_id($msginfo); section_time('gen_mail_id') if $sql_storage; } # compatibility: replace quarantine method 'local:xxx' # with $notify_method when quarantine_to looks like an e-mail address my($notif_m) = c('notify_method'); for my $tuple (@q_tuples) { my($q_method,$q_to,$ccat_name) = @$tuple; $tuple->[0] = $notif_m if $q_method =~ /^local:/i && $q_to =~ /\@/; } my($hdr_edits) = prepare_header_edits_for_quarantine($msginfo); if (@q_tuples) { do_log(4,"do_notify_and_quarantine: quarantine %s", join(',', map($_->[1], @q_tuples))); my(@q_tuples_tmp) = @q_tuples; while (@q_tuples_tmp) { my($q_method,$q_to,$ccat_name) = @{$q_tuples_tmp[0]}; my(@same_method_tuples) = grep($_->[0] eq $q_method, @q_tuples_tmp); @q_tuples_tmp = grep($_->[0] ne $q_method, @q_tuples_tmp); my(@q_to) = unique_list(map($_->[1], @same_method_tuples)); # per-recipient blocking ccat names select snmp counter names my(@snmp_id) = unique_list(map($_->[2], @same_method_tuples)); do_quarantine($msginfo, $hdr_edits, \@q_to, $q_method, @snmp_id); } } if ($archive_any && $archive_transparent) { # transparent archiving retains envelope recipient addresses do_log(4,"do_notify_and_quarantine: transparent archiving"); do_quarantine($msginfo, $hdr_edits, undef, $qar_method, 'Arch'); } } if (!@a_addr) { do_log(4,"skip admin notification, no administrators"); } elsif (!ref($notify_admin_templ_ref) || (ref($notify_admin_templ_ref) eq 'ARRAY' ? !@$notify_admin_templ_ref : $$notify_admin_templ_ref eq '')) { do_log(5,"skip admin notifications - empty template"); } else { # notify per-recipient administrators ll(5) && do_log(5, "Admin notifications to %s; sender: %s", join(',',qquote_rfc2821_local(@a_addr)), $msginfo->sender_smtp); $hdrfrom_admin = expand_variables($hdrfrom_admin); my($mailfrom_admin_q); if (!defined($mailfrom_admin)) { # defaults to email address in hdrfrom_notify_admin $mailfrom_admin_q = (parse_address_list($hdrfrom_admin))[0]; $mailfrom_admin = unquote_rfc2821_local($mailfrom_admin_q); } $mailfrom_admin_q = qquote_rfc2821_local($mailfrom_admin); my($notification) = Amavis::In::Message->new; $notification->rx_time($msginfo->rx_time); # copy the reception time $notification->log_id($msginfo->log_id); # copy log id $notification->partition_tag($msginfo->partition_tag); # same partition_tag $notification->conn_obj($msginfo->conn_obj); $notification->originating(1); $notification->sender($mailfrom_admin); $notification->sender_smtp($mailfrom_admin_q); $notification->auth_submitter($mailfrom_admin_q); $notification->auth_user(c('amavis_auth_user')); $notification->auth_pass(c('amavis_auth_pass')); $notification->recips([@a_addr]); my($notif_m) = c('notify_method'); $_->delivery_method($notif_m) for @{$notification->per_recip_data}; my(@rfc2822_from_admin) = map(unquote_rfc2821_local($_), parse_address_list($hdrfrom_admin)); $notification->rfc2822_from($rfc2822_from_admin[0]); # if ($mailfrom_admin ne '') # { $_->dsn_notify(['NEVER']) for @{$notification->per_recip_data} } my(%mybuiltins) = %builtins; # make a local copy $mybuiltins{'T'} = [qquote_rfc2821_local(@a_addr)]; # used in To: $mybuiltins{'f'} = $hdrfrom_admin; # From: $notification->mail_text( build_mime_entity(expand($notify_admin_templ_ref,\%mybuiltins), $msginfo, undef,undef,0, 1,0) ); # $notification->body_type('7BIT'); my($hdr_edits) = Amavis::Out::EditHeader->new; $notification->header_edits($hdr_edits); mail_dispatch($notification, 'Notif', 0); my($n_smtp_resp, $n_exit_code, $n_dsn_needed) = one_response_for_all($notification, 0); # check status if ($n_smtp_resp =~ /^2/ && !$n_dsn_needed) { # ok } elsif ($n_smtp_resp =~ /^4/) { die "temporarily unable to notify admin: $n_smtp_resp"; } else { do_log(-1, "FAILED to notify admin: %s", $n_smtp_resp); } # $notification->purge; } # recipient notifications my($wrmbc) = cr('warnrecip_maps_by_ccat'); for my $r (@{$msginfo->per_recip_data}) { my($rec) = $r->recip_addr; my($wr); my($notify_recips_templ_ref); my($warnrecip_maps_ref) = $r->setting_by_contents_category($wrmbc); ($wr) = lookup2(0,$rec,$warnrecip_maps_ref) if $warnrecip_maps_ref; if ($wr) { $notify_recips_templ_ref = $r->setting_by_contents_category(cr('notify_recips_templ_by_ccat')); if (!ref($notify_recips_templ_ref) || (ref($notify_recips_templ_ref) eq 'ARRAY' ? !@$notify_recips_templ_ref : $$notify_recips_templ_ref eq '')){ do_log(5,"skip recipient notifications - empty template"); $wr = 0; # do not send empty notifications } elsif (!c('warn_offsite') && !$r->recip_is_local) { do_log(5,"skip recipient notifications - nonlocal recipient"); $wr = 0; # do not notify foreign recipients # } elsif ($r->recip_destiny == D_PASS) { # do_log(5,"skip recipient notifications - mail will be delivered"); # $wr = 0; # do not notify recips which will be getting a message anyway # } elsif ($msginfo->sender eq '') { # (not general enough) # do_log(5,"skip recipient notifications for null sender"); # $wr = 0; } } if ($wr) { # warn recipient my($mailfrom_recip) = $r->setting_by_contents_category(cr('mailfrom_notify_recip_by_ccat')); my($hdrfrom_recip) = $r->setting_by_contents_category(cr('hdrfrom_notify_recip_by_ccat')); $hdrfrom_recip = expand_variables($hdrfrom_recip); my($mailfrom_recip_q); if (!defined($mailfrom_recip)) { # defaults to email address in hdrfrom_notify_recip $mailfrom_recip_q = (parse_address_list($hdrfrom_recip))[0]; $mailfrom_recip = unquote_rfc2821_local($mailfrom_recip_q); } $mailfrom_recip_q = qquote_rfc2821_local($mailfrom_recip); my($notification) = Amavis::In::Message->new; $notification->rx_time($msginfo->rx_time); # copy the reception time $notification->log_id($msginfo->log_id); # copy log id $notification->partition_tag($msginfo->partition_tag); # same partition $notification->conn_obj($msginfo->conn_obj); $notification->originating(1); $notification->sender($mailfrom_recip); $notification->sender_smtp($mailfrom_recip_q); $notification->auth_submitter($mailfrom_recip_q); $notification->auth_user(c('amavis_auth_user')); $notification->auth_pass(c('amavis_auth_pass')); $notification->recips([$rec]); my($notif_m) = c('notify_method'); $_->delivery_method($notif_m) for @{$notification->per_recip_data}; my(@rfc2822_from_recip) = map(unquote_rfc2821_local($_), parse_address_list($hdrfrom_recip)); $notification->rfc2822_from($rfc2822_from_recip[0]); # if ($mailfrom_recip ne '') # { $_->dsn_notify(['NEVER']) for @{$notification->per_recip_data} } my(@b); @b = @{$r->banned_parts} if defined $r->banned_parts; my($b_chopped) = @b > 2; @b = (@b[0,1],'...') if $b_chopped; s/[ \t]{6,}/ ... /g for @b; my(%mybuiltins) = %builtins; # make a local copy $mybuiltins{'banned_parts'} = \@b; # list of banned parts $mybuiltins{'F'} = $r->banning_reason_short; # just one name & comment $mybuiltins{'banning_rule_comment'} = !defined($r->banning_rule_comment) ? undef : unique_ref($r->banning_rule_comment); $mybuiltins{'banning_rule_rhs'} = !defined($r->banning_rule_rhs) ? undef : unique_ref($r->banning_rule_rhs); $mybuiltins{'f'} = $hdrfrom_recip; # From: $mybuiltins{'T'} = qquote_rfc2821_local($rec); # To: $notification->mail_text( build_mime_entity(expand($notify_recips_templ_ref,\%mybuiltins), $msginfo, undef,undef,0, 0,0) ); # $notification->body_type('7BIT'); my($hdr_edits) = Amavis::Out::EditHeader->new; $notification->header_edits($hdr_edits); mail_dispatch($notification, 'Notif', 0); my($n_smtp_resp, $n_exit_code, $n_dsn_needed) = one_response_for_all($notification, 0); # check status if ($n_smtp_resp =~ /^2/ && !$n_dsn_needed) { # ok } elsif ($n_smtp_resp =~ /^4/) { die "temporarily unable to notify recipient rec: $n_smtp_resp"; } else { do_log(-1, "FAILED to notify recipient %s: %s", $rec,$n_smtp_resp); } # $notification->purge; } } do_log(5, "do_notify_and_quarantine - done"); } # Calculate a message body digest; # While at it, also get message size, verify DKIM signatures, check for 8-bit # data, collect entropy, and store original header section since we need it # for the %H macro, and MIME::Tools may modify its copy. # sub get_body_digest($$) { my($msginfo, $alg) = @_; my($remaining_time, $dkim_deadline) = # sanity limit for DKIM verification get_deadline('get_body_digest', 0.5, 8, 30); prolong_timer('digest_pre'); # restart the timer my($fh) = $msginfo->mail_text; $fh->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; my($hctx,$bctx); # choose a message digest: MD5: 128 bits (32 hex), SHA family: 160..512 bits if (uc($alg) eq 'MD5') { $hctx = Digest::MD5->new; $bctx = Digest::MD5->new } else { $hctx = Digest::SHA->new($alg); $bctx = Digest::SHA->new($alg) } my($dkim_verifier); $dkim_verifier = Mail::DKIM::Verifier->new if c('enable_dkim_verification'); # section_time('digest_init'); my($header_size,$body_size) = (0,0); my($h_8bit,$b_8bit) = (0,0); my($orig_header) = []; # array of header fields, with folding and trailing NL my($orig_header_fields) = {}; my($ln); local($1,$2); do_log(5, "get_body_digest: reading header section"); my($sanity_limit) = 4*1024*1024; # 4 MiB header size sanity limit my($dkim_sanity_limit) = 256*1024; # 256 KiB header size sanity limit for ($! = 0; defined($ln=$fh->getline); $! = 0) { # read mail header section last if $ln eq "\n"; $h_8bit = 1 if !$h_8bit && $ln =~ tr/\000-\177//c; $hctx->add($ln); if ($ln =~ /^[ \t]/) { # header field continuation $$orig_header[-1] .= $ln; # with NL } else { # starts a new header field push(@$orig_header,$ln); # with NL if ($ln =~ /^([^: \t]+)[ \t]*:/si) { # remember array index of the last occurrence of each header field $orig_header_fields->{lc($1)} = $#$orig_header; } } chomp($ln); if (!defined $dkim_verifier) { # don't bother } elsif ($header_size > $dkim_sanity_limit) { do_log(-1,"Stopped feeding header to DKIM verifier: ". "%.0f KiB sanity limit exceeded", $dkim_sanity_limit/1024); undef $dkim_verifier; } elsif (Time::HiRes::time > $dkim_deadline) { do_log(-1,"Stopped feeding header to DKIM verifier: deadline exceeded"); undef $dkim_verifier; } else { eval { $dkim_verifier->PRINT($ln."\015\012") or die "Can't write to dkim: $!"; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"Error feeding header line to DKIM verifier: %s",$eval_stat); undef $dkim_verifier; }; } $header_size += length($ln)+2; # size includes CRLF (RFC 1870) # exceeded $sanity_limit will break DKIM signatures, too bad... last if $header_size > $sanity_limit; } defined $ln || $!==0 or # returning EBADF at EOF is a perl bug $!==EBADF ? do_log(0,"Error reading mail header section: $!") : die "Error reading mail header section: $!"; if (defined $ln) { # true or imagined header/body separator line $header_size += 2; # include a separator line in a header section size if (defined $dkim_verifier) { eval { # h/b separator will trigger signature pre-processing in DKIM module $dkim_verifier->PRINT("\015\012") or die "Can't write to dkim: $!"; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"Error feeding h/b separ to DKIM verifier: %s", $eval_stat); undef $dkim_verifier; }; } } $header_size = untaint($header_size); # length(tainted) stays tainted add_entropy($hctx->digest); section_time('digest_hdr'); # a DNS lookup in Mail::DKIM older than 0.30 stops the timer! # The lookup is performed at a header/body separator line or at CLOSE, at # which point signatures become available through the $dkim_verifier object. prolong_timer('digest_hdr'); # restart timer if stopped my(@dkim_signatures); if (defined $ln) { # only read further if not already at end-of-file # don't bother feeding body to DKIM if there are no signature header fields @dkim_signatures = $dkim_verifier->signatures if defined $dkim_verifier; my($feed_dkim) = @dkim_signatures > 0; if ($feed_dkim) { $msginfo->checks_performed({}) if !$msginfo->checks_performed; $msginfo->checks_performed->{D} = 1; } my($len); local($_); do_log(5, "get_body_digest: reading mail body"); while (($len = $fh->read($_,16384)) > 0) { $bctx->add($_); $body_size += $len + tr/\n//; # count \n compensating for CRLF (RFC 1870) $b_8bit = 1 if !$b_8bit && tr/\000-\177//c; if ($feed_dkim) { s{\n}{\015\012}gs; eval { $dkim_verifier->PRINT($_) or die "Can't write to dkim: $!"; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"Error feeding body to DKIM verifier: %s",$eval_stat); undef $dkim_verifier; }; } } defined $len or die "Error reading mail body: $!"; } if (defined $dkim_verifier) { eval { # this will trigger signature verification in the DKIM module $dkim_verifier->CLOSE or die "Can't close dkim_verifier: $!"; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"Error closing DKIM verifier: %s",$eval_stat); undef $dkim_verifier; }; @dkim_signatures = $dkim_verifier->signatures if defined $dkim_verifier; } prolong_timer('digest_body'); # restart timer if stopped my($body_digest) = $bctx->hexdigest; # my($body_digest) = $bctx->b64digest; add_entropy($body_digest); $body_digest = untaint($body_digest) # checked (hex digits, 128..512 bits) if $body_digest =~ /^ [0-9a-fA-F]{32,128} \z/x; # store information obtained if (@dkim_signatures) { if (@dkim_signatures > 50) { # sanity do_log(-1, "Too many DKIM or DK signatures (%d), truncating to 50", scalar(@dkim_signatures)); $#dkim_signatures = 49; } $msginfo->dkim_signatures_all(\@dkim_signatures); } $msginfo->orig_header_fields($orig_header_fields); # stores just pointers $msginfo->orig_header($orig_header); # header section, without separator line $msginfo->orig_header_size($header_size); # size includes a separator line! $msginfo->orig_body_size($body_size); $msginfo->body_digest($body_digest); $msginfo->header_8bit($h_8bit ? 1 : 0); $msginfo->body_8bit($b_8bit ? 1 : 0); # check for 8-bit characters and adjust body type if necessary (RFC 1652) my($bt_orig) = $msginfo->body_type; $bt_orig = !defined($bt_orig) ? '' : uc($bt_orig); if ($h_8bit || $b_8bit) { # just keep original label whatever it is (garbage-in - garbage-out); # keeping 8-bit mail unlabeled might avoid breaking DKIM in transport # (labeling as 8-bit may invoke 8>7 downgrades in MTA, breaking signatures) } elsif ($bt_orig eq '') { # unlabeled on reception $msginfo->body_type('7BIT'); # safe to label } elsif ($bt_orig eq '8BITMIME') { # redundant (quite common) $msginfo->body_type('7BIT'); # turn a redundant 8BITMIME into 7BIT } if (ll(4)) { my($msg_fmt) = ($bt_orig eq '' && $b_8bit) ? "%s, but 8-bit body" : ($bt_orig eq '' && $h_8bit) ? "%s, but 8-bit header" : ($bt_orig eq '7BIT' && ($h_8bit || $b_8bit)) ? "%s inappropriately" : ($bt_orig eq '8BITMIME' && !($h_8bit || $b_8bit)) ? "%s unnecessarily" : "%s, good"; do_log(4, "body type (ESMTP BODY): $msg_fmt (h=%s, b=%s)", $bt_orig eq '' ? 'unlabeled' : "labeled $bt_orig", $h_8bit,$b_8bit); } do_log(3, "body hash: %s", $body_digest); section_time(defined($dkim_verifier) ? 'digest_body_dkim' : 'digest_body'); $body_digest; } sub find_program_path($$) { my($fv_list, $path_list_ref) = @_; $fv_list = [$fv_list] if !ref $fv_list; my($found); for my $fv (@$fv_list) { my(@fv_cmd) = split(' ',$fv); if (!@fv_cmd) { # empty, not available } elsif ($fv_cmd[0] =~ /^\//) { # absolute path my($errn) = stat($fv_cmd[0]) ? 0 : 0+$!; if ($errn == ENOENT) { } elsif ($errn) { do_log(-1, "find_program_path: %s inaccessible: %s", $fv_cmd[0], $!); } elsif (-x _ && !-d _) { $found = join(' ', @fv_cmd) } } elsif ($fv_cmd[0] =~ /\//) { # relative path die "find_program_path: relative paths not implemented: @fv_cmd\n"; } else { # walk through the specified PATH for my $p (@$path_list_ref) { my($errn) = stat("$p/$fv_cmd[0]") ? 0 : 0+$!; if ($errn == ENOENT) { } elsif ($errn) { do_log(-1, "find_program_path: %s/%s inaccessible: %s", $p, $fv_cmd[0], $!); } elsif (-x _ && !-d _) { $found = $p . '/' . join(' ', @fv_cmd); last; } } } last if defined $found; } $found; } sub find_external_programs($) { my($path_list_ref) = @_; for my $f (qw($file $altermime)) { my($g) = $f; $g =~ s/\$/Amavis::Conf::/; my($fv_list) = eval('$' . $g); my($found) = find_program_path($fv_list, $path_list_ref); { no strict 'refs'; $$g = $found } # NOTE: a symbolic reference if (!defined $found) { do_log(0,"No %-19s not using it", "$f,") } else { do_log(0,"Found %-16s at %s%s", $f, $daemon_chroot_dir ne '' ? "(chroot: $daemon_chroot_dir/) " : '', $found); } } # map program name path hints to full paths for decoders my(%any_st); for my $f (@{ca('decoders')}) { next if !defined $f || !ref $f; # empty, skip my($short_type) = $f->[0]; my(@tried,@found); my($any) = 0; for my $d (@$f[2..$#$f]) { # all but the first two elements are programs # allow one level of indirection my($dd) = (ref $d eq 'SCALAR' || ref $d eq 'REF') ? $$d : $d; my($found) = find_program_path($dd, $path_list_ref); if (defined $found) { $any = 1; $d = $dd = $found; push(@found,$dd); } else { push(@tried, !ref($dd) ? $dd : join(", ",@$dd)) if $dd ne ''; undef $d; } } my($is_a_backup) = $any_st{$short_type}; my($ll,$tier) = !$is_a_backup ? (0,'') : (2,' (backup, not used)'); if (@$f <= 2) { # no external programs specified do_log($ll, "Internal decoder for .%-4s%s", $short_type,$tier); undef $f if $is_a_backup; # discard a backup entry } elsif (!$any) { # external programs specified but none found do_log($ll, "No decoder for .%-4s%s", $short_type, !@tried ? '' : ' tried: '.join("; ",@tried)) if !$is_a_backup; undef $f; # release its storage } else { do_log($ll, "Found decoder for .%-4s at %s%s%s", $short_type, $daemon_chroot_dir ne '' ? "(chroot: $daemon_chroot_dir/) " : '', join("; ",@found), $tier); undef $f if $is_a_backup; # discard a backup entry } $any_st{$short_type} = 1 if defined $f; } # map program name hints to full paths - av scanners my($tier) = 'primary'; # primary, secondary, ... av scanners for my $f (@{ca('av_scanners')}, "\000", @{ca('av_scanners_backup')}) { if ($f eq "\000") { # next tier $tier = 'secondary'; } elsif (!defined $f || !ref $f) { # empty, skip } elsif (ref($f->[1]) eq 'CODE') { do_log(0, "Using %s internal av scanner code for %s", $tier,$f->[0]); } else { my($found) = $f->[1] = find_program_path($f->[1], $path_list_ref); if (!defined $found) { do_log(3, "No %s av scanner: %s", $tier, $f->[0]); undef $f; # release its storage } else { do_log(0, "Found %s av scanner %-11s at %s%s", $tier, $f->[0], $daemon_chroot_dir ne '' ? "(chroot: $daemon_chroot_dir/) " : '', $found); } } } for my $f (@{ca('spam_scanners')}) { if (!defined $f || !ref $f) { # empty, skip } elsif ($f->[1] ne 'Amavis::SpamControl::ExtProg') { do_log(5, "Using internal spam scanner code for %s", $f->[0]); } else { my($found) = $f->[2] = find_program_path($f->[2], $path_list_ref); if (!defined $found) { do_log(3, "No spam scanner: %s", $f->[0]); undef $f; # release its storage } else { do_log(0, "Found spam scanner %-11s at %s%s", $f->[0], $daemon_chroot_dir ne '' ? "(chroot: $daemon_chroot_dir/) " : '', $found); } } } } # Fetch remaining modules, all must be loaded before chroot and fork occurs # sub fetch_modules_extra() { my(@modules,@optmodules); if ($extra_code_sql_base) { push(@modules, 'DBI'); for (@lookup_sql_dsn, @storage_sql_dsn) { my(@dsn) = split(/:/,$_->[0],-1); push(@modules, 'DBD::'.$dsn[1]) if uc($dsn[0]) eq 'DBI'; } } push(@modules, qw(Net::LDAP Net::LDAP::Util Net::LDAP::Search Net::LDAP::Bind Net::LDAP::Extension)) if $extra_code_ldap; if (c('bypass_decode_parts') && !grep(exists $policy_bank{$_}{'bypass_decode_parts'} && !$policy_bank{$_}{'bypass_decode_parts'}, keys %policy_bank)) { } else { push(@modules, qw(Convert::TNEF Convert::UUlib Archive::Zip)); # push(@modules, qw(Archive::Tar)); # terrible, don't use it! } if ($extra_code_dkim || c('tls_security_level_in') || c('tls_security_level_out')) { push(@modules, qw(Crypt::OpenSSL::RSA)); } if (c('tls_security_level_in') || c('tls_security_level_out')) { push(@modules, qw(IO::Socket::SSL Net::SSLeay auto::Net::SSLeay::ssl_write_all auto::Net::SSLeay::ssl_read_until auto::Net::SSLeay::dump_peer_certificate)); } push(@modules, qw(Net::DNS::RR::TXT Text::ParseWords auto::Crypt::OpenSSL::RSA::new_public_key)) if $extra_code_dkim; push(@modules, 'Anomy::Sanitizer') if $enable_anomy_sanitizer; Amavis::Boot::fetch_modules('REQUIRED ADDITIONAL MODULES', 1, @modules); push(@optmodules, qw( bytes bytes_heavy.pl utf8 utf8_heavy.pl Encode Encode::Byte Encode::MIME::Header Encode::Unicode::UTF7 Encode::CN Encode::TW Encode::KR Encode::JP unicore::To::Lower.pl unicore::To::Upper.pl unicore::To::Fold.pl unicore::To::Title.pl unicore::To::Digit.pl )); push(@optmodules, $] >= 5.012000 ? qw(unicore::Heavy.pl) : qw(unicore::Canonical.pl unicore::Exact.pl unicore::PVA.pl)); # unicore::lib::Perl::Word.pl unicore::lib::Perl::SpacePer.pl # unicore::lib::Perl::Alnum.pl unicore::lib::Alpha::Y.pl # unicore::lib::Nt::De.pl unicore::lib::Hex::Y.pl push(@optmodules, qw(IO::Socket::INET6 Unix::Getrusage)); push(@optmodules, 'Authen::SASL') if $extra_code_ldap && !grep($_ eq 'Authen::SASL', @modules); push(@optmodules, defined($min_servers) ? 'Net::Server::PreFork' : 'Net::Server::PreForkSimple'); push(@optmodules, @additional_perl_modules); my($missing); $missing = Amavis::Boot::fetch_modules('PRE-COMPILE OPTIONAL MODULES', 0, @optmodules) if @optmodules; do_log(2, 'INFO: no optional modules: %s', join(' ',@$missing)) if ref $missing && @$missing; # require minimal version 0.32, Net::LDAP::Util::escape_filter_value() needed Net::LDAP->VERSION(0.32) if $extra_code_ldap; # needed a working last_insert_id in the past, no longer so but nevertheless: DBI->VERSION(1.43) if $extra_code_sql_base; MIME::Entity->VERSION != 5.419 or die "MIME::Entity 5.419 breaks quoted-printable encoding, ". "please upgrade to 5.420 or later (or use 5.418)"; # load optional modules SAVI and Mail::ClamAV if available and requested if ($extra_code_antivirus) { my($clamav_module_ok); for my $entry (@{ca('av_scanners')}, @{ca('av_scanners_backup')}) { if (ref($entry) ne 'ARRAY') { # none } elsif ($entry->[0] eq 'Sophos SAVI') { if (defined(eval { require SAVI }) && SAVI->VERSION(0.30) && Amavis::AV::sophos_savi_init(@$entry)) {} # ok, loaded else { undef $entry->[1] } # disable entry } elsif ($entry->[0] =~ /^Mail::ClamAV/) { if (!defined($clamav_module_ok)) { $clamav_module_ok = eval { require Mail::ClamAV }; $clamav_module_ok = 0 if !defined $clamav_module_ok; } undef $entry->[1] if !$clamav_module_ok; # disable entry } } } } sub usage() { my $myprogram_name = c('myprogram_name'); return <<"EOD"; Usage: $myprogram_name [-u user] [-g group] [-i instance_name] {-c config_file} [-d log_level,area,...] [-m max_servers] {-p listen_port_or_socket} [-L lock_file] [-P pid_file] [-H home_dir] [-D db_home_dir | -D ''] [-Q quarantine_dir | -Q ''] [-R chroot_dir | -R ''] [-S helpers_home_dir] [-T tempbase_dir] ( [start] | stop | reload | restart | debug | debug-sa | foreground | showkeys {domains} | testkeys {domains} | genrsa file_name [nbits] convert_keysfile file_name ) where area is a SpamAssassin debug area, e.g. all,util,rules,plugin,dkim,dcc or: $myprogram_name (-h | -V) ... show help or version, then exit EOD } # drop privileges # sub drop_priv($$) { my($desired_user,$desired_group) = @_; local($1); my($username,$passwd,$uid,$gid) = $desired_user=~/^(\d+)$/ ? (undef,undef,$1,undef) :getpwnam($desired_user); defined $uid or die "drop_priv: No such username: $desired_user\n"; if ($desired_group eq '') { $desired_group = $gid } # for logging purposes else { $gid = $desired_group=~/^(\d+)$/ ? $1 : getgrnam($desired_group) } defined $gid or die "drop_priv: No such group: $desired_group\n"; $( = $gid; $) = "$gid $gid"; # real and effective GID POSIX::setgid($gid) or die "drop_priv: Can't setgid to $gid: $!"; POSIX::setuid($uid) or die "drop_priv: Can't setuid to $uid: $!"; $> = $uid; $< = $uid; # just in case # print STDERR "desired user=$desired_user ($uid), current: EUID: $> ($<)\n"; # print STDERR "desired group=$desired_group ($gid), current: EGID: $) ($()\n"; $> != 0 or die "drop_priv: Still running as root, aborting\n"; $< != 0 or die "Effective UID changed, but Real UID is 0, aborting\n"; } # # Main program starts here # stir_random(); add_entropy($], @INC, %ENV); delete @ENV{'PATH', 'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; # Read dynamic source code, and logging and notification message templates # from the end of this file (pseudo file handle DATA) # $Amavis::Conf::notify_spam_admin_templ = ''; # not used $Amavis::Conf::notify_spam_recips_templ = ''; # not used do { local($/) = "__DATA__\n"; # set line terminator to this string for ( $extra_code_db, $extra_code_sql_lookup, $extra_code_ldap, $extra_code_in_ampdp, $extra_code_in_smtp, $extra_code_in_courier, $extra_code_out_smtp, $extra_code_out_pipe, $extra_code_out_bsmtp, $extra_code_out_local, $extra_code_p0f, $extra_code_sql_base, $extra_code_sql_log, $extra_code_sql_quar, $extra_code_antivirus, $extra_code_antispam, $extra_code_antispam_extprog, $extra_code_antispam_spamc, $extra_code_antispam_sa, $extra_code_unpackers, $extra_code_dkim, $extra_code_tools) { $_ = ; defined($_) or die "Error reading optional code from the source file: $!"; chomp($_); } binmode(\*Amavis::DATA, ':encoding(UTF-8)') or die "Can't set \*DATA encoding to UTF-8: $!"; for ( $Amavis::Conf::log_short_templ, $Amavis::Conf::log_verbose_templ, $Amavis::Conf::log_recip_templ, $Amavis::Conf::notify_sender_templ, $Amavis::Conf::notify_virus_sender_templ, $Amavis::Conf::notify_virus_admin_templ, $Amavis::Conf::notify_virus_recips_templ, $Amavis::Conf::notify_spam_sender_templ, $Amavis::Conf::notify_spam_admin_templ, $Amavis::Conf::notify_release_templ, $Amavis::Conf::notify_report_templ, $Amavis::Conf::notify_autoresp_templ) { $_ = ; defined($_) or die "Error reading templates from the source file: $!"; chomp($_); } }; # restore line terminator close(\*Amavis::DATA) or die "Error closing *Amavis::DATA: $!"; # close(STDIN) or die "Error closing STDIN: $!"; # note: don't close STDIN just yet to prevent some other file taking up fd 0 STDERR->autoflush(1); { local($1); s/^(.*?)[\r\n]+\z/$1/s # discard trailing NL for ($Amavis::Conf::log_short_templ, $Amavis::Conf::log_verbose_templ, $Amavis::Conf::log_recip_templ); }; $Amavis::Conf::log_templ = $Amavis::Conf::log_short_templ; umask(0027); # set our preferred umask POSIX::setlocale(LC_TIME,"C"); # English dates required in syslog and RFC 5322 # using Net::Server internal mechanism for a restart on HUP $warm_restart = defined $ENV{BOUND_SOCKETS} && $ENV{BOUND_SOCKETS} ne '' ?1:0; # Consider dropping privileges early, before reading a config file. # This is only possible if running under chroot will not be needed. # my($desired_group); # defaults to $desired_user's group my($desired_user); # username or UID if ($> != 0) { $desired_user = $> } # use effective UID if not root # collect and parse command line options my($log_level_override, $max_servers_override); my($myhome_override, $tempbase_override, $helpers_home_override); my($quarantinedir_override, $db_home_override, $daemon_chroot_dir_override); my($lock_file_override, $pid_file_override); my(@listen_sockets_override, $listen_sockets_overridden); my(@argv) = @ARGV; # preserve @ARGV, may modify @argv while (@argv >= 2 && $argv[0] =~ /^-[ugdimcpDHLPQRST]\z/ || @argv >= 1 && $argv[0] =~ /^-/) { my($opt,$val); $opt = shift @argv; $val = shift @argv if $opt !~ /^-[hV-]\z/; # these take no arguments if ($opt eq '--') { last; } elsif ($opt eq '-h') { # -h (help) die "$myversion\n\n" . usage(); } elsif ($opt eq '-V') { # -V (version) die "$myversion\n"; } elsif ($opt eq '-u') { # -u username if ($> == 0) { $desired_user = $val } else { print STDERR "Ignoring option -u when not running as root\n" } } elsif ($opt eq '-g') { # -g group print STDERR "NOTICE: Option -g may not achieve desired result when ". "running as non-root\n" if $> != 0 && $val ne $desired_group; $desired_group = $val; } elsif ($opt eq '-i') { # -i instance_name, may be of use to a .conf file $val =~ /^[a-z0-9._+-]*\z/i or die "Special chars in option -i $val\n"; $instance_name = untaint($val); # not used by amavisd directly } elsif ($opt eq '-d') { # -d log_level or -d SAdbg1,SAdbg2,..,SAdbg3 $log_level_override = untaint($val); } elsif ($opt eq '-m') { # -m max_servers $val =~ /^\+?\d+\z/ or die "Option -m requires a numeric argument\n"; $max_servers_override = untaint($val); } elsif ($opt eq '-c') { # -c config_file push(@config_files, untaint($val)) if $val ne ''; } elsif ($opt eq '-p') { # -p port_or_socket $listen_sockets_overridden = 1; # may disable all sockets by -p '' push(@listen_sockets_override, untaint($val)) if $val ne ''; } elsif ($opt eq '-D') { # -D db_home_dir, empty string turns off db use $db_home_override = untaint($val); } elsif ($opt eq '-H') { # -H home_dir $myhome_override = untaint($val) if $val ne ''; } elsif ($opt eq '-L') { # -L lock_file $lock_file_override = untaint($val) if $val ne ''; } elsif ($opt eq '-P') { # -P pid_file $pid_file_override = untaint($val) if $val ne ''; } elsif ($opt eq '-Q') { # -Q quarantine_dir, empty string disables quarant. $quarantinedir_override = untaint($val); } elsif ($opt eq '-R') { # -R chroot_dir, empty string or '/' avoids chroot $daemon_chroot_dir_override = $val eq '/' ? '' : untaint($val); } elsif ($opt eq '-S') { # -S helpers_home_dir for SA $helpers_home_override = untaint($val) if $val ne ''; } elsif ($opt eq '-T') { # -T tempbase_dir $tempbase_override = untaint($val) if $val ne ''; } else { die "Error in parsing command line options: $opt\n\n" . usage(); } } my($cmd) = lc(shift @argv); if ($cmd !~ /^(?:start|debug|debug-sa|foreground|reload|restart|stop| showkeys?|testkeys?|genrsa|convert_keysfile)?\z/xs) { die "$myversion:\n Unknown command line parameter: $cmd\n\n" . usage(); } elsif (@argv > 0 && $cmd !~ /^(:?showkeys?|testkeys?|genrsa|convert_keysfile)/xs) { die sprintf("$myversion:\n Only one command line parameter allowed: %s\n\n". "%s\n", join(' ',@argv), usage()); } if (!defined($desired_user)) { # early dropping of privileges not requested } elsif ($> != 0 && $< != 0) { # early dropping of privileges not needed } elsif (defined $daemon_chroot_dir_override && $daemon_chroot_dir_override ne '') { # early dropping of privs would prevent later chroot and is to be skipped } else { # drop privileges early if an uid was specified on a command line, option -u drop_priv($desired_user,$desired_group); } if ($cmd eq 'genrsa') { eval $extra_code_tools or die "Problem in Amavis::Tools code: $@"; $extra_code_tools = 1; Amavis::Tools::generate_dkim_private_key(@argv); exit(0); } if ($cmd eq 'convert_keysfile') { eval $extra_code_tools or die "Problem in Amavis::Tools code: $@"; $extra_code_tools = 1; Amavis::Tools::convert_dkim_keys_file(@argv); exit(0); } # these settings must be overridden before and after read_config # because some other settings in a config file may be derived from them $Amavis::Conf::MYHOME = $myhome_override if defined $myhome_override; $Amavis::Conf::TEMPBASE = $tempbase_override if defined $tempbase_override; $Amavis::Conf::QUARANTINEDIR = $quarantinedir_override if defined $quarantinedir_override; $Amavis::Conf::helpers_home = $helpers_home if defined $helpers_home; $Amavis::Conf::daemon_chroot_dir = $daemon_chroot_dir_override if defined $daemon_chroot_dir_override; # some remaining initialization, possibly after dropping privileges by -u, # but before reading configuration file init_local_delivery_aliases(); init_builtin_macros(); $instance_name = '' if !defined $instance_name; # convert arrayref to Amavis::Lookup::RE object, the Amavis::Lookup::RE module # was not yet available during BEGIN phase $Amavis::Conf::map_full_type_to_short_type_re = Amavis::Lookup::RE->new(@$Amavis::Conf::map_full_type_to_short_type_re); # default location of the config file if none specified push(@config_files, '/etc/amavisd.conf') if !@config_files; # Read and evaluate config files, which may override default settings Amavis::Conf::include_config_files(@config_files); Amavis::Conf::supply_after_defaults(); update_current_log_level(); add_entropy($Amavis::Conf::myhostname, $Amavis::Conf::myversion_date); # not needed any longer, reclaim storage undef $Amavis::Conf::log_short_templ; undef $Amavis::Conf::log_verbose_templ; if (defined $desired_user && $daemon_user ne '') { local($1); # compare the config file settings to current UID my($username,$passwd,$uid,$gid) = $daemon_user=~/^(\d+)$/ ? (undef,undef,$1,undef) : getpwnam($daemon_user); ($desired_user eq $daemon_user || $desired_user eq $uid) or warn sprintf("WARN: running under user '%s' (UID=%s), ". "the config file specifies \$daemon_user='%s' (UID=%s)\n", $desired_user, $>, $daemon_user, defined $uid ? $uid : '?'); } if ($> != 0 && $< != 0) { # dropping of privs is not needed } elsif (defined $daemon_chroot_dir && $daemon_chroot_dir ne '') { # dropping of privs now would prevent later chroot and is to be skipped } else { # drop privileges, unless needed for chrooting drop_priv($daemon_user,$daemon_group); } # override certain config file options by command line arguments my(@sa_debug_fac); # list of SA debug facilities if (defined $log_level_override) { for my $item (split(/[ \t]*,[ \t]*/,$log_level_override,-1)) { if ($item =~ /^[+-]?\d+\z/) { $Amavis::Conf::log_level = $item } elsif ($item =~ /^[A-Za-z0-9_-]+\z/) { push(@sa_debug_fac,$item) } } update_current_log_level(); } $Amavis::Conf::MYHOME = $myhome_override if defined $myhome_override; $Amavis::Conf::TEMPBASE = $tempbase_override if defined $tempbase_override; $Amavis::Conf::QUARANTINEDIR = $quarantinedir_override if defined $quarantinedir_override; $Amavis::Conf::helpers_home = $helpers_home if defined $helpers_home; $Amavis::Conf::daemon_chroot_dir = $daemon_chroot_dir_override if defined $daemon_chroot_dir_override; if (defined $db_home_override) { if ($db_home_override =~ /^\s*\z/) { $enable_db = 0 } else { $Amavis::Conf::db_home = $db_home_override } } if (defined $max_servers_override && $max_servers_override ne '') { $Amavis::Conf::max_servers = $max_servers_override; } if ($cmd =~ /^(?:showkeys?|testkeys?)\z/) { # useful for preparing DNS zone files and testing public keys in DNS eval $extra_code_dkim or die "Problem in Amavis::DKIM code: $@"; $extra_code_dkim = 1; Amavis::DKIM::dkim_key_postprocess(); eval $extra_code_tools or die "Problem in Amavis::Tools code: $@"; $extra_code_tools = 1; # release memory occupied by the source code Amavis::Tools::show_or_test_dkim_public_keys($cmd,\@argv); exit(0); } undef $extra_code_tools; # no longer needed for ($unix_socketname, $inet_socket_port) { push(@listen_sockets, ref $_ ? @$_ : $_) if defined $_ && $_ ne ''; } @listen_sockets = @listen_sockets_override if $listen_sockets_overridden; for my $s (@listen_sockets) { # convert to a Net::Server::Proto syntax local($1); if ($s =~ m{^unix:(/\S+)\z}s) { $s = "$1|unix" } elsif ($s =~ m{^inet:(.*)\z}s) { $s = "$1/tcp" } elsif ($s =~ m{^inet6:(.*)\z}s) { $s = "$1/tcp" } elsif ($s =~ m{^/\S+}s) { $s = "$s|unix" } elsif ($s =~ m{^\d+\z}s) { $s = "$s/tcp" } # port number elsif ($s =~ m{^[^/|]+\z}s) { $s = "$s/tcp" } # almost anything goes elsif ($s =~ m{^.+\z}s) { $s = "$s" } # anything goes else { die "Socket specification syntax error: $s\n" } } @listen_sockets > 0 or die "No listen sockets or ports specified\n"; # %modules_basic = %INC; # helps to track missing modules in chroot # compile optional modules if needed if (!$enable_db) { undef $extra_code_db; } else { eval $extra_code_db or die "Problem in Amavis::DB or Amavis::DB::SNMP code: $@"; # release memory occupied by the source code # NOTE: use undef(), keep in mind: http://www.perlmonks.org/?node_id=803515 undef $extra_code_db; $extra_code_db = 1; } { my $any_dkim_verification = scalar grep($policy_bank{$_}{enable_dkim_verification}, keys %policy_bank); my $any_dkim_signing = scalar grep($policy_bank{$_}{enable_dkim_signing}, keys %policy_bank); if (!$any_dkim_verification && !$any_dkim_signing) { undef $extra_code_dkim; } else { eval $extra_code_dkim or die "Problem in Amavis::DKIM code: $@"; # release memory occupied by the source code undef $extra_code_dkim; $extra_code_dkim = 1; } if ($any_dkim_signing) { Amavis::DKIM::dkim_key_postprocess(); } else { # release storage undef %dkim_signing_keys_by_domain; undef @dkim_signing_keys_list; undef @dkim_signing_keys_storage; } } { my(%needed_protocols_in); for my $bank_name (keys %policy_bank) { my($var) = $policy_bank{$bank_name}{'protocol'}; $var = $$var if ref($var) eq 'SCALAR'; # allow one level of indirection $needed_protocols_in{$var} = 1 if defined $var; } # compatibility with older config files unaware of $protocol config variable # $needed_protocols_in{'AM.CL'} = 1 # AM.CL is no longer supported # if grep(m{\|unix\z}i, @listen_sockets) && # !grep($needed_protocols_in{$_}, qw(AM.PDP COURIER)); $needed_protocols_in{'SMTP'} = 1 if grep(m{/(?:tcp|ssleay|ssl)\z}i, @listen_sockets) && !grep($needed_protocols_in{$_}, qw(SMTP LMTP QMQPqq)); if ($needed_protocols_in{'AM.PDP'} || $needed_protocols_in{'AM.CL'}) { eval $extra_code_in_ampdp or die "Problem in the In::AMPDP code: $@"; # release memory occupied by the source code undef $extra_code_in_ampdp; $extra_code_in_ampdp = 1; } else { undef $extra_code_in_ampdp; } if ($needed_protocols_in{'SMTP'} || $needed_protocols_in{'LMTP'}) { eval $extra_code_in_smtp or die "Problem in the In::SMTP code: $@"; # release memory occupied by the source code undef $extra_code_in_smtp; $extra_code_in_smtp = 1; } else { undef $extra_code_in_smtp; } if ($needed_protocols_in{'COURIER'}) { eval $extra_code_in_courier or die "Problem in the In::Courier code: $@"; # release memory occupied by the source code undef $extra_code_in_courier; $extra_code_in_courier = 1; } else { undef $extra_code_in_courier; } if ($needed_protocols_in{'QMQPqq'}) { die "In::QMQPqq code not available" } } if (!@lookup_sql_dsn) { undef $extra_code_sql_lookup } if (!@storage_sql_dsn) { undef $extra_code_sql_log } # sql quarantine depends on sql log undef $extra_code_sql_quar if !defined $extra_code_sql_log; { my(%needed_protocols_out); local($1); for my $bank_name (keys %policy_bank) { for my $method_name (qw( forward_method notify_method resend_method release_method requeue_method os_fingerprint_method virus_quarantine_method banned_files_quarantine_method unchecked_quarantine_method spam_quarantine_method bad_header_quarantine_method clean_quarantine_method archive_quarantine_method )) { local($1); my($var) = $policy_bank{$bank_name}{$method_name}; $var = $$var if ref($var) eq 'SCALAR'; # allow one level of indirection $needed_protocols_out{uc($1)} = 1 if $var =~ /^([a-z][a-z0-9.+-]*):/si; } } if (!$needed_protocols_out{'SMTP'} && !$needed_protocols_out{'LMTP'}) { undef $extra_code_out_smtp } else { eval $extra_code_out_smtp or die "Problem in Amavis::Out::SMTP code: $@"; # release memory occupied by the source code undef $extra_code_out_smtp; $extra_code_out_smtp = 1; } if (!$needed_protocols_out{'PIPE'}) { undef $extra_code_out_pipe } else { eval $extra_code_out_pipe or die "Problem in Amavis::Out::Pipe code: $@"; # release memory occupied by the source code undef $extra_code_out_pipe; $extra_code_out_pipe = 1; } if (!$needed_protocols_out{'BSMTP'}) { undef $extra_code_out_bsmtp } else { eval $extra_code_out_bsmtp or die "Problem in Amavis::Out::BSMTP code: $@"; # release memory occupied by the source code undef $extra_code_out_bsmtp; $extra_code_out_bsmtp = 1; } if (!$needed_protocols_out{'LOCAL'}) { undef $extra_code_out_local } else { eval $extra_code_out_local or die "Problem in Amavis::Out::Local code: $@"; # release memory occupied by the source code undef $extra_code_out_local; $extra_code_out_local = 1; } if (!$needed_protocols_out{'SQL'}) { undef $extra_code_sql_quar } else { # deal with it in the next section } if (!$needed_protocols_out{'P0F'}) { undef $extra_code_p0f } else { eval $extra_code_p0f or die "Problem in OS_Fingerprint code: $@"; # release memory occupied by the source code undef $extra_code_p0f; $extra_code_p0f = 1; } } if (!defined($extra_code_sql_log) && !defined($extra_code_sql_quar) && !defined($extra_code_sql_lookup)) { undef $extra_code_sql_base } else { eval $extra_code_sql_base or die "Problem in Amavis SQL base code: $@"; # release memory occupied by the source code undef $extra_code_sql_base; $extra_code_sql_base = 1; } if (defined $extra_code_sql_log) { eval $extra_code_sql_log or die "Problem in Amavis::SQL::Log code: $@"; # release memory occupied by the source code undef $extra_code_sql_log; $extra_code_sql_log = 1; } if (defined $extra_code_sql_quar) { eval $extra_code_sql_quar or die "Problem in Amavis::SQL::Quarantine code: $@"; # release memory occupied by the source code undef $extra_code_sql_quar; $extra_code_sql_quar = 1; } if (defined $extra_code_sql_lookup) { eval $extra_code_sql_lookup or die "Problem in Amavis SQL lookup code: $@"; # release memory occupied by the source code undef $extra_code_sql_lookup; $extra_code_sql_lookup = 1; } if (!$enable_ldap) { undef $extra_code_ldap } else { eval $extra_code_ldap or die "Problem in Lookup::LDAP code: $@"; # release memory occupied by the source code undef $extra_code_ldap; $extra_code_ldap = 1; } my($bpvcm) = ca('bypass_virus_checks_maps'); if (!@{ca('av_scanners')} && !@{ca('av_scanners_backup')}) { undef $extra_code_antivirus; } elsif (@$bpvcm && !ref($bpvcm->[0]) && $bpvcm->[0]) { # do a simple-minded test to make it easy to turn off virus checks undef $extra_code_antivirus; } else { eval $extra_code_antivirus or die "Problem in antivirus code: $@"; # release memory occupied by the source code undef $extra_code_antivirus; $extra_code_antivirus = 1; } if (!$extra_code_antivirus) # release storage { undef @Amavis::Conf::av_scanners; undef @Amavis::Conf::av_scanners_backup } my(%spam_scanners_used); my($bpscm) = ca('bypass_spam_checks_maps'); if (!@{ca('spam_scanners')}) { undef $extra_code_antispam; } elsif (@$bpscm && !ref($bpscm->[0]) && $bpscm->[0]) { # simple-minded undef $extra_code_antispam; } else { eval $extra_code_antispam or die "Problem in antispam code: $@"; # release memory occupied by the source code undef $extra_code_antispam; $extra_code_antispam = 1; for my $as (@{ca('spam_scanners')}) { next if !ref $as || !defined $as->[1]; my($scanner_name,$module) = @$as; $spam_scanners_used{$module} = 1; } } if (!$extra_code_antispam) { undef @Amavis::Conf::spam_scanners } # load required built-in spam scanning modules if ($spam_scanners_used{'Amavis::SpamControl::ExtProg'}) { eval $extra_code_antispam_extprog or die "Problem in ExtProg code: $@"; # release memory occupied by source code undef $extra_code_antispam_extprog; $extra_code_antispam_extprog = 1; } else { undef $extra_code_antispam_extprog; } if ($spam_scanners_used{'Amavis::SpamControl::SpamdClient'}) { eval $extra_code_antispam_spamc or die "Problem in spamd client code: $@"; # release memory occupied by source code undef $extra_code_antispam_spamc; $extra_code_antispam_spamc = 1; } else { undef $extra_code_antispam_spamc; } if ($spam_scanners_used{'Amavis::SpamControl::SpamAssassin'}) { eval $extra_code_antispam_sa or die "Problem in antispam SA code: $@"; # release memory occupied by the source code undef $extra_code_antispam_sa; $extra_code_antispam_sa = 1; } else { undef $extra_code_antispam_sa; } if (c('bypass_decode_parts') && !grep(exists $policy_bank{$_}{'bypass_decode_parts'} && !$policy_bank{$_}{'bypass_decode_parts'}, keys %policy_bank)) { undef $extra_code_unpackers; } else { eval $extra_code_unpackers or die "Problem in Amavis::Unpackers code: $@"; # release memory occupied by the source code undef $extra_code_unpackers; $extra_code_unpackers = 1; } Amavis::Log::init($do_syslog, $logfile); # initialize logging Amavis::Log::log_to_stderr($cmd eq 'debug' || $cmd eq 'debug-sa' ? 1 : 0); do_log(2, "logging initialized, log level %s, %s", c('log_level'), $do_syslog ? sprintf("syslog: %s.%s",c('syslog_ident'),c('syslog_facility')): $logfile ne '' ? "logfile: $logfile" : "STDERR"); # insist on a FQDN in $myhostname my($myhn) = c('myhostname'); $myhn =~ /[^.]\.[a-zA-Z0-9-]+\z/s || lc($myhn) eq 'localhost' or die <<"EOD"; The value of variable \$myhostname is \"$myhn\", but should have been a fully qualified domain name; perhaps uname(3) did not provide such. You must explicitly assign a FQDN of this host to variable \$myhostname in amavisd.conf, or fix what uname(3) provides as a host's network name! EOD $mail_id_size_bits > 0 && $mail_id_size_bits == int $mail_id_size_bits && $mail_id_size_bits % 24 == 0 or die "\$mail_id_size_bits ($mail_id_size_bits) must be a multiple of 24\n"; eval { my($amavisd_pid); # PID of a currently running amavisd daemon (not our pid) # is amavisd daemon already running? my($pidf) = defined $pid_file_override ? $pid_file_override : $pid_file; $pidf ne '' or die "Config parameter \$pid_file not defined"; my(@stat_list) = lstat($pidf); my($errn) = @stat_list ? 0 : 0+$!; if ($warm_restart) { # skip pid file checking, let Net::Server handle it } elsif ($errn == ENOENT) { die "The amavisd daemon is apparently not running, no PID file $pidf\n" if $cmd =~ /^(?:reload|restart|stop)\z/; } elsif ($errn != 0) { die "PID file $pidf is inaccessible: $!\n"; } elsif (!-f _) { die "PID file $pidf is not a regular file\n"; } else { # determine PID of the currently running amavisd daemon, validate it my($ln); my($lcnt) = 0; my($pidf_h) = IO::File->new; $pidf_h->open($pidf,'<') or die "Can't open PID file $pidf: $!"; for ($! = 0; defined($ln=$pidf_h->getline); $! = 0) { chomp($ln); $lcnt++; last if $lcnt > 100; $amavisd_pid = $ln if $lcnt == 1 && $ln =~ /^\d{1,10}\z/; } defined $ln || $!==0 or die "Error reading from file $pidf: $!"; $pidf_h->close or die "Error closing file $pidf: $!"; if ($lcnt <= 1 && !defined $amavisd_pid) { # treat empty or junk one-line pid file the same as nonexisting pid file die "The amavisd daemon is apparently not running, empty PID file $pidf\n" if $cmd =~ /^(?:reload|restart|stop)\z/; # prevent Net::Server from seeing this crippled file do_log(-1, "removing empty or crippled PID file %s", $pidf); unlink($pidf) or die "Can't remove PID file $pidf: $!"; undef $amavisd_pid; } else { $lcnt <= 1 or die "More than one line in file $pidf"; defined $amavisd_pid or die "Missing process ID in file $pidf"; $amavisd_pid > 1 or die "Invalid PID in file $pidf: [$amavisd_pid]"; } my($mtime) = $stat_list[9]; if (defined $amavisd_pid && defined $mtime) { # got a PID from a file # Is pid file older than system uptime? If so, it should be disregarded, # it must not prevent starting up amavisd after unclean shutdown. my($now) = int(time); my($uptime,$uptime_fmt); # sys uptime in seconds my(@prog_args); my(@progs) = ('/usr/bin/uptime','uptime'); if (lc($^O) eq 'freebsd') { @progs = ('/sbin/sysctl','sysctl'); @prog_args = 'kern.boottime' } my($prog) = find_program_path(\@progs, [split(/:/,$path,-1)] ); if (!defined($prog)) { do_log(1,'No programs: %s',join(", ",@progs)); } else { # obtain system uptime my($proc_fh,$uppid); eval { ($proc_fh,$uppid) = run_command(undef,'/dev/null',$prog,@prog_args); for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { local($1,$2,$3,$4); chomp($ln); if (defined $uptime) {} elsif ($ln =~ /{[^}]*\bsec\s*=\s*(\d+)[^}]*}/) { $uptime= $now-$1 } # amazing how broken reports from uptime(1) soon after boot can be! elsif ($ln =~ /\b up \s+ (?: (\d{1,4}) \s* days? )? [,\s]* (\d{1,2}) : (\d{1,2}) (?: : (\d{1,2}))? (?! \d ) /ix || $ln =~ /\b up (?: \s* \b (\d{1,4}) \s* days? )? (?: [,\s]* \b (\d{1,2}) \s* hrs? )? (?: [,\s]* \b (\d{1,2}) \s* mins? )? (?: [,\s]* \b (\d{1,2}) \s* secs? )? /ix ) { $uptime = (($1*24 + $2)*60 + $3)*60 + $4 } elsif ($ln =~ /\b (\d{1,2}) \s* secs?/ix) { $uptime = $1 } #OpenBSD $uptime_fmt = format_time_interval($uptime); do_log(5,"system uptime %s: %s", $uptime_fmt,$ln); } defined $ln || $!==0 or die "Reading uptime: $!"; my($err)=0; $proc_fh->close or $err = $!; my($child_stat) = defined $uppid && waitpid($uppid,0)>0 ? $? : undef; undef $proc_fh; undef $uppid; proc_status_ok($child_stat,$err) or die "Error running $prog: " . exit_status_str($child_stat,$err) . "\n"; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(1,"uptime: %s", $eval_stat); }; if (defined $proc_fh) { $proc_fh->close } # ignoring status if (defined $uppid) { waitpid($uppid,0) } # ignoring status } if (!defined($uptime)) { do_log(1,'Unable to determine system uptime, will trust PID file'); } elsif ($now-$mtime <= $uptime+70) { do_log(1,'Valid PID file (younger than sys uptime %s)', $uptime_fmt); } else { # must not kill an unrelated process which happens to have the # same pid as amavisd had before a system shutdown or crash undef $amavisd_pid; do_log(1,'Ignoring stale PID file %s, older than system uptime %s', $pidf,$uptime_fmt); } } } if (defined $amavisd_pid) { $amavisd_pid = untaint($amavisd_pid); if (!kill(0,$amavisd_pid)) { # does a process exist? $! == ESRCH or die "Can't send SIG 0 to process [$amavisd_pid]: $!"; undef $amavisd_pid; # process does not exist }; } # act on command line parameter in $cmd if ($cmd =~ /^(?:start|debug|debug-sa|foreground)?\z/) { $daemonize=0 if $cmd eq 'foreground'; $daemonize=0, $DEBUG=1 if $cmd eq 'debug'; $daemonize=0, $sa_debug=1 if $cmd eq 'debug-sa'; } if ($warm_restart) { # a semi-documented Net::Server mechanism for a restart on HUP; # assume we have just been reincarnated by exec as a result of a HUP, # so just ignore the command parameter and let Net::Server do the rest } elsif ($cmd =~ /^(?:start|debug|debug-sa|foreground)?\z/) { !defined($amavisd_pid) or die "The amavisd daemon is already running, PID: [$amavisd_pid]\n"; } elsif ($cmd eq 'reload') { # reload: send a HUP signal to a running daemon defined $amavisd_pid or die "The amavisd daemon is not running\n"; kill('HUP',$amavisd_pid) or $! == ESRCH or die "Can't SIGHUP amavisd[$amavisd_pid]: $!"; my($msg) = "Signalling a SIGHUP to a running daemon [$amavisd_pid]"; do_log(2,"%s",$msg); print STDOUT "$msg\n"; exit(0); } elsif ($cmd =~ /^(?:restart|stop)\z/) { # stop or restart defined $amavisd_pid or die "The amavisd daemon is not running\n"; my($kill_sig_used, $killed_amavisd_pid); eval { # first stop a running daemon $kill_sig_used = 'TERM'; kill($kill_sig_used,$amavisd_pid) or $! == ESRCH or die "Can't SIG$kill_sig_used amavisd[$amavisd_pid]: $!"; my($waited) = 0; my($sigkill_sent) = 0; my($delay) = 1; # seconds for (;;) { # wait for the old running daemon to go away sleep($delay); $waited += $delay; $delay = 5; if (!kill(0,$amavisd_pid)) { # is the old daemon still there? $! == ESRCH or die "Can't send SIG 0 to amavisd[$amavisd_pid]: $!"; $killed_amavisd_pid = $amavisd_pid; # old process is gone, done last; } if ($waited < 60 || $sigkill_sent) { do_log(2,"Waiting for the process [%s] to terminate",$amavisd_pid); print STDOUT "Waiting for the process [$amavisd_pid] to terminate\n"; } else { # use stronger hammer do_log(2,"Sending SIGKILL to amavisd[%s]",$amavisd_pid); print STDERR "Sending SIGKILL to amavisd[$amavisd_pid]\n"; $kill_sig_used = 'KILL'; kill($kill_sig_used,$amavisd_pid) or $! == ESRCH or warn "Can't SIGKILL amavisd[$amavisd_pid]: $!"; $sigkill_sent = 1; } } 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; die "$eval_stat, can't $cmd the process\n"; }; my($msg) = !defined($killed_amavisd_pid) ? undef : "Daemon [$killed_amavisd_pid] terminated by SIG$kill_sig_used"; if ($cmd eq 'stop') { if (defined $msg) { do_log(2,"%s",$msg); print STDOUT "$msg\n" } exit(0); } if (defined $killed_amavisd_pid) { print STDOUT "$msg, waiting for dust to settle...\n"; sleep 5; # wait for the TCP socket to be released } print STDOUT "becoming a new daemon...\n"; } else { die "$myversion: Unknown command line parameter: $cmd\n\n" . usage(); } 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(2,"%s", $eval_stat); die "$eval_stat\n"; }; $daemonize = 0 if $DEBUG; # in case $DEBUG came from a config file # Set path, home and term explictly. Don't trust environment $ENV{PATH} = $path if $path ne ''; $ENV{HOME} = $helpers_home if $helpers_home ne ''; $ENV{TERM} = 'dumb'; $ENV{COLUMNS} = '80'; $ENV{LINES} = '100'; { my($msg) = ''; $msg .= ", instance=$instance_name" if $instance_name ne ''; $msg .= ", nl=".sprintf("\\%03o",ord("\n")) if "\n" ne "\012"; $msg .= ", Unicode aware" if $unicode_aware; for (qw(PERLIO LC_ALL LC_TYPE LC_CTYPE LANG)) { $msg .= sprintf(', %s="%s"', $_,$ENV{$_}) if $ENV{$_} ne '' } do_log(0,"starting.%s %s at %s %s%s", !$warm_restart?'':' (warm)', $0, c('myhostname'), $myversion, $msg); } # report version of Perl and process UID/GID do_log(1, "perl=%s, user=%s, EUID: %s (%s); group=%s, EGID: %s (%s)", $], $desired_user, $>, $<, $desired_group, $), $(); if ($warm_restart) { # a semi-documented Net::Server mechanism to let a restarted process # re-acquire sockets from its predecessor on a HUP my($str) = $ENV{BOUND_SOCKETS}; $str =~ s/\n/, /gs; do_log(1,"warm restart on HUP [%s]: '%s', sockets: %s", $$, join(' ',$0,@ARGV), $str); } # $SIG{USR2} = sub { # my($msg) = Carp::longmess("SIG$_[0] received, backtrace:"); # print STDERR "\n",$msg,"\n"; do_log(-1,"%s",$msg); # }; fetch_modules_extra(); # bring additional modules into memory and compile them $spamcontrol_obj = Amavis::SpamControl->new if $extra_code_antispam; $spamcontrol_obj->init_pre_chroot if $spamcontrol_obj; if ($daemonize) { # log warnings and uncaught errors $SIG{'__DIE__' } = sub { if (!$^S) { my($m) = @_; chomp($m); do_log(-1,"_DIE: %s", $m) } }; $SIG{'__WARN__'} = sub { my($m) = @_; chomp($m); do_log(2,"_WARN: %s",$m) }; } # set up Net::Server configuration my(@bind_to); { # merge port numbers, unix sockets and default binding host address into # a unified list @listen_sockets, which will be passed on to Net::Server # local($1); @bind_to = ref $inet_socket_bind ? @$inet_socket_bind : $inet_socket_bind; $_ = !defined $_ || $_ eq '' ? '*' : /^\[(.*)\]\z/s ? $1 : $_ for @bind_to; @bind_to = ( '*' ) if !@bind_to; my(@merged_listen_sockets); for (@listen_sockets) { # roughly mimic the Net::Server::Proto and Net::Server::Proto::TCP parsing if (m{^/} || m{[/|]unix\z}si) { push(@merged_listen_sockets, $_); # looks like a Unix socket } elsif (m{^\[[^\]]*\]:}s || m{^[^/|:]*:}s) { push(@merged_listen_sockets, $_); # explicit host & port specified } else { # assume port (or service) specification only, supply bind addr for my $bind_addr (@bind_to) { # Cartesian product: bind_addr x port # keep IPv4 addresses without square brackets for the benefit # of non-IPv6 -aware Net::Server push(@merged_listen_sockets, $bind_addr =~ /:/ ? "[$bind_addr]:$_" : "$bind_addr:$_"); } } } @listen_sockets = @merged_listen_sockets; do_log(5,"bind to %s", join(', ',@listen_sockets)); } my($server) = Amavis->new({ # command args to be used after HUP must be untainted, deflt: [$0,@ARGV] # commandline => ['/usr/local/sbin/amavisd','-c',$config_file[0] ], # commandline => [], # disable commandline => [ map(untaint($_), ($0,@ARGV)) ], port => \@listen_sockets, # listen on these sockets (Unix, inet, inet6) host => $bind_to[0], # default bind, redundant, merged to @listen_sockets listen => $listen_queue_size, # undef for a default max_servers => $max_servers, # number of pre-forked children !defined($min_servers) ? () : ( min_servers => $min_servers, min_spare_servers => $min_spare_servers, max_spare_servers => $max_spare_servers), max_requests => $max_requests > 0 ? $max_requests : 2E9, # avoid dflt 1000 user => ($> == 0 || $< == 0) ? $daemon_user : undef, group => ($> == 0 || $< == 0) ? $daemon_group : undef, pid_file => defined $pid_file_override ? $pid_file_override : $pid_file, # socket serialization lockfile lock_file => defined $lock_file_override? $lock_file_override: $lock_file, # serialize => 'flock', # flock, semaphore, pipe background => $daemonize ? 1 : undef, setsid => $daemonize ? 1 : undef, chroot => $daemon_chroot_dir ne '' ? $daemon_chroot_dir : undef, no_close_by_child => 1, leave_children_open_on_hup => 1, # no_client_stdout introduced with Net::Server 0.92, but is broken in 0.92 no_client_stdout => (Net::Server->VERSION >= 0.93 ? 1 : 0), # controls log level for Net::Server internal log messages: # 0=err, 1=warning, 2=notice, 3=info, 4=debug log_level => ($DEBUG || c('log_level') >= 5) ? 4 : 2, log_file => undef, # method will be overridden by a call to do_log() # SSL_cert_file => "$MYHOME/cert/mail-cert.pem", # SSL_key_file => "$MYHOME/cert/mail-key.pem", }); $0 = c('myprogram_name') . ' (master)'; $server->run; # transferring control to Net::Server # shouldn't get here exit 1; # we read text (such as notification templates) from DATA sections # to avoid any interpretations of special characters (e.g. \ or ') by Perl # __DATA__ # package Amavis::DB::SNMP; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform $myversion $myhostname $snmp_contact $snmp_location $nanny_details_level); import Amavis::Util qw(ll do_log snmp_counters_get add_entropy fetch_entropy_bytes); } use BerkeleyDB; use MIME::Base64; use Time::HiRes (); # open existing databases (called by each child process) # sub new { my($class,$db_env) = @_; $! = 0; my($env) = $db_env->get_db_env; defined $env or die "BDB get_db_env (dbS/dbN): $BerkeleyDB::Error, $!."; $! = 0; my($dbs) = BerkeleyDB::Hash->new(-Filename=>'snmp.db', -Env=>$env); defined $dbs or die "BDB no dbS: $BerkeleyDB::Error, $!."; $! = 0; my($dbn) = BerkeleyDB::Hash->new(-Filename=>'nanny.db',-Env=>$env); defined $dbn or die "BDB no dbN: $BerkeleyDB::Error, $!."; bless { 'db_snmp'=>$dbs, 'db_nanny'=>$dbn }, $class; } sub DESTROY { my $self = shift; local($@,$!,$_); my($myactualpid) = $$; if (defined($my_pid) && $myactualpid != $my_pid) { eval { do_log(5,"Amavis::DB::SNMP DESTROY skip, clone [%s] (born as [%s])", $myactualpid, $my_pid) }; } else { eval { do_log(5,"Amavis::DB::SNMP DESTROY called") }; for my $db_name ('db_snmp', 'db_nanny') { my($db) = $self->{$db_name}; if (defined $db) { eval { $db->db_close==0 or die "db_close: $BerkeleyDB::Error, $!."; 1; } or do { $@ = "errno=$!" if $@ eq '' }; if ($@ ne '' && $@ !~ /\bDatabase is already closed\b/) { warn "[$myactualpid] BDB S+N DESTROY INFO ($db_name): $@" } undef $db; } } } } #sub lock_stat($) { # my($label) = @_; # my($s) = qx'/usr/local/bin/db_stat-4.2 -c -h /var/amavis/db | /usr/local/bin/perl -ne \'$a{$2}=$1 if /^(\d+)\s+Total number of locks (requested|released)/; END {printf("%d, %d\n",$a{requested}, $a{requested}-$a{released})}\''; # do_log(0, "lock_stat %s: %s", $label,$s); #} # insert startup time SNMP entry, called from the master process at startup # (a classical subroutine, not a method) # sub put_initial_snmp_data($) { my($db) = @_; my($eval_stat,$interrupt); $interrupt = ''; { my($cursor); my($h1) = sub { $interrupt = $_[0] }; local(@SIG{qw(INT HUP TERM TSTP QUIT ALRM USR1 USR2)}) = ($h1) x 8; eval { # ensure cursor will be unlocked even in case of errors or signals $cursor = $db->db_cursor(DB_WRITECURSOR); # obtain write lock defined $cursor or die "BDB S db_cursor: $BerkeleyDB::Error, $!."; for my $obj (['sysDescr', 'STR', $myversion], ['sysObjectID', 'OID', '1.3.6.1.4.1.15312.2'], # iso.org.dod.internet.private.enterprise.ijs.amavisd-new ['sysUpTime', 'INT', int(time)], # to be converted to TIM # later it must be converted to timeticks (10ms since start) ['sysContact', 'STR', $snmp_contact], ['sysName', 'STR', $myhostname], ['sysLocation', 'STR', $snmp_location], ['sysServices', 'INT', 64], # application ) { my($key,$type,$val) = @$obj; $cursor->c_put($key, sprintf("%s %s",$type,$val), DB_KEYLAST) == 0 or die "BDB S c_put: $BerkeleyDB::Error, $!."; }; $cursor->c_close==0 or die "BDB S c_close: $BerkeleyDB::Error, $!."; undef $cursor; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; $cursor->c_close if defined $cursor; # unlock, ignoring status undef $cursor; }; # restore signal handlers if ($interrupt ne '') { kill($interrupt,$$) } # resignal, ignoring status elsif (defined $eval_stat) { chomp $eval_stat; die "put_initial_snmp_data: BDB S $eval_stat\n"; } } sub update_snmp_variables { my($self) = @_; do_log(5,"updating snmp variables"); my($snmp_var_names_ref) = snmp_counters_get(); my($eval_stat,$interrupt); $interrupt = ''; if (defined $snmp_var_names_ref && @$snmp_var_names_ref) { my($db) = $self->{'db_snmp'}; my($cursor); my($h1) = sub { $interrupt = $_[0] }; local(@SIG{qw(INT HUP TERM TSTP QUIT ALRM USR1 USR2)}) = ($h1) x 8; eval { # ensure cursor will be unlocked even in case of errors or signals $cursor = $db->db_cursor(DB_WRITECURSOR); # obtain write lock defined $cursor or die "db_cursor: $BerkeleyDB::Error, $!."; for my $key (@$snmp_var_names_ref) { my($snmp_var_name,$arg,$type) = ref $key ? @$key : ($key); $type = 'C32' if !defined($type) || $type eq ''; if ($type eq 'C32' || $type eq 'C64') { # a counter if (!defined($arg)) { $arg = 1 } # by default counter increments by 1 elsif ($arg < 0) { $arg = 0 } # counter is supposed to be unsigned } elsif ($type eq 'TIM') { # TimeTicks if ($arg < 0) { $arg = 0 } # non-decrementing } my($val,$flags); local($1); my($stat) = $cursor->c_get($snmp_var_name,$val,DB_SET); if ($stat==0) { # exists, update it (or replace it) if ($type eq 'C32' && $val=~/^C32 (\d+)\z/) { $val = $1+$arg } elsif ($type eq 'C64' && $val=~/^C64 (\d+)\z/) { $val = $1+$arg } elsif ($type eq 'TIM' && $val=~/^TIM (\d+)\z/) { $val = $1+$arg } elsif ($type eq 'INT' && $val=~/^INT ([+-]?\d+)\z/) { $val = $arg } elsif ($type=~/^(STR|OID)\z/ && $val=~/^\Q$type\E (.*)\z/) { if ($snmp_var_name ne 'entropy') { $val = $arg } else { # blend-in entropy $val = $1; add_entropy($val, Time::HiRes::gettimeofday); $val = fetch_entropy_bytes(18); # 18 bytes $val = encode_base64($val,''); # 18*8/6 = 24 chars $val =~ tr{+/}{-_}; # base64 -> RFC 4648 base64url [A-Za-z0-9-_] } } else { do_log(-2,"WARN: variable syntax? %s: %s, clearing", $snmp_var_name,$val); $val = 0; } $flags = DB_CURRENT; } else { # create new entry $stat==DB_NOTFOUND or die "c_get: $BerkeleyDB::Error, $!."; $flags = DB_KEYLAST; $val = $arg; } my($fmt) = $type eq 'C32' ? "%010d" : $type eq 'C64' ? "%020.0f" : $type eq 'INT' ? "%010d" : undef; # format for INT should really be %011d, but keep compatibility for now my($str) = defined($fmt) ? sprintf($fmt,$val) : $val; $cursor->c_put($snmp_var_name, $type.' '.$str, $flags) == 0 or die "c_put: $BerkeleyDB::Error, $!."; } $cursor->c_close==0 or die "c_close: $BerkeleyDB::Error, $!."; undef $cursor; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; if (defined $db) { $cursor->c_close if defined $cursor; # unlock, ignoring status undef $cursor; # if (!defined($eval_stat)) { # my($stat); $db->db_sync(); # not really needed # $stat==0 or warn "BDB S db_sync,status $stat: $BerkeleyDB::Error, $!."; # } } }; # restore signal handlers delete $self->{'cnt'}; if ($interrupt ne '') { kill($interrupt,$$) } # resignal, ignoring status elsif (defined $eval_stat) { chomp $eval_stat; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout die "update_snmp_variables: BDB S $eval_stat\n"; } } sub read_snmp_variables { my($self,@snmp_var_names) = @_; my($eval_stat,$interrupt); $interrupt = ''; my($db) = $self->{'db_snmp'}; my($cursor); my(@values); { my($h1) = sub { $interrupt = $_[0] }; local(@SIG{qw(INT HUP TERM TSTP QUIT ALRM USR1 USR2)}) = ($h1) x 8; eval { # ensure cursor will be unlocked even in case of errors or signals $cursor = $db->db_cursor; # obtain read lock defined $cursor or die "db_cursor: $BerkeleyDB::Error, $!."; for my $cname (@snmp_var_names) { my($val); my($stat) = $cursor->c_get($cname,$val,DB_SET); push(@values, $stat==0 ? $val : undef); $stat==0 || $stat==DB_NOTFOUND or die "c_get: $BerkeleyDB::Error, $!."; } $cursor->c_close==0 or die "c_close: $BerkeleyDB::Error, $!."; undef $cursor; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; if (defined $db) { $cursor->c_close if defined $cursor; # unlock, ignoring status undef $cursor; } }; # restore signal handlers if ($interrupt ne '') { kill($interrupt,$$) } # resignal, ignoring status elsif (defined $eval_stat) { chomp $eval_stat; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout die "read_snmp_variables: BDB S $eval_stat\n"; } for my $val (@values) { if (!defined($val)) {} # keep undefined elsif ($val =~ /^(?:C32|C64) (\d+)\z/) { $val = 0+$1 } elsif ($val =~ /^(?:INT) ([+-]?\d+)\z/) { $val = 0+$1 } elsif ($val =~ /^(?:STR|OID) (.*)\z/) { $val = $1 } else { do_log(-2,"WARN: counter syntax? %s", $val); undef $val } } \@values; } sub register_proc { my($self,$details_level,$reset_timestamp,$state,$task_id) = @_; my($eval_stat); my($interrupt) = ''; if (!defined($state) || $details_level <= $nanny_details_level) { $task_id = '' if !defined $task_id; my($db) = $self->{'db_nanny'}; my($key) = sprintf("%05d",$$); my($cursor); my($val); my($h1) = sub { $interrupt = $_[0] }; local(@SIG{qw(INT HUP TERM TSTP QUIT ALRM USR1 USR2)}) = ($h1) x 8; eval { # ensure cursor will be unlocked even in case of errors or signals $cursor = $db->db_cursor(DB_WRITECURSOR); # obtain write lock defined $cursor or die "db_cursor: $BerkeleyDB::Error, $!."; my($stat) = $cursor->c_get($key,$val,DB_SET); $stat==0 || $stat==DB_NOTFOUND or die "c_get: $BerkeleyDB::Error, $!."; if ($stat==0 && !defined $state) { # remove existing entry $cursor->c_del==0 or die "c_del: $BerkeleyDB::Error, $!."; } elsif (defined $state) { # add new, or update existing entry my($timestamp); local($1); # keep its timestamp when updating existing record $timestamp = $1 if $stat==0 && $val=~/^(\d+(?:\.\d*)?) /s; $timestamp = sprintf("%014.3f", Time::HiRes::time) if !defined($timestamp) || $reset_timestamp; my($new_val) = sprintf("%s %-14s", $timestamp, $state.$task_id); $cursor->c_put($key, $new_val, $stat==0 ? DB_CURRENT : DB_KEYLAST ) == 0 or die "c_put: $BerkeleyDB::Error, $!."; } $cursor->c_close==0 or die "c_close: $BerkeleyDB::Error, $!."; undef $cursor; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; if (defined $db) { $cursor->c_close if defined $cursor; # unlock, ignoring status undef $cursor; # if (!defined($eval_stat)) { # my($stat) = $db->db_sync(); # not really needed # $stat==0 or warn "BDB N db_sync,status $stat: $BerkeleyDB::Error, $!."; # } } }; # restore signal handlers if ($interrupt ne '') { kill($interrupt,$$) } # resignal, ignoring status elsif (defined $eval_stat) { chomp $eval_stat; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout die "register_proc: BDB N $eval_stat\n"; } } 1; # package Amavis::DB; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw($db_home $daemon_chroot_dir); import Amavis::Util qw(untaint ll do_log); } use BerkeleyDB; # create new databases, then close them (called by the parent process) # (called only if $db_home is nonempty) # sub init($$) { my($predelete_nanny, $predelete_snmp) = @_; my($name) = $db_home; $name = "$daemon_chroot_dir $name" if $daemon_chroot_dir ne ''; if ($predelete_nanny || $predelete_snmp) { # delete existing db files first? local(*DIR); opendir(DIR,$db_home) or die "db_init: Can't open directory $name: $!"; # modifying a directory while traversing it can cause surprises, avoid; # avoid slurping the whole directory contents into memory my($f, @rmfiles); while (defined($f = readdir(DIR))) { next if $f eq '.' || $f eq '..'; if ($f eq 'nanny.db') { push(@rmfiles, $f) if $predelete_nanny; } elsif ($f eq 'snmp.db') { push(@rmfiles, $f) if $predelete_snmp; } elsif ($f =~ /^__db\.\d+\z/s) { push(@rmfiles, $f) if $predelete_nanny && $predelete_snmp; } elsif ($f =~ /^(?:cache-expiry|cache)\.db\z/s) { push(@rmfiles, $f); # old databases, no longer used since 2.7.0-pre9 } } closedir(DIR) or die "db_init: Error closing directory $name: $!"; do_log(0, 'Deleting db files %s in %s', join(',',@rmfiles), $name); for my $f (@rmfiles) { my($fname) = $db_home . '/' . untaint($f); unlink($fname) or die "db_init: Can't delete file $fname: $!"; } undef @rmfiles; # release storage } $! = 0; my($env) = BerkeleyDB::Env->new(-Home=>$db_home, -Mode=>0640, -Flags=> DB_CREATE | DB_INIT_CDB | DB_INIT_MPOOL); defined $env or die "BDB can't create db env. at $db_home: $BerkeleyDB::Error, $!."; do_log(0, 'Creating db in %s/; BerkeleyDB %s, libdb %s', $name, BerkeleyDB->VERSION, $BerkeleyDB::db_version); $! = 0; my($dbs) = BerkeleyDB::Hash->new( -Filename=>'snmp.db', -Flags=>DB_CREATE, -Env=>$env ); defined $dbs or die "db_init: BDB no dbS: $BerkeleyDB::Error, $!."; $! = 0; my($dbn) = BerkeleyDB::Hash->new( -Filename=>'nanny.db', -Flags=>DB_CREATE, -Env=>$env ); defined $dbn or die "db_init: BDB no dbN: $BerkeleyDB::Error, $!."; Amavis::DB::SNMP::put_initial_snmp_data($dbs) if $predelete_snmp; for my $db ($dbs, $dbn) { $db->db_close==0 or die "db_init: BDB db_close: $BerkeleyDB::Error, $!."; } } # open an existing databases environment (called by each child process) # sub new { my($class) = @_; my($env); if (defined $db_home) { $! = 0; $env = BerkeleyDB::Env->new( -Home=>$db_home, -Mode=>0640, -Flags=> DB_INIT_CDB | DB_INIT_MPOOL); defined $env or die "BDB can't connect db env. at $db_home: $BerkeleyDB::Error, $!."; } bless \$env, $class; } sub get_db_env { my $self = shift; $$self } 1; __DATA__ # package Amavis::Lookup::SQLfield; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Util qw(ll do_log); import Amavis::Conf qw($trim_trailing_space_in_lookup_result_fields); } # the sub new() is already declared in the always-loaded code section # fieldtype: B=boolean, N=numeric, S=string, # N-: numeric, nonexistent field returns undef without complaint # S-: string, nonexistent field returns undef without complaint # B-: boolean, nonexistent field returns undef without complaint # B0: boolean, nonexistent field treated as false # B1: boolean, nonexistent field treated as true sub lookup_sql_field($$$%) { my($self, $addr, $get_all, %options) = @_; my(@result, @matchingkey, $sql_query, $field); if ($self) { $sql_query = $self->{sql_query}; $field = $self->{fieldname} } $sql_query = $Amavis::sql_lookups if !defined $sql_query; # global default if (!defined $self) { do_log(5, 'lookup_sql_field - no field query object, "%s" no match',$addr); } elsif (!defined $field || $field eq '') { do_log(5, 'lookup_sql_field() - no field name, "%s" no match', $addr); } elsif (!defined $sql_query) { do_log(5, 'lookup_sql_field(%s) - no sql_lookups object, "%s" no match', $field, $addr); } else { my(@result_attr_names) = !ref $field ? ( $field ) : ref $field eq 'ARRAY' ? @$field : ref $field eq 'HASH' ? keys %$field : (); my(%attr_name_to_sqlfield_name) = ref $field eq 'HASH' ? %$field : map( ($_,$_), @result_attr_names); my $fieldtype = $self->{fieldtype}; $fieldtype = 'S-' if !defined $fieldtype; my($res_ref,$mk_ref) = $sql_query->lookup_sql($addr,1, %options, !exists($self->{args}) ? () : (ExtraArguments => $self->{args})); if (!defined $res_ref || !@$res_ref) { ll(5) && do_log(5, 'lookup_sql_field(%s), "%s" no matching records', join(',', map(lc($_) eq lc($attr_name_to_sqlfield_name{$_}) ? $_ : $_ . '/' . $attr_name_to_sqlfield_name{$_}, @result_attr_names)), $addr); } else { my %nosuchfield; for my $ind (0 .. $#$res_ref) { my($any_field_matches, @match_values_by_ind); my $h_ref = $res_ref->[$ind]; my $mk = $mk_ref->[$ind]; for my $result_attr_ind (0 .. $#result_attr_names) { my $result_attr_name = $result_attr_names[$result_attr_ind]; next if !defined $result_attr_name; my $fieldname = $attr_name_to_sqlfield_name{$result_attr_name}; next if !defined $fieldname || $fieldname eq ''; my $match; if (!exists($h_ref->{$fieldname})) { $nosuchfield{$fieldname} = 1; # record found, but no field with that name in the table # fieldtype: B0: boolean, nonexistent field treated as false, # B1: boolean, nonexistent field treated as true if ($fieldtype =~ /^.-/s) { # allowed to not exist # this type is almost universally in use now, continue searching } elsif ($fieldtype =~ /^B1/) { # defaults to true # only used for the 'local' field $match = 1; # nonexistent field treated as 1 } elsif ($fieldtype =~ /^B0/) { # boolean, defaults to false # no longer in use $match = 0; # nonexistent field treated as 0 } else { # treated as 'no match', returns undef } } else { # field exists # fieldtype: B=boolean, N=numeric, S=string $match = $h_ref->{$fieldname}; if (!defined $match) { # NULL field values represented as undef } elsif ($fieldtype =~ /^B/) { # boolean # convert values 'N', 'F', '0', ' ' and "\000" to 0 # to allow value to be used directly as a Perl boolean $match = 0 if $match =~ /^([NnFf ]|0+|\000+)\ *\z/; } elsif ($fieldtype =~ /^N/) { # numeric $match = $match + 0; # convert string into a number } elsif ($fieldtype =~ /^S/) { # string $match =~ s/ +\z// # trim trailing spaces if $trim_trailing_space_in_lookup_result_fields; } } $match_values_by_ind[$result_attr_ind] = $match; $any_field_matches = 1 if defined $match; } ll(5) && do_log(5, 'lookup_sql_field(%s) rec=%d, "%s" result: %s', join(',', map(lc($_) eq lc($attr_name_to_sqlfield_name{$_}) ? $_ : $_ . '/' . $attr_name_to_sqlfield_name{$_}, @result_attr_names)), $ind, $addr, join(', ', map(defined $_ ? '"'.$_.'"' : 'undef', @match_values_by_ind)) ); if ($any_field_matches) { push(@matchingkey, $mk); push(@result, !ref $field ? $match_values_by_ind[0] : { map( ($result_attr_names[$_], $match_values_by_ind[$_]), grep(defined $match_values_by_ind[$_], (0 .. $#result_attr_names) )) } ); last if !$get_all; } } do_log(5, 'lookup_sql_field, no such fields: %s', join(', ', keys %nosuchfield)) if ll(5) && %nosuchfield; } } if (!$get_all) { !wantarray ? $result[0] : ($result[0], $matchingkey[0]) } else { !wantarray ? \@result : (\@result, \@matchingkey) } } 1; # package Amavis::Lookup::SQL; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::Timing qw(section_time); import Amavis::Util qw(untaint snmp_count ll do_log); import Amavis::rfc2821_2822_Tools qw(make_query_keys); import Amavis::Out::SQL::Connection (); } use DBI qw(:sql_types); # return a new Lookup::SQL object to contain DBI handle and prepared selects # sub new { my($class, $conn_h, $clause_name) = @_; if ($clause_name eq '') { undef } else { # $clause_name is a key into %sql_clause of the currently selected # policy bank; one level of indirection is allowed in %sql_clause result, # the resulting SQL clause may include %k, %a, %l, %u, %e, %d placeholders, # to be expanded bless { conn_h => $conn_h, incarnation => 0, clause_name => $clause_name }, $class; } } sub DESTROY { my $self = shift; local($@,$!,$_); eval { do_log(5,"Amavis::Lookup::SQL DESTROY called") }; } sub init { my $self = $_[0]; if ($self->{incarnation} != $self->{conn_h}->incarnation) { # invalidated? $self->{incarnation} = $self->{conn_h}->incarnation; $self->clear_cache; # db handle has changed, invalidate cache } $self; } sub clear_cache { my $self = $_[0]; delete $self->{cache}; } # lookup_sql() performs a lookup for an e-mail address against a SQL map. # If a match is found it returns whatever the query returns (a reference # to a hash containing values of requested fields), otherwise returns undef. # A match aborts further fetching sequence, unless $get_all is true. # # SQL lookups (e.g. for user+foo@example.com) are performed in order # which can be requested by 'ORDER BY' in the SELECT statement, otherwise # the order is unspecified, which is only useful if only specific entries # exist in a database (e.g. only full addresses, not domains). # # The following order is recommended, going from specific to more general: # - lookup for user+foo@example.com # - lookup for user@example.com (only if $recipient_delimiter nonempty) # - lookup for user+foo ('naked lookup' (i.e. no '@'): only if local) # - lookup for user ('naked lookup': local and $recipient_delimiter nonempty) # - lookup for @sub.example.com # - lookup for @.sub.example.com # - lookup for @.example.com # - lookup for @.com # - lookup for @. (catchall) # NOTE: # this is different from hash and ACL lookups in two important aspects: # - a key without '@' implies a mailbox (=user) name, not domain name; # - a naked mailbox name (i.e. no '@' in the query) lookups are only # performed when the e-mail address (usually its domain part) matches # static local_domains* lookups. # # Domain part is always lowercased when constructing a key, # localpart is lowercased unless $localpart_is_case_sensitive is true. # sub lookup_sql($$$%) { my($self, $addr,$get_all,%options) = @_; my(@matchingkey,@result); my($extra_args) = $options{ExtraArguments}; my($sel); my($sql_cl_r) = cr('sql_clause'); my($clause_name) = $self->{clause_name}; $sel = $sql_cl_r->{$clause_name} if defined $sql_cl_r; $sel = $$sel if ref $sel eq 'SCALAR'; # allow one level of indirection if (!defined($sel) || $sel eq '') { ll(4) && do_log(4,"lookup_sql disabled for clause: %s", $clause_name); return(!wantarray ? undef : (undef,undef)); } elsif (!defined $extra_args && exists $self->{cache} && exists $self->{cache}->{$addr}) { # cached ? my($c) = $self->{cache}->{$addr}; @result = @$c if ref $c; @matchingkey = map('/cached/',@result); # will do for now, improve some day # if (!ll(5)) {}# don't bother preparing log report which will not be printed # elsif (!@result) { do_log(5,'lookup_sql (cached): "%s" no match', $addr) } # else { # for my $m (@result) { # do_log(5, "lookup_sql (cached): \"%s\" matches, result=(%s)", # $addr, join(", ", map { sprintf("%s=>%s", $_, # !defined($m->{$_})?'-':'"'.$m->{$_}.'"' # ) } sort keys(%$m) ) ); # } # } if (!$get_all) { return(!wantarray ? $result[0] : ($result[0], $matchingkey[0])); } else { return(!wantarray ? \@result : (\@result, \@matchingkey)); } } my($is_local); # not looked up in SQL and LDAP to avoid recursion! $is_local = Amavis::Lookup::lookup(0,$addr, grep(ref ne 'Amavis::Lookup::SQL' && ref ne 'Amavis::Lookup::SQLfield' && ref ne 'Amavis::Lookup::LDAP' && ref ne 'Amavis::Lookup::LDAPattr', @{ca('local_domains_maps')})); my($keys_ref,$rhs_ref) = make_query_keys($addr, $sql_lookups_no_at_means_domain,$is_local); if (!$sql_allow_8bit_address) { s/[^\040-\176]/?/g for @$keys_ref } my($n) = scalar(@$keys_ref); # number of keys my(@extras_tmp) = !ref $extra_args ? () : @$extra_args; local($1); my(@pos_args); my($sel_taint) = substr($sel,0,0); # taintedness my($datatype) = $sql_allow_8bit_address ? SQL_VARBINARY : SQL_VARCHAR; # substitute %k for a list of keys, %a for unmodified full mail address, # %l for full unmodified localpart, %u for lowercased username (a localpart # without extension), %e for lowercased extension, %d for lowercased domain, # and ? for each extra argument $sel =~ s{ ( %[kaluedL] | \? ) } { push(@pos_args, $1 eq '%k' ? map([$_,$datatype], @$keys_ref) : $1 eq '%a' ? [$rhs_ref->[0], $datatype] #full addr : $1 eq '%l' ? [$rhs_ref->[1], $datatype] #localpart : $1 eq '%u' ? [$rhs_ref->[2], $datatype] #username : $1 eq '%e' ? [$rhs_ref->[3], $datatype] #extension : $1 eq '%d' ? [$rhs_ref->[4], $datatype] #domain #*** (%L is experimental, incomplete) : $1 eq '%L' ? [($is_local?'1':'0'), SQL_BOOLEAN] #is local : shift @extras_tmp), $1 eq '%k' ? join(',', ('?') x $n) : '?' }gxe; $sel = untaint($sel) . $sel_taint; # keep original clause taintedness ll(4) && do_log(4,"lookup_sql %s \"%s\", query args: %s", $clause_name, $addr, join(', ', map(!ref $_ ? '"'.$_.'"' : '['.join(',',@$_).']', @pos_args)) ); ll(4) && do_log(4,"lookup_sql select: %s", $sel); my($a_ref,$found); my($match) = {}; my($conn_h) = $self->{conn_h}; $conn_h->begin_work_nontransaction; # (re)connect if not connected my($driver) = $conn_h->driver_name; # only available when connected if ($driver eq 'Pg') { $datatype = { pg_type => DBD::Pg::PG_BYTEA() }; for (@pos_args) { $_->[1] = $datatype if ref($_) && $_->[1]==SQL_VARBINARY } } for (@pos_args) { if (ref $_) { $_->[0] = untaint($_->[0]) } else { $_ = untaint($_) } } eval { snmp_count('OpsSqlSelect'); $conn_h->execute($sel,@pos_args); # do the query # fetch query results while ( defined($a_ref=$conn_h->fetchrow_arrayref($sel)) ) { my(@names) = @{$conn_h->sth($sel)->{NAME_lc}}; $match = {}; @$match{@names} = @$a_ref; if ($clause_name eq 'sel_policy' && !exists $match->{'local'} && defined $match->{'email'} && $match->{'email'} eq '@.') { # UGLY HACK to let a catchall (@.) imply that field 'local' has # a value undef (NULL) when that field is not present in the # database. This overrides B1 fieldtype default by an explicit # undef for '@.', causing a fallback to static lookup tables. # The purpose is to provide a useful default for local_domains # lookup if the field 'local' is not present in the SQL table. # NOTE: field names 'local' and 'email' are hardwired here!!! push(@names,'local'); $match->{'local'} = undef; do_log(5, 'lookup_sql: "%s" matches catchall, local=>undef', $addr); } push(@result, {%$match}); # copy hash push(@matchingkey, join(", ", map { sprintf("%s=>%s", $_, !defined($match->{$_})?'-':'"'.$match->{$_}.'"' ) } @names)); last if !$get_all; } $conn_h->finish($sel) if defined $a_ref; # only if not all read 1; } or do { my($err) = $@ ne '' ? $@ : "errno=$!"; chomp $err; do_log(-1, "lookup_sql: %s, %s, %s", $err, $DBI::err, $DBI::errstr); die $err if $err =~ /^timed out\b/; # resignal timeout die $err; }; if (!ll(4)) { # don't bother preparing log report which will not be printed } elsif (!@result) { do_log(4,'lookup_sql, "%s" no match', $addr); } else { do_log(4,'lookup_sql(%s) matches, result=(%s)', $addr,$_) for @matchingkey; } # save for future use, but only within processing of this message $self->{cache}->{$addr} = \@result; section_time('lookup_sql'); if (!$get_all) { !wantarray ? $result[0] : ($result[0], $matchingkey[0]) } else { !wantarray ? \@result : (\@result, \@matchingkey) } } 1; __DATA__ #^L package Amavis::LDAP::Connection; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; use Net::LDAP; use Net::LDAP::Util; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION $have_sasl $ldap_sys_default $have_inet6); $VERSION = '2.303'; @ISA = qw(Exporter); $have_sasl = eval { require Authen::SASL }; $have_inet6 = eval { require IO::Socket::INET6 }; import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::Util qw(ll do_log); import Amavis::Timing qw(section_time); $ldap_sys_default = { hostname => 'localhost', localaddr => undef, port => undef, # 389 or 636, default provided by Net::LDAP scheme => undef, # 'ldaps' or 'ldap', depending on hostname inet6 => $have_inet6 ? 1 : 0, version => 3, timeout => 120, deref => 'find', bind_dn => undef, bind_password => undef, tls => 0, verify => 'none', sslversion => 'tlsv1', clientcert => undef, clientkey => undef, cafile => undef, capath => undef, sasl => 0, sasl_mech => undef, # space-separated list of mech names sasl_auth_id => undef, }; } sub new { my($class,$default) = @_; my($self) = bless { ldap => undef }, $class; $self->{incarnation} = 1; for (qw(hostname localaddr port scheme inet6 version timeout base scope deref bind_dn bind_password tls verify sslversion clientcert clientkey cafile capath sasl sasl_mech sasl_auth_id)) { # replace undefined attributes with user values or defaults $self->{$_} = $default->{$_} if !defined($self->{$_}); $self->{$_} = $ldap_sys_default->{$_} if !defined($self->{$_}); } if (!defined $self->{scheme}) { $self->{scheme} = $self->{hostname} =~ /^ldaps/i ? 'ldaps' : 'ldap'; } $self; } sub ldap { # get/set ldap handle my $self = shift; !@_ ? $self->{ldap} : ($self->{ldap}=shift); } sub DESTROY { my $self = shift; local($@,$!,$_); eval { do_log(5,"Amavis::LDAP::Connection DESTROY called") }; eval { $self->disconnect_from_ldap }; } sub incarnation { my $self = shift; $self->{incarnation} } sub in_transaction { 0 } sub begin_work { my $self = shift; do_log(5,"ldap begin_work"); $self->ldap or $self->connect_to_ldap; } sub connect_to_ldap { my $self = shift; my($bind_err,$start_tls_err); do_log(3,"Connecting to LDAP server"); my $hostlist = ref $self->{hostname} eq 'ARRAY' ? join(", ",@{$self->{hostname}}) : $self->{hostname}; do_log(4,"connect_to_ldap: trying %s", $hostlist); my $ldap = Net::LDAP->new($self->{hostname}, localaddr => $self->{localaddr}, port => $self->{port}, scheme => $self->{scheme}, inet6 => $self->{inet6}, version => $self->{version}, timeout => $self->{timeout}, ); if (!$ldap) { # connect failed do_log(-1,"connect_to_ldap: unable to connect to host %s", $hostlist); } else { do_log(3,"connect_to_ldap: connected to %s", $hostlist); if ($self->{tls}) { # TLS required my($mesg) = $ldap->start_tls(verify => $self->{verify}, sslversion => $self->{sslversion}, clientcert => $self->{clientcert}, clientkey => $self->{clientkey}, cafile => $self->{cafile}, capath => $self->{capath}); if ($mesg->code) { # start TLS failed my($err) = $mesg->error_name; do_log(-1,"connect_to_ldap: start TLS failed: %s", $err); $self->ldap(undef); $start_tls_err = 1; } else { # started TLS do_log(3,"connect_to_ldap: TLS version %s enabled", $mesg); } } if ($self->{bind_dn} || $self->{sasl}) { # bind required my($sasl); my($passw) = $self->{bind_password}; if ($self->{sasl}) { # using SASL to authenticate? $have_sasl or die "connect_to_ldap: SASL requested but no Authen::SASL"; $sasl = Authen::SASL->new(mechanism => $self->{sasl_mech}, callback => { user => $self->{sasl_auth_id}, pass => $passw } ); } my($mesg) = $ldap->bind($self->{bind_dn}, $sasl ? (sasl => $sasl) : defined $passw ? (password => $passw) : ()); $passw = 'X' x length($passw) if defined $passw; # can't hurt if ($mesg->code) { # bind failed my($err) = $mesg->error_name; do_log(-1,"connect_to_ldap: bind failed: %s", $err); $self->ldap(undef); $bind_err = 1; } else { # bind succeeded do_log(3,"connect_to_ldap: bind %s succeeded", $self->{bind_dn}); } } } $self->ldap($ldap); $self->{incarnation}++; $ldap or die "connect_to_ldap: unable to connect"; if ($start_tls_err) { die "connect_to_ldap: start TLS failed" } if ($bind_err) { die "connect_to_ldap: bind failed" } section_time('ldap-connect'); $self; } sub disconnect_from_ldap { my $self = shift; if ($self->ldap) { do_log(4,"disconnecting from LDAP"); $self->ldap->disconnect; $self->ldap(undef); } } sub do_search { my($self,$base,$scope,$filter) = @_; my($result,$error_name); $self->ldap or die "do_search: ldap not available"; do_log(5,'lookup_ldap: searching base="%s", scope="%s", filter="%s"', $base, $scope, $filter); eval { $result = $self->{ldap}->search(base => $base, scope => $scope, filter => $filter, deref => $self->{deref}, ); if ($result->code) { $error_name = $result->error_name; if ($error_name eq 'LDAP_NO_SUCH_OBJECT') { # probably alright, e.g. a foreign %d do_log(4, 'do_search failed in "%s": %s', $base, $error_name); } else { die $error_name."\n"; } } 1; } or do { my($err) = $@ ne '' ? $@ : "errno=$!"; chomp $err; die $err if $err =~ /^timed out\b/; # resignal timeout if ($err !~ /^LDAP_/) { die "do_search: $err"; } elsif ($error_name !~ /^LDAP_(?:BUSY|UNAVAILABLE|UNWILLING_TO_PERFORM| TIMEOUT|SERVER_DOWN|CONNECT_ERROR|OTHER)\z/x) { die "do_search: failed: $error_name\n"; } else { # LDAP related error, worth retrying do_log(0, "NOTICE: do_search: trying again: %s", $error_name); $self->disconnect_from_ldap; $self->connect_to_ldap; $self->ldap or die "do_search: reconnect failed"; do_log(5, 'lookup_ldap: searching (again) base="%s", scope="%s", filter="%s"', $base, $scope, $filter); eval { $result = $self->{ldap}->search(base => $base, scope => $scope, filter => $filter, deref => $self->{deref}, ); if ($result->code) { die $result->error_name, "\n"; } 1; } or do { my($err) = $@ ne '' ? $@ : "errno=$!"; chomp $err; $self->disconnect_from_ldap; die $err if $err =~ /^timed out\b/; # resignal timeout die "do_search: failed again, $err"; }; }; }; $result; } 1; # package Amavis::Lookup::LDAPattr; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Util qw(ll do_log); import Amavis::Conf qw($trim_trailing_space_in_lookup_result_fields); } # the sub new() is already declared in the always-loaded code section # attrtype: B=boolean, N=numeric, S=string, L=list # N-: numeric, nonexistent field returns undef without complaint # S-: string, nonexistent field returns undef without complaint # L-: list, nonexistent field returns undef without complaint # B-: boolean, nonexistent field returns undef without complaint # B0: boolean, nonexistent field treated as false # B1: boolean, nonexistent field treated as true sub lookup_ldap_attr($$$%) { my($self, $addr, $get_all, %options) = @_; my(@result, @matchingkey, $ldap_query, $attr); if ($self) { $ldap_query = $self->{ldap_query}; $attr = $self->{attrname} } $ldap_query = $Amavis::ldap_lookups if !defined $ldap_query; # global dflt if (!defined $self) { do_log(5, 'lookup_ldap_attr - no attr query object, "%s" no match',$addr); } elsif (!defined $attr || $attr eq '') { do_log(5, 'lookup_ldap_attr() - no attribute name, "%s" no match', $addr); } elsif (!defined $ldap_query) { do_log(5, 'lookup_ldap_attr(%s) - no ldap_lookups object, "%s" no match', $attr, $addr); } else { # result attribute names are case-sensitive # LDAP attribute names are case-INsensitive my(@result_attr_names) = !ref $attr ? ( $attr ) : ref $attr eq 'ARRAY' ? @$attr : ref $attr eq 'HASH' ? keys %$attr : (); my(%attr_name_to_ldapattr_name) = ref $attr eq 'HASH' ? %$attr : map( ($_,$_), @result_attr_names); my $attrtype = $self->{attrtype}; $attrtype = 'S-' if !defined $attrtype; my($res_ref,$mk_ref) = $ldap_query->lookup_ldap($addr,1, %options, !exists($self->{args}) ? () : (ExtraArguments => $self->{args})); if (!defined $res_ref || !@$res_ref) { ll(5) && do_log(5, 'lookup_ldap_attr(%s), "%s" no matching entries', join(',', map(lc($_) eq lc($attr_name_to_ldapattr_name{$_}) ? $_ : $_ . '/' . lc($attr_name_to_ldapattr_name{$_}), @result_attr_names)), $addr); } else { my %nosuchattr; for my $ind (0 .. $#$res_ref) { my($any_attr_matches, @match_values_by_ind); my $h_ref = $res_ref->[$ind]; my $mk = $mk_ref->[$ind]; for my $result_attr_ind (0 .. $#result_attr_names) { my $result_attr_name = $result_attr_names[$result_attr_ind]; next if !defined $result_attr_name; my $attrname = $attr_name_to_ldapattr_name{$result_attr_name}; next if !defined $attrname || $attrname eq ''; my $match; if (!exists($h_ref->{lc $attrname})) { $nosuchattr{$attrname} = 1; # LDAP entry found, but no attribute with that name in it if ($attrtype =~ /^.-/s) { # allowed to not exist # this type is almost universally in use now, continue searching } elsif ($attrtype =~ /^B1/) { # defaults to true # only used for the 'local' attr $match = 1; # nonexistent attribute treated as 1 } elsif ($attrtype =~ /^B0/) { # boolean, defaults to false # no longer in use $match = 0; # nonexistent attribute treated as 0 } else { # treated as 'no match', returns undef } } else { # attribute exists # attrtype: B=boolean, N=numeric, S=string $match = $h_ref->{lc $attrname}; if (!defined $match) { # NULL attribute values represented as undef } elsif ($attrtype =~ /^B/) { # boolean $match = $match eq 'TRUE' ? 1 : 0; # convert TRUE|FALSE to 1|0 } elsif ($attrtype =~ /^N/) { # numeric $match = $match + 0; # unify different numeric forms } elsif ($attrtype =~ /^S/) { # string $match =~ s/ +\z// # trim trailing spaces if $trim_trailing_space_in_lookup_result_fields; } elsif ($self->{attrtype} =~ /^L/) { # list #$match = join(', ',@$match); } } $match_values_by_ind[$result_attr_ind] = $match; $any_attr_matches = 1 if defined $match; } ll(5) && do_log(5, 'lookup_ldap_attr(%s) rec=%d, "%s" result: %s', join(',', map(lc($_) eq lc($attr_name_to_ldapattr_name{$_}) ? $_ : $_ . '/' . lc($attr_name_to_ldapattr_name{$_}), @result_attr_names)), $ind, $addr, join(', ', map(defined $_ ? '"'.$_.'"' : 'undef', @match_values_by_ind)) ); if ($any_attr_matches) { push(@matchingkey, $mk); push(@result, !ref $attr ? $match_values_by_ind[0] : { map( ($result_attr_names[$_], $match_values_by_ind[$_]), grep(defined $match_values_by_ind[$_], (0 .. $#result_attr_names) )) } ); last if !$get_all; } } do_log(5, 'lookup_ldap_attr, no such attrs: %s', join(', ', keys %nosuchattr)) if ll(5) && %nosuchattr; } } if (!$get_all) { !wantarray ? $result[0] : ($result[0], $matchingkey[0]) } else { !wantarray ? \@result : (\@result, \@matchingkey) } } 1; # package Amavis::Lookup::LDAP; use strict; use re 'taint'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION $ldap_sys_default @ldap_attrs @mv_ldap_attrs); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::Timing qw(section_time); import Amavis::Util qw(untaint snmp_count ll do_log); import Amavis::rfc2821_2822_Tools qw(make_query_keys split_address); import Amavis::LDAP::Connection (); $ldap_sys_default = { base => undef, scope => 'sub', query_filter => '(&(objectClass=amavisAccount)(mail=%m))', }; @ldap_attrs = qw(amavisLocal amavisMessageSizeLimit amavisVirusLover amavisSpamLover amavisUncheckedLover amavisBannedFilesLover amavisBadHeaderLover amavisBypassVirusChecks amavisBypassSpamChecks amavisBypassBannedChecks amavisBypassHeaderChecks amavisSpamTagLevel amavisSpamTag2Level amavisSpamKillLevel amavisSpamDsnCutoffLevel amavisSpamQuarantineCutoffLevel amavisSpamSubjectTag amavisSpamSubjectTag2 amavisSpamModifiesSubj amavisVirusQuarantineTo amavisSpamQuarantineTo amavisBannedQuarantineTo amavisUncheckedQuarantineTo amavisBadHeaderQuarantineTo amavisCleanQuarantineTo amavisArchiveQuarantineTo amavisAddrExtensionVirus amavisAddrExtensionSpam amavisAddrExtensionBanned amavisAddrExtensionBadHeader amavisWarnVirusRecip amavisWarnBannedRecip amavisWarnBadHeaderRecip amavisVirusAdmin amavisNewVirusAdmin amavisSpamAdmin amavisBannedAdmin amavisBadHeaderAdmin amavisBannedRuleNames amavisDisclaimerOptions amavisForwardMethod amavisSaUserConf amavisSaUserName amavisBlacklistSender amavisWhitelistSender ); @mv_ldap_attrs = qw(amavisBlacklistSender amavisWhitelistSender); } sub new { my($class,$default,$conn_h) = @_; my($self) = bless {}, $class; $self->{conn_h} = $conn_h; $self->{incarnation} = 0; for (qw(base scope query_filter)) { # replace undefined attributes with config values or defaults $self->{$_} = $default->{$_} unless defined($self->{$_}); $self->{$_} = $ldap_sys_default->{$_} unless defined($self->{$_}); } $self; } sub DESTROY { my $self = shift; local($@,$!,$_); eval { do_log(5,"Amavis::Lookup::LDAP DESTROY called") }; } sub init { my $self = $_[0]; if ($self->{incarnation} != $self->{conn_h}->incarnation) { # invalidated? $self->{incarnation} = $self->{conn_h}->incarnation; $self->clear_cache; # db handle has changed, invalidate cache } $self; } sub clear_cache { my $self = $_[0]; delete $self->{cache}; } sub lookup_ldap($$$%) { my($self,$addr,$get_all,%options) = @_; my(@result,@matchingkey,@tmp_result,@tmp_matchingkey); if (exists $self->{cache} && exists $self->{cache}->{$addr}) { # cached? my($c) = $self->{cache}->{$addr}; @result = @$c if ref $c; @matchingkey = map('/cached/',@result); # will do for now, improve some day # if (!ll(5)) { # # don't bother preparing log report which will not be printed # } elsif (!@result) { # do_log(5,'lookup_ldap (cached): "%s" no match', $addr); # } else { # for my $m (@result) { # do_log(5, 'lookup_ldap (cached): "%s" matches, result=(%s)', # $addr, join(", ", map { sprintf("%s=>%s", $_, # !defined($m->{$_})?'-':'"'.$m->{$_}.'"' # ) } sort keys(%$m) ) ); # } # } if (!$get_all) { return(!wantarray ? $result[0] : ($result[0], $matchingkey[0])); } else { return(!wantarray ? \@result : (\@result, \@matchingkey)); } } my($is_local); # not looked up in SQL and LDAP to avoid recursion! $is_local = Amavis::Lookup::lookup(0,$addr, grep(ref ne 'Amavis::Lookup::SQL' && ref ne 'Amavis::Lookup::SQLfield' && ref ne 'Amavis::Lookup::LDAP' && ref ne 'Amavis::Lookup::LDAPattr', @{ca('local_domains_maps')})); my($keys_ref,$rhs_ref,@keys); ($keys_ref,$rhs_ref) = make_query_keys($addr, $ldap_lookups_no_at_means_domain,$is_local); @keys = @$keys_ref; unshift(@keys, '<>') if $addr eq ''; # a hack for a null return path $_ = untaint($_) for @keys; # untaint keys $_ = Net::LDAP::Util::escape_filter_value($_) for @keys; # process %m my $filter = $self->{query_filter}; my @filter_attr; my $expanded_filter = ''; for my $t ($filter =~ /\G( \( [^(=]+ = %m \) | [ \t0-9A-Za-z]+ | . )/gsx) { if ($t !~ m{ \( ([^(=]+) = %m \) }sx) { $expanded_filter .= $t } else { push(@filter_attr, $1); $expanded_filter .= '(|' . join('', map("($1=$_)", @keys)) . ')'; } } $filter = $expanded_filter; # process %d my($base) = $self->{base}; if ($base =~ /%d/) { my($localpart,$domain) = split_address($addr); if ($domain) { $domain = untaint($domain); $domain = lc($domain); local($1); $domain =~ s/^\@?(.*?)\.*\z/$1/s; $base =~ s/%d/&Net::LDAP::Util::escape_dn_value($domain)/ge; } } # build hash of keys and array position my(%xref); my($key_num) = 0; $xref{$_} = $key_num++ for @keys; # do_log(4,'lookup_ldap "%s", query keys: %s, base: %s, filter: %s', $addr,join(', ',map("\"$_\"",@keys)),$self->{base},$self->{query_filter}); my($conn_h) = $self->{conn_h}; $conn_h->begin_work; # (re)connect if not connected eval { snmp_count('OpsLDAPSearch'); my(@entry); my($search_obj) = $conn_h->do_search($base, $self->{scope}, $filter); @entry = $search_obj->entries if $search_obj && !$search_obj->code; my(%mv_ldap_attrs) = map((lc($_), 1), @mv_ldap_attrs); for my $entry (@entry) { my($match) = {}; $match->{dn} = $entry->dn; for my $attr (@ldap_attrs) { my($value); do_log(9,'lookup_ldap: reading attribute "%s" from object', $attr); $attr = lc($attr); if ($mv_ldap_attrs{$attr}) { # multivalued $value = $entry->get_value($attr, asref => 1); } else { $value = $entry->get_value($attr); } $match->{$attr} = $value if defined $value; } my $pos; for my $attr (@filter_attr) { my $value = scalar($entry->get_value($attr)); if (defined $value) { if (!exists $match->{'amavislocal'} && $value eq '@.') { # NOTE: see lookup_sql $match->{'amavislocal'} = undef; do_log(5, 'lookup_ldap: "%s" matches catchall, amavislocal=>undef', $addr); } $pos = $xref{$value}; last; } } my $key_str = join(", ",map {sprintf("%s=>%s",$_,!defined($match->{$_})? '-':'"'.$match->{$_}.'"')} keys(%$match)); push(@tmp_result, [$pos,{%$match}]); # copy hash push(@tmp_matchingkey, [$pos,$key_str]); last if !$get_all; } 1; } or do { my($err) = $@ ne '' ? $@ : "errno=$!"; chomp $err; do_log(-1,"lookup_ldap: %s", $err); die $err; }; @result = map($_->[1], sort {$a->[0] <=> $b->[0]} @tmp_result); @matchingkey = map($_->[1], sort {$a->[0] <=> $b->[0]} @tmp_matchingkey); if (!ll(4)) { # don't bother preparing log report which will not be printed } elsif (!@result) { do_log(4,'lookup_ldap, "%s" no match', $addr); } else { do_log(4,'lookup_ldap(%s) matches, result=(%s)',$addr,$_) for @matchingkey; } # save for future use, but only within processing of this message $self->{cache}->{$addr} = \@result; section_time('lookup_ldap'); if (!$get_all) { !wantarray ? $result[0] : ($result[0], $matchingkey[0]) } else { !wantarray ? \@result : (\@result, \@matchingkey) } } 1; __DATA__ # package Amavis::In::AMPDP; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::Util qw(ll do_log debug_oneshot untaint snmp_counters_init snmp_count proto_encode proto_decode orcpt_encode switch_to_my_time switch_to_client_time am_id new_am_id add_entropy rmdir_recursively); import Amavis::Lookup qw(lookup lookup2); import Amavis::Lookup::IP qw(lookup_ip_acl); import Amavis::Timing qw(section_time); import Amavis::rfc2821_2822_Tools; import Amavis::In::Message; import Amavis::In::Connection; import Amavis::IO::Zlib; import Amavis::Out::EditHeader qw(hdr); import Amavis::Out qw(mail_dispatch); import Amavis::Notify qw(msg_from_quarantine); } use subs @EXPORT; use Errno qw(ENOENT EACCES); use IO::File (); use Time::HiRes (); use Digest::MD5; use MIME::Base64; sub new($) { my($class) = @_; bless {}, $class } # used with sendmail milter and traditional (non-SMTP) MTA interface, # but also to request a message release from a quarantine # sub process_policy_request($$$$) { my($self, $sock, $conn, $check_mail, $old_amcl) = @_; # $sock: connected socket from Net::Server # $conn: information about client connection # $check_mail: subroutine ref to be called with file handle my(%attr); $0 = sprintf("%s (ch%d-P-idle)", c('myprogram_name'), $Amavis::child_invocation_count); ll(5) && do_log(5, "process_policy_request: %s, %s, fileno=%s", $old_amcl, c('myprogram_name'), fileno($sock)); if ($old_amcl) { # Accept a single request from traditional amavis helper program. # Receive TEMPDIR/SENDER/RCPTS/LDA/LDAARGS from client # Simple protocol: \2 means LDA follows; \3 means EOT (end of transmission) die "process_policy_request: old AM.CL protocol is no longer supported\n"; } else { # new amavis helper protocol AM.PDP or a Postfix policy server # for Postfix policy server see Postfix docs SMTPD_POLICY_README my(@response); local($1,$2,$3); local($/) = "\012"; # set line terminator to LF (Postfix idiosyncrasy) my($ln); # can accept multiple tasks switch_to_client_time("start receiving AM.PDP data"); $conn->appl_proto('AM.PDP'); for ($! = 0; defined($ln=$sock->getline); $! = 0) { my($end_of_request) = $ln =~ /^\015?\012\z/ ? 1 : 0; # end of request? switch_to_my_time($end_of_request ? 'rx entire AM.PDP request' : 'rx AM.PDP line'); $0 = sprintf("%s (ch%d-P)", c('myprogram_name'), $Amavis::child_invocation_count); Amavis::Timing::init(); snmp_counters_init(); # must not use \r and \n, not the same as \015 and \012 on some platforms if ($end_of_request) { # end of request section_time('got data'); eval { my($msginfo,$bank_names_ref) = preprocess_policy_query(\%attr,$conn); $Amavis::MSGINFO = $msginfo; # ugly my($req) = lc($attr{'request'}); @response = $req eq 'smtpd_access_policy' ? postfix_policy($msginfo,\%attr) : $req =~ /^(?:release|requeue|report)\z/ ? dispatch_from_quarantine($msginfo, $req, $req eq 'report' ? 'abuse' : 'miscategorized') : check_ampdp_policy($msginfo,$check_mail,0,$bank_names_ref); undef $Amavis::MSGINFO; # release global reference 1; } or do { my($err) = $@ ne '' ? $@ : "errno=$!"; chomp $err; do_log(-2, "policy_server FAILED: %s", $err); @response = (proto_encode('setreply','450','4.5.0',"Failure: $err"), proto_encode('return_value','tempfail'), proto_encode('exit_code',sprintf("%d",EX_TEMPFAIL))); die $err if $err =~ /^timed out\b/; # resignal timeout # last; }; $sock->print( join('', map($_."\015\012", (@response,'')) )) or die "Can't write response to socket: $!, fileno=".fileno($sock); %attr = (); @response = (); do_log(2, "%s", Amavis::Timing::report()); } elsif ($ln =~ /^ ([^=\000\012]*?) (=|:[ \t]*) ([^\012]*?) \015?\012 \z/xsi) { my($attr_name) = proto_decode($1); my($attr_val) = proto_decode($3); if (!exists $attr{$attr_name}) { $attr{$attr_name} = $attr_val; } else { $attr{$attr_name} = [ $attr{$attr_name} ] if !ref $attr{$attr_name}; push(@{$attr{$attr_name}}, $attr_val); } my($known_attr) = scalar(grep($_ eq $attr_name, qw( request protocol_state version_client protocol_name helo_name client_name client_address client_port client_source sender recipient delivery_care_of queue_id partition_tag mail_id secret_id quar_type mail_file tempdir tempdir_removed_by policy_bank requested_by) )); do_log(!$known_attr?1:3, "policy protocol: %s=%s", $attr_name,$attr_val); } else { do_log(-1, "policy protocol: INVALID AM.PDP ATTRIBUTE LINE: %s", $ln); } $0 = sprintf("%s (ch%d-P-idle)", c('myprogram_name'), $Amavis::child_invocation_count); switch_to_client_time("receiving AM.PDP data"); } defined $ln || $!==0 or die "Read from client socket FAILED: $!"; switch_to_my_time('end of AM.PDP session'); }; $0 = sprintf("%s (ch%d-P)", c('myprogram_name'), $Amavis::child_invocation_count); } # Based on given query attributes describing a message to be checked or # released, return a new Amavis::In::Message object with filled-in information # sub preprocess_policy_query($$) { my($attr_ref,$conn) = @_; my($now) = Time::HiRes::time; my($msginfo) = Amavis::In::Message->new; $msginfo->rx_time($now); $msginfo->log_id(am_id()); $msginfo->conn_obj($conn); add_entropy(%$attr_ref); # amavisd -> amavis-helper protocol query consists of any number of # the following lines, the response is terminated by an empty line. # The 'request=AM.PDP' is a required first field, the order of # remaining fields is arbitrary, but multivalued attributes such as # 'recipient' must retain their relative order. # Required AM.PDP fields are: request, tempdir, sender, recipient(s) # request=AM.PDP # version_client=n (currently ignored) # tempdir=/var/amavis/amavis-milter-MWZmu9Di # tempdir_removed_by=client (tempdir_removed_by=server is a default) # mail_file=/var/amavis/am.../email.txt (defaults to tempdir/email.txt) # sender= # recipient= # recipient= # recipient= # delivery_care_of=server (client or server, client is a default) # queue_id=qid # protocol_name=ESMTP # helo_name=host.example.com # client_address=10.2.3.4 # client_port=45678 # client_name=host.example.net # client_source=LOCAL/REMOTE/[UNAVAILABLE] # (matches local_header_rewrite_clients, see Postfix XFORWARD_README) # policy_bank=SMTP_AUTH,TLS,ORIGINATING,MYNETS,... # Required 'release' or 'requeue' or 'report' fields are: request, mail_id # request=release (or request=requeue, or request=report) # mail_id=xxxxxxxxxxxx # secret_id=xxxxxxxxxxxx (authorizes a release/report) # partition_tag=xx (required if mail_id is not unique) # quar_type=x F/Z/B/Q/M (defaults to Q or F) # file/zipfile/bsmtp/sql/mailbox # mail_file=... (optional: overrides automatics; $QUARANTINEDIR prepended) # requested_by= (optional: lands in Resent-From:) # sender= (optional: replaces envelope sender) # recipient= (optional: replaces envelope recips) # recipient= # recipient= my(@recips); my(@bank_names); exists $attr_ref->{'request'} or die "Missing 'request' field"; my($ampdp) = $attr_ref->{'request'} =~ /^(?:AM\.CL|AM\.PDP|release|requeue|report)\z/i; @bank_names = grep($_ ne '', map { s/^[ \t]+//; s/[ \t]+\z//; $_ } split(/,/, $attr_ref->{'policy_bank'})) if exists $attr_ref->{'policy_bank'}; my($d_co) = $attr_ref->{'delivery_care_of'}; my($td_rm) = $attr_ref->{'tempdir_removed_by'}; $msginfo->client_delete(defined($td_rm) && lc($td_rm) eq 'client' ? 1 : 0); $msginfo->queue_id($attr_ref->{'queue_id'}) if exists $attr_ref->{'queue_id'}; $msginfo->client_proto($attr_ref->{'protocol_name'}) if exists $attr_ref->{'protocol_name'}; if (exists $attr_ref->{'client_address'}) { my($cl_ip) = $attr_ref->{'client_address'}; $cl_ip =~ s/^IPv6://i; $msginfo->client_addr(lc $cl_ip); } $msginfo->client_port($attr_ref->{'client_port'}) if exists $attr_ref->{'client_port'}; $msginfo->client_name($attr_ref->{'client_name'}) if exists $attr_ref->{'client_name'}; $msginfo->client_source($attr_ref->{'client_source'}) if exists $attr_ref->{'client_source'} && uc($attr_ref->{'client_source'}) ne '[UNAVAILABLE]'; $msginfo->client_helo($attr_ref->{'helo_name'}) if exists $attr_ref->{'helo_name'}; # $msginfo->body_type('8BITMIME'); $msginfo->requested_by(unquote_rfc2821_local($attr_ref->{'requested_by'})) if exists $attr_ref->{'requested_by'}; if (exists $attr_ref->{'sender'}) { my($sender) = $attr_ref->{'sender'}; $sender = '<'.$sender.'>' if $sender !~ /^<.*>\z/; $msginfo->sender_smtp($sender); $sender = unquote_rfc2821_local($sender); $msginfo->sender($sender); } if (exists $attr_ref->{'recipient'}) { my($r) = $attr_ref->{'recipient'}; @recips = (); for my $addr (!ref($r) ? $r : @$r) { my($addr_quo) = $addr; my($addr_unq) = unquote_rfc2821_local($addr); $addr_quo = '<'.$addr_quo.'>' if $addr_quo !~ /^<.*>\z/; my($recip_obj) = Amavis::In::Message::PerRecip->new; $recip_obj->recip_addr($addr_unq); $recip_obj->recip_addr_smtp($addr_quo); $recip_obj->dsn_orcpt(orcpt_encode($addr_quo)); $recip_obj->recip_destiny(D_PASS); # default is Pass $recip_obj->delivery_method('') if !defined($d_co) || lc($d_co) eq 'client'; push(@recips,$recip_obj); } $msginfo->per_recip_data(\@recips); } if (!exists $attr_ref->{'tempdir'}) { my($tempdir) = Amavis::TempDir->new; $tempdir->prepare_dir; $msginfo->mail_tempdir($tempdir->path); # Save the Amavis::TempDir object from destruction by keeping a ref to it # in $msginfo. When $msginfo is destroyed, the temporary directory will be # automatically destroyed too. This is specific to AM.PDP requests without # a working directory provided by a caller, and different from usual # SMTP sessions which keep a per-process permanent reference to an # Amavis::TempDir object, which makes keeping it in mail_tempdir_obj # not necessary. $msginfo->mail_tempdir_obj($tempdir); } else { local($1,$2); my($tempdir) = $attr_ref->{tempdir}; $tempdir =~ m{^ (?: \Q$TEMPBASE\E | \Q$MYHOME\E ) (?: / (?! \.\. (?:\z|/)) [A-Za-z0-9_.-]+ )* / [A-Za-z0-9_.-]+ \z}xso or die "Suspicious temporary directory name '$tempdir'"; $msginfo->mail_tempdir(untaint($tempdir)); } my($quar_type); if (!$ampdp) { # don't bother with filenames } elsif ($attr_ref->{'request'} =~ /^(?:release|requeue|report)\z/i) { exists $attr_ref->{'mail_id'} or die "Missing 'mail_id' field"; $msginfo->partition_tag($attr_ref->{'partition_tag'}); # may be undef my($mail_id) = $attr_ref->{'mail_id'}; # amavisd almost-base64: 62 +, 63 - (in use up to 2.6.4, dropped in 2.7.0) # RFC 4648 base64: 62 +, 63 / (not used here) # RFC 4648 base64url: 62 -, 63 _ $mail_id =~ m{^ [A-Za-z0-9] [A-Za-z0-9_+-]* ={0,2} \z}xs or die "Invalid mail_id '$mail_id'"; $msginfo->mail_id(untaint($mail_id)); if (!exists($attr_ref->{'secret_id'}) || $attr_ref->{'secret_id'} eq '') { die "Secret_id is required, but missing" if c('auth_required_release'); } else { # version 2.7.0 and later uses RFC 4648 base64url and id=b64(md5(sec)), # versions before 2.7.0 used almost-base64 and id=b64(md5(b64(sec))) { # begin block, 'last' exits it my($secret_b64) = $attr_ref->{'secret_id'}; $secret_b64 = '' if !defined $secret_b64; if (index($secret_b64,'+') < 0) { # new or undetermined format local($_) = $secret_b64; tr{-_}{+/}; # revert base64url to base64 my($secret_bin) = decode_base64($_); my($id_new_b64) = Digest::MD5->new->add($secret_bin)->b64digest; $id_new_b64 = substr($id_new_b64, 0, 12); $id_new_b64 =~ tr{+/}{-_}; # base64 -> RFC 4648 base64url last if $id_new_b64 eq $mail_id; # exit enclosing block } if (index($secret_b64,'_') < 0) { # old or undetermined format my($id_old_b64) = Digest::MD5->new->add($secret_b64)->b64digest; $id_old_b64 = substr($id_old_b64, 0, 12); $id_old_b64 =~ tr{/}{-}; # base64 -> almost-base64 last if $id_old_b64 eq $mail_id; # exit enclosing block } die "Secret_id $secret_b64 does not match mail_id $mail_id"; }; # end block, 'last' arrives here } $quar_type = $attr_ref->{'quar_type'}; if (!defined($quar_type) || $quar_type eq '') { # choose some reasonable default (simpleminded) $quar_type = c('spam_quarantine_method') =~ /^sql:/i ? 'Q' : 'F'; } my($fn) = $mail_id; if ($quar_type eq 'F' || $quar_type eq 'Z') { $QUARANTINEDIR ne '' or die "Config variable \$QUARANTINEDIR is empty"; if ($attr_ref->{'mail_file'} ne '') { $fn = $attr_ref->{'mail_file'}; $fn =~ m{^[A-Za-z0-9][A-Za-z0-9/_.+-]*\z}s && $fn !~ m{\.\.(/|\z)} or die "Unsafe filename '$fn'"; $fn = $QUARANTINEDIR.'/'.untaint($fn); } else { # automatically guess a filename - simpleminded if ($quarantine_subdir_levels < 1) { $fn = "$QUARANTINEDIR/$fn" } else { my($subd) = substr($fn,0,1); $fn = "$QUARANTINEDIR/$subd/$fn" } $fn .= '.gz' if $quar_type eq 'Z'; } } $msginfo->mail_text_fn($fn); } elsif (!exists $attr_ref->{'mail_file'}) { $msginfo->mail_text_fn($msginfo->mail_tempdir . '/email.txt'); } else { # SECURITY: just believe the supplied file name, blindly untainting it $msginfo->mail_text_fn(untaint($attr_ref->{'mail_file'})); } if ($ampdp && $msginfo->mail_text_fn ne '') { my($fh); my($fname) = $msginfo->mail_text_fn; my($releasing) = $attr_ref->{'request'}=~ /^(?:release|requeue|report)\z/i; new_am_id('rel-'.$msginfo->mail_id) if $releasing; if ($releasing && $quar_type eq 'Q') { # releasing from SQL do_log(5, "preprocess_policy_query: opening in sql: %s", $msginfo->mail_id); my($obj) = $Amavis::sql_storage; $Amavis::extra_code_sql_quar && $obj or die "SQL quarantine code not enabled (3)"; my($conn_h) = $obj->{conn_h}; my($sql_cl_r) = cr('sql_clause'); my($sel_msg) = $sql_cl_r->{'sel_msg'}; my($sel_quar) = $sql_cl_r->{'sel_quar'}; if (!defined($msginfo->partition_tag) && defined($sel_msg) && $sel_msg ne '') { do_log(5, "preprocess_policy_query: missing partition_tag in request,". " fetching msgs record for mail_id=%s", $msginfo->mail_id); # find a corresponding partition_tag if missing from a release request $conn_h->begin_work_nontransaction; #(re)connect if necessary $conn_h->execute($sel_msg, untaint($msginfo->mail_id)); my($a_ref); my($cnt) = 0; my($partition_tag); while ( defined($a_ref=$conn_h->fetchrow_arrayref($sel_msg)) ) { $cnt++; $partition_tag = $a_ref->[0] if !defined $partition_tag; ll(5) && do_log(5, "release: got msgs record for mail_id=%s: %s", $msginfo->mail_id, join(', ',@$a_ref)); } $conn_h->finish($sel_msg) if defined $a_ref; # only if not all read $cnt <= 1 or die "Multiple ($cnt) records with same mail_id exist, ". "specify a partition_tag in the AM.PDP request"; if ($cnt < 1) { do_log(0, "release: no records with msgs.mail_id=%s in a database, ". "trying to read from a quar. anyway", $msginfo->mail_id); } $msginfo->partition_tag($partition_tag); # could still be undef/NULL ! } ll(5) && do_log(5, "release: opening mail_id=%s, partition_tag=%s", $msginfo->mail_id, $msginfo->partition_tag); $conn_h->begin_work_nontransaction; # (re)connect if not connected $fh = Amavis::IO::SQL->new; $fh->open($conn_h, $sel_quar, untaint($msginfo->mail_id), 'r', untaint($msginfo->partition_tag)) or die "Can't open sql obj for reading: $!"; 1; } else { # mail checking or releasing from a file do_log(5, "preprocess_policy_query: opening mail '%s'", $fname); # set new amavis message id new_am_id( ($fname =~ m{amavis-(milter-)?([^/ \t]+)}s ? $2 : undef) ) if !$releasing; # file created by amavis helper program or other client, just open it my(@stat_list) = lstat($fname); my($errn) = @stat_list ? 0 : 0+$!; if ($errn == ENOENT) { die "File $fname does not exist" } elsif ($errn) { die "File $fname inaccessible: $!" } elsif (!-f _) { die "File $fname is not a plain file" } add_entropy(@stat_list); if ($fname =~ /\.gz\z/) { $fh = Amavis::IO::Zlib->new; $fh->open($fname,'rb') or die "Can't open gzipped file $fname: $!"; } else { # $msginfo->msg_size(0 + (-s _)); # underestimates the RFC 1870 size $fh = IO::File->new; $fh->open($fname,'<') or die "Can't open file $fname: $!"; binmode($fh,':bytes') or die "Can't cancel :utf8 mode: $!"; } } $msginfo->mail_text($fh); # save file handle to object $msginfo->log_id(am_id()); } if ($ampdp) { do_log(1, "Request: %s %s %s: %s -> %s", $attr_ref->{'request'}, $attr_ref->{'mail_id'}, $msginfo->mail_tempdir, $msginfo->sender_smtp, join(',', map($_->recip_addr_smtp, @recips)) ); } else { do_log(1, "Request: %s(%s): %s %s %s: %s[%s] <%s> -> <%s>", @$attr_ref{qw(request protocol_state mail_id protocol_name queue_id client_name client_address sender recipient)}); } ($msginfo, \@bank_names); } sub check_ampdp_policy($$$$) { my($msginfo,$check_mail,$old_amcl,$bank_names_ref) = @_; my($smtp_resp, $exit_code, $preserve_evidence); my(%baseline_policy_bank) = %current_policy_bank; # do some sanity checks before deciding to call check_mail() if (!ref($msginfo->per_recip_data) || !defined($msginfo->mail_text)) { $smtp_resp = '450 4.5.0 Incomplete request'; $exit_code = EX_TEMPFAIL; } else { # loading a policy bank can affect subsequent c(), cr() and ca() results, # so it is necessary to load each policy bank in the right order and soon # after information becomes available; general principle is that policy # banks are loaded in order in which information becomes available: # interface/socket, client IP, SMTP session info, sender, ... my($cl_ip) = $msginfo->client_addr; my($cl_src) = $msginfo->client_source; my($cl_ip_mynets, $policy_name_requested); { my($cl_ip_tmp) = $cl_ip; # treat unknown client IP addr as 0.0.0.0, from "This" Network, RFC 1700 $cl_ip_tmp = '0.0.0.0' if !defined($cl_ip) || $cl_ip eq ''; my(@cp) = @{ca('client_ipaddr_policy')}; do_log(-1,"\@client_ipaddr_policy must contain pairs, ". "number of elements is not even") if @cp % 2 != 0; while (@cp) { my($lookup_table) = shift(@cp); my($policy_name) = shift(@cp); if (lookup_ip_acl($cl_ip_tmp, $lookup_table)) { if (defined $policy_name && $policy_name ne '') { $policy_name_requested = $policy_name; $cl_ip_mynets = 1 if $policy_name eq 'MYNETS'; # compatibility } last; } } } if (($cl_ip_mynets?1:0) > ($msginfo->originating?1:0)) { $current_policy_bank{'originating'} = $cl_ip_mynets; # compatibility } if (defined $policy_name_requested && defined $policy_bank{$policy_name_requested}) { Amavis::load_policy_bank($policy_name_requested,$msginfo); } for my $bank_name (@$bank_names_ref) { # additional banks from the request if (defined $policy_bank{$bank_name}) { Amavis::load_policy_bank(untaint($bank_name),$msginfo) } } $msginfo->originating(c('originating')); my($sender) = $msginfo->sender; if (defined $policy_bank{'MYUSERS'} && $sender ne '' && $msginfo->originating && lookup2(0,$sender, ca('local_domains_maps'))) { Amavis::load_policy_bank('MYUSERS',$msginfo); $msginfo->originating(c('originating')); # may have changed by a p.b.load } my $debrecipm = ca('debug_recipient_maps'); if (lookup2(0, $sender, ca('debug_sender_maps')) || @$debrecipm && grep(lookup2(0, $_->recip_addr, $debrecipm), @{$msginfo->per_recip_data})) { debug_oneshot(1); } # check_mail() expects open file on $fh, need not be rewound Amavis::check_mail_begin_task(); ($smtp_resp, $exit_code, $preserve_evidence) = &$check_mail($msginfo,0); my($fh) = $msginfo->mail_text; my($tempdir) = $msginfo->mail_tempdir; $fh->close or die "Error closing temp file: $!" if $fh; undef $fh; $msginfo->mail_text(undef); my($errn) = $tempdir eq '' ? ENOENT : (stat($tempdir) ? 0 : 0+$!); if ($tempdir eq '' || $errn == ENOENT) { # do nothing } elsif ($msginfo->client_delete) { do_log(4, "AM.PDP: deletion of %s is client's responsibility", $tempdir); } elsif ($preserve_evidence) { do_log(-1,"AM.PDP: tempdir is to be PRESERVED: %s", $tempdir); } else { my($fname) = $msginfo->mail_text_fn; do_log(4, "AM.PDP: tempdir and file being removed: %s, %s", $tempdir,$fname); unlink($fname) or die "Can't remove file $fname: $!" if $fname ne ''; # must step out of the directory which is about to be deleted, # otherwise rmdir can fail (e.g. on Solaris) chdir($TEMPBASE) or die "Can't chdir to $TEMPBASE: $!"; rmdir_recursively($tempdir); } } # amavisd -> amavis-helper protocol response consists of any number of # the following lines, the response is terminated by an empty line: # version_server=2 # log_id=xxx # delrcpt= # addrcpt= # delheader=hdridx hdr_head # chgheader=hdridx hdr_head hdr_body # insheader=hdridx hdr_head hdr_body # addheader=hdr_head hdr_body # replacebody=new_body (not implemented) # quarantine=reason (currently never used, supposed to call # smfi_quarantine, placing message on hold) # return_value=continue|reject|discard|accept|tempfail # setreply=rcode xcode message # exit_code=n my(@response); my($rcpt_deletes,$rcpt_count)=(0,0); push(@response, proto_encode('version_server', '2')); push(@response, proto_encode('log_id', $msginfo->log_id)); for my $r (@{$msginfo->per_recip_data}) { $rcpt_count++; $rcpt_deletes++ if $r->recip_done; } local($1,$2,$3); if ($smtp_resp=~/^([1-5]\d\d) ([245]\.\d{1,3}\.\d{1,3})(?: |\z)(.*)\z/s) { push(@response, proto_encode('setreply', $1,$2,$3)) } if ( $exit_code == EX_TEMPFAIL) { push(@response, proto_encode('return_value','tempfail')); } elsif ($exit_code == EX_NOUSER) { # reject the whole message push(@response, proto_encode('return_value','reject')); } elsif ($exit_code == EX_UNAVAILABLE) { # reject the whole message push(@response, proto_encode('return_value','reject')); } elsif ($exit_code == 99 || $rcpt_deletes >= $rcpt_count) { $exit_code = 99; # let MTA discard the message, it was already handled here push(@response, proto_encode('return_value','discard')); } elsif (grep($_->delivery_method ne '', @{$msginfo->per_recip_data})) { # explicit forwarding by us die "Not all recips done, but explicit forwarding"; # just in case } else { # EX_OK for my $r (@{$msginfo->per_recip_data}) { # modified recipient addresses? my($newaddr) = $r->recip_final_addr; if ($r->recip_done) { # delete push(@response, proto_encode('delrcpt', $r->recip_addr_smtp)) if defined $r->recip_addr; # if in the original list, not always_bcc } elsif ($newaddr ne $r->recip_addr) { # modify, e.g. adding extension push(@response, proto_encode('delrcpt', $r->recip_addr_smtp)) if defined $r->recip_addr; # if in the original list, not always_bcc push(@response, proto_encode('addrcpt', qquote_rfc2821_local($newaddr))); } } my($hdr_edits) = $msginfo->header_edits; if ($hdr_edits) { # any added or modified header fields? local($1,$2); my($field_name,$edit,$field_body); while ( ($field_name,$edit) = each %{$hdr_edits->{edit}} ) { $field_body = $msginfo->get_header_field_body($field_name,0); # first if (!defined($field_body)) { # such header field does not exist or is not available, do nothing } else { # edit the first occurrence chomp($field_body); my($orig_field_body) = $field_body; for my $e (@$edit) { # possibly multiple (iterative) edits if (!defined($e)) { $field_body = undef; last } # delete existing my($new_fbody,$verbatim) = &$e($field_name,$field_body); if (!defined($new_fbody)) { $field_body = undef; last } # delete my($curr_head) = $verbatim ? ($field_name . ':' . $new_fbody) : hdr($field_name, $new_fbody, 0); chomp($curr_head); $curr_head .= "\n"; $curr_head =~ /^([^:]*?)[ \t]*:(.*)\z/s; $field_body = $2; chomp($field_body); # carry to next iteration } if (!defined($field_body)) { push(@response, proto_encode('delheader','1',$field_name)); } elsif ($field_body ne $orig_field_body) { # sendmail insertes a space after a colon, remove ours $field_body =~ s/^[ \t]//; push(@response, proto_encode('chgheader','1', $field_name,$field_body)); } } } my($hdridx) = c('prepend_header_fields_hdridx'); # milter insertion index $hdridx = 0 if !defined($hdridx) || $hdridx < 0; $hdridx = sprintf("%d",$hdridx); # convert to string # prepend header fields one at a time, topmost field last for my $hf (map(ref $hdr_edits->{$_} ? reverse @{$hdr_edits->{$_}} : (), qw(addrcvd prepend)) ) { if ($hf =~ /^([^:]*?)[ \t]*:[ \t]*(.*?)$/s) { push(@response, proto_encode('insheader',$hdridx,$1,$2)) } } # append header fields for my $hf (map(ref $hdr_edits->{$_} ? @{$hdr_edits->{$_}} : (), qw(append)) ) { if ($hf =~ /^([^:]*?)[ \t]*:[ \t]*(.*?)$/s) { push(@response, proto_encode('addheader',$1,$2)) } } } if ($old_amcl) { # milter via old amavis helper program # warn if there is anything that should be done but MTA is not capable of # (or a helper program cannot pass the request) for (grep(/^(delrcpt|addrcpt)=/, @response)) { do_log(-1, "WARN: MTA can't do: %s", $_) } if ($rcpt_deletes && $rcpt_count-$rcpt_deletes > 0) { do_log(-1, "WARN: ACCEPT THE WHOLE MESSAGE, ". "MTA-in can't do selective recips deletion"); } } push(@response, proto_encode('return_value','continue')); } push(@response, proto_encode('exit_code',sprintf("%d",$exit_code))); ll(2) && do_log(2, "mail checking ended: %s", join("\n",@response)); %current_policy_bank = %baseline_policy_bank; # restore bank settings @response; } # just a proof-of-concept, experimental # sub postfix_policy($$) { my($msginfo,$attr_ref) = @_; my(@response); if ($attr_ref->{'request'} ne 'smtpd_access_policy') { die("unknown 'request' value: " . $attr_ref->{'request'}); } else { @response = 'action=DUNNO'; } @response; } sub dispatch_from_quarantine($$$) { my($msginfo,$request_type,$feedback_type) = @_; my($err); eval { # feed information to a msginfo object, possibly replacing it $msginfo = msg_from_quarantine($msginfo,$request_type,$feedback_type); mail_dispatch($msginfo,0,1); # re-send the original mail or report 1; } or do { $err = $@ ne '' ? $@ : "errno=$!"; chomp $err; do_log(0, "WARN: dispatch_from_quarantine failed: %s",$err); die $err if $err =~ /^timed out\b/; # resignal timeout }; my(@response); my($per_recip_data) = $msginfo->per_recip_data; if (!defined($per_recip_data) || !@$per_recip_data) { push(@response, proto_encode('setreply','250','2.5.0', "No recipients, nothing to do")); } else { for my $r (@$per_recip_data) { local($1,$2,$3); my($smtp_s,$smtp_es,$msg); my($resp) = $r->recip_smtp_response; if ($err ne '') { ($smtp_s,$smtp_es,$msg) = ('450', '4.5.0', "ERROR: $err") } elsif ($resp =~ /^([1-5]\d\d) ([245]\.\d{1,3}\.\d{1,3})(?: |\z)(.*)\z/s) { ($smtp_s,$smtp_es,$msg) = ($1,$2,$3) } elsif ($resp =~ /^(([1-5])\d\d)(?: |\z)(.*)\z/s) { ($smtp_s,$smtp_es,$msg) = ($1, "$2.0.0" ,$3) } else { ($smtp_s,$smtp_es,$msg) = ('450', '4.5.0', "Unexpected: $resp") } push(@response, proto_encode('setreply',$smtp_s,$smtp_es,$msg)); } } @response; } 1; __DATA__ # package Amavis::In::SMTP; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::Util qw(ll do_log untaint am_id new_am_id snmp_counters_init orcpt_encode xtext_decode debug_oneshot waiting_for_client prolong_timer switch_to_my_time switch_to_client_time sanitize_str rmdir_recursively add_entropy setting_by_given_contents_category); import Amavis::Lookup qw(lookup lookup2); import Amavis::Lookup::IP qw(lookup_ip_acl); import Amavis::Timing qw(section_time); import Amavis::rfc2821_2822_Tools; import Amavis::TempDir; import Amavis::In::Message; import Amavis::In::Connection; } use Errno qw(ENOENT EACCES); use MIME::Base64; use Time::HiRes (); #use IO::Socket::SSL; BEGIN { # due to dynamic loading runs only after config files have been read my($tls_security_level) = c('tls_security_level_in'); $tls_security_level = 0 if !defined($tls_security_level) || lc($tls_security_level) eq 'none'; if ($tls_security_level) { defined $smtpd_tls_cert_file && $smtpd_tls_cert_file ne '' or die '$tls_security_level is enabled '. 'but $smtpd_tls_cert_file is not provided'."\n"; defined $smtpd_tls_key_file && $smtpd_tls_key_file ne '' or die '$tls_security_level is enabled '. 'but $smtpd_tls_key_file is not provided'."\n"; } } sub new($) { my($class) = @_; my($self) = bless {}, $class; undef $self->{sock}; # SMTP socket $self->{proto} = undef; # SMTP / ((ESMTP / LMTP) (A | S | SA)? ) $self->{smtp_outbuf} = undef; # SMTP responses buffer for PIPELINING undef $self->{pipelining}; # may we buffer responses? undef $self->{session_closed_normally}; # closed properly with QUIT $self->{within_data_transfer} = 0; $self->{smtp_inpbuf} = ''; # SMTP input buffer $self->{tempdir} = Amavis::TempDir->new; # TempDir object $self; } sub DESTROY { my $self = shift; local($@,$!,$_); my($myactualpid) = $$; eval { if (defined($my_pid) && $myactualpid != $my_pid) { do_log(5,"Skip closing SMTP session in a clone [%s] (born as [%s])", $myactualpid, $my_pid); } elsif (ref($self->{sock}) && ! $self->{session_closed_normally}) { my($msg) = "421 4.3.2 Service shutting down, closing channel"; $msg .= ", during waiting for input from client" if waiting_for_client(); $msg .= ", sig: " . join(',', keys %Amavisd::got_signals) if %Amavisd::got_signals; $self->smtp_resp(1,$msg); } 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; eval { do_log(1,"SMTP shutdown: %s", $eval_stat) }; }; } sub readline { my($self) = @_; my($ifh) = $self->{sock}; for (;;) { local($1); return $1 if $self->{smtp_inpbuf} =~ s/^(.*?\015\012)//s; my($nbytes) = $ifh->sysread($self->{smtp_inpbuf}, 16384, length($self->{smtp_inpbuf})); if ($nbytes) { do_log(5, "smtp readline: read %d bytes into buffer, new size: %d", $nbytes, length($self->{smtp_inpbuf})); } elsif (defined $nbytes) { do_log(5, "smtp readline: EOF"); $! = 0; # eof, no error last; } else { do_log(5, "smtp readline: error: %s", !$self->{ssl_active} ? $! : $ifh->errstr.", $!"); last; } } undef; } # Efficiently copy mail text from an SMTP socket to a file, converting # CRLF to a local filesystem newlines \n, and handling dot-destuffing. # Should be called just after the DATA command has been responded to, # stops reading at a CRLF DOT CRLF or eof. Does not report stuffing errors. # sub copy_smtp_data { my($self, $size_limit) = @_; my($ifh) = $self->{sock}; my($ofh) = $self->{tempdir}->fh; my($buff) = $self->{smtp_inpbuf}; # work with a local copy # assumes to be called right after a DATA my($eof) = 0; my($at_the_beginning) = 1; my($size) = 0; my($oversized) = 0; my($errno,$nreads,$nwrites,$j); my($smtpd_t_o) = c('smtpd_timeout'); while (!$eof) { # alarm should apply per-line, but we are dealing with whole chunks here alarm($smtpd_t_o); $nreads = $ifh->sysread($buff, 32768, length($buff)); if ($nreads) { do_log(5, "smtp copy: read %d bytes into buffer, new size: %d", $nreads, length($buff)); } elsif (defined $nreads) { $eof = 1; do_log(5, "smtp copy: EOF"); } else { $eof = 1; $errno = !$self->{ssl_active} ? $! : $ifh->errstr.", $!"; do_log(5, "smtp copy: error: %s", $errno); } if ($at_the_beginning && substr($buff,0,3) eq ".\015\012") { # a preceeding \015\012 is implied, although no longer in the buffer substr($buff,0,3) = ''; $self->{within_data_transfer} = 0; last; } elsif ( ($j=index($buff,"\015\012.\015\012")) >= 0 ) { # last chunk my($carry) = substr($buff,$j+5); # often empty substr($buff,$j+2) = ''; # ditch the dot and the rest $size += length($buff); if (!$oversized) { $buff =~ s/\015\012\.?/\n/gs; for (my $ofs = 0; $ofs < length($buff); $ofs += $nwrites) { $nwrites = syswrite($ofh, $buff, length($buff)-$ofs, $ofs); defined $nwrites or die "Error writing to mail file: $!"; } if ($size_limit && $size > $size_limit) { do_log(1,"Message size exceeded %d B", $size_limit); $oversized = 1; } } $buff = $carry; $self->{within_data_transfer} = 0; last; } my($carry) = ''; if ($eof) { # flush whatever is in the buffer, no more data coming } elsif ($at_the_beginning && ($buff eq ".\015" || $buff eq '.' || $buff eq '')) { $carry = $buff; $buff = ''; } elsif (substr($buff,-4,4) eq "\015\012.\015") { substr($buff,-4,4) = ''; $carry = "\015\012.\015"; } elsif (substr($buff,-3,3) eq "\015\012.") { substr($buff,-3,3) = ''; $carry = "\015\012."; } elsif (substr($buff,-2,2) eq "\015\012") { substr($buff,-2,2) = ''; $carry = "\015\012"; } elsif (substr($buff,-1,1) eq "\015") { substr($buff,-1,1) = ''; $carry = "\015"; } if ($buff ne '') { $at_the_beginning = 0; # message size is defined in RFC 1870, includes CRLF but no stuffed dots # NOTE: we overshoot here by the number of stuffed dots, for performance $size += length($buff); if (!$oversized) { # The RFC 5321 is quite clear, leading "." characters in # SMTP are stripped regardless of the following character. # Some MTAs only trim "." when the next character is also # a ".", but this violates the RFC. $buff =~ s/\015\012\.?/\n/gs; # this is quite fast for (my $ofs = 0; $ofs < length($buff); $ofs += $nwrites) { $nwrites = syswrite($ofh, $buff, length($buff)-$ofs, $ofs); defined $nwrites or die "Error writing to mail file: $!"; } if ($size_limit && $size > $size_limit) { do_log(1,"Message size exceeded %d B, ". "skipping further input", $size_limit); $nwrites = syswrite($ofh, "\n***TRUNCATED***\n"); defined $nwrites or die "Error writing to mail file: $!"; $oversized = 1; } } } $buff = $carry; } do_log(5, "smtp copy: %d bytes still buffered at end", length($buff)); $self->{smtp_inpbuf} = $buff; # put a local copy back into object !$self->{within_data_transfer} or die "Connection broken during DATA: ". (!$self->{ssl_active} ? $! : $ifh->errstr.", $!"); # return a message size and an indication of exceeded size limit ($size,$oversized); } sub preserve_evidence { # preserve temporary files etc in case of trouble my $self = shift; !$self->{tempdir} ? undef : $self->{tempdir}->preserve(@_); } sub authenticate($$$) { my($state,$auth_mech,$auth_resp) = @_; my($result,$newchallenge); if ($auth_mech eq 'ANONYMOUS') { # RFC 2245 $result = [$auth_resp,undef]; } elsif ($auth_mech eq 'PLAIN') { # RFC 2595, "user\0authname\0pass" if (!defined($auth_resp)) { $newchallenge = '' } else { $result = [ (split(/\000/,$auth_resp,-1))[0,2] ] } } elsif ($auth_mech eq 'LOGIN' && !defined $state) { $newchallenge = 'Username:'; $state = []; } elsif ($auth_mech eq 'LOGIN' && @$state==0) { push(@$state, $auth_resp); $newchallenge = 'Password:'; } elsif ($auth_mech eq 'LOGIN' && @$state==1) { push(@$state, $auth_resp); $result = $state; } # CRAM-MD5:RFC 2195, DIGEST-MD5:RFC 2831 ($state,$result,$newchallenge); } # Accept an SMTP or LMTP connect (which can do any number of transactions) # and call content checking for each message received # sub process_smtp_request($$$$) { my($self, $sock, $lmtp, $conn, $check_mail) = @_; # $sock: connected socket from Net::Server # $lmtp: greet as a LMTP server instead of (E)SMTP # $conn: information about client connection # $check_mail: subroutine ref to be called with file handle my($msginfo,$authenticated,$auth_user,$auth_pass); $self->{sock} = $sock; $self->{pipelining} = 0; # may we buffer responses? $self->{smtp_outbuf} = []; # SMTP responses buffer for PIPELINING $self->{session_closed_normally} = 0; # closed properly with QUIT? $self->{ssl_active} = 0; # session upgraded to SSL my($tls_security_level) = c('tls_security_level_in'); $tls_security_level = 0 if !defined($tls_security_level) || lc($tls_security_level) eq 'none'; my($myheloname); # $myheloname = c('myhostname'); # $myheloname = 'localhost'; # $myheloname = '[127.0.0.1]'; for ($conn->socket_ip) { # just aliasing, not a loop $myheloname = defined($_) && $_ ne '' ? "[$_]" : '[localhost]'; } new_am_id(undef, $Amavis::child_invocation_count, undef); my($initial_am_id) = 1; my($sender_unq,$sender_quo,@recips); my($got_rcpt); my($max_recip_size_limit); # maximum of per-recipient message size limits my($terminating,$aborting,$eof,$voluntary_exit); my(%xforward_args); my($seq) = 0; my(%baseline_policy_bank) = %current_policy_bank; $conn->appl_proto($self->{proto} = $lmtp ? 'LMTP' : 'SMTP'); # system-wide message size limit, if any my($final_oversized_destiny) = setting_by_given_contents_category( CC_OVERSIZED, cr('final_destiny_by_ccat')); my($message_size_limit) = c('smtpd_message_size_limit'); if ($enforce_smtpd_message_size_limit_64kb_min && $message_size_limit && $message_size_limit < 65536) { $message_size_limit = 65536 } # RFC 5321 requires at least 64k my($smtpd_greeting_banner_tmp) = c('smtpd_greeting_banner'); $smtpd_greeting_banner_tmp =~ s{ \$ (?: \{ ([^\}]+) \} | ([a-zA-Z](?:[a-zA-Z0-9_-]*[a-zA-Z0-9])?\b) ) } { { 'helo-name' => $myheloname, 'myhostname' => c('myhostname'), 'version' => $myversion, 'version-id' => $myversion_id, 'version-date' => $myversion_date, 'product' => $myproduct_name, 'protocol' => $lmtp?'LMTP':'ESMTP' }->{lc($1.$2)} }egx; $self->smtp_resp(1,"220 $smtpd_greeting_banner_tmp"); section_time('SMTP greeting'); # each call to smtp_resp starts a $smtpd_timeout timeout to tame slow clients $0 = sprintf("%s (ch%d-idle)", c('myprogram_name'), $Amavis::child_invocation_count); Amavis::Timing::go_idle(4); local($_); local($/) = "\012"; # input line terminator set to LF for ($! = 0; defined($_ = $self->readline); $! = 0) { $0 = sprintf("%s (ch%d-%s)", c('myprogram_name'), $Amavis::child_invocation_count, am_id()); Amavis::Timing::go_busy(5); # the ball is now in our courtyard, (re)start our timer; # each of our smtp responses will switch back to a $smtpd_timeout timer { # a block is used as a 'switch' statement - 'last' will exit from it my($cmd) = $_; do_log(4, "%s< %s", $self->{proto},$cmd); if (!/^ [ \t]* ( [A-Za-z] [A-Za-z0-9]* ) (?: [ \t]+ (.*?) )? [ \t]* \015 \012 \z /xs) { $self->smtp_resp(1,"500 5.5.2 Error: bad syntax", 1, $cmd); last; }; $_ = uc($1); my($args) = $2; switch_to_my_time("rx SMTP $_"); # (causes holdups in Postfix, it doesn't retry immediately; better set max_use) # $Amavis::child_task_count >= $max_requests # exceeded max_requests # && /^(?:HELO|EHLO|LHLO|DATA|NOOP|QUIT|VRFY|EXPN|TURN)\z/ && do { # # pipelining checkpoints; # # in case of multiple-transaction protocols (e.g. SMTP, LMTP) # # we do not like to keep running indefinitely at the MTA's mercy # my($msg) = "Closing transmission channel ". # "after $Amavis::child_task_count transactions, $_"; # do_log(2,"%s",$msg); $self->smtp_resp(1,"421 4.3.0 ".$msg); #flush! # $terminating=1; last; # }; $tls_security_level && lc($tls_security_level) ne 'may' && !$self->{ssl_active} && !/^(?:NOOP|EHLO|STARTTLS|QUIT)\z/ && do { $self->smtp_resp(1,"530 5.7.0 Must issue a STARTTLS command first", 1,$cmd); last; }; # lc($tls_security_level) eq 'verify' && !/^QUIT\z/ && do { # $self->smtp_resp(1,"554 5.7.0 Command refused due to lack of security", # 1,$cmd); # last; # }; /^NOOP\z/ && do { $self->smtp_resp(1,"250 2.0.0 Ok $_"); last }; #flush! /^QUIT\z/ && do { if ($args ne '') { $self->smtp_resp(1,"501 5.5.4 Error: QUIT does not accept arguments", 1,$cmd); #flush } else { my($smtpd_quit_banner_tmp) = c('smtpd_quit_banner'); $smtpd_quit_banner_tmp =~ s{ \$ (?: \{ ([^\}]+) \} | ([a-zA-Z](?:[a-zA-Z0-9_-]*[a-zA-Z0-9])?\b) ) } { { 'helo-name' => $myheloname, 'myhostname' => c('myhostname'), 'version' => $myversion, 'version-id' => $myversion_id, 'version-date' => $myversion_date, 'product' => $myproduct_name, 'protocol' => $lmtp?'LMTP':'ESMTP' }->{lc($1.$2)} }egx; $self->smtp_resp(1,"221 2.0.0 $smtpd_quit_banner_tmp"); #flush! $terminating = 1; } last; }; /^(?:RSET|HELO|EHLO|LHLO|STARTTLS)\z/ && do { # explicit or implicit session reset $sender_unq = $sender_quo = undef; @recips = (); $got_rcpt = 0; undef $max_recip_size_limit; undef $msginfo; # forget previous %current_policy_bank = %baseline_policy_bank; # restore bank settings %xforward_args = (); if (/^(?:RSET|STARTTLS)\z/ && $args ne '') { $self->smtp_resp(1,"501 5.5.4 Error: $_ does not accept arguments", 1,$cmd); } elsif (/^RSET\z/) { $self->smtp_resp(0,"250 2.0.0 Ok $_"); } elsif (/^STARTTLS\z/) { # RFC 3207 (ex RFC 2487) if ($self->{ssl_active}) { $self->smtp_resp(1,"554 5.5.1 Error: TLS already active"); } elsif (!$tls_security_level) { $self->smtp_resp(1,"502 5.5.1 Error: command not available"); } else { $self->smtp_resp(1,"220 2.0.0 Ready to start TLS"); #flush! IO::Socket::SSL->start_SSL($sock, SSL_server => 1, SSL_session_cache => 2, SSL_error_trap => sub { my($sock,$msg)=@_; do_log(-2,"Error on socket: %s",$msg) }, SSL_passwd_cb => sub { 'example' }, SSL_key_file => $smtpd_tls_key_file, SSL_cert_file => $smtpd_tls_cert_file, ) or die "Error upgrading socket to SSL: ". IO::Socket::SSL::errstr(); if ($self->{smtp_inpbuf} ne '') { do_log(-1, "STARTTLS pipelining violation attempt, sanitized"); $self->{smtp_inpbuf} = ''; # ditch any buffered data } $self->{ssl_active} = 1; ll(3) && do_log(3,"smtpd TLS cipher: %s", $sock->get_cipher); section_time('SMTP starttls'); } } elsif (/^HELO\z/) { $self->{pipelining} = 0; $lmtp = 0; $conn->appl_proto($self->{proto} = 'SMTP'); $self->smtp_resp(0,"250 $myheloname"); $conn->smtp_helo($args); section_time('SMTP HELO'); } elsif (/^(?:EHLO|LHLO)\z/) { $self->{pipelining} = 1; $lmtp = /^LHLO\z/ ? 1 : 0; $conn->appl_proto($self->{proto} = $lmtp ? 'LMTP' : 'ESMTP'); my(@ehlo_keywords) = ( 'VRFY', 'PIPELINING', # RFC 2920 !defined($message_size_limit) ? 'SIZE' # RFC 1870 : sprintf('SIZE %d',$message_size_limit), 'ENHANCEDSTATUSCODES', # RFC 2034, RFC 3463 '8BITMIME', # RFC 1652 'DSN', # RFC 3461 !$tls_security_level || $self->{ssl_active} ? () : 'STARTTLS', # RFC 3207 (ex RFC 2487) !@{ca('auth_mech_avail')} ? () # RFC 4954 (ex RFC 2554) : join(' ','AUTH',@{ca('auth_mech_avail')}), 'XFORWARD NAME ADDR PORT PROTO HELO IDENT SOURCE' ); my(%smtpd_discard_ehlo_keywords) = map((uc($_),1), @{ca('smtpd_discard_ehlo_keywords')}); @ehlo_keywords = grep(/^([A-Za-z0-9]+)/ && !$smtpd_discard_ehlo_keywords{uc($1)}, @ehlo_keywords); $self->smtp_resp(1,"250 $myheloname\n" . join("\n",@ehlo_keywords)); #flush! $conn->smtp_helo($args); section_time("SMTP $_"); }; last; }; /^XFORWARD\z/ && do { # Postfix extension if (defined $sender_unq) { $self->smtp_resp(0,"503 5.5.1 Error: XFORWARD not allowed ". "within transaction",1,$cmd); last; } my($bad); for (split(' ',$args)) { if (!/^( [A-Za-z0-9] [A-Za-z0-9-]* ) = ( [\041-\176]{0,255} )\z/xs) { $self->smtp_resp(0,"501 5.5.4 Syntax error in XFORWARD parameters", 1, $cmd); $bad = 1; last; } else { my($name,$val) = (uc($1), $2); if ($name =~ /^(?:NAME|ADDR|PORT|PROTO|HELO|IDENT|SOURCE)\z/) { $val = undef if uc($val) eq '[UNAVAILABLE]'; # Postfix since vers 2.3 (20060610) uses xtext-encoded (RFC 3461) # strings in XCLIENT and XFORWARD attribute values, previous # versions sent plain text with neutered special characters. # The IDENT option is available since postfix 2.8.0 . $val = xtext_decode($val) if defined $val && $val =~ /\+([0-9a-fA-F]{2})/; $xforward_args{$name} = $val; } else { $self->smtp_resp(0,"501 5.5.4 XFORWARD command parameter ". "error: $name=$val",1,$cmd); $bad = 1; last; } } } $self->smtp_resp(1,"250 2.5.0 Ok $_") if !$bad; last; }; /^HELP\z/ && do { $self->smtp_resp(0,"214 2.0.0 See $myproduct_name home page at:\n". "http://www.ijs.si/software/amavisd/"); last; }; /^AUTH\z/ && @{ca('auth_mech_avail')} && do { # RFC 4954 (ex RFC 2554) if ($args !~ /^([^ ]+)(?: ([^ ]*))?\z/is) { $self->smtp_resp(0,"501 5.5.2 Syntax: AUTH mech [initresp]",1,$cmd); last; } my($auth_mech,$auth_resp) = (uc($1), $2); if ($authenticated) { $self->smtp_resp(0,"503 5.5.1 Error: session already authenticated", 1,$cmd); } elsif (defined $sender_unq) { $self->smtp_resp(0,"503 5.5.1 Error: AUTH not allowed within ". "transaction",1,$cmd); } elsif (!grep(uc($_) eq $auth_mech, @{ca('auth_mech_avail')})) { $self->smtp_resp(0,"504 5.5.4 Error: requested authentication ". "mechanism not supported",1,$cmd); } else { my($state,$result,$challenge); if ($auth_resp eq '=') { $auth_resp = '' } # zero length elsif ($auth_resp eq '') { $auth_resp = undef } for (;;) { if ($auth_resp !~ m{^[A-Za-z0-9+/]*=*\z}) { $self->smtp_resp(0,"501 5.5.2 Authentication failed: ". "malformed authentication response",1,$cmd); last; } else { $auth_resp = decode_base64($auth_resp) if $auth_resp ne ''; ($state,$result,$challenge) = authenticate($state, $auth_mech, $auth_resp); if (ref($result) eq 'ARRAY') { $self->smtp_resp(0,"235 2.7.0 Authentication succeeded"); $authenticated = 1; ($auth_user,$auth_pass) = @$result; do_log(2,"AUTH %s, user=%s", $auth_mech,$auth_user); #auth_resp last; } elsif (defined $result && !$result) { $self->smtp_resp(0,"535 5.7.8 Authentication credentials ". "invalid", 1, $cmd); last; } } # server challenge or ready prompt $self->smtp_resp(1,"334 ".encode_base64($challenge,'')); $! = 0; $auth_resp = $self->readline; defined $auth_resp or die "Error reading auth resp: ". (!$self->{ssl_active} ? $! : $sock->errstr.", $!"); switch_to_my_time('rx AUTH challenge reply'); do_log(5, "%s< %s", $self->{proto},$auth_resp); $auth_resp =~ s/\015?\012\z//; if (length($auth_resp) > 12288) { # RFC 4954 $self->smtp_resp(0,"500 5.5.6 Authentication exchange ". "line is too long"); last; } elsif ($auth_resp eq '*') { $self->smtp_resp(0,"501 5.7.1 Authentication aborted"); last; } } } last; }; /^VRFY\z/ && do { if ($args eq '') { $self->smtp_resp(1,"501 5.5.2 Syntax: VRFY address", 1,$cmd); #flush! } else { # RFC 2505 $self->smtp_resp(1,"252 2.0.0 Argument not checked", 0,$cmd); #flush! } last; }; /^MAIL\z/ && do { # begin new SMTP transaction if (defined $sender_unq) { $self->smtp_resp(0,"503 5.5.1 Error: nested MAIL command", 1, $cmd); last; } if (!$authenticated && c('auth_required_inp') && @{ca('auth_mech_avail')} ) { $self->smtp_resp(0,"530 5.7.0 Authentication required", 1, $cmd); last; } # begin SMTP transaction my($now) = Time::HiRes::time; if (!$seq) { # the first connect section_time('SMTP pre-MAIL'); } else { # establish a new time reference for each transaction Amavis::Timing::init(); snmp_counters_init(); } $seq++; if (!$initial_am_id) { new_am_id(undef, $Amavis::child_invocation_count, $seq); # enter 'in transaction' state $Amavis::snmp_db->register_proc( 1,1,'m',am_id()) if defined $Amavis::snmp_db; } $initial_am_id = 0; Amavis::check_mail_begin_task(); $self->{tempdir}->prepare_dir; $self->{tempdir}->prepare_file; $msginfo = Amavis::In::Message->new; $msginfo->rx_time($now); $msginfo->log_id(am_id()); $msginfo->conn_obj($conn); my($cl_ip) = $xforward_args{'ADDR'}; my($cl_port)= $xforward_args{'PORT'}; my($cl_src) = $xforward_args{'SOURCE'}; # local_header_rewrite_clients $cl_port = undef if $cl_port !~ /^\d{1,9}\z/ || $cl_port > 65535; my($cl_ip_mynets, $policy_name_requested); $cl_ip =~ s/^IPv6://i if defined $cl_ip; { my($cl_ip_tmp) = $cl_ip; # treat unknown client IP address as 0.0.0.0, # from "This" Network, RFC 1700 $cl_ip_tmp = '0.0.0.0' if !defined($cl_ip) || $cl_ip eq ''; my(@cp) = @{ca('client_ipaddr_policy')}; do_log(-1,"\@client_ipaddr_policy must contain pairs, ". "number of elements is not even") if @cp % 2 != 0; while (@cp) { my($lookup_table) = shift(@cp); my($policy_name) = shift(@cp); if (lookup_ip_acl($cl_ip_tmp, $lookup_table)) { if (defined $policy_name && $policy_name ne '') { $policy_name_requested = $policy_name; $cl_ip_mynets = 1 if $policy_name eq 'MYNETS'; # compatibility } last; } } } if (($cl_ip_mynets?1:0) > ($msginfo->originating?1:0)) { $current_policy_bank{'originating'} = $cl_ip_mynets; # compatibility } if (defined $policy_name_requested && defined $policy_bank{$policy_name_requested}) { Amavis::load_policy_bank($policy_name_requested,$msginfo); } $msginfo->originating(c('originating')); $msginfo->client_addr(lc $cl_ip); # ADDR $msginfo->client_port($cl_port); # PORT $msginfo->client_source($cl_src); # SOURCE $msginfo->client_name($xforward_args{'NAME'}); $msginfo->client_helo($xforward_args{'HELO'}); $msginfo->client_proto($xforward_args{'PROTO'}); $msginfo->queue_id($xforward_args{'IDENT'}); # $msginfo->body_type('7BIT'); # presumed, unless explicitly declared %xforward_args = (); # reset values for the next transaction if ($self->{ssl_active}) { $msginfo->tls_cipher($sock->get_cipher); $conn->appl_proto($self->{proto}.'S') # RFC 3848 if $self->{proto} =~ /^(LMTP|ESMTP)\z/i; } my($submitter); if ($authenticated) { $msginfo->auth_user($auth_user); $msginfo->auth_pass($auth_pass); $conn->appl_proto($self->{proto}.'A') # RFC 3848 if $self->{proto} =~ /^(LMTP|ESMTP)S?\z/i; } elsif (c('auth_reauthenticate_forwarded') && c('amavis_auth_user') ne '') { $msginfo->auth_user(c('amavis_auth_user')); $msginfo->auth_pass(c('amavis_auth_pass')); # $submitter = quote_rfc2821_local(c('mailfrom_notify_recip')); # $submitter = expand_variables($submitter) if defined $submitter; } local($1,$2); if ($args !~ /^FROM: [ \t]* ( < (?: " (?: \\. | [^\\"] ){0,999} " | [^"\@ \t] )* (?: \@ (?: \[ (?: \\. | [^\]\\] ){0,999} \] | [^\[\]\\> \t] )* )? > ) (?: [ \t]+ (.+) )? \z/isx ) { $self->smtp_resp(0,"501 5.5.2 Syntax: MAIL FROM:
",1,$cmd); last; } my($addr,$opt) = ($1,$2); my($size,$dsn_ret,$dsn_envid); my($msg); my($msg_nopenalize) = 0; for (split(' ',$opt)) { if (!/^ ( [A-Za-z0-9] [A-Za-z0-9-]* ) = ( [\041-\074\076-\176]+ ) \z/xs) { # printable, not '=' or SP $msg = "501 5.5.4 Syntax error in MAIL FROM parameters"; } else { my($name,$val) = (uc($1),$2); if ($name eq 'SIZE' && $val=~/^\d{1,20}\z/) { # RFC 1870 if (!defined($size)) { $size = untaint($val) } else { $msg = "501 5.5.4 Syntax error in MAIL parameter: $name" } } elsif ($name eq 'BODY' && $val=~/^(?:7BIT|8BITMIME)\z/i) { $msginfo->body_type(uc($val)); } elsif ($name eq 'RET') { # RFC 3461 if (!defined($dsn_ret)) { $dsn_ret = uc($val) } else { $msg = "501 5.5.4 Syntax error in MAIL parameter: $name" } } elsif ($name eq 'ENVID') { # RFC 3461, value encoded as xtext if (!defined($dsn_envid)) { $dsn_envid = $val } else { $msg = "501 5.5.4 Syntax error in MAIL parameter: $name" } } elsif ($name eq 'AUTH') { # RFC 4954 (ex RFC 2554) my($s) = xtext_decode($val); # encoded as xtext: RFC 3461 do_log(5, "MAIL command, %s, submitter: %s", $authenticated,$s); if (defined $submitter) { # authorized identity $msg = "504 5.5.4 MAIL command duplicate param.: $name=$val"; } elsif (!@{ca('auth_mech_avail')}) { do_log(3, "MAIL command parameter AUTH supplied, but ". "authentication capability not announced, ignored"); $submitter = '<>'; # mercifully ignore invalid parameter for the benefit of # running amavisd as a Postix pre-queue smtp proxy filter # $msg = "503 5.7.4 Error: authentication disabled"; } else { $submitter = $s; } } else { $msg = "504 5.5.4 MAIL command parameter error: $name=$val"; } } last if defined $msg; } if (!defined($msg) && defined $dsn_ret && $dsn_ret!~/^(FULL|HDRS)\z/) { $msg = "501 5.5.4 Syntax error in MAIL parameter RET: $dsn_ret"; } if (!defined($msg) && defined($size) && $message_size_limit && $size > $message_size_limit && $final_oversized_destiny == D_REJECT) { $msg = "552 5.3.4 Declared message size ($size B) ". "exceeds fixed size limit"; $msg_nopenalize = 1; do_log(0, "%s REJECT 'MAIL FROM': %s", $self->{proto},$msg); } if (!defined($msg)) { $sender_quo = $addr; $sender_unq = unquote_rfc2821_local($addr); $addr = $1 if $addr =~ /^<(.*)>\z/s; my($requoted) = qquote_rfc2821_local($sender_unq); do_log(2, "address modified (sender): %s -> %s", $sender_quo, $requoted) if $requoted ne $sender_quo; if (defined $policy_bank{'MYUSERS'} && $sender_unq ne '' && $msginfo->originating && lookup2(0,$sender_unq, ca('local_domains_maps'))) { Amavis::load_policy_bank('MYUSERS',$msginfo); $msginfo->originating(c('originating')); # may have changed } debug_oneshot( lookup2(0,$sender_unq, ca('debug_sender_maps')) ? 1 : 0, $self->{proto} . "< $cmd"); # $submitter = $addr if !defined($submitter); # RFC 4954/RFC 2554: MAY $submitter = '<>' if !defined($msginfo->auth_user); $msginfo->auth_submitter($submitter); $msginfo->msg_size(0+$size) if defined $size; if (defined $dsn_ret || defined $dsn_envid) { # keep ENVID in xtext-encoded form $msginfo->dsn_ret($dsn_ret) if defined $dsn_ret; $msginfo->dsn_envid($dsn_envid) if defined $dsn_envid; } $msg = "250 2.1.0 Sender $sender_quo OK"; }; $self->smtp_resp(0,$msg, !$msg_nopenalize && $msg=~/^5/ ? 1 : 0, $cmd); last; }; /^RCPT\z/ && do { if (!defined($sender_unq)) { $self->smtp_resp(0,"503 5.5.1 Need MAIL command before RCPT",1,$cmd); @recips = (); $got_rcpt = 0; last; } $got_rcpt++; local($1,$2); if ($args !~ /^TO: [ \t]* ( < (?: " (?: \\. | [^\\"] ){0,999} " | [^"\@ \t] )* (?: \@ (?: \[ (?: \\. | [^\]\\] ){0,999} \] | [^\[\]\\> \t] )* )? > ) (?: [ \t]+ (.+) )? \z/isx ) { $self->smtp_resp(0,"501 5.5.2 Syntax: RCPT TO:
",1,$cmd); last; } my($addr,$opt) = ($1,$2); my($notify,$orcpt); my($msg); my($msg_nopenalize) = 0; for (split(' ',$opt)) { if (!/^ ( [A-Za-z0-9] [A-Za-z0-9-]* ) = ( [\041-\074\076-\176]+ ) \z/xs) { # printable, not '=' or SP $msg = "501 5.5.4 Syntax error in RCPT parameters"; } else { my($name,$val) = (uc($1),$2); if ($name eq 'NOTIFY') { # RFC 3461 if (!defined($notify)) { $notify = $val } else { $msg = "501 5.5.4 Syntax error in RCPT parameter $name" } } elsif ($name eq 'ORCPT') { # RFC 3461, value encoded as xtext if (!defined($orcpt)) { $orcpt = $val } else { $msg = "501 5.5.4 Syntax error in RCPT parameter $name" } } else { $msg = "555 5.5.4 RCPT command parameter unrecognized: $name"; # 504 5.5.4 RCPT command parameter not implemented: # 504 5.5.4 RCPT command parameter error: # 555 5.5.4 RCPT command parameter unrecognized: } } last if defined $msg; } my($addr_unq) = unquote_rfc2821_local($addr); my($requoted) = qquote_rfc2821_local($addr_unq); if ($requoted ne $addr) { # check for valid canonical quoting do_log(0, "WARN: address modified (recip): %s -> %s", $addr, $requoted); # RFC 3461: If no ORCPT parameter was present in the RCPT command # when the message was received, an ORCPT parameter MAY be added # to the RCPT command when the message is relayed. If an ORCPT # parameter is added by the relaying MTA, it MUST contain the # recipient address from the RCPT command used when the message # was received by that MTA $orcpt = orcpt_encode($addr) if !defined $orcpt; } if (lookup2(0,$addr_unq, ca('debug_recipient_maps'))) { debug_oneshot(1, $self->{proto} . "< $cmd"); } my($recip_size_limit); my($mslm) = ca('message_size_limit_maps'); $recip_size_limit = lookup2(0,$addr_unq,$mslm) if @$mslm; if ($enforce_smtpd_message_size_limit_64kb_min && $recip_size_limit && $recip_size_limit < 65536) { $recip_size_limit = 65536 } # RFC 5321 requires at least 64k if ($recip_size_limit > $max_recip_size_limit) { $max_recip_size_limit = $recip_size_limit } my($mail_size) = $msginfo->msg_size; if (!defined($msg) && defined($notify)) { my(@v) = split(/,/,uc($notify),-1); if (grep(!/^(NEVER|SUCCESS|FAILURE|DELAY)\z/, @v)) { $msg = "501 5.5.4 Error in RCPT parameter NOTIFY, ". "illegal value: $notify"; } elsif (grep($_ eq 'NEVER', @v) && grep($_ ne 'NEVER', @v)) { $msg = "501 5.5.4 Error in RCPT parameter NOTIFY, ". "illegal combination of values: $notify"; } elsif (!@v) { $msg = "501 5.5.4 Error in RCPT parameter NOTIFY, ". "missing value: $notify"; } $notify = \@v; # replace a string with a listref of items } if (!defined($msg) && defined($mail_size) && $recip_size_limit && $mail_size > $recip_size_limit && $final_oversized_destiny == D_REJECT) { $msg = "552 5.3.4 Declared message size ($mail_size B) ". "exceeds size limit for recipient $addr"; $msg_nopenalize = 1; do_log(0, "%s REJECT 'RCPT TO': %s", $self->{proto},$msg); } if (!defined($msg) && $got_rcpt > $smtpd_recipient_limit) { $msg = "452 4.5.3 Too many recipients"; } if (!defined($msg)) { my($recip_obj) = Amavis::In::Message::PerRecip->new; $recip_obj->recip_addr($addr_unq); $recip_obj->recip_addr_smtp($addr); $recip_obj->recip_destiny(D_PASS); # default is Pass $recip_obj->dsn_notify($notify) if defined $notify; $recip_obj->dsn_orcpt($orcpt) if defined $orcpt; push(@recips,$recip_obj); $msg = "250 2.1.5 Recipient $addr OK"; } $self->smtp_resp(0,$msg, !$msg_nopenalize && $msg=~/^5/ ? 1 : 0, $cmd); last; }; /^DATA\z/ && $args ne '' && do { $self->smtp_resp(1,"501 5.5.4 Error: DATA does not accept arguments", 1,$cmd); #flush last; }; /^DATA\z/ && !@recips && do { if (!defined($sender_unq)) { $self->smtp_resp(1,"503 5.5.1 Need MAIL command before DATA",1,$cmd); } elsif (!$got_rcpt) { $self->smtp_resp(1,"503 5.5.1 Need RCPT command before DATA",1,$cmd); } elsif ($lmtp) { # RFC 2033 requires 503 code! $self->smtp_resp(1,"503 5.1.1 Error (DATA): no valid recipients", 0,$cmd); #flush! } else { $self->smtp_resp(1,"554 5.1.1 Error (DATA): no valid recipients", 0,$cmd); #flush! } last; }; # /^DATA\z/ && uc($msginfo->body_type) eq "BINARYMIME" && do { # RFC 3030 # $self->smtp_resp(1,"503 5.5.1 DATA is incompatible with BINARYMIME", # 0,$cmd); #flush! # last; # }; /^DATA\z/ && do { # set timer to the initial value, MTA timer starts here if ($message_size_limit) { # enforce system-wide size limit if (!$max_recip_size_limit || $max_recip_size_limit > $message_size_limit) { $max_recip_size_limit = $message_size_limit; } } my($complete); my($size) = 0; my($oversized) = 0; my($eval_stat); eval { $msginfo->sender($sender_unq); $msginfo->sender_smtp($sender_quo); $msginfo->per_recip_data(\@recips); ll(1) && do_log(1, "%s:%s:%s %s: %s -> %s%s Received: %s", $conn->appl_proto, $conn->socket_ip eq $inet_socket_bind?'':'['.$conn->socket_ip.']', $conn->socket_port, $self->{tempdir}->path, $sender_quo, join(',', map($_->recip_addr_smtp, @{$msginfo->per_recip_data})), join('', !defined $msginfo->msg_size ? () : # RFC 1870 ' SIZE='.$msginfo->msg_size, !defined $msginfo->body_type ? () : ' BODY='.$msginfo->body_type, !defined $msginfo->auth_submitter || $msginfo->auth_submitter eq '<>' ? (): ' AUTH='.$msginfo->auth_submitter, !defined $msginfo->dsn_ret ? () : ' RET='.$msginfo->dsn_ret, !defined $msginfo->dsn_envid ? () : ' ENVID='.xtext_decode($msginfo->dsn_envid), ), make_received_header_field($msginfo,0) ); # pipelining checkpoint $self->smtp_resp(1,"354 End data with ."); #flush! $self->{within_data_transfer} = 1; $Amavis::snmp_db->register_proc( # data transfering state 2,0,'d',am_id()) if defined $Amavis::snmp_db; section_time('SMTP pre-DATA-flush') if $self->{pipelining}; $self->{tempdir}->empty(0); switch_to_client_time('receiving data'); my($stuffing_err); my($fh) = $self->{tempdir}->fh; # the copy_smtp_data() will use syswrite, flush buffer just in case $fh->flush or die "Can't flush mail file: $!"; if (!$max_recip_size_limit || $final_oversized_destiny == D_PASS) { # no message size limit enforced ($size,$oversized) = $self->copy_smtp_data(undef); } else { # enforce size limit do_log(5,"enforcing size limit %s during DATA", $max_recip_size_limit); ($size,$oversized) = $self->copy_smtp_data($max_recip_size_limit); }; switch_to_my_time('rx data-end'); $complete = !$self->{within_data_transfer}; $eof = 1 if !$complete; # normal data termination, eof on socket, timeout, fatal error do_log(4, "%s< .", $self->{proto}) if $complete; do_log(2, "INFO: dot-stuffing error (only one leading dot): %s", $stuffing_err) if defined $stuffing_err; $fh->flush or die "Can't flush mail file: $!"; # On some systems you have to do a seek whenever you # switch between reading and writing. Amongst other things, # this may have the effect of calling stdio's clearerr(3). $fh->seek(0,1) or die "Can't seek on file: $!"; section_time('SMTP DATA'); 1; } or do { # end eval $eval_stat = $@ ne '' ? $@ : "errno=$!"; }; if ($eval_stat ne '' || !$complete || # err or connection broken ($oversized && $final_oversized_destiny == D_REJECT)) { chomp $eval_stat; # on error, either send: '421 Shutting down', # or: '451 Aborted, error in processing' and NOT shut down! if ($oversized && $eval_stat eq '' && !$self->{within_data_transfer}) { my($msg) = "552 5.3.4 Message size ($size B) exceeds size limit"; do_log(0, "%s REJECT: %s", $self->{proto},$msg); $self->smtp_resp(0,$msg, 0,$cmd); } elsif (!$self->{within_data_transfer}) { my($msg) = "Error in processing: " . !$complete && $eval_stat eq '' ? 'incomplete' : $eval_stat; do_log(-2, "%s TROUBLE: 451 4.5.0 %s", $self->{proto},$msg); $self->smtp_resp(1,"451 4.5.0 $msg"); ### $aborting = $msg; } else { $aborting = "Connection broken during data transfer" if $eof; $aborting .= ', ' if $aborting ne '' && $eval_stat ne ''; $aborting .= $eval_stat; $aborting .= " during waiting for input from client" if $eval_stat =~ /^timed out\b/ && waiting_for_client(); $aborting = '???' if $aborting eq ''; do_log($eval_stat ne '' ? -1 : 3, "%s ABORTING: %s", $self->{proto},$aborting); } } else { # all OK # According to RFC 1047 it is not a good idea to do lengthy # processing here, but we do not have much choice, amavis has no # queueing mechanism and cannot accept responsibility for delivery. # # check contents before responding # check_mail() expects an open file handle in $msginfo->mail_text, # need not be rewound $msginfo->mail_tempdir($self->{tempdir}->path); $msginfo->mail_text_fn($self->{tempdir}->path . '/email.txt'); $msginfo->mail_text($self->{tempdir}->fh); # # RFC 1870: The message size is defined as the number of octets, # including CR-LF pairs, but not counting the SMTP DATA command's # terminating dot or doubled (stuffing) dots my($declared_size) = $msginfo->msg_size; # RFC 1870 if (!defined($declared_size)) { } elsif ($size > $declared_size) { # shouldn't happen with decent MTA do_log(4,"Actual message size %s B greater than the ". "declared %s B", $size,$declared_size); } elsif ($size < $declared_size) { # not unusual, but permitted do_log(4,"Actual message size %d B less than the declared %d B", $size,$declared_size); } $msginfo->msg_size(untaint($size)); # store actual RFC 1870 mail size # some fatal errors are not catchable by eval (like exceeding virtual # memory), but may still allow processing to continue in a DESTROY or # END method; turn on trouble flag here to allow DESTROY to deal with # such a case correctly, then clear the flag after content checking # if everything turned out well $self->{tempdir}->preserve(1); my($smtp_resp, $exit_code, $preserve_evidence) = &$check_mail($msginfo,$lmtp); # do all the contents checking $self->{tempdir}->preserve(0) if !$preserve_evidence; # clear if ok prolong_timer('check done'); if ($smtp_resp =~ /^4/) { # ok, not-done recipients are to be expected, do not check } elsif (grep(!$_->recip_done && $_->delivery_method ne '', @{$msginfo->per_recip_data})) { die "TROUBLE: (MISCONFIG?) not all recipients done"; } elsif (grep(!$_->recip_done && $_->delivery_method eq '', @{$msginfo->per_recip_data})) { die "NOT ALL RECIPIENTS DONE, EMPTY DELIVERY_METHOD!"; # do_log(0, "NOT ALL RECIPIENTS DONE, EMPTY DELIVERY_METHOD!"); } section_time('SMTP pre-response'); if (!$lmtp) { # smtp do_log(3, 'sending SMTP response: "%s"', $smtp_resp); $self->smtp_resp(0, $smtp_resp); } else { # lmtp my($bounced) = $msginfo->dsn_sent; # 1=bounced, 2=suppressed for my $r (@{$msginfo->per_recip_data}) { my($resp) = $r->recip_smtp_response; my($recip_quoted) = $r->recip_addr_smtp; if ($resp=~/^[24]/) { # success or tempfail, no need to change status } elsif ($bounced && $bounced == 1) { # genuine bounce # a non-delivery notifications was already sent by us, so # MTA must not bounce it again; turn status into a success $resp = sprintf("250 2.5.0 Ok %s, DSN was sent (%s)", $recip_quoted, $resp); } elsif ($bounced) { # fake bounce - bounce was suppressed $resp = sprintf("250 2.5.0 Ok %s, DSN suppressed (%s)", $recip_quoted, $resp); } elsif ($resp=~/^5/ && $r->recip_destiny != D_REJECT) { # just in case, if the bounce suppression scheme did not work $resp = sprintf("250 2.5.0 Ok %s, DSN suppressed_2 (%s)", $recip_quoted, $resp); } do_log(3, 'LMTP response for %s: "%s"', $recip_quoted, $resp); $self->smtp_resp(0, $resp); } } $self->smtp_resp_flush; # optional, but nice to report timing right section_time('SMTP response'); }; # end all OK $self->{tempdir}->clean; my $msg_size = $msginfo->msg_size; $sender_unq = $sender_quo = undef; @recips = (); $got_rcpt = 0; undef $max_recip_size_limit; undef $msginfo; # forget previous %current_policy_bank = %baseline_policy_bank; # restore bank settings %xforward_args = (); # report elapsed times by section for each transaction # (the time for a QUIT remains unaccounted for) do_log(2, "size: %d, %s", $msg_size, Amavis::Timing::report()); Amavis::Timing::init(); snmp_counters_init(); $Amavis::last_task_completed_at = Time::HiRes::time; last; }; # DATA /^(?:EXPN|TURN|ETRN|SEND|SOML|SAML)\z/ && do { $self->smtp_resp(1,"502 5.5.1 Error: command $_ not implemented", 0,$cmd); last; }; # catchall (unknown commands): #flush! $self->smtp_resp(1,"500 5.5.2 Error: command $_ not recognized", 1,$cmd); }; # end of 'switch' block if ($terminating || defined $aborting) { # exit SMTP-session loop $voluntary_exit = 1; last; } # RFC 2920 requires a flush whenever the local TCP input buffer is # emptied. Since we can't check it (unless we use sysread & select), # we should do a flush here to be in compliance. To be improved some day. $self->smtp_resp_flush; $0 = sprintf("%s (ch%d-%s-idle)", c('myprogram_name'), $Amavis::child_invocation_count, am_id()); Amavis::Timing::go_idle(6); } # end of loop my($errn,$errs); if (!$voluntary_exit) { $eof = 1; if (!defined($_)) { $errn = 0+$!; $errs = !$self->{ssl_active} ? "$!" : $sock->errstr.", $!"; } } # come here when: QUIT is received, eof or err on socket, or we need to abort $0 = sprintf("%s (ch%d)", c('myprogram_name'), $Amavis::child_invocation_count); alarm(0); do_log(4,"SMTP session over, timer stopped"); Amavis::Timing::go_busy(7); # flush just in case, session might have been disconnected eval { $self->smtp_resp_flush; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(1, "flush failed: %s", $eval_stat); }; my($msg) = defined $aborting && !$eof ? "ABORTING the session: $aborting" : defined $aborting ? $aborting : !$terminating ? "client broke the connection without a QUIT ($errs)" : ''; if ($msg eq '') { # ok } elsif ($aborting) { do_log(-1, "%s: NOTICE: %s", $self->{proto},$msg); } else { do_log( 3, "%s: notice: %s", $self->{proto},$msg); } if (defined $aborting && !$eof) { $self->smtp_resp(1,"421 4.3.2 Service shutting down, ".$aborting) } $self->{session_closed_normally} = 1; # Net::Server closes connection after child_finish_hook } # sends an SMTP response consisting of a 3-digit code and an optional message; # slow down evil clients by delaying response on permanent errors # sub smtp_resp($$$;$$) { my($self, $flush,$resp, $penalize,$line) = @_; if ($penalize) { # PENALIZE syntax errors? do_log(0, "%s: %s; smtp_resp: %s", $self->{proto},$resp,$line); # sleep 1; # section_time('SMTP penalty wait'); } push(@{$self->{smtp_outbuf}}, @{wrap_smtp_resp(sanitize_str($resp,1))}); $self->smtp_resp_flush if $flush || !$self->{pipelining} || @{$self->{smtp_outbuf}} > 200; } sub smtp_resp_flush($) { my $self = shift; if (ref($self->{smtp_outbuf}) && @{$self->{smtp_outbuf}}) { if (ll(4)) { for my $resp (@{$self->{smtp_outbuf}}) { do_log(4, "%s> %s", $self->{proto},$resp) }; } my($sock) = $self->{sock}; my($stat) = $sock->print(join('', map($_."\015\012", @{$self->{smtp_outbuf}}))); @{$self->{smtp_outbuf}} = (); # prevent printing again even if error $stat or die "Error writing an SMTP response to the socket: ". (!$self->{ssl_active} ? $! : $sock->errstr.", $!"); # put a ball in client's courtyard, start his timer switch_to_client_time('smtp response sent'); } } 1; __DATA__ # package Amavis::In::Courier; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { die "Code not available for module Amavis::In::Courier" } 1; __DATA__ # package Amavis::Out::SMTP::Protocol; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform); import Amavis::Util qw(ll do_log min max minmax); } use Errno qw(EIO EINTR EAGAIN ECONNRESET); use Time::HiRes (); use IO::Socket; use IO::Socket::UNIX; use IO::Socket::INET; #use IO::Socket::SSL; BEGIN { use vars qw($have_inet6); $have_inet6 = eval { require IO::Socket::INET6 }; } sub init { my $self = shift; delete $self->{domain}; delete $self->{supports}; $self->{pipelining} = 0; } sub new { my($class,$socket_specs,%arg) = @_; my($self) = bless {}, $class; $self->{at_line_boundary} = 1; $self->{dotstuffing} = 1; # defaults to on $self->{dotstuffing} = 0 if defined $arg{DotStuffing} && !$arg{DotStuffing}; $self->{strip_cr} = 1; # sanitizing bare CR defaults to on $self->{strip_cr} = 0 if defined $arg{StripCR} && !$arg{StripCR}; $self->{io} = Amavis::IO::RW->new($socket_specs, Eol => "\015\012", %arg); $self->init; $self; } sub close { my $self = $_[0]; $self->{io}->close; } sub DESTROY { my $self = $_[0]; local($@,$!,$_); eval { $self->close }; } sub ehlo_response_parse { my($self,$smtp_resp) = @_; delete $self->{domain}; delete $self->{supports}; my(@ehlo_lines) = split(/\n/,$smtp_resp,-1); my($bad); my($first) = 1; local($1,$2); for my $el (@ehlo_lines) { if ($first) { if ($el =~ /^(\d{3})(?:[ \t]+(.*))?\z/) { $self->{domain} = $2 } elsif (!defined($bad)) { $bad = $el } $first = 0; } elsif ($el =~ /^([A-Z0-9][A-Z0-9-]*)(?:[ =](.*))?\z/i) { $self->{supports}{uc($1)} = defined($2) ? $2 : ''; } elsif (!defined($bad)) { $bad = $el } } $self->{pipelining} = defined $self->{supports}{'PIPELINING'} ? 1 : 0; do_log(0, "Bad EHLO kw %s ignored in %s, socket %s", $bad, $smtp_resp, $self->socketname) if defined $bad; 1; } sub domain { my $self = $_[0]; $self->{domain} } sub supports { my($self,$keyword) = @_; $self->{supports}{uc($keyword)} } *print = \&datasend; # alias name for datasend sub datasend { my $self = shift; my($buff) = @_ == 1 ? $_[0] : join('',@_); do_log(-1,"WARN: Unicode string passed to datasend") if Encode::is_utf8($buff); # always false on tainted, Perl bug #32687 # do_log(5,"smtp print %d bytes>", length($buff)); $buff =~ tr/\r//d if $self->{strip_cr}; # sanitize bare CR if necessary $buff =~ s{\n}{\015\012}gs; # CR/LF are never split across a buffer boundary if ($self->{dotstuffing}) { $buff =~ s{\015\012\.}{\015\012..}gs; # dot stuffing $self->{io}->print('.') if substr($buff,0,1) eq '.' && $self->{at_line_boundary}; } $self->{io}->print($buff); $self->{at_line_boundary} = $self->{io}->at_line_boundary; $self->{io}->out_buff_large ? $self->flush : 1; } sub socketname { my $self = shift; $self->{io}->socketname(@_) } sub protocol { my $self = shift; $self->{io}->protocol(@_) } sub timeout { my $self = shift; $self->{io}->timeout(@_) } sub ssl_active { my $self = shift; $self->{io}->ssl_active(@_) } sub ssl_upgrade { my $self = shift; $self->{io}->ssl_upgrade(@_) } sub last_io_event_timestamp { my $self = shift; $self->{io}->last_io_event_timestamp(@_) } sub eof { my $self = shift; $self->{io}->eof(@_) } sub flush { my $self = shift; $self->{io}->flush(@_) } sub dataend { my($self) = @_; if (!$self->{at_line_boundary}) { $self->datasend("\n"); } if ($self->{dotstuffing}) { $self->{dotstuffing} = 0; $self->datasend(".\n"); $self->{dotstuffing} = 1; } $self->{io}->out_buff_large ? $self->flush : 1; } sub command { my($self,$command,@args) = @_; my($line) = $command =~ /:\z/ ? $command.join(' ',@args) : join(' ',$command,@args); do_log(3,"smtp cmd> %s", $line); $self->datasend($line."\n"); $self->{at_line_boundary} = 1; # RFC 2920: comands that can appear anywhere in a pipelined command group # RSET, MAIL FROM, SEND FROM, SOML FROM, SAML FROM, RCPT TO, (data) if (!$self->{pipelining} || $self->{io}->out_buff_large || $command !~ /^(?:RSET|MAIL|SEND|SOML|SAML|RCPT)\b/is) { return $self->flush; } 1; } sub smtp_response { my($self) = @_; my($resp) = ''; my($line,$code,$enh); my($first) = 1; for (;;) { $line = $self->{io}->get_response_line; last if !defined $line; # eof, error, timeout my($line_complete) = $line =~ s/\015\012\z//s; $line .= ' INCOMPLETE' if !$line_complete; my($more); local($1,$2,$3); $line =~ s/^(\d{3}) (-|\ |\z) (?: ([245] \. \d{1,3} \. \d{1,3}) (\ |\z) )?//xs; if ($first) { $code = $1; $enh = $3; $first = 0 } else { $resp .= "\n" } $resp .= $line; $more = $2 eq '-'; last if !$more || !$line_complete; } !defined $code ? undef : $code . (defined $enh ? " $enh" : '') . ' '. $resp; } sub helo { my $self = shift; $self->init; $self->command("HELO",@_) } sub ehlo { my $self = shift; $self->init; $self->command("EHLO",@_) } sub lhlo { my $self = shift; $self->init; $self->command("LHLO",@_) } sub noop { my $self = shift; $self->command("NOOP",@_) } sub rset { my $self = shift; $self->command("RSET",@_) } sub auth { my $self = shift; $self->command("AUTH",@_) } sub data { my $self = shift; $self->command("DATA",@_) } sub quit { my $self = shift; $self->command("QUIT",@_) } sub mail { my($self,$reverse_path,%params) = @_; my(@mail_parameters) = map { my($v)=$params{$_}; defined($v) ? "$_=$v" : "$_" } (keys %params); $self->command("MAIL FROM:", $reverse_path, @mail_parameters); } sub recipient { my($self,$forward_path,%params) = @_; my(@rcpt_parameters) = map { my($v)=$params{$_}; defined($v) ? "$_=$v" : "$_" } (keys %params); $self->command("RCPT TO:", $forward_path, @rcpt_parameters); } 1; package Amavis::Out::SMTP::Session; # provides a mechanism for SMTP session caching use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&rundown_stale_sessions); import Amavis::Conf qw(:platform c cr ca $smtp_connection_cache_enable); import Amavis::Util qw(min max minmax ll do_log snmp_count); } use subs @EXPORT_OK; use vars qw(%sessions_cache); use Time::HiRes (); sub new { my($class, $socket_specs, $deadline, $wildcard_implied_host, $wildcard_implied_port) = @_; my($self); my($cache_key); my($found_cached) = 0; for my $proto_sockname (ref $socket_specs ? @$socket_specs : $socket_specs) { $cache_key = $proto_sockname; local($1,$2,$3,$4); if ($proto_sockname =~ # deal with dynamic destinations (wildcards) /^([a-z][a-z0-9.+-]*) : (?: \[ ([^\]]*) \] | ([^:]*) ) : ([^:]*)/sx) { my($peeraddress) = defined $2 ? $2 : $3; my($peerport) = $4; $peeraddress = $wildcard_implied_host if $peeraddress eq '*'; $peerport = $wildcard_implied_port if $peerport eq '*'; $cache_key = sprintf("%s:[%s]:%s", $1, $peeraddress, $peerport); } if (exists $sessions_cache{$cache_key}) { $found_cached = 1; last } } if ($found_cached) { $self = $sessions_cache{$cache_key}; $self->{deadline} = $deadline; do_log(3, "smtp session reuse (%s), %d transactions so far", $cache_key, $self->{transaction_count}); } else { do_log(3, "smtp session: setting up a new session"); $cache_key = undef; $self = bless { socket_specs => $socket_specs, socketname => undef, protocol => undef, smtp_handle => undef, deadline => $deadline, timeout => undef, in_xactn => 0, transaction_count => 0, state => 'down', established_at_time => undef, wildcard_implied_host => $wildcard_implied_host, wildcard_implied_port => $wildcard_implied_port, }, $class; } $self->establish_or_refresh; if (!defined $cache_key) { # newly established session $cache_key = sprintf("%s:%s", $self->protocol, $self->socketname); $sessions_cache{$cache_key} = $self; } $self; } sub smtp_handle { my $self = shift; !@_ ? $self->{handle} : ($self->{handle}=shift) } sub socketname { my $self = shift; !@_ ? $self->{socketname} :($self->{socketname}=shift) } sub protocol { my $self = shift; !@_ ? $self->{protocol} : ($self->{protocol}=shift) } sub session_state { my $self = shift; !@_ ? $self->{state} : ($self->{state}=shift) } sub in_smtp_transaction { my $self = shift; !@_ ? $self->{in_xactn} : ($self->{in_xactn}=shift) } sub established_at_time { my $self = shift; !@_ ? $self->{established_at_time} : ($self->{established_at_time}=shift) } sub transaction_begins { my $self = $_[0]; !$self->in_smtp_transaction or die "smtp session: transaction_begins, but already active"; $self->in_smtp_transaction(1); } sub transaction_begins_unconfirmed { my $self = $_[0]; snmp_count('OutConnTransact'); $self->{transaction_count}++; !$self->in_smtp_transaction or die "smtp session: transaction_begins_unconfirmed, but already active"; $self->in_smtp_transaction(undef); } sub transaction_ends { my $self = $_[0]; $self->in_smtp_transaction(0); } sub transaction_ends_unconfirmed { my $self = $_[0]; # if already 0 then keep it, otherwise undefine $self->in_smtp_transaction(undef) if $self->in_smtp_transaction; } sub timeout { my $self = shift; if (@_) { my($timeout) = shift; $self->{timeout} = $timeout; $self->{handle}->timeout($timeout) if defined $self->{handle}; # do_log(5, "smtp session, timeout set to %s", $timeout); } $self->{timeout}; } sub supports { my($self,$keyword) = @_; defined $self->{handle} ? $self->{handle}->supports($keyword) : undef; } sub smtp_response { my $self = $_[0]; defined $self->{handle} ? $self->{handle}->smtp_response : undef; } sub quit { my $self = $_[0]; my($smtp_handle) = $self->smtp_handle; if (defined $smtp_handle) { $self->session_state('quitsent'); snmp_count('OutConnQuit'); $smtp_handle->quit; #flush! QUIT } } sub close { my($self,$keep_connected) = @_; my($msg); my($smtp_handle) = $self->smtp_handle; if (defined($smtp_handle) && $smtp_handle->eof) { $msg = 'already disconnected'; $keep_connected = 0; } else { $msg = $keep_connected ? 'keeping connection' : 'disconnecting'; } do_log(3, "Amavis::Out::SMTP::Session close, %s", $msg); if (!$keep_connected) { if (defined $smtp_handle) { $smtp_handle->close or do_log(1, "Error closing Amavis::Out::SMTP::Protocol obj"); $self->in_smtp_transaction(0); $self->established_at_time(undef); $self->smtp_handle(undef); $self->session_state('down'); } if (defined $self->socketname) { my($cache_key) = sprintf("%s:%s", $self->protocol, $self->socketname); delete $sessions_cache{$cache_key} if exists $sessions_cache{$cache_key}; } } 1; } sub rundown_stale_sessions($) { my($close_all) = @_; for my $cache_key (keys %sessions_cache) { my($smtp_session) = $sessions_cache{$cache_key}; my($smtp_handle) = $smtp_session->smtp_handle; my($established_at_time) = $smtp_session->established_at_time; my($last_event_time); $last_event_time = $smtp_handle->last_io_event_timestamp if $smtp_handle; my($now) = Time::HiRes::time; if ($close_all || !$smtp_connection_cache_enable || !defined($last_event_time) || $now - $last_event_time >= 30) { ll(3) && do_log(3,"smtp session rundown%s%s%s, %s, state %s", $close_all ? ' all sessions' : $smtp_connection_cache_enable ? ' stale sessions' : ', cache off', !defined($last_event_time) ? '' : sprintf(", idle %.1f s", $now - $last_event_time), !defined($established_at_time) ? '' : sprintf(", since %.1f s ago", $now - $established_at_time), $cache_key, $smtp_session->session_state); if ($smtp_session->session_state ne 'down' && $smtp_session->session_state ne 'quitsent' && (!defined($last_event_time) || $now - $last_event_time <= 55)) { do_log(3,"smtp session rundown, sending QUIT"); eval { $smtp_session->quit }; #flush! QUIT } if ($smtp_session->session_state eq 'quitsent') { # collect response $smtp_session->timeout(5); my($smtp_resp) = eval { $smtp_session->smtp_response }; $smtp_resp = '' if !defined $smtp_resp; do_log(3,"smtp resp to QUIT: %s", $smtp_resp); $smtp_resp =~ /^2/ or do_log(3,"Negative SMTP resp. to QUIT: %s",$smtp_resp); } if ($smtp_session->session_state ne 'down') { do_log(3,"smtp session rundown, closing session %s", $cache_key); $smtp_session->close(0) or do_log(-2, "Error closing smtp session %s", $cache_key); } } } } sub establish_or_refresh { my($self) = @_; # Timeout should be more than MTA normally takes to check DNS and RBL, # which may take a minute or more in case of unreachable DNS server. # Specifying shorter timeout will cause alarm to terminate the wait # for SMTP status line prematurely, resulting in status code 000. # RFC 5321 (ex RFC 2821) section 4.5.3.2 requires timeout to be # at least 5 minutes my($smtp_connect_timeout) = 35; my($smtp_helo_timeout) = 300; my($smtp_starttls_timeout) = 300; my($smtp_handle) = $self->smtp_handle; my($smtp_resp); my($last_event_time); $last_event_time = $smtp_handle->last_io_event_timestamp if $smtp_handle; my($now) = Time::HiRes::time; do_log(5,"establish_or_refresh, state: %s", $self->session_state); die "panic, still in SMTP transaction" if $self->in_smtp_transaction; if (defined($smtp_handle) && $self->session_state ne 'down' && $self->session_state ne 'quitsent') { # if session has been idling for some time, check with a low-cost NOOP # whether the session is still alive - reconnecting now is cheap; # note that NOOP is non-pipelinable, MTA must respond immediately if (defined($last_event_time) && $now - $last_event_time <= 18) { snmp_count('OutConnReuseRecent'); do_log(3,"smtp session most likely still valid (short idle %.1f s)", $now - $last_event_time); } else { # Postfix default smtpd idle timeout is 60 s eval { $self->timeout(15); $smtp_handle->noop; #flush! $smtp_resp = $self->smtp_response; # fetch response to NOOP do_log(3,"smtp resp to NOOP (idle %.1f s): %s", $now - $last_event_time, $smtp_resp); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(3,"smtp NOOP failed (idle %.1f s): %s", $now - $last_event_time, $eval_stat); $smtp_resp = ''; }; if ($smtp_resp =~ /^2/) { snmp_count('OutConnReuseRefreshed'); } else { snmp_count('OutConnReuseFail'); $self->close(0) or do_log(-1, "Error closing smtp session"); } } } if ($self->session_state eq 'down' || $self->session_state eq 'quitsent') { if (defined $smtp_handle) { $smtp_handle->close or do_log(-2, "Error closing Amavis::Out::SMTP::Protocol obj"); } my($localaddr) = c('local_client_bind_address'); # IP assigned to socket snmp_count('OutConnNew'); $smtp_handle = Amavis::Out::SMTP::Protocol->new( $self->{socket_specs}, LocalAddr => $localaddr, Timeout => 35, WildcardImpliedHost => $self->{wildcard_implied_host}, WildcardImpliedPort => $self->{wildcard_implied_port}); $self->smtp_handle($smtp_handle); defined $smtp_handle # don't change die text, it is referred to elsewhere or die sprintf("Can't connect to %s", !ref $self->{socket_specs} ? $self->{socket_specs} : join(", ",@$self->{socket_specs}) ); $self->socketname($smtp_handle->socketname); $self->protocol($smtp_handle->protocol); $self->session_state('connected'); $self->established_at_time(time); $self->timeout($smtp_connect_timeout); $smtp_resp = $self->smtp_response; # fetch greeting do_log(3,"smtp greeting: %s", $smtp_resp); $smtp_resp=~/^2/ or die "Negative greeting: $smtp_resp"; } if ($self->session_state eq 'connected') { my($lmtp) = lc($self->protocol) eq 'lmtp' ? 1 : 0; # RFC 2033 my($deadline) = $self->{deadline}; my($tls_security_level) = c('tls_security_level_out'); $tls_security_level = 0 if !defined($tls_security_level) || lc($tls_security_level) eq 'none'; my($heloname) = c('localhost_name'); # host name used in EHLO/HELO/LHLO $heloname = 'localhost' if $heloname eq ''; for (1..2) { # send EHLO/LHLO/HELO $self->timeout(max(60,min($smtp_helo_timeout, $deadline - Time::HiRes::time))); if ($lmtp) { $smtp_handle->lhlo($heloname) } #flush! else { $smtp_handle->ehlo($heloname) } #flush! $smtp_resp = $self->smtp_response; # fetch response to EHLO/LHLO do_log(3,"smtp resp to %s: %s", $lmtp?'LHLO':'EHLO', $smtp_resp); if ($smtp_resp =~ /^2/) { # good } elsif ($lmtp) { # no fallback possible $smtp_resp=~/^2/ or die "Negative SMTP resp. to LHLO: $smtp_resp"; } else { # fallback to HELO $smtp_handle->helo($heloname); #flush! $smtp_resp = $self->smtp_response; # fetch response to HELO do_log(3,"smtp resp to HELO: %s", $smtp_resp); $smtp_resp=~/^2/ or die "Negative SMTP resp. to HELO: $smtp_resp"; } $self->session_state('ehlo'); $smtp_handle->ehlo_response_parse($smtp_resp); my($tls_capable) = defined($self->supports('STARTTLS')); # RFC 3207 ll(5) && do_log(5, "tls active=%d, capable=%s, sec_level=%s", $smtp_handle->ssl_active, $tls_capable, $tls_security_level); if ($smtp_handle->ssl_active) { last; # done } elsif (!$tls_capable && $tls_security_level && lc($tls_security_level) ne 'may') { die "MTA does not offer STARTTLS, ". "but TLS is required: \"$tls_security_level\""; } elsif (!$tls_capable || !$tls_security_level) { last; # not offered and not mandated } else { $self->timeout(max(60,min($smtp_starttls_timeout, $deadline - Time::HiRes::time))); $smtp_handle->command('STARTTLS'); #flush! $smtp_resp = $self->smtp_response; # fetch response to STARTTLS do_log(3,"smtp resp to STARTTLS: %s", $smtp_resp); if ($smtp_resp !~ /^2/) { (!$tls_security_level || lc($tls_security_level) eq 'may') or die "Negative SMTP resp. to STARTTLS: $smtp_resp"; } else { $smtp_handle->ssl_upgrade or die "Error upgrading socket to SSL"; $self->session_state('connected'); } } } } $self; } 1; package Amavis::Out::SMTP; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT = qw(&mail_via_smtp); import Amavis::Conf qw(:platform c cr ca $smtp_connection_cache_enable); import Amavis::Util qw(untaint min max minmax ll do_log snmp_count xtext_encode xtext_decode prolong_timer get_deadline collect_equal_delivery_recips); import Amavis::Timing qw(section_time); import Amavis::rfc2821_2822_Tools; import Amavis::Lookup qw(lookup lookup2); import Amavis::Out::EditHeader; } use Time::HiRes qw(time); # use Authen::SASL; # simple OO wrapper around Mail::DKIM::Signer to provide a method 'print' # and to convert \n to CRLF # sub new_dkim_wrapper { my($class, $handle,$strip_cr) = @_; bless { handle => $handle, strip_cr => $strip_cr }, $class; } sub close { 1 } sub print { my $self = shift; my($buff) = @_ == 1 ? $_[0] : join('',@_); $buff =~ tr/\r//d if $self->{strip_cr}; $buff =~ s{\n}{\015\012}gs; $self->{handle}->PRINT($buff); } # Add a log_id to the SMTP status text, insert a fabricated RFC 3463 enhanced # status code if missing in an MTA response. # sub enhance_smtp_response($$$$$) { my($smtp_resp,$am_id,$mta_id,$dflt_enhcode,$cmd_name) = @_; local($1,$2,$3,$4); my($resp_msg); my($resp_code,$resp_more,$resp_enhcode) = ('451', ' ', '4.5.0'); if (!defined($smtp_resp) || $smtp_resp eq '') { $smtp_resp = sprintf('No resp. to %s', $cmd_name); } elsif ($smtp_resp !~ /^[245]\d{2}/) { $smtp_resp = sprintf('Bad resp. to %s: %s', $cmd_name,$smtp_resp); } elsif ($smtp_resp=~/^ (\d{3}) (\ |-|\z) [ \t]* ([245] \. \d{1,3} \. \d{1,3})? \s* (.*) \z/xs) { ($resp_code, $resp_more, $resp_enhcode, $resp_msg) = ($1, $2, $3, $4); if (!defined $resp_enhcode && $resp_code =~ /^[245]/) { my($c) = substr($resp_code,0,1); $resp_enhcode = $dflt_enhcode; $resp_enhcode =~ s/^\d*/$c/; } } sprintf("%s%s%s from MTA(%s): %s", $resp_code, $resp_more, $resp_enhcode, $mta_id, $smtp_resp); } # Send mail using SMTP - single transaction # (e.g. forwarding original mail or sending notification) # May throw exception (die) if temporary failure (4xx) or other problem # # Multiple transactions may be necessary, either due to different delivery # methods (IP address, port, SMTP vs. LMTP) or due to '452 Too many recipients' # sub mail_via_smtp(@) { my($msginfo, $initial_submission, $dsn_per_recip_capable, $filter) = @_; # # RFC 2033: LMTP protocol MUST NOT be used on the TCP port 25 # # $initial_submission can be treated as a boolean, but for more detailed # needs it can be any of: false: 0 # or true: 'Quar', 'Dsn', 'Notif', 'AV', 'Arf' my($which_section) = 'fwd_init'; my($logmsg) = sprintf("%s from %s", $initial_submission?'SEND':'FWD', $msginfo->sender_smtp); my($per_recip_data_ref, $proto_sockname) = collect_equal_delivery_recips($msginfo, $filter, qr/^(?:smtp|lmtp):/i); if (!$per_recip_data_ref || !@$per_recip_data_ref) { do_log(5, "%s, nothing to do", $logmsg); return 1; } my($proto_sockname_displ) = !ref $proto_sockname ? $proto_sockname : '(' . join(', ',@$proto_sockname) . ')'; my(@per_recip_data) = @$per_recip_data_ref; undef $per_recip_data_ref; ll(4) && do_log(4, "about to connect to %s, %s -> %s", $proto_sockname_displ, $logmsg, join(',', qquote_rfc2821_local( map($_->recip_final_addr, @per_recip_data)) )); my($am_id) = $msginfo->log_id; my($dsn_envid) = $msginfo->dsn_envid; my($dsn_ret) = $msginfo->dsn_ret; my($msg) = $msginfo->mail_text; # a file handle or a MIME::Entity object my($relayhost, $protocol, $lmtp, $mta_id, @snmp_vars); my($smtp_session, $smtp_handle, $smtp_resp, $smtp_response); my($any_valid_recips) = 0; my($any_tempfail_recips) = 0; my($pipelining) = 0; my($any_valid_recips_and_data_sent) = 0; my($in_datasend_mode) = 0; my($mimetransport8bit_capable) = 0; # RFC 1652 my($dsn_capable) = 0; my($auth_capable) = 0; my(%from_options); # RFC 5321 (ex RFC 2821), section 4.5.3.2. Timeouts my($smtp_connect_timeout) = 35; my($smtp_helo_timeout) = 300; my($smtp_starttls_timeout) = 300; my($smtp_xforward_timeout) = 300; my($smtp_mail_timeout) = 300; my($smtp_rcpt_timeout) = 300; my($smtp_data_init_timeout) = 120; my($smtp_data_xfer_timeout) = 180; my($smtp_data_done_timeout) = 600; my($smtp_quit_timeout) = 10; # 300 my($smtp_rset_timeout) = 20; # can appear anywhere in a pipelined command group: # RSET, MAIL FROM, SEND FROM, SOML FROM, SAML FROM, RCPT TO, data # can only appear as the last command in a pipelined group: --> flush # EHLO, DATA, VRFY, EXPN, TURN, QUIT, NOOP, # AUTH(RFC 4954), STARTTLS(RFC 3207), and all unknown commands # needed to implement dynamic_destination: a '*' in place of a host or port my($wildcard_implied_host, $wildcard_implied_port); my($conn) = $msginfo->conn_obj; if ($conn) { my($host) = $conn->client_ip; $wildcard_implied_host = $host if defined($host) && $host ne ''; my($port) = $conn->socket_port; $wildcard_implied_port = $port+1 if defined($port) && $port =~ /^\d+\z/; } my($remaining_time, $deadline) = get_deadline($which_section, 1, 0); alarm(0); # stop the timer my($err); eval { $which_section = 'fwd-connect'; $smtp_session = Amavis::Out::SMTP::Session->new($proto_sockname, $deadline, $wildcard_implied_host, $wildcard_implied_port) or die "Can't establish an SMTP/LMTP session with $proto_sockname_displ"; $smtp_handle = $smtp_session->smtp_handle; if ($smtp_handle) { $relayhost = $smtp_handle->socketname; $protocol = $smtp_handle->protocol; $lmtp = lc($protocol) eq 'lmtp' ? 1 : 0; # RFC 2033 $mta_id = sprintf("%s:%s", $protocol, $relayhost); @snmp_vars = !$initial_submission ? ('', 'Relay', 'Proto'.uc($protocol), 'Proto'.uc($protocol).'Relay') : ('', 'Submit', 'Proto'.uc($protocol), 'Proto'.uc($protocol).'Submit', 'Submit'.$initial_submission); snmp_count('OutMsgs'.$_) for @snmp_vars; } $dsn_capable = c('propagate_dsn_if_possible') && defined($smtp_session->supports('DSN')); # RFC 3461 $mimetransport8bit_capable = defined($smtp_session->supports('8BITMIME')); # RFC 1652 $pipelining = defined($smtp_session->supports('PIPELINING')); # RFC 2920 do_log(3,"No announced PIPELINING support by MTA?") if !$pipelining; ll(5) && do_log(5,"Remote host presents itself as: %s%s%s", $smtp_handle->domain, $dsn_capable ? ', handles DSN' : '', $pipelining ? ', handles PIPELINING' : ''); if ($lmtp && !$pipelining) # RFC 2033 requirements { die "A LMTP server implementation MUST implement PIPELINING" } if ($lmtp && !defined($smtp_session->supports('ENHANCEDSTATUSCODES'))) { die "A LMTP server implementation MUST implement ENHANCEDSTATUSCODES" } section_time($which_section); $which_section = 'fwd-xforward'; my($cl_ip) = $msginfo->client_addr; if (defined $cl_ip && $cl_ip ne '' && defined($smtp_session->supports('XFORWARD'))) { $cl_ip = 'IPv6:'.$cl_ip if $cl_ip =~ /:.*:/ && $cl_ip !~ /^IPv6:/i; my(%xfwd_supp_opt) = map((uc($_),1), split(' ',$smtp_session->supports('XFORWARD'))); my(@params) = map { my($n,$v) = @$_; # Postfix since version 20060610 uses xtext-encoded (RFC 3461) # strings in XCLIENT and XFORWARD attribute values, previous # versions expected plain text with neutered special characters; # see README_FILES/XFORWARD_README if (defined $v && $v ne '') { $v =~ s/[^\041-\176]/?/g; # isprint $v =~ s/[<>()\\";\@]/?/g; # other chars that are special in hdrs # postfix/src/smtpd/smtpd.c NEUTER_CHARACTERS $v = xtext_encode($v); $v = substr($v,0,255) if length($v) > 255; # chop xtext, not nice } !defined $v || $v eq '' || !$xfwd_supp_opt{$n} ? () : ("$n=$v") } ( ['ADDR',$cl_ip], ['NAME',$msginfo->client_name], ['PORT',$msginfo->client_port], ['PROTO',$msginfo->client_proto], ['HELO',$msginfo->client_helo], ['SOURCE',$msginfo->client_source], ['IDENT',$msginfo->queue_id] ); $smtp_session->timeout( max(60,min($smtp_xforward_timeout,$deadline-time()))); $smtp_handle->command('XFORWARD',@params); #flush! $smtp_resp = $smtp_session->smtp_response; # fetch response to XFORWARD do_log(3,"smtp resp to XFORWARD: %s", $smtp_resp); $smtp_resp=~/^2/ or do_log(-1,"Negative SMTP resp. to XFORWARD: %s", $smtp_resp); section_time($which_section); } $which_section = 'fwd-auth'; my($auth_user) = $msginfo->auth_user; my($mechanisms) = $smtp_session->supports('AUTH'); if (!c('auth_required_out')) { do_log(3,"AUTH not needed, user='%s', MTA offers '%s'", $auth_user,$mechanisms); } elsif ($mechanisms eq '') { do_log(3,"INFO: MTA does not offer AUTH capability, user='%s'", $auth_user); } elsif (!defined $auth_user) { do_log(0,"INFO: AUTH needed for submission but AUTH data not available"); } else { do_log(3,"INFO: authenticating %s, server supports AUTH %s", $auth_user,$mechanisms); $auth_capable = 1; # my($sasl) = Authen::SASL->new( # 'callback' => { 'user' => $auth_user, 'authname' => $auth_user, # 'pass' => $msginfo->auth_pass }); # $smtp_handle->auth($sasl) or die "sending AUTH, user=$auth_user\n";#flush do_log(0,"Sorry, AUTH not supported in this version of amavisd!"); section_time($which_section); } $which_section = 'fwd-mail-from'; $smtp_session->timeout(max(60,min($smtp_mail_timeout,$deadline-time()))); my($fetched_mail_resp) = 0; my($fetched_rcpt_resp) = 0; my($data_command_accepted) = 0; if ($initial_submission && $dsn_capable && !defined($dsn_envid)) { # ENVID identifies transaction, not a message $dsn_envid = xtext_encode(sprintf("AM.%s.%s@%s", $msginfo->mail_id || $msginfo->log_id, iso8601_utc_timestamp(time), c('myhostname'))); } my($submitter) = $msginfo->auth_submitter; my($btype) = $msginfo->body_type; $from_options{'BODY'} = uc($btype) if $mimetransport8bit_capable && defined($btype) && $btype ne ''; $from_options{'RET'} = $dsn_ret if $dsn_capable && defined $dsn_ret; $from_options{'ENVID'} = $dsn_envid if $dsn_capable && defined $dsn_envid; $from_options{'AUTH'} = xtext_encode($submitter) # RFC 4954 (ex RFC 2554) if $auth_capable && defined($submitter) && $submitter ne '' && $submitter ne '<>'; my($faddr) = $msginfo->sender_smtp; $smtp_handle->mail($faddr, %from_options); # MAIL FROM # consider the transaction state unknown until we see a response $smtp_session->transaction_begins_unconfirmed; # also counts transactions if (!$pipelining) { $smtp_resp = $smtp_session->smtp_response; $fetched_mail_resp = 1; $smtp_resp = '' if !defined $smtp_resp; my($ok) = $smtp_resp =~ /^2/; do_log($ok?3:1, "smtp resp to MAIL: %s", $smtp_resp); if ($ok) { $smtp_session->transaction_begins; # transaction is active } else { # transaction state unchanged, consider it unknown my($smtp_resp_ext) = enhance_smtp_response($smtp_resp,$am_id,$mta_id, '.1.0','MAIL FROM'); for my $r (@per_recip_data) { next if $r->recip_done; $r->recip_remote_mta($relayhost); $r->recip_remote_mta_smtp_response($smtp_resp); $r->recip_smtp_response($smtp_resp_ext); $r->recip_done(2); } } } section_time($which_section) if !$pipelining; # otherwise it just shows 0 $which_section = 'fwd-rcpt-to'; $smtp_session->timeout(max(60,min($smtp_rcpt_timeout,$deadline-time()))); my($skipping_resp); my(@per_recip_data_rcpt_sent); for my $r (@per_recip_data) { # send recipient addresses next if $r->recip_done; if (defined $skipping_resp) { $r->recip_smtp_response($skipping_resp); $r->recip_done(2); next; } # prepare to send a RCPT TO command my($raddr) = qquote_rfc2821_local($r->recip_final_addr); if (!$dsn_capable) { $smtp_handle->recipient($raddr); # a barebones RCPT TO command } else { # include dsn options with a RCPT TO command my(@dsn_notify); # implies a default when the list is empty my($dn) = $r->dsn_notify; @dsn_notify = @$dn if $dn && $msginfo->sender ne ''; # if nondefault if (c('terminate_dsn_on_notify_success')) { # we want to handle option SUCCESS locally if (grep($_ eq 'SUCCESS', @dsn_notify)) { # strip out SUCCESS @dsn_notify = grep($_ ne 'SUCCESS', @dsn_notify); @dsn_notify = ('NEVER') if !@dsn_notify; do_log(3,"stripped out SUCCESS, result: NOTIFY=%s", join(',',@dsn_notify)); } } my(%rcpt_options); $rcpt_options{'NOTIFY'} = join(',', map(uc($_),@dsn_notify)) if @dsn_notify; $rcpt_options{'ORCPT'} = $r->dsn_orcpt if defined $r->dsn_orcpt; $smtp_handle->recipient($raddr, %rcpt_options); # RCPT TO } push(@per_recip_data_rcpt_sent, $r); # remember which recips were sent if (!$pipelining) { # must fetch responses to RCPT TO right away $smtp_resp = $smtp_session->smtp_response; $fetched_rcpt_resp = 1; $smtp_resp = '' if !defined $smtp_resp; $r->recip_remote_mta($relayhost); $r->recip_remote_mta_smtp_response($smtp_resp); my($smtp_resp_ext) = enhance_smtp_response($smtp_resp,$am_id,$mta_id, '.1.0','RCPT TO'); $r->recip_smtp_response($smtp_resp_ext); # preliminary response my($ok) = $smtp_resp =~ /^2/; do_log($ok?3:1, "smtp resp to RCPT (%s): %s", $raddr,$smtp_resp); if ($ok) { $any_valid_recips++ } else { if ($smtp_resp =~ /^452/) { # too many recipients - see RFC 5321 do_log(-1, 'Only %d recips sent in one go: "%s"', $any_valid_recips, $smtp_resp) if !defined $skipping_resp; $skipping_resp = enhance_smtp_response($smtp_resp,$am_id,$mta_id, '.5.3','RCPT TO'); } elsif ($smtp_resp =~ /^4/) { $any_tempfail_recips++; } $r->recip_done(2); # got a negative response to RCPT TO } } } section_time($which_section) if !$pipelining; # otherwise it just shows 0 my($what_cmd); if (!@per_recip_data_rcpt_sent || # no recipients were sent $fetched_rcpt_resp && !$any_valid_recips) { # no recipients accepted # it is known there are no valid recipients, don't go into DATA section do_log(0,"no valid recipients, skip data transfer"); $smtp_session->timeout($smtp_rset_timeout); $what_cmd = 'RSET'; $smtp_handle->rset; # send a RSET $smtp_session->transaction_ends_unconfirmed; } elsif ($fetched_rcpt_resp && # no pipelining $any_tempfail_recips && !$dsn_per_recip_capable) { # we must not proceed if mail did not came in as LMTP, # or we would generate mail duplicates on each delivery attempt do_log(-1,"mail_via_smtp: DATA skipped, tempfailed recips: %s", $any_tempfail_recips); $smtp_session->timeout($smtp_rset_timeout); $what_cmd = 'RSET'; $smtp_handle->rset; # send a RSET $smtp_session->transaction_ends_unconfirmed; } else { # pipelining, or we know we got a clearance to proceed $which_section = 'fwd-data-cmd'; # pipelining in effect, or we have at least one valid recipient, go DATA $smtp_session->timeout( max(60,min($smtp_data_init_timeout,$deadline-time()))); $smtp_handle->data; #flush! DATA $in_datasend_mode = 1; # DATA command was sent (but not yet confirmed) if (!$fetched_mail_resp) { # pipelining in effect, late response to MAIL $which_section = 'fwd-mail-pip'; $smtp_session->timeout(max(60,min($smtp_mail_timeout, $deadline-time()))); $smtp_resp = $smtp_session->smtp_response; $fetched_mail_resp = 1; $smtp_resp = '' if !defined $smtp_resp; my($ok) = $smtp_resp =~ /^2/; do_log($ok?3:1, "smtp resp to MAIL (pip): %s", $smtp_resp); if ($ok) { $smtp_session->transaction_begins; # transaction is active } else { # transaction state unchanged, consider it unknown my($smtp_resp_ext) = enhance_smtp_response($smtp_resp,$am_id,$mta_id, '.1.0','MAIL FROM'); for my $r (@per_recip_data) { next if $r->recip_done; $r->recip_remote_mta($relayhost); $r->recip_remote_mta_smtp_response($smtp_resp); $r->recip_smtp_response($smtp_resp_ext); $r->recip_done(2); } } section_time($which_section); } if (!$fetched_rcpt_resp) { # pipelining in effect, late response to RCPT $which_section = 'fwd-rcpt-pip'; $smtp_session->timeout(max(60,min($smtp_rcpt_timeout, $deadline-time()))); for my $r (@per_recip_data_rcpt_sent) { # only for those actually sent $smtp_resp = $smtp_session->smtp_response; $fetched_rcpt_resp = 1; $smtp_resp = '' if !defined $smtp_resp; my($raddr) = qquote_rfc2821_local($r->recip_final_addr); my($ok) = $smtp_resp =~ /^2/; do_log($ok?3:1, "smtp resp to RCPT (pip) (%s): %s", $raddr,$smtp_resp); next if $r->recip_done; # shouldn't happen $r->recip_remote_mta($relayhost); $r->recip_remote_mta_smtp_response($smtp_resp); my($smtp_resp_ext) = enhance_smtp_response($smtp_resp,$am_id,$mta_id, '.1.0','RCPT TO'); $r->recip_smtp_response($smtp_resp_ext); # preliminary response if ($ok) { $any_valid_recips++ } else { if ($smtp_resp =~ /^452/) { # too many recipients - see RFC 5321 do_log(-1, 'Only %d recips sent in one go: "%s"', $any_valid_recips, $smtp_resp); $skipping_resp = enhance_smtp_response($smtp_resp,$am_id,$mta_id, '.5.3','RCPT TO'); } elsif ($smtp_resp =~ /^4/) { $any_tempfail_recips++; } $r->recip_done(2); # got a negative response to RCPT TO } } section_time($which_section); } $which_section = 'fwd-data-chkpnt' if $pipelining; $smtp_session->timeout( max(60,min($smtp_data_init_timeout,$deadline-time()))); $smtp_resp = $smtp_session->smtp_response; # fetch response to DATA $smtp_resp = '' if !defined $smtp_resp; do_log(3,"smtp resp to DATA: %s", $smtp_resp); section_time($which_section); $data_command_accepted = $smtp_resp=~/^3/ ? 1 : 0; if (!$data_command_accepted) { do_log(0,"Negative SMTP resp. to DATA: %s", $smtp_resp); $in_datasend_mode = 0; $smtp_session->timeout($smtp_rset_timeout); $what_cmd = 'RSET'; $smtp_handle->rset; # send a RSET $smtp_session->transaction_ends_unconfirmed; } elsif (!$any_valid_recips) { # pipelining and no recipients, in DATA do_log(2,"Too late, DATA accepted but no valid recips, send dummy"); $which_section = 'fwd-dummydata-end'; $smtp_session->timeout( max(60,min($smtp_data_done_timeout,$deadline-time()))); $what_cmd = 'data-dot'; $smtp_handle->dataend; # . as required by RFC 2920: if a DATA # command was accepted the SMTP client should send a single dot $in_datasend_mode = 0; $smtp_session->transaction_ends_unconfirmed; } elsif ($any_tempfail_recips && !$dsn_per_recip_capable) { # pipelining # we must not proceed if mail did not came in as LMTP, # or we would generate mail duplicates on each delivery attempt do_log(2,"Too late, DATA accepted but tempfailed recips, bail out"); die "Bail out, DATA accepted but tempfailed recips, not a LMTP input"; } else { # all ok so far, we are in a DATA state and must send contents $which_section = 'fwd-data-hdr'; if (defined($msg) && !$msg->isa('MIME::Entity')) { $msg->seek($msginfo->skip_bytes, 0) or die "mail_via_smtp: Can't rewind mail file: $!"; } my($hdr_edits) = $msginfo->header_edits; $hdr_edits = Amavis::Out::EditHeader->new if !$hdr_edits; $smtp_session->timeout( max(60,min($smtp_data_xfer_timeout,$deadline-time()))); my($received_cnt) = $hdr_edits->write_header($msg,$smtp_handle,!$initial_submission); if ($received_cnt > 100) { # loop detection required by RFC 5321 (ex RFC 2821) section 6.3 # Do not modify the signal text, it gets matched elsewhere! die "Too many hops: $received_cnt 'Received:' header fields\n"; } $which_section = 'fwd-data-contents'; if (!defined($msg)) { # empty mail } elsif ($msg->isa('MIME::Entity')) { $msg->print_body($smtp_handle); } else { my($nbytes,$buff); while (($nbytes=$msg->read($buff,16384)) > 0) { $smtp_handle->datasend($buff) } defined $nbytes or die "Error reading: $!"; } section_time($which_section); $which_section = 'fwd-data-end'; $smtp_session->timeout( max(60,min($smtp_data_done_timeout,$deadline-time()))); $what_cmd = 'data-dot'; $smtp_handle->dataend; # . $in_datasend_mode = 0; $smtp_session->transaction_ends_unconfirmed; $any_valid_recips_and_data_sent = 1; section_time($which_section) if !$pipelining; # otherwise it shows 0 } } if ($pipelining && !$smtp_connection_cache_enable) { do_log(5,"smtp connection_cache disabled, sending QUIT"); $smtp_session->quit; #flush! QUIT # can't be sure until we see a response, consider uncertain just in case $smtp_session->transaction_ends_unconfirmed; } $which_section = 'fwd-rundown-1'; $smtp_resp = undef; if (!defined $what_cmd) { # not expecting a response? } elsif ($what_cmd ne 'data-dot') { # must be a response to a RSET $smtp_resp = $smtp_session->smtp_response; # fetch a response $smtp_resp = '' if !defined $smtp_resp; do_log(3,"smtp resp to %s: %s", $what_cmd,$smtp_resp); if ($smtp_resp =~ /^2/) { $smtp_session->transaction_ends if $what_cmd eq 'RSET'; } else { die "Negative SMTP response to $what_cmd: $smtp_resp"; } } else { # get response(s) to data-dot # replace success responses to RCPT TO commands with a final response my $first = 1; my $anyfail = 0; my $anysucc = 0; for my $r (@per_recip_data_rcpt_sent) { # only for those actually sent if ($lmtp || $first) { $first = 0; my($raddr) = qquote_rfc2821_local($r->recip_final_addr); $raddr .= ', etc.' if !$lmtp && @per_recip_data > 1; $smtp_resp = $smtp_session->smtp_response; # resp to data-dot $smtp_resp = '' if !defined $smtp_resp; do_log(3,"smtp resp to %s (%s): %s", $what_cmd,$raddr,$smtp_resp); if ($smtp_resp =~ /^2/) { $anysucc = 1; } else { $anyfail = 1; do_log(0,"Negative SMTP response to %s (%s): %s", $what_cmd,$raddr,$smtp_resp); } } next if $r->recip_done; # skip those that failed at earlier stages $r->recip_remote_mta($relayhost); $r->recip_remote_mta_smtp_response($smtp_resp); my($smtp_resp_ext) = enhance_smtp_response($smtp_resp,$am_id,$mta_id, '.6.0','data-dot'); $smtp_response = $smtp_resp_ext if !defined $smtp_response; $r->recip_smtp_response($smtp_resp_ext); $r->recip_done(2); $r->recip_mbxname($r->recip_final_addr) if $smtp_resp =~ /^2/; } if ($first) { # fetch an uncollected response # fetch unprocessed response if all recipients were rejected # but we nevertheless somehow entered a data transfer mode # (i.e. if an SMTP server failed to reject a DATA command). # RFC 2033: when there have been no successful RCPT commands in the # mail transaction, the DATA command MUST fail with a 503 reply code $smtp_resp = $smtp_session->smtp_response; # resp to data-dot $smtp_resp = '' if !defined $smtp_resp; if ($smtp_resp =~ /^2/) { $anysucc = 1 } else { $anyfail = 1 } do_log(3,"smtp resp to _dummy_ data %s: %s", $what_cmd,$smtp_resp); } if ($anysucc && !$anyfail) { # we are certain all went fine and a transaction is definitely over $smtp_session->transaction_ends; } } # if ($pipelining) {} # QUIT was already sent # elsif (!$smtp_connection_cache_enable) { # $smtp_session->quit; #flush! QUIT # # can't be sure until we see a response, consider uncertain just in case # $smtp_session->transaction_ends_unconfirmed; # } # if ($smtp_session->session_state eq 'quitsent') { # $smtp_session->timeout($smtp_quit_timeout); # $smtp_resp = $smtp_session->smtp_response; # $smtp_resp = '' if !defined $smtp_resp; # do_log(3,"smtp resp to QUIT: %s", $smtp_resp); # if ($smtp_resp =~ /^2/) { # $smtp_session->transaction_ends; # } else { # $smtp_session->transaction_ends_unconfirmed; # do_log(0,"Negative SMTP resp. to QUIT: %s", $smtp_resp); # } # } my($keep_session) = $smtp_session->session_state ne 'quitsent'; if ($keep_session && !defined($smtp_session->in_smtp_transaction)) { do_log(2,"SMTP transaction state uncertain, closing just in case"); $keep_session = 0; } $smtp_session->close($keep_session) or die "Error closing Amavis::Out::SMTP::Session"; undef $smtp_handle; undef $smtp_session; 1; # some unusual error conditions _are_ captured by eval, but fail to set $@ } or do { $err = $@ ne '' ? $@ : "errno=$!" }; my($saved_section_name) = $which_section; $which_section = 'fwd-end-chkpnt'; if ($err ne '') { chomp $err; $err = ' ' if $err eq '' } # careful chomp do_log(2,"mail_via_smtp: session failed: %s", $err) if $err ne ''; prolong_timer($which_section); # restart timer # terminate the SMTP session if still alive if (!defined($smtp_session)) { # already closed normally } elsif ($in_datasend_mode) { # We are aborting SMTP session. Data transfer mode must NOT be terminated # with a dataend (dot), otherwise recipient will receive a chopped-off mail # (and possibly be receiving it over and over again during each MTA retry. do_log(-1, "mail_via_smtp: NOTICE: aborting SMTP session, %s", $err); $smtp_session->close(0); # abruptly terminate SMTP session, ignore status } else { do_log(5,"smtp session done, sending QUIT"); eval { $smtp_session->timeout(1); # don't wait for too long $smtp_session->quit; #flush! # send a QUIT regardless of success so far $smtp_session->transaction_ends_unconfirmed; for (my($cnt)=0; ; $cnt++) { # curious if there are any pending responses my($smtp_resp) = $smtp_session->smtp_response; last if !defined $smtp_resp; do_log(0,"discarding unprocessed reply: %s", $smtp_resp); if ($cnt > 20) { do_log(-1,"aborting, discarding many replies"); last } } } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1, "mail_via_smtp: error during QUIT: %s", $eval_stat); }; $smtp_session->close(0); # terminate SMTP session, ignore status } undef $smtp_handle; undef $smtp_session; # prepare final smtp response and log abnormal events for my $r (@per_recip_data) { my($resp) = $r->recip_smtp_response; $smtp_response = $resp if !defined($smtp_response) || $resp =~ /^4/ && $smtp_response !~ /^4/ || $resp !~ /^2/ && $smtp_response !~ /^[45]/; } if ($err eq '') { # no errors } elsif ($err =~ /^timed out\b/ || $err =~ /: Timeout\z/) { $smtp_response = sprintf("450 4.4.2 Timed out during %s, MTA(%s), id=%s", $saved_section_name, $mta_id, $am_id); } elsif ($err =~ /^Can't connect\b/) { $smtp_response = sprintf("450 4.4.1 %s, MTA(%s), id=%s", $err, $mta_id, $am_id); } elsif ($err =~ /^Too many hops\b/) { $smtp_response = sprintf("554 5.4.6 Reject: %s, id=%s", $err, $am_id); } else { $smtp_response = sprintf("451 4.5.0 From MTA(%s) during %s (%s): id=%s", $mta_id, $saved_section_name, $err, $am_id); } # NOTE: $initial_submission argument is typically treated as a boolean # but a value of 'AV' is supplied by av_smtp_client to allow a forwarding # method to distinguish it from ordinary submissions my($ll) = ($smtp_response =~ /^2/ || $initial_submission eq 'AV') ? 1 : -1; ll($ll) && do_log($ll, "%s -> %s,%s %s", $logmsg, join(',', qquote_rfc2821_local( map($_->recip_final_addr, @per_recip_data))), join(' ', map { my($v)=$from_options{$_}; defined($v)?"$_=$v":"$_" } (keys %from_options)), $smtp_response); if (defined $smtp_response) { $msginfo->dsn_passed_on($dsn_capable && $smtp_response=~/^2/ && !c('terminate_dsn_on_notify_success') ? 1 : 0); for my $r (@per_recip_data) { # attach an SMTP response to each recip that was not already processed if (!$r->recip_done) { # mark it as done $r->recip_smtp_response($smtp_response); $r->recip_done(2); $r->recip_mbxname($r->recip_final_addr) if $smtp_response =~ /^2/; } elsif ($any_valid_recips_and_data_sent && $r->recip_smtp_response =~ /^452/) { # 'undo' the RCPT TO '452 Too many recipients' situation, # mail needs to be transferred in more than one transaction $r->recip_smtp_response(undef); $r->recip_done(undef); } } if ($smtp_response =~ /^2/) { snmp_count('OutMsgsDelivers'); my($size) = $msginfo->msg_size; snmp_count( ['OutMsgsSize'.$_, $size, 'C64'] ) for @snmp_vars; } elsif ($smtp_response =~ /^4/) { snmp_count('OutMsgsAttemptFails'); } elsif ($smtp_response =~ /^5/) { snmp_count('OutMsgsRejects'); } } section_time($which_section); die $err if $err =~ /^timed out\b/; # resignal timeout 1; } 1; __DATA__ # package Amavis::Out::Pipe; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT = qw(&mail_via_pipe); import Amavis::Conf qw(:platform c cr ca); import Amavis::Util qw(untaint min max minmax ll do_log snmp_count collect_equal_delivery_recips); import Amavis::ProcControl qw(exit_status_str proc_status_ok kill_proc run_command_consumer); import Amavis::Timing qw(section_time); import Amavis::rfc2821_2822_Tools; import Amavis::Out::EditHeader; } use Errno qw(ENOENT EACCES ESRCH); use POSIX qw(WIFEXITED WIFSIGNALED WIFSTOPPED WEXITSTATUS WTERMSIG WSTOPSIG); # Send mail using external mail submission program 'sendmail' or its lookalike # (also available with Postfix and Exim) - used for forwarding original mail # or sending notifications or quarantining. May throw exception (die) on # temporary failure (4xx) or other problem. # sub mail_via_pipe(@) { my($msginfo, $initial_submission, $dsn_per_recip_capable, $filter) = @_; my(@snmp_vars) = !$initial_submission ? ('', 'Relay', 'ProtoPipe', 'ProtoPipeRelay') : ('', 'Submit', 'ProtoPipe', 'ProtoPipeSubmit', 'Submit'.$initial_submission); snmp_count('OutMsgs'.$_) for @snmp_vars; my($logmsg) = sprintf("%s via PIPE: %s", ($initial_submission?'SEND':'FWD'), $msginfo->sender_smtp); my($per_recip_data_ref, $proto_sockname) = collect_equal_delivery_recips($msginfo, $filter, qr/^pipe:/i); if (!$per_recip_data_ref || !@$per_recip_data_ref) { do_log(5, "%s, nothing to do", $logmsg); return 1; } $proto_sockname = $proto_sockname->[0] if ref $proto_sockname; ll(1) && do_log(1, "delivering to %s, %s -> %s", $proto_sockname, $logmsg, join(',', qquote_rfc2821_local( map($_->recip_final_addr, @$per_recip_data_ref)) )); # just use the first one, ignoring failover alternatives local($1); $proto_sockname =~ /^pipe:(.*)\z/si or die "Bad fwd method syntax: ".$proto_sockname; my($pipe_args) = $1; $pipe_args =~ s/^flags=\S*\s*//i; # flags are currently ignored, q implied $pipe_args =~ s/^argv=//i; my(@pipe_args) = split(' ',$pipe_args); my(@command) = shift(@pipe_args); my($dsn_capable) = c('propagate_dsn_if_possible'); # assume, unless disabled $dsn_capable = 0 if $command[0] !~ /sendmail/; # a hack, don't use -N or -V if ($dsn_capable) { # DSN is supported since Postfix 2.3 # notify options are per-recipient, yet a command option -N applies to all my($common_list); my($not_all_the_same) = 0; for my $r (@{$msginfo->per_recip_data}) { my($dsn_notify) = $r->dsn_notify; my $d; if ($msginfo->sender eq '') { $d = 'NEVER'; } elsif (!$dsn_notify) { $d = 'DELAY,FAILURE'; # sorted } else { $d = uc(join(',', sort @$dsn_notify)); # normalize order } if (!defined($common_list)) { $common_list = $d } elsif ($d ne $common_list) { $not_all_the_same = 1 } } if ($common_list=~/\bSUCCESS\b/ && c('terminate_dsn_on_notify_success')) { # strip out option SUCCESS, we want to handle it locally my(@dsn_notify) = grep($_ ne 'SUCCESS', split(/,/,$common_list)); @dsn_notify = ('NEVER') if !@dsn_notify; $common_list = join(',',@dsn_notify); do_log(3,"stripped out SUCCESS, result: NOTIFY=%s",$common_list); } if ($not_all_the_same || $msginfo->sender eq '') {} # leave at default elsif ($common_list eq "DELAY,FAILURE") {} # leave at default else { unshift(@pipe_args, '-N', $common_list) } unshift(@pipe_args, '-V', $msginfo->dsn_envid) if defined $msginfo->dsn_envid; # but there is no mechanism to specify ORCPT or RET } for (@pipe_args) { # The sendmail command line expects addresses quoted as per RFC 822. # "funny user"@some.domain # For compatibility with Sendmail, the Postfix sendmail command line also # accepts address formats that are legal in RFC 822 mail header section: # Funny Dude <"funny user"@some.domain> # Although addresses passed as args to sendmail submission program # should not be <...> bracketed, for some reason original sendmail # issues a warning on null reverse-path, but gladly accepty <>. # As this is not strictly wrong, we comply to make it happy. # NOTE: the -fsender is not allowed, -f and sender must be separate args! my($null_ret_path) = '<>'; # some sendmail lookalikes want '<>', others '' # Courier sendmail accepts '' but not '<>' for null reverse path $null_ret_path = '' if $Amavis::extra_code_in_courier; if (/^\$\{sender\}\z/i) { push(@command, $msginfo->sender eq '' ? $null_ret_path : do { local($_)=$msginfo->sender_smtp; s/^\z//; untaint($_) }); } elsif (/^\$\{recipient\}\z/i) { push(@command, map { $_ eq '' ? $null_ret_path : untaint(quote_rfc2821_local($_)) } map($_->recip_final_addr, @$per_recip_data_ref)); } else { push(@command, $_); } } my($msg) = $msginfo->mail_text; # a file handle or a MIME::Entity object if (defined($msg) && !$msg->isa('MIME::Entity')) { $msg->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; } do_log(5, "mail_via_pipe running command: %s", join(' ', @command)); local $SIG{CHLD} = 'DEFAULT'; local $SIG{PIPE} = 'IGNORE'; # don't signal on a write to a widowed pipe my($proc_fh,$pid) = run_command_consumer(undef,undef,@command); my($hdr_edits) = $msginfo->header_edits; $hdr_edits = Amavis::Out::EditHeader->new if !$hdr_edits; my($received_cnt) = $hdr_edits->write_header($msg,$proc_fh,!$initial_submission); if ($received_cnt > 100) { # loop detection required by RFC 5321 section 6.3 # deal with it later, for now just skip the body } elsif (!defined($msg)) { # empty mail } elsif ($msg->isa('MIME::Entity')) { $msg->print_body($proc_fh); } else { my($nbytes,$buff); while (($nbytes=$msg->read($buff,16384)) > 0) { $proc_fh->print($buff) or die "Submitting mail text failed: $!" } defined $nbytes or die "Error reading: $!"; } $proc_fh->flush or die "Can't flush pipe to a mail submission program: $!"; my($smtp_response); if ($received_cnt > 100) { # loop detection required by RFC 5321 section 6.3 do_log(-2, "Too many hops: %d 'Received:' header fields", $received_cnt); kill_proc($pid,$command[0],10,$proc_fh,'too many hops') if defined $pid; $proc_fh->close; undef $proc_fh; undef $pid; # and ignore status $smtp_response = "554 5.4.6 Reject: " . "Too many hops: $received_cnt 'Received:' header fields"; } else { my($err) = 0; $proc_fh->close or $err=$!; my($child_stat) = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; # sendmail program (Postfix variant) can return the following exit codes: # EX_OK(0), EX_DATAERR, EX_SOFTWARE, EX_TEMPFAIL, EX_NOUSER, EX_UNAVAILABLE if (proc_status_ok($child_stat,$err, EX_OK)) { $smtp_response = "250 2.6.0 Ok"; # submitted to MTA snmp_count('OutMsgsDelivers'); my($size) = $msginfo->msg_size; snmp_count( ['OutMsgsSize'.$_, $size, 'C64'] ) for @snmp_vars; } elsif (proc_status_ok($child_stat,$err, EX_TEMPFAIL)) { $smtp_response = "450 4.5.0 Temporary failure submitting message"; snmp_count('OutMsgsAttemptFails'); } elsif (proc_status_ok($child_stat,$err, EX_NOUSER)) { $smtp_response = "554 5.1.1 Recipient unknown"; snmp_count('OutMsgsRejects'); } elsif (proc_status_ok($child_stat,$err, EX_UNAVAILABLE)) { $smtp_response = "554 5.5.0 Mail submission service unavailable"; snmp_count('OutMsgsRejects'); } else { $smtp_response = "451 4.5.0 Failed to submit a message: ". exit_status_str($child_stat,$err); snmp_count('OutMsgsAttemptFails'); } ll(3) && do_log(3,"mail_via_pipe %s, %s, %s", $command[0], exit_status_str($child_stat,$err), $smtp_response); } $smtp_response .= ", id=" . $msginfo->log_id; for my $r (@$per_recip_data_ref) { next if $r->recip_done; $r->recip_smtp_response($smtp_response); $r->recip_done(2); $r->recip_mbxname($r->recip_final_addr) if $smtp_response =~ /^2/; } $msginfo->dsn_passed_on($dsn_capable && $smtp_response=~/^2/ && !c('terminate_dsn_on_notify_success') ? 1 : 0); section_time('fwd-pipe'); 1; } 1; __DATA__ # package Amavis::Out::BSMTP; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT = qw(&mail_via_bsmtp); import Amavis::Conf qw(:platform $QUARANTINEDIR c cr ca); import Amavis::Util qw(untaint min max minmax ll do_log snmp_count collect_equal_delivery_recips); import Amavis::Timing qw(section_time); import Amavis::rfc2821_2822_Tools; import Amavis::Out::EditHeader; } use Errno qw(ENOENT EACCES); use IO::File qw(O_CREAT O_EXCL O_WRONLY); # store message in a BSMTP format # # RFC 2442: Application/batch-SMTP material is generated by a specially # modified SMTP client operating without a corresponding SMTP server. # The client simply assumes a successful response to all commands it issues. # The resulting content then consists of the collected output from the SMTP # client. # sub mail_via_bsmtp(@) { my($msginfo, $initial_submission, $dsn_per_recip_capable, $filter) = @_; my(@snmp_vars) = !$initial_submission ? ('', 'Relay', 'ProtoBSMTP', 'ProtoBSMTPRelay') : ('', 'Submit', 'ProtoBSMTP', 'ProtoBSMTPSubmit', 'Submit'.$initial_submission); snmp_count('OutMsgs'.$_) for @snmp_vars; my($logmsg) = sprintf("%s via BSMTP: %s", ($initial_submission?'SEND':'FWD'), $msginfo->sender_smtp); my($per_recip_data_ref, $proto_sockname) = collect_equal_delivery_recips($msginfo, $filter, qr/^bsmtp:/i); if (!$per_recip_data_ref || !@$per_recip_data_ref) { do_log(5, "%s, nothing to do", $logmsg); return 1; } $proto_sockname = $proto_sockname->[0] if ref $proto_sockname; ll(1) && do_log(1, "delivering to %s, %s -> %s", $proto_sockname, $logmsg, join(',', qquote_rfc2821_local( map($_->recip_final_addr, @$per_recip_data_ref)) )); # just use the first one, ignoring failover alternatives local($1); $proto_sockname =~ /^bsmtp:(.*)\z/si or die "Bad fwd method syntax: ".$proto_sockname; my($bsmtp_file_final) = $1; my($mbxname); my($s) = $msginfo->sender; # sanitized sender name for use in a filename $s =~ tr/a-zA-Z0-9@._+-/=/c; $s = substr($s,0,100)."..." if length($s) > 100+3; $s =~ s/\@/_at_/g; $s =~ s/^(\.{0,2})\z/_$1/; $bsmtp_file_final =~ s{%(.)} { $1 eq 'b' ? $msginfo->body_digest : $1 eq 'P' ? $msginfo->partition_tag : $1 eq 'm' ? $msginfo->mail_id||'' : $1 eq 'n' ? $msginfo->log_id : $1 eq 's' ? untaint($s) # a hack, avoid using %s : $1 eq 'i' ? iso8601_timestamp($msginfo->rx_time,1) #,'-') : $1 eq '%' ? '%' : '%'.$1 }egs; # prepend directory if not specified my($bsmtp_file_final_to_show) = $bsmtp_file_final; $bsmtp_file_final = $QUARANTINEDIR."/".$bsmtp_file_final if $QUARANTINEDIR ne '' && $bsmtp_file_final !~ m{^/}; my($bsmtp_file_tmp) = $bsmtp_file_final . ".tmp"; my($msg) = $msginfo->mail_text; # a scalar reference, or a file handle if (defined($msg) && !$msg->isa('MIME::Entity')) { $msg->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; } my($mp); my($err); eval { my($errn) = lstat($bsmtp_file_tmp) ? 0 : 0+$!; if ($errn == ENOENT) {} # good, no file, as expected elsif ($errn==0 && (-f _ || -l _)) { die "File $bsmtp_file_tmp already exists, refuse to overwrite" } else { die "File $bsmtp_file_tmp exists??? Refuse to overwrite it, $!" } $mp = IO::File->new; # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] $mp->open($bsmtp_file_tmp, untaint(O_CREAT|O_EXCL|O_WRONLY), 0640) or die "Can't create BSMTP file $bsmtp_file_tmp: $!"; binmode($mp,':bytes') or die "Can't set :bytes, $!"; # RFC 2442: Since no SMTP server is present the client must be prepared # to make certain assumptions about which SMTP extensions can be used. # The generator MAY assume that ESMTP [RFC 1869 (obsoleted by RFC 5321)] # facilities are available, that is, it is acceptable to use the EHLO # command and additional parameters on MAIL FROM and RCPT TO. If EHLO # is used MAY assume that the 8bitMIME [RFC 1652], SIZE [RFC 1870], and # NOTARY [RFC 1891] extensions are available. In particular, NOTARY # SHOULD be used. (nowadays called DSN) $mp->printf("EHLO %s\n", c('localhost_name')) or die "print failed (EHLO): $!"; my($btype) = $msginfo->body_type; # RFC 1652: need "8bit Data"? (RFC 2045) $btype = '' if !defined $btype; my($dsn_envid) = $msginfo->dsn_envid; my($dsn_ret) = $msginfo->dsn_ret; $mp->printf("MAIL FROM:%s\n", join(' ', $msginfo->sender_smtp, $btype ne '' ? ('BODY='.uc($btype)) : (), defined $dsn_ret ? ('RET='.$dsn_ret) : (), defined $dsn_envid ? ('ENVID='.$dsn_envid) : () ), ) or die "print failed (MAIL FROM): $!"; for my $r (@$per_recip_data_ref) { my(@dsn_notify); # implies a default when the list is empty my($dn) = $r->dsn_notify; @dsn_notify = @$dn if $dn && $msginfo->sender ne ''; # if nondefault if (@dsn_notify && c('terminate_dsn_on_notify_success')) { # we want to handle option SUCCESS locally if (grep($_ eq 'SUCCESS', @dsn_notify)) { # strip out SUCCESS @dsn_notify = grep($_ ne 'SUCCESS', @dsn_notify); @dsn_notify = ('NEVER') if !@dsn_notify; do_log(3,"stripped out SUCCESS, result: NOTIFY=%s", join(',',@dsn_notify)); } } $mp->printf("RCPT TO:%s\n", join(' ', qquote_rfc2821_local($r->recip_final_addr), @dsn_notify ? ('NOTIFY='.join(',',@dsn_notify)) : (), defined $r->dsn_orcpt ? ('ORCPT='.$r->dsn_orcpt) : () ), ) or die "print failed (RCPT TO): $!"; } $mp->print("DATA\n") or die "print failed (DATA): $!"; my($hdr_edits) = $msginfo->header_edits; $hdr_edits = Amavis::Out::EditHeader->new if !$hdr_edits; my($received_cnt)= $hdr_edits->write_header($msg,$mp,!$initial_submission); if ($received_cnt > 100) { # loop detection required by RFC 5321 sect. 6.3 die "Too many hops: $received_cnt 'Received:' header fields"; } elsif (!defined($msg)) { # empty mail } elsif ($msg->isa('MIME::Entity')) { $msg->print_body($mp); } else { my($ln); for ($! = 0; defined($ln=$msg->getline); $! = 0) { $mp->print($ln=~/^\./ ?(".",$ln) :$ln) or die "print failed-data: $!"; } defined $ln || $!==0 or die "Error reading: $!"; } $mp->print(".\n") or die "print failed (final dot): $!"; # $mp->print("QUIT\n") or die "print failed (QUIT): $!"; $mp->close or die "Error closing BSMTP file $bsmtp_file_tmp: $!"; undef $mp; rename($bsmtp_file_tmp, $bsmtp_file_final) or die "Can't rename BSMTP file to $bsmtp_file_final: $!"; $mbxname = $bsmtp_file_final; 1; } or do { $err = $@ ne '' ? $@ : "errno=$!" }; my($smtp_response); if ($err eq '') { $smtp_response = "250 2.6.0 Ok, queued as BSMTP $bsmtp_file_final_to_show"; snmp_count('OutMsgsDelivers'); my($size) = $msginfo->msg_size; snmp_count( ['OutMsgsSize'.$_, $size, 'C64'] ) for @snmp_vars; } else { chomp $err; unlink($bsmtp_file_tmp) or do_log(-2,"Can't delete half-finished BSMTP file %s: %s", $bsmtp_file_tmp, $!); $mp->close if defined $mp; # ignore status if ($err =~ /too many hops\b/i) { $smtp_response = "554 5.4.6 Reject: $err"; snmp_count('OutMsgsRejects'); } else { $smtp_response = "451 4.5.0 Writing $bsmtp_file_tmp failed: $err"; snmp_count('OutMsgsAttemptFails'); } die $err if $err =~ /^timed out\b/; # resignal timeout } $smtp_response .= ", id=" . $msginfo->log_id; $msginfo->dsn_passed_on($smtp_response=~/^2/ && !c('terminate_dsn_on_notify_success') ? 1 : 0); for my $r (@$per_recip_data_ref) { next if $r->recip_done; $r->recip_smtp_response($smtp_response); $r->recip_done(2); $r->recip_mbxname($mbxname) if $mbxname ne '' && $smtp_response =~ /^2/; } section_time('fwd-bsmtp'); 1; } 1; __DATA__ # package Amavis::Out::Local; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&mail_to_local_mailbox); import Amavis::Conf qw(:platform c cr ca $QUARANTINEDIR $quarantine_subdir_levels); import Amavis::Util qw(snmp_count ll do_log untaint unique_list collect_equal_delivery_recips); import Amavis::Timing qw(section_time); import Amavis::rfc2821_2822_Tools; import Amavis::Out::EditHeader; } use Errno qw(ENOENT EACCES); use Fcntl qw(LOCK_SH LOCK_EX LOCK_UN); #use File::Spec; use IO::File qw(O_RDONLY O_WRONLY O_RDWR O_APPEND O_CREAT O_EXCL); use subs @EXPORT_OK; # Deliver to local mailboxes only, ignore the rest: either to directory # (maildir style), or file (Unix mbox). (normally used as a quarantine method) # sub mail_to_local_mailbox(@) { my($msginfo, $initial_submission, $dsn_per_recip_capable, $filter) = @_; # note that recipients of a message being delivered to a quarantine # are typically not the original envelope recipients, but are pseudo # address provided to do_quarantine() based on @quarantine_to_maps; # nevertheless, we do the usual collect_equal_delivery_recips() ritual # here too for consistency # my($logmsg) = sprintf("%s via LOCAL: %s", ($initial_submission?'SEND':'FWD'), $msginfo->sender_smtp); my($per_recip_data_ref, $proto_sockname) = collect_equal_delivery_recips($msginfo, $filter, qr/^local:/i); if (!$per_recip_data_ref || !@$per_recip_data_ref) { do_log(5, "%s, nothing to do", $logmsg); return 1; } my(@per_recip_data) = @$per_recip_data_ref; undef $per_recip_data_ref; $proto_sockname = $proto_sockname->[0] if ref $proto_sockname; ll(4) && do_log(4, "delivering to %s, %s -> %s", $proto_sockname, $logmsg, join(',', qquote_rfc2821_local( map($_->recip_final_addr, @per_recip_data)) )); # just use the first one, ignoring failover alternatives local($1); $proto_sockname =~ /^local:(.*)\z/si or die "Bad local method syntax: ".$proto_sockname; my($via_arg) = $1; my(@snmp_vars) = !$initial_submission ? ('', 'Relay', 'ProtoLocal', 'ProtoLocalRelay') : ('', 'Submit','ProtoLocal', 'ProtoLocalSubmit', 'Submit'.$initial_submission); snmp_count('OutMsgs'.$_) for @snmp_vars; my($msg) = $msginfo->mail_text; # a file handle or a MIME::Entity object my($sender) = $msginfo->sender; for my $r (@per_recip_data) { # determine a mailbox file for each recipient # each recipient gets his own copy; these are not the original message's # recipients but are mailbox addresses, typically telling where a message # to be quarantined is to be stored my($recip) = $r->recip_final_addr; # %local_delivery_aliases emulates aliases map - this would otherwise # be done by MTA's local delivery agent if we gave the message to MTA. # This way we keep interface compatible with other mail delivery # methods. The hash value may be a ref to a pair of fixed strings, # or a subroutine ref (which must return such pair) to allow delayed # (lazy) evaluation when some part of the pair is not yet known # at initialization time. # If no matching entry is found quarantining is skipped. my($mbxname, $suggested_filename); my($localpart,$domain) = split_address($recip); my($ldar) = cr('local_delivery_aliases'); # a ref to a hash my($alias) = $ldar->{$localpart}; if (ref($alias) eq 'ARRAY') { ($mbxname, $suggested_filename) = @$alias; } elsif (ref($alias) eq 'CODE') { # lazy (delayed) evaluation ($mbxname, $suggested_filename) = &$alias; } elsif ($alias ne '') { ($mbxname, $suggested_filename) = ($alias, undef); } elsif (!exists $ldar->{$localpart}) { do_log(3, "no key '%s' in %s, using a default", $localpart, '%local_delivery_aliases'); ($mbxname, $suggested_filename) = ($QUARANTINEDIR, undef); } if (!defined($mbxname) || $mbxname eq '' || $recip eq '') { my($why) = !exists $ldar->{$localpart} ? 1 : $alias eq '' ? 2 : 3; do_log(2, "skip local delivery(%s): <%s> -> <%s>", $why,$sender,$recip); my($smtp_response) = "250 2.6.0 Ok, skip local delivery($why)"; $smtp_response .= ", id=" . $msginfo->log_id; $r->recip_smtp_response($smtp_response); $r->recip_done(2); next; } my($ux); # is it a UNIX-style mailbox? my($errn) = stat($mbxname) ? 0 : 0+$!; if ($errn == ENOENT) { $ux = 1; # $mbxname is a UNIX-style mailbox (new file) } elsif ($errn != 0) { die "Can't access a mailbox file or directory $mbxname: $!"; } elsif (-f _) { $ux = 1; # $mbxname is a UNIX-style mailbox (existing file) } elsif (!-d _) { die "Mailbox is neither a file nor a directory: $mbxname"; } else { # a directory $ux = 0; # $mbxname is a directory (amavis/maildir style mailbox) my($explicitly_suggested_filename) = $suggested_filename ne ''; if ($suggested_filename eq '') { $suggested_filename = $via_arg ne '' ? $via_arg : '%m' } my($mail_id) = $msginfo->mail_id; if (!defined($mail_id)) { do_log(-1, "mail_to_local_mailbox: mail_id still undefined!?"); $mail_id = ''; } $suggested_filename =~ s{%(.)} { $1 eq 'b' ? $msginfo->body_digest : $1 eq 'P' ? $msginfo->partition_tag : $1 eq 'm' ? $mail_id : $1 eq 'n' ? $msginfo->log_id : $1 eq 'i' ? iso8601_timestamp($msginfo->rx_time,1) #,'-') : $1 eq '%' ? '%' : '%'.$1 }egs; # $mbxname = File::Spec->catfile($mbxname, $suggested_filename); $mbxname = "$mbxname/$suggested_filename"; if ($quarantine_subdir_levels>=1 && !$explicitly_suggested_filename) { # using a subdirectory structure to disperse quarantine files local($1,$2); my($subdir) = substr($mail_id, 0, 1); $subdir=~/^[A-Z0-9]\z/i or die "Unexpected first char: $subdir"; $mbxname =~ m{^ (.*/)? ([^/]+) \z}sx; my($path,$fname) = ($1,$2); # $mbxname = File::Spec->catfile($path, $subdir, $fname); $mbxname = "$path$subdir/$fname"; # resulting full filename my($errn) = stat("$path$subdir") ? 0 : 0+$!; # only test for ENOENT, other errors will be detected later on access if ($errn == ENOENT) { # check/prepare a set of subdirectories do_log(2, "checking/creating quarantine subdirs under %s", $path); for my $d ('A'..'Z','a'..'z','0'..'9') { $errn = stat("$path$d") ? 0 : 0+$!; if ($errn == ENOENT) { mkdir("$path$d", 0750) or die "Can't create dir $path$d: $!"; } } } } } # save location where mail should be stored, prepend a mailbox style $r->recip_mbxname(($ux ? 'mbox' : 'maildir') . ':' . $mbxname); } # # now go ahead and store a message to predetermined files in recip_mbxname; # iterate by groups of recipients with the same mailbox name # @per_recip_data = grep(!$_->recip_done, @per_recip_data); while (@per_recip_data) { my($mbxname) = $per_recip_data[0]->recip_mbxname; # first mailbox name # collect all recipient which have the same mailbox file as the first one my(@recips_with_same_mbx) = grep($_->recip_mbxname eq $mbxname, @per_recip_data); @per_recip_data = grep($_->recip_mbxname ne $mbxname, @per_recip_data); # retrieve mailbox style and a filename local($1,$2); $mbxname =~ /^([^:]*):(.*)\z/; my($ux) = $1 eq 'mbox' ? 1 : 0; $mbxname = $2; my(@recips) = map($_->recip_final_addr, @recips_with_same_mbx); @recips = unique_list(\@recips); my($smtp_response); { # a block is used as a 'switch' statement - 'last' will exit from it do_log(1,"local delivery: %s -> %s, mbx=%s", $msginfo->sender_smtp, join(", ",@recips), $mbxname); my($eval_stat); my($mp,$pos,$pid); my($errn) = stat($mbxname) ? 0 : 0+$!; section_time('stat-mbx'); local $SIG{CHLD} = 'DEFAULT'; local $SIG{PIPE} = 'IGNORE'; # don't signal on a write to a widowed pipe eval { # try to open the mailbox file for writing if (!$ux) { # one mail per file, will create specified file if ($errn == ENOENT) { # good, no file, as expected } elsif ($errn != 0) { die "File $mbxname not accessible, refuse to write: $!"; } elsif (!-f _) { die "File $mbxname is not a regular file, refuse to write"; } else { die "File $mbxname already exists, refuse to overwrite"; } if ($mbxname =~ /\.gz\z/) { $mp = Amavis::IO::Zlib->new; # ?how to request an exclusive access? $mp->open($mbxname,'wb') or die "Can't create gzip file $mbxname: $!"; } else { $mp = IO::File->new; # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] $mp->open($mbxname, untaint(O_CREAT|O_EXCL|O_WRONLY), 0640) or die "Can't create file $mbxname: $!"; binmode($mp,':bytes') or die "Can't cancel :utf8 mode: $!"; } } else { # append to a UNIX-style mailbox # deliver only to non-executable regular files if ($errn == ENOENT) { # if two processes try creating the same new UNIX-style mailbox # file at the same time, one will tempfail at this point, with # its mail delivery to be retried later by MTA $mp = IO::File->new; # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] $mp->open($mbxname, untaint(O_CREAT|O_EXCL|O_WRONLY), 0640) or die "Can't create file $mbxname: $!"; } elsif ($errn==0 && !-f _) { die "Mailbox $mbxname is not a regular file, refuse to deliver"; } elsif (-x _ || -X _) { die "Mailbox file $mbxname is executable, refuse to deliver"; } else { $mp = IO::File->new; # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] $mp->open($mbxname, untaint(O_APPEND|O_WRONLY), 0640) or die "Can't append to $mbxname: $!"; } binmode($mp,':bytes') or die "Can't cancel :utf8 mode: $!"; flock($mp,LOCK_EX) or die "Can't lock mailbox file $mbxname: $!"; $mp->seek(0,2) or die "Can't position mailbox file to its tail: $!"; $pos = $mp->tell; # remember where we started } section_time('open-mbx'); if (defined($msg) && !$msg->isa('MIME::Entity')) { $msg->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; $smtp_response = $eval_stat =~ /^timed out\b/ ? "450 4.4.2" : "451 4.5.0"; $smtp_response .= " Local delivery(1) to $mbxname failed: $eval_stat"; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout }; last if defined $eval_stat; # exit block, not the loop my($failed) = 0; $eval_stat = undef; eval { # if things fail from here on, try to restore mailbox state if ($ux) { # a null return path may not appear in the 'From ' delimiter line my($snd) = $sender eq '' ? 'MAILER-DAEMON' # as in sendmail & Postfix : $msginfo->sender_smtp; $mp->printf("From %s %s\n", $snd, scalar(localtime($msginfo->rx_time)) ) # English date! or die "Can't write to $mbxname: $!"; } my($hdr_edits) = $msginfo->header_edits; if (!$hdr_edits) { $hdr_edits = Amavis::Out::EditHeader->new; $msginfo->header_edits($hdr_edits); } $hdr_edits->delete_header('Return-Path'); $hdr_edits->prepend_header('Delivered-To', join(', ',@recips)); $hdr_edits->prepend_header('Return-Path', $msginfo->sender_smtp); my($received_cnt) = $hdr_edits->write_header($msg,$mp,!$initial_submission); if ($received_cnt > 110) { # loop detection required by RFC 5321 (ex RFC 2821) section 6.3 # Do not modify the signal text, it gets matched elsewhere! die "Too many hops: $received_cnt 'Received:' header fields\n"; } if (!$ux) { # do it in blocks for speed if we can my($nbytes,$buff); while (($nbytes=$msg->read($buff,16384)) > 0) { $mp->print($buff) or die "Can't write to $mbxname: $!" } defined $nbytes or die "Error reading: $!"; } else { # for UNIX-style mailbox delivery: escape 'From ' # mail(1) and elm(1) recognize /^From / as a message delimiter # only after a blank line, which is correct. Other MUAs like mutt, # thunderbird, kmail and pine need all /^From / lines escaped. my($ln); my($blank_line) = 1; for ($! = 0; defined($ln=$msg->getline); $! = 0) { $mp->print('>') or die "Can't write to $mbxname: $!" if $ln=~/^From /; # escape all "From " lines # if $blank_line && $ln=~/^From /; # escape only after blank line $mp->print($ln) or die "Can't write to $mbxname: $!"; $blank_line = $ln eq "\n"; } defined $ln || $!==0 or die "Error reading: $!"; } # must append an empty line for a Unix mailbox format $mp->print("\n") or die "Can't write to $mbxname: $!" if $ux; 1; } or do { # trouble $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; if ($ux && defined($pos)) { $mp->flush or die "Can't flush file $mbxname: $!"; $can_truncate or do_log(-1, "Truncating a mailbox file will most likely fail"); # try to restore UNIX-style mailbox to previous size; # Produces a fatal error if truncate isn't implemented on the system $mp->truncate($pos) or die "Can't truncate file $mbxname: $!"; } $failed = 1; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout }; if ($ux) { $mp->flush or die "Can't flush mailbox file $mbxname: $!"; flock($mp,LOCK_UN) or die "Can't unlock mailbox $mbxname: $!"; } $mp->close or die "Error closing $mbxname: $!"; undef $mp; if (!$failed) { $smtp_response = "250 2.6.0 Ok, delivered to $mbxname"; snmp_count('OutMsgsDelivers'); my($size) = $msginfo->msg_size; snmp_count( ['OutMsgsSize'.$_, $size, 'C64'] ) for @snmp_vars; } elsif ($@ =~ /^timed out\b/) { $smtp_response = "450 4.4.2 Local delivery to $mbxname timed out"; snmp_count('OutMsgsAttemptFails'); } elsif ($@ =~ /too many hops\b/i) { $smtp_response = "554 5.4.6 Rejected delivery to mailbox $mbxname: $@"; snmp_count('OutMsgsRejects'); } else { $smtp_response = "451 4.5.0 Local delivery to mailbox $mbxname ". "failed: $@"; snmp_count('OutMsgsAttemptFails'); } } # end of block, 'last' within the block brings us here do_log(-1, "%s", $smtp_response) if $smtp_response !~ /^2/; $smtp_response .= ", id=" . $msginfo->log_id; for my $r (@recips_with_same_mbx) { $r->recip_smtp_response($smtp_response); $r->recip_done(2); $r->recip_mbxname($smtp_response =~ /^2/ ? $mbxname : undef); } } section_time('save-to-local-mailbox'); } 1; __DATA__ # package Amavis::OS_Fingerprint; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Util qw(ll do_log); } use Errno qw(EINTR EAGAIN); use Socket; use IO::Socket::UNIX; use IO::Socket::INET; use Time::HiRes (); sub new { my($class, $service_method,$timeout, $src_ip,$src_port, $dst_ip,$dst_port, $nonce) = @_; local($1,$2,$3); my($type,$service_host,$service_port,$service_path); if ($service_method =~ m{^p0f: (?: \[ ([^\]]*) \] | ([^:]*) ) : ([^:]*) }six) { $type = "p0f-analyzer"; ($service_host, $service_port) = ($1.$2, $3); } elsif ($service_method =~ m{^p0f: ( / [^ ]+ ) \z}six) { # looks like a unix socket $type = "p0f"; $service_path = $1; } else { die "Bad p0f method syntax: $service_method" } $dst_ip = '0.0.0.0' if !defined $dst_ip; # our MTA's IP address $dst_port = defined $dst_port ? 0+$dst_port : 0; # our MTA port, usually 25 $src_port = defined $src_port ? 0+$src_port : 0; # remote client's port no. do_log(4,"Fingerprint query: [%s]:%s %s (%s) %s", $src_ip,$src_port,$nonce,$type,$service_method); my($sock); my($query); my($query_sent) = 0; if ($type eq "p0f-analyzer") { # send a UDP query to p0f-analyzer $query = '['.$src_ip.']' . ($src_port==0 ? '' : ':'.$src_port); $sock = IO::Socket::INET->new(Type=>SOCK_DGRAM, Proto=>'udp'); $sock or die "Can't create INET socket: $!"; my($hisiaddr); $hisiaddr = inet_aton($service_host) or die "Fingerprint bad IP address: $service_host"; my($hispaddr) = scalar(sockaddr_in($service_port, $hisiaddr)); # bypass send method in IO::Socket to be able to retrieve # status/errno directly from 'send', not from 'getpeername': defined send($sock, "$query $nonce", 0, $hispaddr) or die "Fingerprint - send error: $!"; $query_sent = 1; } elsif ($type eq "p0f") { # contact p0f directly if ($src_ip !~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\z/s) { do_log(5,"Fingerprint - SRC addr not an IPv4: %s", $src_ip); } elsif ($dst_ip !~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\z/s) { do_log(5,"Fingerprint - DST addr not an IPv4: %s", $dst_ip); } else { # connect to a Unix socket where p0f is listening $sock = IO::Socket::UNIX->new(Type=>SOCK_STREAM, Peer=>$service_path); if (!$sock) { do_log(-1, "Can't connect to p0f socket %s: %s", $service_path,$!); } else { # send a query directly to a p0f service my($QUERY_MAGIC) = 0x0defaced; my($QTYPE_FINGERPRINT) = 1; my($src_n,$dst_n); $src_n = inet_aton($src_ip) or die "Fingerprint bad IP addr: $src_ip"; $dst_n = inet_aton($dst_ip) or die "Fingerprint bad IP addr: $dst_ip"; my($j)=0; $j = ($j*7 ^ ord($_)) & 0xffffffff for split(//,$nonce); $nonce = $j; # convert a string into a 32-bit integer $query = pack("LLLa4a4SS", $QUERY_MAGIC, $QTYPE_FINGERPRINT, $nonce, $src_n, $dst_n, $src_port, $dst_port); my($nwrite) = syswrite($sock,$query); if (defined $nwrite && $nwrite==length($query)) { $query_sent = 1 } else { do_log(-1, "Error writing to p0f %s: %s", $service_path,$!) } } } } return undef if !$query_sent; bless { sock=>$sock, wait_until=>(Time::HiRes::time + $timeout), query=>$query, nonce=>$nonce, type=>$type }, $class; } sub collect_response { my($self) = @_; my($timeout) = $self->{wait_until} - Time::HiRes::time; if ($timeout < 0) { $timeout = 0 }; my($type) = $self->{type}; my($sock) = $self->{sock}; my($resp,$nfound,$inbuf); my($rin,$rout); $rin = ''; vec($rin,fileno($sock),1) = 1; while ($nfound=select($rout=$rin, undef,undef,$timeout)) { my($rv) = $type eq "p0f-analyzer" ? $sock->recv($inbuf,1024,0) : $sock->sysread($inbuf,1024); if (!defined $rv) { if ($!==EAGAIN || $!==EINTR) { Time::HiRes::sleep(0.1); # slow down, just in case } else { die "Fingerprint - error reading from socket: $!"; } } elsif ($type eq "p0f" && $rv < 1) { # sysread returns 0 at eof last; } elsif ($type eq "p0f-analyzer") { local($1,$2,$3); if ($inbuf =~ /^([^ ]*) ([^ ]*) (.*)\015\012\z/) { my($r_query,$r_nonce,$r_resp) = ($1,$2,$3); if ($r_query eq $self->{query} && $r_nonce eq $self->{nonce}) { $resp = $r_resp }; } do_log(4,"Fingerprint collect: max_wait=%.3f, %.35s... => %s", $timeout,$inbuf,$resp); $timeout = 0; } elsif ($type eq "p0f") { # # default struct alignments # my($magic, $id, $r_status, $genre, $detail, $dist, $link, $tos, # $fw, $nat, $real, $dmy1, $masq_score, $masq_flags, $dmy2, $uptime) = # unpack ("L L C Z20 Z40 c Z30 Z30 C C C C s S S l", $inbuf); # properly packed struct my($magic, $id, $r_status, $genre, $detail, $dist, $link, $tos, $fw, $nat, $real, $masq_score, $masq_flags, $uptime) = unpack ("L L C Z20 Z40 c Z30 Z30 C C C s S l", $inbuf); my($QUERY_MAGIC) = 0x0defaced; $magic == $QUERY_MAGIC or die "Bad response magic"; if ($r_status == 1) { do_log(-1, "Fingerprint - malformed query"); } elsif ($r_status == 0 && $id != $self->{nonce}) { do_log(-1, "Fingerprint - nonce mismatch: %s", $id); } elsif ($r_status == 2) { do_log(1, "Fingerprint - no matching entry in the p0f cache"); } elsif ($r_status == 0) { $resp = sprintf("%s%s%s%s%s%s, (%s%s)", ($genre eq '' ? 'UNKNOWN' : $genre), ($detail eq '' ? '' : " $detail"), (!$fw ? '' : " (firewall!)"), (!$nat ? '' : $nat==1 ? " (NAT!)" : " (NAT$nat!)"), ($tos eq '' ? '' : " [tos $tos]"), $uptime == -1 ? '' : " (up: $uptime hrs)", ($dist == -1 ? '' : "distance $dist, "), ($link eq '' ? '' : "link: $link")); } else { do_log(-1, "Fingerprint - invalid reply type: %s", $r_status); } do_log(4,"Fingerprint collect: max_wait=%.3f => %s", $timeout,$resp); $timeout = 0; } } defined $nfound or die "Fingerprint - select on socket failed: $!"; if ($type eq "p0f") { $sock->close or die "Error closing socket: $!" } $resp; } 1; __DATA__ #^L package Amavis::Out::SQL::Connection; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform c cr ca); import Amavis::Util qw(ll do_log); import Amavis::Timing qw(section_time); } use DBI qw(:sql_types); # one object per connection (normally exactly one) to a database server; # connection need not exist at all times, stores info on how to connect; # when connected it holds a database handle # sub new { my($class, @dsns) = @_; # a list of DSNs to try connecting to sequentially bless { dbh=>undef, sth=>undef, incarnation=>1, in_transaction=>0, dsn_list=>\@dsns, dsn_current=>undef }, $class; } sub dsn_current { # get/set information on currently connected data set name my $self = shift; !@_ ? $self->{dsn_current} : ($self->{dsn_current}=shift); } sub dbh { # get/set database handle my $self = shift; !@_ ? $self->{dbh} : ($self->{dbh}=shift); } sub sth { # get/set statement handle my $self = shift; my $clause = shift; !@_ ? $self->{sth}{$clause} : ($self->{sth}{$clause}=shift); } sub dbh_inactive { # get/set dbh "InactiveDestroy" attribute my $self = shift; my $dbh = $self->dbh; if (!$dbh) { undef } else { !@_ ? $dbh->{'InactiveDestroy'} : ($dbh->{'InactiveDestroy'}=shift) } } sub DESTROY { my $self = shift; local($@,$!,$_); eval { do_log(5,"Amavis::Out::SQL::Connection DESTROY called") }; eval { $self->disconnect_from_sql }; } # returns current connection version; works like cache versioning/invalidation: # SQL statement handles need to be rebuilt and caches cleared when SQL # connection is re-established and a new database handle provided # sub incarnation { my $self = shift; $self->{incarnation} } sub in_transaction { my $self = shift; !@_ ? $self->{in_transaction} : ($self->{in_transaction}=shift) } # returns DBD driver name such as 'Pg', 'mysql'; or undef if unknown # sub driver_name { my $self = shift; my $dbh = $self->dbh; $dbh or die "sql driver_name: dbh not available"; !$dbh->{Driver} ? undef : $dbh->{Driver}->{Name}; } # DBI method wrappers: # sub begin_work { my $self = shift; do_log(5,"sql begin transaction"); # DBD::mysql man page: if you detect an error while changing # the AutoCommit mode, you should no longer use the database handle. # In other words, you should disconnect and reconnect again $self->dbh or $self->connect_to_sql; my($stat); my($eval_stat); eval { $stat = $self->dbh->begin_work(@_); 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; }; if (defined $eval_stat || !$stat) { do_log(-1,"sql begin transaction failed, ". "probably disconnected by server, reconnecting (%s)", $eval_stat); $self->disconnect_from_sql; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout $self->connect_to_sql; $self->dbh->begin_work(@_); } $self->in_transaction(1); }; sub begin_work_nontransaction { my $self = shift; do_log(5,"sql begin, nontransaction"); $self->dbh or $self->connect_to_sql; }; sub commit { my $self = shift; do_log(5,"sql commit"); $self->in_transaction(0); my($dbh) = $self->dbh; $dbh or die "commit: dbh not available"; $dbh->commit(@_); my($rv_err,$rv_str) = ($dbh->err, $dbh->errstr); do_log(2,"sql commit status: err=%s, errstr=%s", $rv_err,$rv_str) if defined $rv_err; ($rv_err,$rv_str); # potentially useful to see non-fatal errors }; sub rollback { my $self = shift; do_log(5,"sql rollback"); $self->in_transaction(0); $self->dbh or die "rollback: dbh not available"; eval { $self->dbh->rollback(@_); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"sql rollback error, reconnecting (%s)", $eval_stat); $self->disconnect_from_sql; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout $self->connect_to_sql; # $self->dbh->rollback(@_); # too late now, hopefully implied in disconnect }; }; sub last_insert_id { # no longer used my $self = shift; $self->dbh or die "last_insert_id: dbh not available"; $self->dbh->last_insert_id(@_); }; sub fetchrow_arrayref { my($self,$clause,@args) = @_; $self->dbh or die "fetchrow_arrayref: dbh not available"; my($sth) = $self->sth($clause); $sth or die "fetchrow_arrayref: statement handle not available"; $sth->fetchrow_arrayref(@args); }; sub finish { my($self,$clause,@args) = @_; $self->dbh or die "finish: dbh not available"; my($sth) = $self->sth($clause); $sth or die "finish: statement handle not available"; $sth->finish(@args); }; sub execute { my($self,$clause,@args) = @_; $self->dbh or die "sql execute: dbh not available"; my($sth) = $self->sth($clause); # fetch cached st. handle or prepare new if ($sth) { do_log(5,"sql: executing clause: %s", $clause); } else { do_log(4,"sql: preparing and executing: %s", $clause); $sth = $self->dbh->prepare($clause); $self->sth($clause,$sth); $sth or die "sql: prepare failed: ".$DBI::errstr; } my($rv_err,$rv_str); eval { for my $j (0..$#args) { # arg can be a scalar or [val,type] or [val,\%attr] $sth->bind_param($j+1, !ref($args[$j]) ? $args[$j] : @{$args[$j]}); } $sth->execute; $rv_err = $sth->err; $rv_str = $sth->errstr; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; # man DBI: ->err code is typically an integer but you should not assume so # $DBI::errstr is normally already contained in $eval_stat my($sqlerr) = $sth ? $sth->err : $DBI::err; my($sqlstate) = $sth ? $sth->state : $DBI::state; my($msg) = sprintf("err=%s, %s, %s", $sqlerr, $sqlstate, $eval_stat); if (!$sth) { die "sql execute (no handle): ".$msg; } elsif (! ($sqlerr eq '2006' || $sqlerr eq '2013' || # MySQL ($sqlerr == -1 && $sqlstate eq 'S1000') || # PostgreSQL 7 ($sqlerr == 7 && $sqlstate =~ /^(S8|08|57)...\z/i) )) { #PgSQL # libpq-fe.h: ExecStatusType PGRES_FATAL_ERROR=7 eval { $self->disconnect_from_sql }; # better safe than sorry die "sql exec: $msg\n"; } else { # Server has gone away; Lost connection to... # MySQL: 2006, 2013; PostgreSQL: 7 if ($self->in_transaction) { eval { $self->disconnect_from_sql }; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout die "sql execute failed within transaction, $msg"; } else { # try one more time do_log(0,"NOTICE: reconnecting in response to: %s", $msg); eval { $self->disconnect_from_sql }; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout $self->connect_to_sql; $self->dbh or die "sql execute: reconnect failed"; do_log(4,"sql: preparing and executing (again): %s", $clause); $sth = $self->dbh->prepare($clause); $self->sth($clause,$sth); $sth or die "sql: prepare (reconnected) failed: ".$DBI::errstr; $rv_err = $rv_str = undef; eval { for my $j (0..$#args) { # a scalar or [val,type] or [val,\%attr] $sth->bind_param($j+1, !ref($args[$j]) ? $args[$j] : @{$args[$j]}); } $sth->execute; $rv_err = $sth->err; $rv_str = $sth->errstr; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; $msg = sprintf("err=%s, %s, %s", $DBI::err,$DBI::state,$eval_stat); $self->disconnect_from_sql; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout die "sql execute failed again, $msg"; }; } } }; # $rv_err: undef indicates success, "" indicates an 'information', # "0" indicates a 'warning', true indicates an error do_log(2,"sql execute status: err=%s, errstr=%s", $rv_err,$rv_str) if defined $rv_err; ($rv_err,$rv_str); # potentially useful to see non-fatal errors } # Connect to a database. Take a list of database connection # parameters and try each until one succeeds. # -- based on code from Ben Ransford 2002-09-22 # sub connect_to_sql { my $self = shift; # a list of DSNs to try connecting to sequentially my($dbh); my(@dsns) = @{$self->{dsn_list}}; do_log(3,"Connecting to SQL database server"); for my $tmpdsn (@dsns) { my($dsn, $username, $password) = @$tmpdsn; do_log(4,"connect_to_sql: trying '%s'", $dsn); $dbh = DBI->connect($dsn, $username, $password, {PrintError => 0, RaiseError => 0, Taint => 1, AutoCommit => 1} ); if ($dbh) { $self->dsn_current($dsn); do_log(3,"connect_to_sql: '%s' succeeded", $dsn); last; } do_log(-1,"connect_to_sql: unable to connect to DSN '%s': %s", $dsn, $DBI::errstr); } $self->dbh($dbh); delete($self->{sth}); $self->in_transaction(0); $self->{incarnation}++; $dbh or die "connect_to_sql: unable to connect to any dataset"; $dbh->{'RaiseError'} = 1; # $dbh->{mysql_auto_reconnect} = 1; # questionable benefit # $dbh->func(30000,'busy_timeout'); # milliseconds (SQLite) eval { $dbh->do("SET NAMES 'utf8'"); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(2,"connect_to_sql: SET NAMES 'utf8' failed: %s", $eval_stat); }; section_time('sql-connect'); $self; } sub disconnect_from_sql($) { my $self = shift; $self->in_transaction(0); if ($self->dbh) { do_log(4,"disconnecting from SQL"); $self->dbh->disconnect; $self->dbh(undef); } delete($self->{sth}); $self->dsn_current(undef); } 1; __DATA__ #^L package Amavis::Out::SQL::Log; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::rfc2821_2822_Tools; import Amavis::Util qw(ll do_log min max minmax snmp_count add_entropy untaint safe_decode safe_encode sanitize_str orcpt_decode ccat_split ccat_maj); import Amavis::Lookup qw(lookup lookup2); import Amavis::Out::SQL::Connection (); } use Encode; # Perl 5.8 UTF-8 support use DBI qw(:sql_types); sub new { my($class,$conn_h) = @_; bless { conn_h=>$conn_h, incarnation=>0 }, $class; } sub DESTROY { my $self = shift; local($@,$!,$_); eval { do_log(5,"Amavis::Out::SQL::Log DESTROY called") }; } # find an existing e-mail address record or insert one, returning its id; # may return undef if 'sel_adr' or 'ins_adr' SQL clauses are not defined; # sub find_or_save_addr { my($self,$addr,$partition_tag,$keep_localpart_case) = @_; my($id); my($existed) = 0; my($localpart,$domain); my($naddr) = untaint($addr); if ($naddr ne '') { # normalize address (lowercase, 7-bit, max 255 ch...) ($localpart,$domain) = split_address($naddr); $domain =~ s/[^\040-\176]/?/gs; $domain = lc $domain; if (!$keep_localpart_case && !c('localpart_is_case_sensitive')) { $localpart = lc($localpart); } local($1); $domain = $1 if $domain=~/^\@?(.*?)\.*\z/s; # chop leading @ and tr. dots $naddr = $localpart.'@'.$domain; $naddr = substr($naddr,0,255) if length($naddr) > 255; # avoid UTF-8 SQL trouble, legitimate RFC 5321 addresses only need 7 bits $naddr =~ s/[^\040-\176]/?/g if !$sql_allow_8bit_address; # SQL character strings disallow zero octets, and also disallow any other # octet values and sequences of octet values that are invalid according to # the database's selected character set encoding } my($conn_h) = $self->{conn_h}; my($sql_cl_r) = cr('sql_clause'); my($sel_adr) = $sql_cl_r->{'sel_adr'}; my($ins_adr) = $sql_cl_r->{'ins_adr'}; if (!defined($sel_adr) || $sel_adr eq '') { # no way to query a database, behave as if no record was found do_log(5,"find_or_save_addr: sel_adr query disabled, %s", $naddr); } else { $conn_h->begin_work_nontransaction; #(re)connect if necessary, autocommit my($datatype) = SQL_VARCHAR; if ($sql_allow_8bit_address) { my($driver) = $conn_h->driver_name; # only available when connected $datatype = $driver eq 'Pg' ? { pg_type => DBD::Pg::PG_BYTEA() } : SQL_VARBINARY; } $conn_h->execute($sel_adr, $partition_tag, [$naddr,$datatype]); my($a_ref,$a2_ref); if (defined($a_ref=$conn_h->fetchrow_arrayref($sel_adr))) { # exists? $id = $a_ref->[0]; $conn_h->finish($sel_adr); $existed = 1; } elsif (!defined($ins_adr) || $ins_adr eq '') { # record does not exist, insertion is not allowed do_log(5,"find_or_save_addr: ins_adr insertion disabled, %s", $naddr); } else { # does not exist, attempt to insert a new e-mail address record my($invdomain); # domain with reversed fields, chopped to 255 characters $invdomain = join('.', reverse split(/\./,$domain,-1)); $invdomain = substr($invdomain,0,255) if length($invdomain) > 255; $conn_h->begin_work_nontransaction; # (re)connect if not connected my($eval_stat); eval { $conn_h->execute($ins_adr, $partition_tag, [$naddr,$datatype], $invdomain); 1 } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; # INSERT may have failed because of race condition with other processes; # try the SELECT again, it will most likely succeed this time; # SELECT after INSERT also avoids the need for a working last_insert_id() $conn_h->begin_work_nontransaction; # (re)connect if not connected # try select again, regardless of the success of INSERT $conn_h->execute($sel_adr, $partition_tag, [$naddr,$datatype]); if ( defined($a2_ref=$conn_h->fetchrow_arrayref($sel_adr)) ) { $id = $a2_ref->[0]; $conn_h->finish($sel_adr); add_entropy($id); if (!defined($eval_stat)) { # status of the INSERT do_log(5,"find_or_save_addr: record inserted, id=%s, %s", $id,$naddr); } else { $existed = 1; chomp $eval_stat; do_log(5,"find_or_save_addr: found on a second attempt, ". "id=%s, %s, (first attempt: %s)", $id,$naddr,$eval_stat); die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout } } else { # still does not exist $id = $existed = undef; if (defined $eval_stat) { # status of the INSERT chomp $eval_stat; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout }; die "find_or_save_addr: failed to insert addr $naddr: $eval_stat"; } } } ($id, $existed); } # find a penpals record which proves that a local user sid really sent a # mail to a recipient rid some time ago. Returns an interval time in seconds # since the last such mail was sent by our local user to a specified recipient # (or undef if information is not available). If @$message_id_list is a # nonempty list of Message-IDs as found in References header field, the query # also provides previous outgoing messages with a matching Message-ID but # possibly to recipients different from what the mail was originally sent to. # sub penpals_find { my($self, $sid,$rid,$message_id_list, $now) = @_; my($a_ref,$found,$age,$send_time,$ref_mail_id,$ref_subj,$ref_mid,$ref_rid); my($conn_h) = $self->{conn_h}; my($sql_cl_r) = cr('sql_clause'); my($sel_penpals) = $sql_cl_r->{'sel_penpals'}; my($sel_penpals_msgid) = $sql_cl_r->{'sel_penpals_msgid'}; if (defined($sel_penpals_msgid) && @$message_id_list && defined($sid)) { # list of refs to Message-ID is nonempty, try reference or recipient match my($n) = scalar(@$message_id_list); # number of keys my(@args) = ($sid,$rid); my(@pos_args); local($1); my($sel_taint) = substr($sel_penpals_msgid,0,0); # taintedness $sel_penpals_msgid =~ s{ ( %m | \? ) } # substitute %m for keys and ? for next arg { push(@pos_args, $1 eq '%m' ? (map { my($s)=$_; $s=~s/[^\040-\176]/?/gs; $s } @$message_id_list) : shift @args), $1 eq '%m' ? join(',', ('?') x $n) : '?' }gxe; # keep original clause taintedness $sel_penpals_msgid = untaint($sel_penpals_msgid) . $sel_taint; $_ = untaint($_) for @pos_args; # untaint arguments do_log(4, "penpals: query args: %s", join(', ',@pos_args)); do_log(4, "penpals: %s", $sel_penpals_msgid); $conn_h->begin_work_nontransaction; # (re)connect if not connected $conn_h->execute($sel_penpals_msgid,@pos_args); snmp_count('PenPalsAttempts'); snmp_count('PenPalsAttemptsMid'); if (!defined($a_ref=$conn_h->fetchrow_arrayref($sel_penpals_msgid))) { snmp_count('PenPalsMisses'); } else { ($send_time, $ref_mail_id, $ref_subj, $ref_mid, $ref_rid) = @$a_ref; $found = 1; $conn_h->finish($sel_penpals_msgid); my($rid_match) = defined $ref_rid && defined $rid && $rid eq $ref_rid; my($mid_match) = grep($ref_mid eq $_, @$message_id_list); my($t) = $mid_match && $rid_match ? 'MidRid' : # $mid_match && !defined($rid) ? 'MidNullRPath' : $mid_match ? 'Mid' : $rid_match ? 'Rid' : 'none'; snmp_count('PenPalsHits'.$t); snmp_count('PenPalsHits'); ll(4) && do_log(4, "penpals: MATCH ON %s: %s", $t, join(", ",@$a_ref)); } } if (!$found && defined($sel_penpals) && defined($rid) && defined($sid)) { # list of Message-ID references not given, try matching on recipient only $conn_h->begin_work_nontransaction; # (re)connect if not connected $conn_h->execute($sel_penpals,untaint($sid),untaint($rid)); snmp_count('PenPalsAttempts'); snmp_count('PenPalsAttemptsRid'); if (!defined($a_ref=$conn_h->fetchrow_arrayref($sel_penpals))) { # exists? snmp_count('PenPalsMisses'); } else { ($send_time, $ref_mail_id, $ref_subj) = @$a_ref; $found = 1; $conn_h->finish($sel_penpals); snmp_count('PenPalsHitsRid'); snmp_count('PenPalsHits'); ll(4) && do_log(4, "penpals: MATCH ON RID(%s): %s", $rid, join(", ",@$a_ref)); } } if (!$found) { ll(4) && do_log(4, "penpals: (%s,%s) not found%s", $sid,$rid, !@$message_id_list ? '' : ' refs: '.join(", ",@$message_id_list)); } else { $age = max(0, $now - $send_time); do_log(3, "penpals: (%s,%s) %s age %.3f days", $sid,$rid, $ref_mail_id, $age/(24*60*60)); } ($age, $ref_mail_id, $ref_subj); } sub save_info_preliminary { my($self, $msginfo) = @_; my($mail_id) = $msginfo->mail_id; defined $mail_id or die "save_info_preliminary: mail_id still undefined"; my($partition_tag) = $msginfo->partition_tag; my($sid,$existed,$sender_smtp); local($1); $sender_smtp = $msginfo->sender_smtp; $sender_smtp =~ s/^<(.*)>\z/$1/s; # find an existing e-mail address record for sender, or insert a new one ($sid,$existed) = $self->find_or_save_addr($sender_smtp,$partition_tag); if (defined $sid) { $msginfo->sender_maddr_id($sid); # there is perhaps 30-50% chance the sender address is already in the db snmp_count('SqlAddrSenderAttempts'); snmp_count($existed ? 'SqlAddrSenderHits' : 'SqlAddrSenderMisses'); do_log(4,"save_info_preliminary %s, sender id: %s, %s, %s", $mail_id, $sid, $sender_smtp, $existed ? 'exists' : 'new' ); } # find existing address records for recipients, or insert them for my $r (@{$msginfo->per_recip_data}) { my $addr_smtp = $r->recip_addr_smtp; if (defined $addr_smtp) { $addr_smtp =~ s/^<(.*)>\z/$1/s; $addr_smtp =~ s/(\@[^@]+)\z/lc $1/se; # lowercase just a domain part } my $orig_addr = $r->dsn_orcpt; # RCPT command ORCPT option, RFC 3461 if (defined $orig_addr) { $orig_addr = orcpt_decode($orig_addr); $orig_addr =~ s/(\@[^@]+)\z/lc $1/se; # lowercase just a domain part } else { $orig_addr = $addr_smtp; } my($rid, $o_rid, $existed); if ($addr_smtp ne '') { ($rid,$existed) = $self->find_or_save_addr($addr_smtp,$partition_tag); # there is perhaps 90-100% chance the recipient addr is already in the db if (defined $rid) { $r->recip_maddr_id($rid); snmp_count('SqlAddrRecipAttempts'); snmp_count($existed ? 'SqlAddrRecipHits' : 'SqlAddrRecipMisses'); do_log(4,"save_info_preliminary %s, recip id: %s, %s%s, %s", $mail_id, $rid, $addr_smtp, $orig_addr eq $addr_smtp ? '' : " (ORCPT $orig_addr)", $existed ? 'exists' : 'new'); } } ## currently disabled, probably not worth saving into SQL, rarely useful # if ($orig_addr ne '' && lc($orig_addr) ne lc($addr_smtp)) { # # don't bother saving as a separate record for just a case change # ($o_rid,$existed) = $self->find_or_save_addr($orig_addr,$partition_tag,1); # if (defined $o_rid) { # $r->recip_maddr_id_orig($o_rid); # snmp_count('SqlAddrRecipAttempts'); # snmp_count($existed ? 'SqlAddrRecipHits' : 'SqlAddrRecipMisses'); # do_log(4,"save_info_preliminary %s, o_recip id: %s, %s, %s", # $mail_id, $o_rid, $orig_addr, $existed ? 'exists' : 'new'); # } # } } my($conn_h) = $self->{conn_h}; my($sql_cl_r) = cr('sql_clause'); my($ins_msg) = $sql_cl_r->{'ins_msg'}; if (!defined($ins_msg) || $ins_msg eq '') { do_log(4,"save_info_preliminary: ins_msg undef, not saving"); } elsif (!defined($sid)) { do_log(4,"save_info_preliminary: sid undef, not saving"); } else { $conn_h->begin_work; # SQL transaction starts eval { # MySQL does not like a standard iso8601 delimiter 'T' or a timezone # when data type of msgs.time_iso is TIMESTAMP (instead of a string) my($time_iso) = $timestamp_fmt_mysql && $conn_h->driver_name eq 'mysql' ? iso8601_utc_timestamp($msginfo->rx_time,1,'') : iso8601_utc_timestamp($msginfo->rx_time); # insert a placeholder msgs record with sender information $conn_h->execute($ins_msg, $partition_tag, $msginfo->mail_id, $msginfo->secret_id, $msginfo->log_id, int($msginfo->rx_time), $time_iso, untaint($sid), c('policy_bank_path'), untaint($msginfo->client_addr), 0+untaint($msginfo->msg_size), untaint(substr(c('myhostname'),0,255))); $conn_h->commit; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; if ($conn_h->in_transaction) { eval { $conn_h->rollback; do_log(1,"save_info_preliminary: rollback done"); 1; } or do { $@ = "errno=$!" if $@ eq ''; chomp $@; do_log(1,"save_info_preliminary: rollback %s", $@); die $@ if $@ =~ /^timed out\b/; # resignal timeout }; } do_log(-1, "WARN save_info_preliminary: %s", $eval_stat); die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout return 0; }; } 1; } sub save_info_final { my($self, $msginfo,$dsn_sent) = @_; my($mail_id) = $msginfo->mail_id; defined $mail_id or die "save_info_final: mail_id still undefined"; my($sid) = $msginfo->sender_maddr_id; my($conn_h) = $self->{conn_h}; my($sql_cl_r) = cr('sql_clause'); my($ins_msg) = $sql_cl_r->{'ins_msg'}; my($upd_msg) = $sql_cl_r->{'upd_msg'}; my($ins_rcp) = $sql_cl_r->{'ins_rcp'}; if ($ins_msg eq '' || $upd_msg eq '' || $ins_rcp eq '') { # updates disabled } elsif (!defined($sid)) { # sender not in table maddr, msgs record was not inserted by preliminary } else { $conn_h->begin_work; # SQL transaction starts eval { my(%content_short_name) = ( # as written to a SQL record CC_VIRUS,'V', CC_BANNED,'B', CC_UNCHECKED,'U', CC_SPAM,'S', CC_SPAMMY,'Y', CC_BADH.",2",'M', CC_BADH,'H', CC_OVERSIZED,'O', CC_MTA,'T', CC_CLEAN,'C', CC_CATCHALL,'?'); my($min_spam_level, $max_spam_level) = minmax(map($_->spam_level, @{$msginfo->per_recip_data})); # insert per-recipient records into table msgrcpt my($r_seq_num) = 0; # can serve as a component of a primary key for my $r (@{$msginfo->per_recip_data}) { $r_seq_num++; my($rid) = $r->recip_maddr_id; next if !defined $rid; # e.g. always_bcc, or table 'maddr' is disabled my($o_rid) = $r->recip_maddr_id_orig; # may be undef my($spam_level) = $r->spam_level; my($dest,$resp) = ($r->recip_destiny, $r->recip_smtp_response); my($d) = $resp=~/^4/ ? 'TEMPFAIL' : ($dest==D_BOUNCE && $resp=~/^5/) ? 'BOUNCE' : ($dest!=D_BOUNCE && $resp=~/^5/) ? 'REJECT' : ($dest==D_PASS && ($resp=~/^2/ || !$r->recip_done)) ? 'PASS' : ($dest==D_DISCARD) ? 'DISCARD' : '?'; my($r_content_type) = $r->setting_by_contents_category(\%content_short_name); for ($r_content_type) { $_ = ' ' if !defined $_ || /^ *\z/ } $resp = substr($resp,0,255) if length($resp) > 255; $resp =~ s/[^\040-\176]/?/gs; # just in case, only need 7 bit printbl # avoid op '?:' on tainted operand in args list, see PR [perl #81028] my($recip_local_yn) = $r->recip_is_local ? 'Y' : 'N'; my($blacklisted_yn) = $r->recip_blacklisted_sender ? 'Y' : 'N'; my($whitelisted_yn) = $r->recip_whitelisted_sender ? 'Y' : 'N'; $conn_h->execute($ins_rcp, $msginfo->partition_tag, $mail_id, $sql_schema_version < 2.007000 ? untaint($rid) : ($r_seq_num, untaint($rid), $recip_local_yn, $r_content_type), # untaint(defined $o_rid ? $o_rid : $rid), substr($d,0,1), ' ', $blacklisted_yn, $whitelisted_yn, 0+untaint($spam_level), untaint($resp) ); # int($msginfo->rx_time), # untaint($r->user_policy_id), } my($q_to) = $msginfo->quarantined_to; # ref to a list of quar. locations if (!defined($q_to) || !@$q_to) { $q_to = undef } else { $q_to = $q_to->[0]; # keep only the first quarantine location $q_to =~ s{^\Q$QUARANTINEDIR\E/}{}; # strip directory name } my($m_id) = $msginfo->get_header_field_body('message-id'); $m_id = join(' ',parse_message_id($m_id)) if $m_id ne ''; # strip CFWS my($subj) = $msginfo->get_header_field_body('subject'); my($from) = $msginfo->get_header_field_body('from'); # raw full field my($rfc2822_from) = $msginfo->rfc2822_from; # undef, scalar or listref my($rfc2822_sender) = $msginfo->rfc2822_sender; # undef or scalar $rfc2822_from = join(', ',@$rfc2822_from) if ref $rfc2822_from; my($os_fp) = $msginfo->client_os_fingerprint; $_ = !defined($_) ? '' :untaint($_) for ($subj,$from,$m_id,$q_to,$os_fp); for ($subj,$from) { # character set decoding, sanitation chomp; s/\n(?=[ \t])//gs; s/^[ \t]+//s; s/[ \t]+\z//s; # unfold, trim eval { # convert to UTF-8 octets, truncate to 255 bytes local($1); my($chars) = safe_decode('MIME-Header',$_); # logical characters my($octets) = safe_encode('UTF-8',$chars); # bytes, UTF-8 encoded if (length($octets) > 255 && $octets =~ /^ (.{0,255}) (?= [\x00-\x7F\xC0-\xFF] | \z )/xs) { $octets = $1; # cleanly chop a UTF-8 byte sequence, RFC 3629 $chars = safe_decode('UTF-8',$octets); # convert back to chars } # man DBI: Drivers should accept [unicode and non-unicode] strings # and, if required, convert them to the character set of the # database being used. Similarly, when fetching from the database # character data that isn't iso-8859-1 the driver should convert # it into UTF-8. # $_ = $chars; 1; # pass logical characters to SQL $_ = $octets; 1; # pass bytes to SQL, works better } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(1,"save_info_final INFO: header field ". "not decodable, keeping raw bytes: %s", $eval_stat); $_ = substr($_,0,255) if length($_) > 255; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout }; } for ($m_id,$q_to,$os_fp) { # truncate to 255 ch, ensure 7-bit characters $_ = substr($_,0,255) if length($_) > 255; s/[^\040-\176]/?/gs; # only use 7 bit printable, compatible with UTF-8 } my($content_type) = $msginfo->setting_by_contents_category(\%content_short_name); my($checks_performed) = $msginfo->checks_performed; $checks_performed = !ref $checks_performed ? '' : join('', grep($checks_performed->{$_}, qw(V S H B F P D))); my($quar_type) = $msginfo->quar_type; for ($quar_type,$content_type) { $_ = ' ' if !defined $_ || /^ *\z/ } $min_spam_level = 0 if !defined $min_spam_level; $max_spam_level = 0 if !defined $max_spam_level; my($orig) = $msginfo->originating ? 'Y' : 'N'; ll(4) && do_log(4,"save_info_final %s, orig=%s, chks=%s, cont.ty=%s, ". "q.type=%s, q.to=%s, dsn=%s, score=%s, ". "Message-ID: %s, From: '%s', Subject: '%s'", $mail_id, $orig, $checks_performed, $content_type, $quar_type, $q_to, $dsn_sent, $min_spam_level, $m_id, sanitize_str($from), sanitize_str($subj)); # update message record with additional information $conn_h->execute($upd_msg, $content_type, $quar_type, $q_to, $dsn_sent, 0+untaint($min_spam_level), $m_id, $from, $subj, untaint($msginfo->client_addr), # we may have a better info now $sql_schema_version < 2.007000 ? () : $orig, $msginfo->partition_tag, $mail_id); # $os_fp, $rfc2822_sender, $rfc2822_from, $checks_performed, ... # SQL_CHAR, SQL_VARCHAR, SQL_VARBINARY, SQL_BLOB, SQL_INTEGER, SQL_FLOAT, # SQL_TIMESTAMP, SQL_TYPE_TIMESTAMP_WITH_TIMEZONE, ... $conn_h->commit; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; if ($conn_h->in_transaction) { eval { $conn_h->rollback; do_log(1,"save_info_final: rollback done"); 1; } or do { $@ = "errno=$!" if $@ eq ''; chomp $@; do_log(1,"save_info_final: rollback %s", $@); die $@ if $@ =~ /^timed out\b/; # resignal timeout }; } do_log(-1, "WARN save_info_final: %s", $eval_stat); die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout return 0; } } 1; } 1; __DATA__ # package Amavis::IO::SQL; # an IO wrapper around SQL for inserting/retrieving mail text # to/from a database use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Util qw(ll do_log untaint min max minmax); } use Errno qw(ENOENT EACCES EIO); use DBI qw(:sql_types); # use DBD::Pg; sub new { my($class) = shift; my($self) = bless {}, $class; if (@_) { $self->open(@_) or return undef } $self; } sub open { my $self = shift; eval { $self->close } if exists $self->{conn_h}; @$self{qw(conn_h clause dbkey mode partition_tag maxbuf rx_time)} = @_; my($conn_h) = $self->{conn_h}; $self->{buf} = ''; $self->{chunk_ind} = $self->{pos} = $self->{bufpos} = $self->{eof} = 0; my($driver); my($eval_stat); eval { $driver = $conn_h->driver_name; 1 } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat }; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout if ($self->{mode} eq 'w') { # open for write access ll(4) && do_log(4,"Amavis::IO::SQL::open %s drv=%s (%s); key=%s, p_tag=%s", $self->{mode}, $driver, $self->{clause}, $self->{dbkey}, $self->{partition_tag}); } else { # open for read access $eval_stat = undef; eval { $conn_h->execute($self->{clause}, $self->{partition_tag},$self->{dbkey}); 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat }; my($ll) = $eval_stat ne '' ? -1 : 4; do_log($ll,"Amavis::IO::SQL::open %s drv=%s (%s); key=%s, p_tag=%s, s: %s", $self->{mode}, $driver, $self->{clause}, $self->{dbkey}, $self->{partition_tag}, $eval_stat) if ll($ll); if ($eval_stat ne '') { if ($eval_stat =~ /^timed out\b/) { die $eval_stat } # resignal timeout else { die "Amavis::IO::SQL::open $driver SELECT error: $eval_stat" } $! = EIO; return undef; # not reached } $eval_stat = undef; eval { # fetch the first chunk; if missing treat it as a file-not-found my($a_ref) = $conn_h->fetchrow_arrayref($self->{clause}); if (!defined($a_ref)) { $self->{eof} = 1 } else { $self->{buf} = $a_ref->[0]; $self->{chunk_ind}++ } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; if ($eval_stat =~ /^timed out\b/) { die $eval_stat } # resignal timeout else { die "Amavis::IO::SQL::open $driver read error: $eval_stat" } $! = EIO; return undef; # not reached }; if ($self->{eof}) { # no records, make it look like a missing file do_log(0,"Amavis::IO::SQL::open key=%s, p_tag=%s: no such record", $self->{dbkey}, $self->{partition_tag}); $! = ENOENT; # No such file or directory return undef; } } $self; } sub DESTROY { my $self = shift; local($@,$!,$_); my($myactualpid) = $$; if (ref $self && $self->{conn_h}) { eval { $self->close or die "Error closing: $!"; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; warn "[$myactualpid] Amavis::IO::SQL::close error: $eval_stat"; }; delete $self->{conn_h}; } } sub close { my $self = shift; my($eval_stat); eval { if ($self->{mode} eq 'w') { $self->flush or die "Can't flush: $!"; } elsif ($self->{conn_h} && $self->{clause} && !$self->{eof}) { # reading, closing before eof was reached $self->{conn_h}->finish($self->{clause}) or die "Can't finish: $!"; }; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; }; delete @$self{ qw(conn_h clause dbkey mode maxbuf rx_time buf chunk_ind pos bufpos eof) }; if (defined $eval_stat) { chomp $eval_stat; if ($eval_stat =~ /^timed out\b/) { die $eval_stat } # resignal timeout else { die "Error closing, $eval_stat" } $! = EIO; return undef; # not reached } 1; } sub seek { my($self,$pos,$whence) = @_; $whence == 0 or die "Only absolute seek is supported on sql i/o"; $pos >= 0 or die "Can't seek to a negative absolute position on sql i/o"; ll(5) && do_log(5, "Amavis::IO::SQL::seek mode=%s, pos=%s", $self->{mode}, $pos); $self->{mode} ne 'w' or die "Seek to $whence,$pos on sql i/o only supported for read mode"; if ($pos < $self->{pos}) { if (!$self->{eof} && $self->{chunk_ind} <= 1) { # still in the first chunk, just reset pos $self->{pos} = $self->{bufpos} = 0; # reset } else { # beyond the first chunk, restart the query from the beginning my($con,$clause,$key,$mode,$partition_tag,$maxb,$rx_time) = @$self{qw(conn_h clause dbkey mode partition_tag maxbuf rx_time)}; $self->close or die "seek: error closing, $!"; $self->open($con,$clause,$key,$mode,$partition_tag,$maxb,$rx_time) or die "seek: reopen failed: $!"; } } my($skip) = $pos - $self->{pos}; if ($skip > 0) { my($s); my($nbytes) = $self->read($s,$skip); # acceptable for small skips defined $nbytes or die "seek: error skipping $skip bytes on sql i/o: $!"; } 1; # seek is supposed to return 1 upon success, 0 otherwise } sub read { # SCALAR,LENGTH,OFFSET my $self = shift; my($req_len) = $_[1]; my($offset) = $_[2]; my($conn_h) = $self->{conn_h}; my($a_ref); ll(5) && do_log(5, "Amavis::IO::SQL::read, %d, %d", $self->{chunk_ind}, $self->{bufpos}); eval { while (!$self->{eof} && length($self->{buf})-$self->{bufpos} < $req_len) { $a_ref = $conn_h->fetchrow_arrayref($self->{clause}); if (!defined($a_ref)) { $self->{eof} = 1 } else { $self->{buf} .= $a_ref->[0]; $self->{chunk_ind}++ } } 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; # we can't stash an arbitrary error message string into $!, # which forces us to use 'die' to properly report an error if ($eval_stat =~ /^timed out\b/) { die $eval_stat } # resignal timeout else { die "read: sql select failed, $eval_stat" } $! = EIO; return undef; # not reached }; my($nbytes); if (!defined($offset) || $offset == 0) { $_[0] = substr($self->{buf}, $self->{bufpos}, $req_len); $nbytes = length($_[0]); } else { my($buff) = substr($self->{buf}, $self->{bufpos}, $req_len); substr($_[0],$offset) = $buff; $nbytes = length($buff); } $self->{bufpos} += $nbytes; $self->{pos} += $nbytes; if ($self->{bufpos} > 0 && $self->{chunk_ind} > 1) { # discard used-up part of the buf unless at ch.1, which may still be useful ll(5) && do_log(5,"read: moving on by %d chars", $self->{bufpos}); $self->{buf} = substr($self->{buf},$self->{bufpos}); $self->{bufpos} = 0; } $nbytes; # eof: 0, error: undef } sub getline { my $self = shift; my($conn_h) = $self->{conn_h}; ll(5) && do_log(5, "Amavis::IO::SQL::getline, chunk %d, pos %d", $self->{chunk_ind}, $self->{bufpos}); my($a_ref,$line); my($ind) = -1; eval { while (!$self->{eof} && ($ind=index($self->{buf},"\n",$self->{bufpos})) < 0) { $a_ref = $conn_h->fetchrow_arrayref($self->{clause}); if (!defined($a_ref)) { $self->{eof} = 1 } else { $self->{buf} .= $a_ref->[0]; $self->{chunk_ind}++ } } 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; if ($eval_stat =~ /^timed out\b/) { die $eval_stat } # resignal timeout else { die "getline: reading sql select results failed, $eval_stat" } $! = EIO; return undef; # not reached }; if ($ind < 0 && $self->{eof}) # imply a NL before eof if missing { $self->{buf} .= "\n"; $ind = index($self->{buf}, "\n", $self->{bufpos}) } $ind >= 0 or die "Programming error, NL not found"; if (length($self->{buf}) > $self->{bufpos}) { # nonempty buffer? $line = substr($self->{buf}, $self->{bufpos}, $ind+1-$self->{bufpos}); my($nbytes) = length($line); $self->{bufpos} += $nbytes; $self->{pos} += $nbytes; if ($self->{bufpos} > 0 && $self->{chunk_ind} > 1) { # discard used part of the buf unless at ch.1, which may still be useful ll(5) && do_log(5,"getline: moving on by %d chars", $self->{bufpos}); $self->{buf} = substr($self->{buf},$self->{bufpos}); $self->{bufpos} = 0; } } # eof: undef, $! zero; error: undef, $! nonzero $! = 0; $line eq '' ? undef : $line; } sub flush { my $self = shift; return if $self->{mode} ne 'w'; my($msg); my($conn_h) = $self->{conn_h}; while (length($self->{buf}) > 0) { my($ind) = $self->{chunk_ind} + 1; ll(4) && do_log(4, "sql flush: key: (%s, %d), p_tag=%s, rx_t=%d, size=%d", $self->{dbkey}, $ind, $self->{partition_tag}, $self->{rx_time}, min(length($self->{buf}),$self->{maxbuf})); eval { my($driver) = $conn_h->driver_name; $conn_h->execute($self->{clause}, $self->{partition_tag}, $self->{dbkey}, $ind, # int($self->{rx_time}), [ untaint(substr($self->{buf},0,$self->{maxbuf})), $driver eq 'Pg' ? { pg_type => DBD::Pg::PG_BYTEA() } : SQL_BLOB ] ); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; $msg = $eval_stat; }; last if defined $msg; substr($self->{buf},0,$self->{maxbuf}) = ''; $self->{chunk_ind} = $ind; } if (defined $msg) { chomp $msg; if ($msg =~ /^timed out\b/) { die $msg } # resignal timeout else { $msg = "flush: sql inserting text failed, $msg"; die $msg; # we can't stash an arbitrary error message string into $!, # which forces us to use 'die' to properly report an error } $! = EIO; return undef; # not reached } 1; } sub print { my $self = shift; $self->{mode} eq 'w' or die "Can't print, not opened for writing"; my($buff_ref) = @_ == 1 ? \$_[0] : \join('',@_); my($len) = length($$buff_ref); my($nbytes); my($conn_h) = $self->{conn_h}; if ($len <= 0) { $nbytes = "0 but true" } else { $self->{buf} .= $$buff_ref; $self->{pos} += $len; $nbytes = $len; while (length($self->{buf}) >= $self->{maxbuf}) { my($ind) = $self->{chunk_ind} + 1; ll(4) && do_log(4, "sql print: key: (%s, %d), p_tag=%s, size=%d", $self->{dbkey}, $ind, $self->{partition_tag}, $self->{maxbuf}); eval { my($driver) = $conn_h->driver_name; $conn_h->execute($self->{clause}, $self->{partition_tag}, $self->{dbkey}, $ind, # int($self->{rx_time}), [ untaint(substr($self->{buf},0,$self->{maxbuf})), $driver eq 'Pg' ? { pg_type => DBD::Pg::PG_BYTEA() } : SQL_BLOB ] ); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; # we can't stash an arbitrary error message string into $!, # which forces us to use 'die' to properly report an error if ($eval_stat =~ /^timed out\b/) { die $eval_stat } # resignal timeout else { die "print: sql inserting mail text failed, $eval_stat" } $! = EIO; return undef; # not reached }; substr($self->{buf},0,$self->{maxbuf}) = ''; $self->{chunk_ind} = $ind; } } $nbytes; } sub printf { shift->print(sprintf(shift,@_)) } 1; #^L package Amavis::Out::SQL::Quarantine; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT = qw(&mail_via_sql); import Amavis::Conf qw(:platform c cr ca $sql_quarantine_chunksize_max); import Amavis::rfc2821_2822_Tools qw(qquote_rfc2821_local); import Amavis::Util qw(ll do_log snmp_count collect_equal_delivery_recips); import Amavis::Timing qw(section_time); import Amavis::Out::SQL::Connection (); } use subs @EXPORT; use DBI qw(:sql_types); sub mail_via_sql { my($conn_h, $msginfo, $initial_submission, $dsn_per_recip_capable, $filter) = @_; my(@snmp_vars) = !$initial_submission ? ('', 'Relay', 'ProtoSQL', 'ProtoSQLRelay') : ('', 'Submit', 'ProtoSQL', 'ProtoSQLSubmit', 'Submit'.$initial_submission); snmp_count('OutMsgs'.$_) for @snmp_vars; my($logmsg) = sprintf("%s via SQL (%s): %s", ($initial_submission?'SEND':'FWD'), $conn_h->dsn_current, $msginfo->sender_smtp); my($per_recip_data_ref, $proto_sockname) = collect_equal_delivery_recips($msginfo, $filter, qr/^sql:/i); if (!$per_recip_data_ref || !@$per_recip_data_ref) { do_log(5, "%s, nothing to do", $logmsg); return 1; } my($mail_id) = $msginfo->mail_id; defined $mail_id or die "mail_via_sql: mail_id still undefined"; $proto_sockname = $proto_sockname->[0] if ref $proto_sockname; ll(1) && do_log(1, "delivering to %s, %s -> %s, mail_id %s", $proto_sockname, $logmsg, join(',', qquote_rfc2821_local( map($_->recip_final_addr, @$per_recip_data_ref)) ), $mail_id); my($msg) = $msginfo->mail_text; # a scalar reference, or a file handle if (defined($msg) && !$msg->isa('MIME::Entity')) { $msg->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; } my($err,$smtp_response); eval { my($sql_cl_r) = cr('sql_clause'); $conn_h->begin_work; # SQL transaction starts eval { my($mp) = Amavis::IO::SQL->new; $mp->open($conn_h, $sql_cl_r->{'ins_quar'}, $msginfo->mail_id, 'w', $msginfo->partition_tag, $sql_quarantine_chunksize_max, $msginfo->rx_time) or die "Can't open Amavis::IO::SQL object: $!"; my($hdr_edits) = $msginfo->header_edits; $hdr_edits = Amavis::Out::EditHeader->new if !$hdr_edits; my($received_cnt) = $hdr_edits->write_header($msg,$mp,!$initial_submission); if ($received_cnt > 100) { # loop detection required by RFC 5321 sect 6.2 die "Too many hops: $received_cnt 'Received:' header fields"; } elsif (!defined($msg)) { # empty mail } elsif ($msg->isa('MIME::Entity')) { $msg->print_body($mp); } else { my($nbytes,$buff); while (($nbytes=$msg->read($buff,16384)) > 0) { $mp->print($buff) or die "Can't write to SQL storage: $!" } defined $nbytes or die "Error reading: $!"; } $mp->close or die "Error closing Amavis::IO::SQL object: $!"; $conn_h->commit; 1; } or do { my($err) = $@ ne '' ? $@ : "errno=$!"; chomp $err; my($msg) = $err; $msg = "writing mail text to SQL failed: $msg"; do_log(0,"%s",$msg); if ($conn_h->in_transaction) { eval { $conn_h->rollback; do_log(1,"mail_via_sql: rollback done"); 1; } or do { $@ = "errno=$!" if $@ eq ''; chomp $@; do_log(1,"mail_via_sql: rollback %s", $@); die $@ if $@ =~ /^timed out\b/; # resignal timeout }; } die $err if $err =~ /^timed out\b/; # resignal timeout die $msg; }; 1; } or do { $err = $@ ne '' ? $@ : "errno=$!" }; if ($err eq '') { $smtp_response = "250 2.6.0 Ok, Stored to sql db as mail_id $mail_id"; snmp_count('OutMsgsDelivers'); my($size) = $msginfo->msg_size; snmp_count( ['OutMsgsSize'.$_, $size, 'C64'] ) for @snmp_vars; } else { chomp $err; if ($err =~ /too many hops\b/i) { $smtp_response = "554 5.4.6 Reject: $err"; snmp_count('OutMsgsRejects'); } else { $smtp_response = "451 4.5.0 Storing to sql db as mail_id $mail_id failed: $err"; snmp_count('OutMsgsAttemptFails'); } die $err if $err =~ /^timed out\b/; # resignal timeout } $smtp_response .= ", id=" . $msginfo->log_id; for my $r (@$per_recip_data_ref) { next if $r->recip_done; $r->recip_smtp_response($smtp_response); $r->recip_done(2); if ($smtp_response =~ /^2/) { my($mbxname) = $mail_id; my($p_tag) = $msginfo->partition_tag; $mbxname .= '[' . $p_tag . ']' if defined($p_tag) && $p_tag ne '' && $p_tag ne '0'; $r->recip_mbxname($mbxname); } } section_time('fwd-sql'); 1; } 1; __DATA__ # package Amavis::AV; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars c cr ca); import Amavis::Util qw(ll untaint min max minmax unique_list do_log add_entropy proto_decode rmdir_recursively prolong_timer get_deadline); import Amavis::ProcControl qw(exit_status_str proc_status_ok run_command run_as_subprocess collect_results collect_results_structured); import Amavis::Lookup qw(lookup lookup2); import Amavis::Timing qw(section_time); import Amavis::Out qw(mail_dispatch); import Amavis::rfc2821_2822_Tools qw(one_response_for_all); } use subs @EXPORT_OK; use vars @EXPORT; use POSIX qw(WIFEXITED WIFSIGNALED WIFSTOPPED WEXITSTATUS WTERMSIG WSTOPSIG); use Errno qw(EPIPE ENOTCONN ENOENT EACCES EINTR EAGAIN ECONNRESET); use Socket; use IO::Socket; use IO::Socket::UNIX; use IO::Socket::INET; use Time::HiRes (); BEGIN { use vars qw($have_inet6); $have_inet6 = eval { require IO::Socket::INET6 }; } use vars qw(%st_socket_created %st_sock); # keep persistent state (per-socket) sub clamav_module_init($) { my($av_name) = @_; # each child should reinitialize clamav module to reload databases my($clamav_version) = Mail::ClamAV->VERSION; my($dbdir) = Mail::ClamAV::retdbdir(); my($clamav_obj) = Mail::ClamAV->new($dbdir); ref $clamav_obj or die "$av_name: Can't load db from $dbdir: $Mail::ClamAV::Error"; $clamav_obj->buildtrie; $clamav_obj->maxreclevel($MAXLEVELS) if $MAXLEVELS > 0; $clamav_obj->maxfiles($MAXFILES) if $MAXFILES > 0; $clamav_obj->maxfilesize($MAX_EXPANSION_QUOTA || 30*1024*1024); if ($clamav_version >= 0.12) { $clamav_obj->maxratio($MAX_EXPANSION_FACTOR); # $clamav_obj->archivememlim(0); # limit memory usage for bzip2 (0/1) } do_log(3,"clamav_module_init: %s init", $av_name); section_time('clamav_module_init'); ($clamav_obj,$clamav_version); } # called from sub ask_clamav or ask_daemon, should not run as a subprocess # use vars qw($clamav_obj $clamav_version); sub clamav_module_internal_pre($) { my($av_name) = @_; if (!defined $clamav_obj) { ($clamav_obj,$clamav_version) = clamav_module_init($av_name); # first time } elsif ($clamav_obj->statchkdir) { # db reload needed? do_log(2, "%s: reloading virus database", $av_name); ($clamav_obj,$clamav_version) = clamav_module_init($av_name); } } # called from sub ask_clamav or ask_daemon, may be called directly # or in a subprocess # sub clamav_module_internal($@) { my($query, $bare_fnames,$names_to_parts,$tempdir, $av_name) = @_; $query = join(' ',@$query) if ref $query; my($fname) = "$tempdir/parts/$query"; # file to be checked my($part) = $names_to_parts->{$query}; # get corresponding parts object my($options) = 0; # bitfield of options to Mail::ClamAV::scan my($opt_archive,$opt_mail); if ($clamav_version < 0.12) { $opt_archive = &Mail::ClamAV::CL_ARCHIVE; $opt_mail = &Mail::ClamAV::CL_MAIL; } else { # >= 0.12, reflects renamed flags in libclamav 0.80 $opt_archive = &Mail::ClamAV::CL_SCAN_ARCHIVE; $opt_mail = &Mail::ClamAV::CL_SCAN_MAIL; } # see clamav.h for standard options enabled by CL_SCAN_STDOPT $options |= &Mail::ClamAV::CL_SCAN_STDOPT if $clamav_version >= 0.13; $options |= $opt_archive; # turn on ARCHIVE $options &= ~$opt_mail; # turn off MAIL if (ref($part) && ($part->type_short eq 'MAIL' || lc($part->type_declared) eq 'message/rfc822')) { do_log(2, "%s: $query - enabling option CL_MAIL", $av_name); $options |= $opt_mail; # turn on MAIL } my($ret) = $clamav_obj->scan(untaint($fname), $options); my($output,$status); if ($ret->virus) { $status = 1; $output = "INFECTED: $ret" } elsif ($ret->clean) { $status = 0; $output = "CLEAN" } else { $status = 2; $output = $ret->error.", errno=".$ret->errno } ($status,$output); # return synthesised status and a result string } # subroutine available for calling from @av_scanners list entries; # it has the same args and returns as run_av() below # sub ask_clamav { my($bare_fnames,$names_to_parts,$tempdir, $av_name) = @_; clamav_module_internal_pre($av_name); # must not run as a subprocess # my(@results) = ask_av(\&clamav_module_internal, @_); # invoke directly my($proc_fh,$pid) = run_as_subprocess(\&ask_av, \&clamav_module_internal,@_); my($results_ref,$child_stat) = collect_results_structured($proc_fh,$pid,$av_name,200*1024); !$results_ref ? () : @$results_ref; } my($savi_obj); sub sophos_savi_init { my($av_name, $command) = @_; my(@savi_bool_options) = qw( GrpArchiveUnpack GrpSelfExtract GrpExecutable GrpInternet GrpMSOffice GrpMisc !GrpDisinfect !GrpClean EnableAutoStop FullSweep FullPdf Xml ); $savi_obj = SAVI->new; ref $savi_obj or die "$av_name: Can't create SAVI object, err=$savi_obj"; my($status) = $savi_obj->load_data; !defined($status) or die "$av_name: Failed to load SAVI virus data " . $savi_obj->error_string($status) . " ($status)"; my($version) = $savi_obj->version; ref $version or die "$av_name: Can't get SAVI version, err=$version"; do_log(2,"%s init: Version %s (engine %d.%d) recognizing %d viruses", $av_name, $version->string, $version->major, $version->minor, $version->count); my($error); if ($MAXLEVELS > 0) { $error = $savi_obj->set('MaxRecursionDepth', $MAXLEVELS); !defined $error or die "$av_name: error setting MaxRecursionDepth: err=$error"; } $error = $savi_obj->set('NamespaceSupport', 3); # new with Sophos 3.67 !defined $error or do_log(-1,"%s: error setting NamespaceSupport: err=%s",$av_name,$error); for (@savi_bool_options) { my($value) = /^!/ ? 0 : 1; s/^!+//; $error = $savi_obj->set($_, $value); !defined $error or die "$av_name: Error setting $_: err=$error"; } section_time('sophos_savi_init'); 1; } sub sophos_savi_stale { defined $savi_obj && $savi_obj->stale; } # run by a master(!) process, invoked from a hook run_n_children_hook # sub sophos_savi_reload { if (defined $savi_obj) { do_log(3,"sophos_savi_reload: about to reload SAVI data"); eval { my($status) = $savi_obj->load_data; do_log(-1,"sophos_savi_reload: failed to load SAVI virus data %s (%s)", $savi_obj->error_string($status), $status) if defined $status; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"sophos_savi_reload failed: %s", $eval_stat); }; my($version) = $savi_obj->version; if (!ref($version)) { do_log(-1,"sophos_savi_reload: Can't get SAVI version: %s", $version); } else { do_log(2,"Updated SAVI data: Version %s (engine %d.%d) ". "recognizing %d viruses", $version->string, $version->major, $version->minor, $version->count); } } } # to be called from sub sophos_savi # sub sophos_savi_internal { my($query, $bare_fnames,$names_to_parts,$tempdir, $av_name,$command,$args) = @_; $query = join(' ',@$query) if ref $query; my($fname) = "$tempdir/parts/$query"; # file to be checked if (!c('bypass_decode_parts')) { my($part) = $names_to_parts->{$query}; # get corresponding parts object my($mime_option_value) = 0; if (ref($part) && ($part->type_short eq 'MAIL' || lc($part->type_declared) eq 'message/rfc822')) { do_log(2, "%s: $query - enabling option Mime", $av_name); $mime_option_value = 1; } my($error) = $savi_obj->set('Mime', $mime_option_value); !defined $error or die sprintf("%s: Error %s option Mime: err=%s", $av_name, $mime_option_value ? 'setting' : 'clearing', $error); } my($output,$status); $!=0; my($result) = $savi_obj->scan($fname); if (!ref($result)) { # error my($msg) = "error scanning file $fname, " . $savi_obj->error_string($result) . " ($result)"; # ignore $! ? if ( !grep($result == $_, (514,527,530,538,549)) ) { $status = 2; $output = "ERROR $query: $msg"; } else { # don't panic on non-fatal (encrypted, corrupted, partial) $status = 0; $output = "CLEAN $query: $msg"; } do_log(5,"%s: %s", $av_name,$output); } elsif ($result->infected) { $status = 1; $output = join(", ", $result->viruses) . " FOUND"; } else { $status = 0; $output = "CLEAN $query"; } ($status,$output); # return synthesised status and a result string } # implements client side of the Sophost SSSP protocol # sub sophos_sssp_internal { my($query, $bare_fnames,$names_to_parts,$tempdir, $av_name,$command,$args) = @_; my($query_template, $socket_specs) = !$args ? () : @$args; # short timeout for connect and sending a request prolong_timer('sophos_sssp_connect', undef, undef, 10); my($remaining_time, $deadline) = get_deadline('sophos_sssp_internal'); # section_time('sssp-pre'); my($sssp_handle) = Amavis::IO::RW->new($socket_specs, Eol => "\015\012", Timeout => 10); defined $sssp_handle or die "Can't connect to savdid"; # section_time('sssp-conn'); my($ln); local($1); $ln = $sssp_handle->get_response_line; # greeting defined $ln && $ln ne '' or die "sssp no greeting"; do_log(5,"sssp greeting %s", $ln); $ln =~ m{^OK\s+SSSP/(\d+.*)\015\012\z}s or die "sssp bad greeting '$ln'"; # section_time('sssp-greet'); # # Use the SSSP OPTIONS request only if necessary, it is cheaper to have the # # options set in the configuration file. If a client has needs different # # from other clients, create another channel tailored for that client. # # # $sssp_handle->print("SSSP/1.0 OPTIONS\015\012". # "savists:zipdecompression 1\015\012". # "output: brief\015\012\015\012") # or die "Error writing to socket"; # $sssp_handle->flush or die "Error flushing socket"; # $ln = $sssp_handle->get_response_line; # defined $ln && $ln ne '' or die "sssp no response to OPTIONS"; # do_log(5,"sssp response to OPTIONS: %s", $ln); # $ln =~ /^ACC\s+(\S*)/ or die "sssp OPTIONS request not accepted"; # while (defined($ln = $sssp_handle->get_response_line)) { # last if $ln eq "\015\012"; # do_log(5,"sssp result of OPTIONS: %s", $ln); # } # # section_time('sssp-opts'); my($output) = ''; # normal timeout for reading a response prolong_timer('sophos_sssp_scan'); $sssp_handle->timeout(max(2, $deadline - Time::HiRes::time)); for my $fname (!ref($query) ? $query : @$query) { my($fname_enc) = $fname; $fname_enc =~ s/([%\000-\040\177\377])/sprintf("%%%02X",ord($1))/egs; $sssp_handle->print("SSSP/1.0 SCANDIRR $fname_enc\015\012") or die "Error writing to socket"; $sssp_handle->flush or die "Error flushing socket"; $ln = $sssp_handle->get_response_line; defined $ln && $ln ne '' or die "sssp no response to SCANDIRR"; do_log(5,"sssp response to SCANDIRR: %s", $ln); # section_time('sssp-scan-ack'); $ln =~ /^ACC\s+(\S*)/ or die "sssp SCANDIRR request not accepted"; while (defined($ln = $sssp_handle->get_response_line)) { last if $ln eq "\015\012"; do_log(3,"sssp result: %s", $ln); $output .= $ln if length($output) < 10000; } } $output = proto_decode($output); # section_time('sssp-scan-result'); $sssp_handle->print("BYE\015\012") or die "Error writing to socket"; $sssp_handle->flush or die "Error flushing socket"; $sssp_handle->timeout(max(2, $deadline - Time::HiRes::time)); while (defined($ln = $sssp_handle->get_response_line)) { do_log(5,"sssp response to BYE: %s", $ln); last if $ln eq "\015\012" || $ln =~ /^BYE/; } # section_time('sssp-bye'); $sssp_handle->close or do_log(-1, "sssp - error closing session: $!"); # section_time('sssp-close'); (0,$output); # return synthesised status and a result string } # implements client side of the AVIRA SAVAPI3 protocol # sub avira_savapi_internal { my($query, $bare_fnames,$names_to_parts,$tempdir, $av_name,$command,$args) = @_; my($query_template, $socket_specs, $product_id) = !$args ? () : @$args; # short timeout for connect and sending a request prolong_timer('avira_savapi_connect', undef, undef, 10); my($remaining_time, $deadline) = get_deadline('avira_savapi_internal'); # section_time('savapi-pre'); my($savapi_handle) = Amavis::IO::RW->new($socket_specs, Eol => "\012", Timeout => 10); defined $savapi_handle or die "Can't connect to savapi daemon"; # section_time('savapi-conn'); my($ln); local($1); $ln = $savapi_handle->get_response_line; # greeting defined $ln && $ln ne '' or die "savapi no greeting"; do_log(5,"savapi greeting %s", $ln); $ln =~ m{^100 SAVAPI:(\d+.*)\012\z}s or die "savapi bad greeting '$ln'"; # section_time('savapi-greet'); $remaining_time = int(max(2, $deadline - Time::HiRes::time + 0.5)); for my $cmd ("SET PRODUCT $product_id", "SET SCAN_TIMEOUT $remaining_time", "SET CWD $tempdir/parts", ) { # consider: "SET MAILBOX_SCAN 1", "SET ARCHIVE_SCAN 1", "SET HEUR_LEVEL 2" $savapi_handle->print($cmd."\012") or die "Error writing '$cmd' to socket"; $savapi_handle->flush or die "Error flushing socket"; $ln = $savapi_handle->get_response_line; defined $ln && $ln ne '' or die "savapi: no response to $cmd"; do_log(5,"savapi response to '%s': %s", $cmd,$ln); $ln =~ /^100/ or die "savapi: $cmd request not accepted: $ln"; } # section_time('savapi-settings'); # set a normal timeout for reading a response prolong_timer('avira_savapi_scan'); $savapi_handle->timeout(max(2, $deadline - Time::HiRes::time)); my($keep_one_success); my($output) = ''; for my $fname (!ref($query) ? $query : @$query) { $savapi_handle->print("SCAN $fname\012") # files only, no directories or die "Error writing to socket"; $savapi_handle->flush or die "Error flushing socket"; while (defined($ln = $savapi_handle->get_response_line)) { if ($ln =~ /^200/) { # clean $keep_one_success = $ln if !defined $keep_one_success; } else { $output .= $ln if length($output) < 10000; # sanity limit } last if $ln =~ /^([0125-9]\d\d|300|319).*\012/; # terminal status # last if $ln =~ !/^(310|420|421|422|430).*\012/; # nonterminal status } } $output = $keep_one_success if $output eq '' && defined $keep_one_success; do_log(5,"savapi result: %s", $output); # section_time('savapi-scan-result'); $savapi_handle->print("QUIT\012") or die "Error writing to socket"; $savapi_handle->flush or die "Error flushing socket"; $savapi_handle->close or do_log(-1, "savapi - error closing session: $!"); # section_time('savapi-close'); (0,$output); # return synthesised status and a result string } # implements client side of the ClamAV clamd protocol # sub clamav_clamd_internal { my($query, $bare_fnames,$names_to_parts,$tempdir, $av_name,$command,$args) = @_; my($query_template, $socket_specs, $product_id) = !$args ? () : @$args; # short timeout for connect prolong_timer('clamav_connect', undef, undef, 10); my($remaining_time, $deadline) = get_deadline('clamav_internal'); my($clamav_handle) = Amavis::IO::RW->new($socket_specs, Eol => "\000", Timeout => 10); defined $clamav_handle or die "Can't connect to a clamd daemon"; # set a normal timeout prolong_timer('clamav_scan'); $clamav_handle->timeout(max(2, $deadline - Time::HiRes::time)); $clamav_handle->print("zIDSESSION\0") or die "Error writing 'zIDSESSION' to a clamd socket: $!"; my(%requests, %requests_filename, %requests_timestamp, $end_sent); my($req_id, $requests_pending) = (0,0); my $requests_remaining = !ref $query ? 1 : scalar @$query; my $keep_one_success; my $output = ''; while ($requests_remaining > 0 || $requests_pending > 0) { my $throttling = $requests_pending >= 8; if ($throttling) { $clamav_handle->flush or die "Error flushing socket: $!"; do_log(5,'clamav: throttling: pending %d, remaining %d', $requests_pending, $requests_remaining); } elsif ($requests_remaining > 0 && !$throttling) { my $fname = !ref $query ? $query : $query->[$req_id]; $req_id++; $requests_remaining--; $requests{$req_id} = 'INITIATING'; $requests_filename{$req_id} = $fname; do_log(5,'clamav: sending contents of %s', $fname); $clamav_handle->print("zINSTREAM\0") or die "Error writing 'zINSTREAM' to a clamd socket: $!"; $requests{$req_id} = 'OPEN'; my $fh = IO::File->new; $fh->open($fname,'<') or die "Can't open file $fname: $!"; binmode($fh,':bytes') or die "Can't cancel :utf8 mode: $!"; my($nbytes,$buff); $buff = pack('N',0); while (($nbytes=$fh->read($buff, 16384-4, 4)) > 0) { substr($buff,0,4) = pack('N',$nbytes); # 32 bits len -> 4 bytes $clamav_handle->print($buff) or die "Error writing $nbytes bytes to a clamd socket: $!"; $requests{$req_id} = 'SENDING'; } my $eod = pack('N',0); # length zero indicates end of data if ($requests_remaining <= 0) { $eod .= "zEND\0"; $end_sent = 1 } $clamav_handle->print($eod) or die "Error writing end-of-data to a clamd socket: $!"; # $clamav_handle->flush or die "Error flushing socket: $!"; $requests_timestamp{$req_id} = Time::HiRes::time; $requests{$req_id} = 'SENT'; $requests_pending++; $fh->close or die "Error closing file $fname: $!"; # do_log(5,'clamav: finished sending %s', $fname); } my $ln; while ($requests_pending > 0 && ( !$requests_remaining || $throttling || $clamav_handle->response_line_available ) && defined($ln = $clamav_handle->get_response_line)) { my $rx_time = Time::HiRes::time; # do_log(5,'clamav: got response %s', $ln); local($1,$2); if ($ln !~ /^(\d+):\s*(.*?)\000\z/s) { do_log(-1,'clamav: unparseable response %s', $ln); next; } my($id,$resp) = ($1,$2); if (!defined $requests{$id}) { do_log(-1,'clamav: bogus id %s in response ignored: %s', $id, $ln); } elsif ($requests{$id} eq 'DONE') { do_log(-1,'clamav: duplicate result for id %s: %s', $id, $ln); } else { do_log(5,'clamav: request id %s on %s took %.1f ms', $id, $requests_filename{$id}, 1000 * ($rx_time - $requests_timestamp{$id})); if ($requests{$id} ne 'SENT') { do_log(2,'clamav: result based on partial data, state %s: %s', $requests{$id}, $ln); } $ln =~ s/\000\z/\n/s; $ln =~ s/^\Q$id\E:\s*stream:\s*/$requests_filename{$id}: /s; if ($resp =~ /\bOK\z/) { # clean $keep_one_success = $ln if !defined $keep_one_success; } else { $output .= $ln if length($output) < 10000; # sanity limit } $requests{$id} = 'DONE'; $requests_pending-- if $requests_pending > 0; delete $requests_filename{$id}; delete $requests_timestamp{$id}; if ($resp =~ /\bFOUND\z/ && $requests_remaining > 0 && c('first_infected_stops_scan')) { do_log(2,'clamav: first infected stops scan'); $requests_remaining = 0; } } } } $output = $keep_one_success if $output eq '' && defined $keep_one_success; do_log(5,'clamav: result: %s', $output); if (!$end_sent) { $clamav_handle->print("zEND\0") or die "Error writing 'zEND' to a clamd socket: $!"; } $clamav_handle->close or do_log(-1, "clamav - error closing session: $!"); (0,$output); # return synthesised status and a result string } sub av_smtp_client($$$$) { my($msginfo,$av_name,$av_test_method,$av_test_recip) = @_; $av_test_recip = 'dummy@localhost' if !defined $av_test_recip; my($test_msg) = Amavis::In::Message->new; $test_msg->rx_time($msginfo->rx_time); # copy the reception time $test_msg->log_id($msginfo->log_id); # use the same log_id $test_msg->partition_tag($msginfo->partition_tag); # same partition_tag $test_msg->conn_obj($msginfo->conn_obj); $test_msg->mail_id($msginfo->mail_id); # use the same mail_id $test_msg->body_type($msginfo->body_type); # use the same BODY= type $test_msg->header_8bit($msginfo->header_8bit); $test_msg->body_8bit($msginfo->body_8bit); $test_msg->body_digest($msginfo->body_digest); # copy original digest $test_msg->dsn_ret($msginfo->dsn_ret); $test_msg->dsn_envid($msginfo->dsn_envid); $test_msg->sender($msginfo->sender); # original sender $test_msg->sender_smtp($msginfo->sender_smtp); $test_msg->auth_submitter($msginfo->sender_smtp); $test_msg->auth_user(c('amavis_auth_user')); $test_msg->auth_pass(c('amavis_auth_pass')); $test_msg->recips([$av_test_recip]); # made-up recipient $_->delivery_method($av_test_method) for @{$test_msg->per_recip_data}; $test_msg->originating(0); # disables DKIM signing $test_msg->mail_text($msginfo->mail_text); # the original mail contents $test_msg->skip_bytes($msginfo->skip_bytes); # NOTE: $initial_submission argument is typically treated as a boolean # but here a value of 2 is supplied to allow a forwarding method to # distinguish it from ordinary submissions mail_dispatch($test_msg, 'AV', 0); my($smtp_resp, $exit_code, $dsn_needed) = one_response_for_all($test_msg, 0); # check status do_log(2, "av_smtp_client %s: %s, %s", $av_name,$av_test_method,$smtp_resp); (0, $smtp_resp); } # same args and returns as run_av() below, # but prepended by a $query, which is a string to be sent to the daemon. # Handles both UNIX, INET and INET6 domain sockets. # More than one socket may be specified for redundancy, they will be tried # one after the other until one succeeds. # sub ask_daemon_internal { my($query, # expanded query template, often a command and a file or dir name $bare_fnames,$names_to_parts,$tempdir, $av_name,$command,$args, $sts_clean,$sts_infected,$how_to_get_names, # regexps ) = @_; my($query_template_orig,$socket_specs) = @$args; my($output) = ''; $socket_specs = [ $socket_specs ] if !ref($socket_specs); my($remaining_time, $deadline) = get_deadline('ask_daemon_internal_connect_pre'); my($max_retries) = 2 * @$socket_specs; my($retries) = 0; # Sophie, Trophie and fpscand can accept multiple requests per session # and return a single line response each time my($multisession) = $av_name =~ /\b(Sophie|Trophie|fpscand)\b/i ? 1 : 0; for (;;) { # gracefully handle cases when av process times out or restarts # short timeout for connect and sending a request prolong_timer('ask_daemon_internal_connect', undef, undef, 10); @$socket_specs or die "panic, no sockets specified!?"; # sanity # try the first one in the current list my($socketname) = $socket_specs->[0]; my($sock) = $st_sock{$socketname}; my($eval_stat); eval { if (!$st_socket_created{$socketname}) { ll(3) && do_log(3, "%s: Connecting to socket %s %s%s", $av_name, $daemon_chroot_dir, $socketname, !$retries ? '' : ", retry #$retries" ); $sock = Amavis::IO::RW->new($socketname, Timeout => 10); $st_sock{$socketname} = $sock; defined $sock or die "Can't connect to socket $socketname\n"; $st_socket_created{$socketname} = 1; } $query = join(' ',@$query) if ref $query; ll(3) && do_log(3,"%s: Sending %s to socket %s", $av_name, $query, $socketname); $sock->print($query) or die "Error writing to socket $socketname\n"; $sock->flush or die "Error flushing socket $socketname\n"; # normal timeout for reading a response prolong_timer('ask_daemon_internal_scan'); $sock->timeout(max(2, $deadline - Time::HiRes::time)); if ($multisession) { # depends on TCP segment boundaries, unreliable my($nread) = $sock->read($output,16384); defined($nread) or die "Error reading from $socketname: $!\n"; # and keep the socket open } else { # single request/response per connection my($buff) = ''; for (;;) { my($nread) = $sock->read($buff,16384); if (!defined($nread)) { die "Error reading from $socketname: $!\n"; } elsif ($nread < 1) { last; # sysread returns 0 at eof } else { # successful read $output .= $buff if length($output) < 100000; # sanity } } $sock->close or die "Error closing socket $socketname\n"; $st_sock{$socketname} = $sock = undef; $st_socket_created{$socketname} = 0; } $output ne '' or die "Empty result from $socketname\n"; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; }; prolong_timer('ask_daemon_internal'); last if $eval_stat eq ''; # mission accomplished # error handling (most interesting error codes are EPIPE and ENOTCONN) chomp $eval_stat; my($err) = "$!"; my($errn) = 0+$!; if (Time::HiRes::time >= $deadline) { die "ask_daemon_internal: Exceeded allowed time"; } ++$retries <= $max_retries or die "Too many retries to talk to $socketname ($eval_stat)"; if ($retries <= 1 && $errn == EPIPE) { # common, don't cause concern do_log(2,"%s broken pipe (don't worry), retrying (%d)", $av_name,$retries); } else { do_log( ($retries > 1 ? -1 : 1), "%s: %s, retrying (%d)", $av_name,$eval_stat,$retries); if ($retries % @$socket_specs == 0) { # every time the list is exhausted my($dly) = min(20, 1 + 5 * ($retries/@$socket_specs - 1)); do_log(3,"%s: sleeping for %s s", $av_name,$dly); sleep($dly); # slow down a possible runaway } } if ($st_socket_created{$socketname}) { # prepare for a retry, implicit close throgh DESTROY ignoring status $st_sock{$socketname} = $sock = undef; $st_socket_created{$socketname} = 0; } # leave good socket as the first entry in the list # so that it will be tried first when needed again if (@$socket_specs > 1) { push(@$socket_specs, shift @$socket_specs); # circular shift left } } (0,$output); # return synthesised status and a result string } # subroutine is available for calling from @av_scanners list entries; # it has the same args and returns as run_av() below. # Based on an implied protocol, or on an explicitly specified protocol name # in the second element of array @$args, it determines a subroutine needed # to implement the required protocol (defaulting to &ask_daemon_internal) # and replaces $command in the argument list by this subroutine reference, # then calls run_av with adjusted arguments. So, its main purpose is to map # a protocol name (a string) into an internal code reference. # sub ask_daemon { my($bare_fnames,$names_to_parts,$tempdir, $av_name,$command,$args, $sts_clean,$sts_infected,$how_to_get_names) = @_; my($av_method,$av_protocol); local($1); # determine a protocol name from the second element of array @$args $av_method = $args->[1] if $args && @$args >= 2; $av_method = $av_method->[0] if ref $av_method; $av_protocol = lc($1) if defined $av_method && $av_method =~ /^([a-z][a-z0-9.+-]*):/si; my($code); my($run_spawned) = 0; if (!defined $av_protocol) { # for compatibility with old style socket specification with # no protocol (scheme) field, equivalent to a former call to ask_av() # Sophie, Trophie, ClamAV-clamd, OpenAntiVirus, AVG, # F-Prot fpscand, F-Prot f-protd, DrWebD, avast, ESET NOD32SS $code = \&ask_daemon_internal; } elsif ($av_protocol =~ /^(simple|sophie|trophie)\z/) { # same as default, but with an explicit protocol prefix $code = \&ask_daemon_internal; } elsif ($av_protocol eq 'sssp') { # Sophos SSSP $code = \&sophos_sssp_internal; } elsif ($av_protocol eq 'savapi') { # Avira SAVAPI3 $code = \&avira_savapi_internal; } elsif ($av_protocol eq 'clamd') { # ClamAV clamd protocol $code = \&clamav_clamd_internal; } elsif ($av_protocol eq 'smtp' || $av_protocol eq 'lmtp') { $code = sub { av_smtp_client($Amavis::MSGINFO, $av_name, $av_method, $args->[2]) }; } elsif ($av_protocol eq 'savi-perl') { # using SAVI-Perl perl module if (@_ < 3+6) { # supply default arguments for backwards compatibility $args = ['*']; $sts_clean = [0]; $sts_infected = [1]; $how_to_get_names = qr/^(.*) FOUND$/m; } $code = \&sophos_savi_internal; } elsif ($av_protocol eq 'clamav-perl') { # using Mail::ClamAV perl module clamav_module_internal_pre($av_name); # must not run as a subprocess $code = \&clamav_module_internal; $run_spawned = 1; } ll(5) && do_log(5, "ask_daemon: proto=%s, spawn=%s, (%s) %s", !defined $av_protocol ? 'DFLT' : $av_protocol, $run_spawned, $av_name, $av_method); ref $code or die "Unsupported AV protocol name: $av_method"; $command = $code; # reassemble arguments, after possibly being modified my(@run_av_args) = ($bare_fnames,$names_to_parts,$tempdir, $av_name,$command,$args, $sts_clean,$sts_infected,$how_to_get_names); my(@results); if (!$run_spawned) { @results = run_av(@run_av_args); # invoke directly } else { my($proc_fh,$pid) = run_as_subprocess(\&ask_av, @run_av_args); my($results_ref,$child_stat) = collect_results_structured($proc_fh,$pid,$av_name,200*1024); @results = @$results_ref if $results_ref; } @results; # ($scan_status,$output,$virusnames) } # for compatibility with pre-2.6.0 versions of amavisd-new and # old @av_scanners entries; use ask_daemon and/or run_av instead sub ask_av(@) { my($code, @run_av_args) = @_; $run_av_args[4] = $code; # replaces $command with a supplied $code run_av(@run_av_args); } # Call a virus scanner and parse its output. # Returns a triplet, or dies in case of failure. # The first element of the triplet has the following semantics: # - true if virus found, # - 0 if no viruses found, # - undef if it did not complete its job; # the second element is a string, the text as provided by the virus scanner; # the third element is ref to a list of virus names found (if any). # (it is guaranteed the list will be nonempty if virus was found) # # If there is at least one glob character '*' present in a query template, the # subroutine will traverse supplied files (@$bare_fnames) and call a supplied # subroutine or program for each file to be scanned, summarizing the final # av scan result. If there are no glob characters in a template, the result # is a single call to a supplied subroutine or program, which will presumably # traverse a directory by itself. # sub run_av(@) { my($bare_fnames, # a ref to a list of filenames to scan (basenames) $names_to_parts, # ref to a hash that maps base file names to parts object $tempdir, # temporary directory # n-tuple from an @av_scanners list entry starts here $av_name, $command, $args, $sts_clean, # a ref to a list of status values, or a regexp $sts_infected, # a ref to a list of status values, or a regexp $how_to_get_names, # ref to sub, or a regexp to get list of virus names $pre_code, $post_code, # routines to be invoked before and after av ) = @_; my($scan_status,@virusnames,$error_str); my($output) = ''; return (0,$output,\@virusnames) if !defined($bare_fnames) || !@$bare_fnames; my($query_template, $socket_specs); my($av_protocol) = ''; if (!ref $args) { $query_template = $args; } else { ($query_template, $socket_specs) = @$args; $socket_specs = $socket_specs->[0] if ref $socket_specs; if (defined $socket_specs) { local($1); $av_protocol = lc($1) if $socket_specs =~ /^([a-z][a-z0-9.+-]*):/si; } } my($one_at_a_time) = 0; $one_at_a_time = 1 if ref $command && $av_protocol !~ /^(?:sssp|savapi|clamd)\z/; my(@query_template) = $one_at_a_time ? $query_template # treat it as one arg : split(' ',$query_template); # shell-like my($bare_fnames_last) = $#{$bare_fnames}; do_log(5,"run_av (%s): query template(%s,%d): %s", $av_name,$one_at_a_time,$bare_fnames_last,$query_template); my($remaining_time, $deadline) = prolong_timer('run_av_pre'); my($cwd) = "$tempdir/parts"; chdir($cwd) or die "Can't chdir to $cwd: $!"; &$pre_code(@_) if defined $pre_code; # a '{}' will be replaced by a directory name, '{}/*' and '*' by file names local($1); my(@query_expanded) = map($_ eq '*' || $_ eq '{}/*' ? [] : m{^ \{ \} ( / .* )? \z}xs ? "$tempdir/parts$1" : $_, @query_template); my($eval_stat); eval { for (my($k) = 0; $k <= $bare_fnames_last; ) { # traverse fnames in chunks my(@processed_filenames); my($arglist_size) = 0; # size of a command with its arguments so far for ($command,@query_expanded) { $arglist_size+=length($_)+1 if !ref $_ } for (@query_expanded) { @$_ = () if ref $_ } # reset placeholder lists while ($k <= $bare_fnames_last) { # traverse fnames individually my($f) = $bare_fnames->[$k]; my($multi) = 0; if ($one_at_a_time) { # glob templates may be substrings anywhere local($1); @query_expanded = @query_template; # start afresh s{ ( {} (?: / \* )? | \* ) } { $1 eq '{}' ? "$tempdir/parts" : $1 eq '{}/*' ? ($multi=1,"$tempdir/parts/$f") : $1 eq '*' ? ($multi=1,$f) : $1 }gesx for @query_expanded; } else { # collect as many filename arguments as suitable, but at least one my($arg_size) = 0; for (@query_template) { if ($_ eq '{}/*') { $arg_size += length("$tempdir/parts/$f") + 1 } elsif ($_ eq '*') { $arg_size += length($f) + 1 } } # do_log(5,"run_av arglist size: %d + %d", $arglist_size,$arg_size); if (@processed_filenames && $arglist_size + $arg_size > 4000) { # POSIX requires 4 kB as a minimum buffer size for program args last; # enough collected for now, the rest on the next iteration } # exact matching on command arguments, no substring matches for my $j (0..$#query_template) { if (ref($query_expanded[$j])) { # placeholders collecing fnames my($arg) = $query_template[$j]; my($repl) = $arg eq '{}/*' ? "$tempdir/parts/$f" : $arg eq '*' ? $f : undef; $multi = 1; push(@{$query_expanded[$j]}, untaint($repl)); $arglist_size += length($repl) + 1; } } } $k = $multi ? $k+1 : $bare_fnames_last+1; push(@processed_filenames, $multi ? $f : "$tempdir/parts"); last if $one_at_a_time; } # now that arguments have been expanded, invoke the scanner my($child_stat,$t_status,$t_output); prolong_timer('run_av_scan'); # restart timer if (ref $command) { my(@q) = map(ref $_ ? @$_ : $_, @query_expanded); ll(3) && do_log(3, "run_av Using (%s): (code) %s", $av_name, join(' ',@q)); # call subroutine directly, passing all our arguments to it ($t_status,$t_output) = &$command(!@q ? '' : @q==1 ? $q[0] : \@q, @_); prolong_timer('run_av_3'); # restart timer $child_stat = 0; # no spawned process, just declare success do_log(4,"run_av (%s) result: %s", $av_name,$t_output); } else { my($proc_fh,$pid); my($results_ref); my($eval_stat2); eval { my(@q) = map(ref $_ ? @$_ : $_, @query_expanded); ll(3) && do_log(3,"run_av Using (%s): %s %s", $av_name,$command,join(' ',@q)); ($proc_fh,$pid) = run_command(undef, '&1', $command, @q); ($results_ref,$child_stat) = collect_results($proc_fh,$pid, $av_name,200*1024); 1; } or do { $eval_stat2 = $@ ne '' ? $@ : "errno=$!" }; undef $proc_fh; undef $pid; $error_str = exit_status_str($child_stat,0); $t_status = WEXITSTATUS($child_stat) if defined $child_stat; prolong_timer('run_av_4'); # restart timer if (defined $eval_stat2) { chomp $eval_stat2; $error_str = $eval_stat2; do_log(-1, "run_av (%s): %s", $av_name,$eval_stat2); } if (defined $results_ref) { $t_output = $$results_ref; undef $results_ref } chomp($t_output); my($t_output_trimmed) = $t_output; $t_output_trimmed =~ s/\r\n/\n/gs; local($1); $t_output_trimmed =~ s/([ \t\n\r])[ \t\n\r]{4,}/$1.../gs; $t_output_trimmed = "..." . substr($t_output_trimmed,-800) if length($t_output_trimmed) > 800; do_log(3, "run_av: %s %s, %s", $command,$error_str,$t_output_trimmed); } # The ""=~/x{0}/ serves as an explicit default for an empty regexp, # providing a workaround for braindamaged Perl, where an empty regexp # implies a reuse of last-used nonempty regular expression if (!defined($child_stat) || !WIFEXITED($child_stat)) { # leave $scan_status undefined, indicating an error } elsif (defined $sts_infected && ( ref($sts_infected) eq 'ARRAY' ? (grep($_==$t_status, @$sts_infected)) : ""=~/x{0}/ && $t_output=~/$sts_infected/m)) { # is infected # test for infected first, in case both expressions match $scan_status = 1; # 'true' indicates virus found my(@t_virusnames) = ref($how_to_get_names) eq 'CODE' ? &$how_to_get_names($t_output) : ""=~/x{0}/ && $t_output=~/$how_to_get_names/gm; @t_virusnames = grep(defined $_, @t_virusnames); push(@virusnames, @t_virusnames); $output .= $t_output . "\n"; do_log(2,"run_av (%s): %s INFECTED: %s", $av_name, join(' ',@processed_filenames), join(', ',@t_virusnames) ); } elsif (!defined($sts_clean)) { # clean, but inconclusive # by convention: undef $sts_clean means result is inconclusive, # file appears clean, but continue scanning with other av scanners, # the current scanner does not want to vouch for it; useful for a # scanner like jpeg checker which tests for one vulnerability only do_log(3,"run_av (%s): CLEAN, but inconclusive", $av_name); } elsif (ref($sts_clean) eq 'ARRAY' ? (grep($_==$t_status, @$sts_clean)) : ""=~/x{0}/ && $t_output=~/$sts_clean/m) { # is clean # 'false' (but defined) indicates no viruses $scan_status = 0 if !$scan_status; # no viruses, no errors do_log(3,"run_av (%s): CLEAN", $av_name); } else { # $error_str = "unexpected $error_str, output=\"$t_output_trimmed\""; $error_str = "unexpected $error_str, output=\"$t_output\""; do_log(-1,"run_av (%s) FAILED - %s", $av_name,$error_str); last; # error, bail out } die "Exceeded allowed time\n" if time >= $deadline; } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; &$post_code(@_) if defined $post_code; @virusnames = ('') if $scan_status && !@virusnames; # ensure nonempty list do_log(3,"run_av (%s) result: clean", $av_name) if defined($scan_status) && !$scan_status; chdir($tempdir) or die "Can't chdir to $tempdir: $!"; if (defined $eval_stat) { prolong_timer('run_av_5'); # restart timer die "run_av error: $eval_stat\n"; } if (!defined($scan_status) && defined($error_str)) { die "$command $error_str"; # die is more informative than a return value } ($scan_status, $output, \@virusnames); } sub virus_scan($$) { my($msginfo,$firsttime) = @_; my($tempdir) = $msginfo->mail_tempdir; my($scan_status,$output,@virusname,@detecting_scanners); my($anyone_done) = 0; my($anyone_tried) = 0; my($bare_fnames_ref,$names_to_parts); my($j); my($tier) = 'primary'; for my $av (@{ca('av_scanners')}, "\000", @{ca('av_scanners_backup')}) { next if !defined $av; if ($av eq "\000") { # 'magic' separator between lists last if $anyone_done; do_log(-1,"WARN: all %s virus scanners failed, considering backups", $tier); $tier = 'secondary'; next; } next if !ref $av || !defined $av->[1]; if (!defined $bare_fnames_ref) { # first time: collect file names to scan my($parts_root) = $msginfo->parts_root; ($bare_fnames_ref,$names_to_parts) = files_to_scan("$tempdir/parts",$parts_root); if (!@$bare_fnames_ref) { do_log(2, "Not calling virus scanners, no files to scan in %s/parts", $tempdir); } else { do_log(5, "Calling virus scanners, %d files to scan in %s/parts", scalar(@$bare_fnames_ref), $tempdir); } } my($scanner_name,$command) = @$av; $anyone_tried = 1; my($this_status,$this_output,$this_vn); if (!@$bare_fnames_ref) { # no files to scan? ($this_status,$this_output,$this_vn) = (0, '', []); # declare clean } else { # call virus scanner do_log(5, "invoking av-scanner %s", $scanner_name); eval { ($this_status,$this_output,$this_vn) = ref $command eq 'CODE' ? &$command($bare_fnames_ref,$names_to_parts,$tempdir, @$av) : run_av($bare_fnames_ref,$names_to_parts,$tempdir, @$av); 1; } or do { my($err) = $@ ne '' ? $@ : "errno=$!"; chomp $err; $err = sprintf("%s av-scanner FAILED: %s", $scanner_name, $err); do_log(-1, "%s", $err); $this_status = undef; }; } $anyone_done = 1 if defined $this_status; $j++; section_time("AV-scan-$j"); if ($this_status && $this_vn && @$this_vn) { @$this_vn = unique_list($this_vn); # virus is reported by this scanner; is it for real, or is it just spam? my(@spam_hits); my($vnts) = ca('virus_name_to_spam_score_maps'); @spam_hits = # map each reported virus name to spam score or to undef map(scalar(lookup2(0,$_,$vnts)), @$this_vn) if ref $vnts; if (@spam_hits && !grep(!defined($_), @spam_hits)) { # all defined # AV scanner did trigger, but all provided names are actually spam! my(%seen); for my $r (@{$msginfo->per_recip_data}) { my($spam_tests) = $r->spam_tests; if (defined $spam_tests) { local($1,$2); for (split(/,/, join(',',map($$_,@$spam_tests)))) { $seen{$1} = $2 if /^AV\.([^=]*)=([0-9.+-]+)\z/; } } } my(@vnms,@hits); # remove already detected virus names and duplicates from the list for my $j (0..$#$this_vn) { my($vname) = $this_vn->[$j]; if (!exists($seen{$vname})) { push(@vnms,$vname); push(@hits,$spam_hits[$j]); $seen{$vname} = $spam_hits[$j]; # keep only one copy } } @$this_vn = @vnms; @spam_hits = @hits; if (!@spam_hits) { do_log(2,"Turning AV infection into a spam report, ". "name already accounted for"); } else { my($spam_level) = max(@spam_hits); my($spam_tests) = join(',', map(sprintf("AV:%s=%s", $this_vn->[$_], $spam_hits[$_]), (0..$#$this_vn) )); for my $r (@{$msginfo->per_recip_data}) { $r->spam_level( ($r->spam_level || 0) + $spam_level ); if (!defined($r->spam_tests)) { $r->spam_tests([ \$spam_tests ]); } else { push(@{$r->spam_tests}, \$spam_tests); } } my($spam_report) = $spam_tests; my($spam_summary) = sprintf("AV scanner %s reported spam (not infection):\n%s\n", $scanner_name, join(',',@$this_vn)); do_log(2,"Turning AV infection into a spam report: score=%s, %s", $spam_level, $spam_tests); if (defined($msginfo->spam_report)||defined($msginfo->spam_summary)){ $spam_report = $msginfo->spam_report . ', ' . $spam_report if $msginfo->spam_report ne ''; $spam_summary = $msginfo->spam_summary . "\n\n" . $spam_summary if $msginfo->spam_summary ne ''; } $msginfo->spam_report($spam_report); $msginfo->spam_summary($spam_summary); } $this_status = 0; @$this_vn = (); # TURN OFF ALERT for this AV scanner! } } if ($this_status) { # a virus detected by this scanner, really! (not spam) push(@detecting_scanners, $scanner_name); if (!@virusname) { # store results of the first scanner detecting # @virusname = map(sprintf('[%s] %s',$scanner_name,$_), @$this_vn); @virusname = @$this_vn; $scan_status = $this_status; $output = $this_output; } last if c('first_infected_stops_scan'); # stop now if we found a virus? } elsif (!defined($scan_status)) { # tentatively keep regardless of status $scan_status = $this_status; $output = $this_output; } } if (ll(2) && @virusname && @detecting_scanners) { my(@ds) = @detecting_scanners; for (@ds) { s/,/;/ } # facilitates parsing do_log(2, "virus_scan: (%s), detected by %d scanners: %s", join(', ',@virusname), scalar(@ds), join(', ',@ds)); } $output =~ s{\Q$tempdir\E/parts/?}{}gs if defined $output; # hide path info if (!$anyone_tried) { die "NO VIRUS SCANNERS AVAILABLE\n" } elsif (!$anyone_done) { die "ALL VIRUS SCANNERS FAILED\n" } ($scan_status, $output, \@virusname, \@detecting_scanners); # return a quad } # return a ref to a list of files to be scanned in a given directory # sub files_to_scan($$) { my($dir,$parts_root) = @_; my($names_to_parts) = {}; # a hash that maps base file names # to Amavis::Unpackers::Part object # traverse decomposed parts tree breadth-first, match it to actual files for (my($part), my(@unvisited)=($parts_root); @unvisited and $part=shift(@unvisited); push(@unvisited,@{$part->children})) { $names_to_parts->{$part->base_name} = $part if $part ne $parts_root } my($bare_fnames_ref) = []; my(%bare_fnames); # traverse parts directory and check for actual files local(*DIR); opendir(DIR,$dir) or die "Can't open directory $dir: $!"; # modifying a directory while traversing it can cause surprises, avoid; # avoid slurping the whole directory contents into memory my($f, @rmfiles, @rmdirs); while (defined($f = readdir(DIR))) { next if $f eq '.' || $f eq '..'; my($fname) = $dir . '/' . $f; my(@stat_list) = lstat($fname); my($errn) = @stat_list ? 0 : 0+$!; next if $errn == ENOENT; if ($errn) { die "files_to_scan: file $fname inaccessible: $!" } add_entropy(@stat_list); if (!-r _) { # attempting to gain read access to the file do_log(3,"files_to_scan: attempting to gain read access to %s", $fname); chmod(0750,untaint($fname)) or die "files_to_scan: Can't change protection on $fname: $!"; $errn = lstat($fname) ? 0 : 0+$!; if ($errn) { die "files_to_scan: file $fname inaccessible: $!" } if (!-r _) { die "files_to_scan: file $fname not readable" } } if (!-f _ || !exists $names_to_parts->{$f}) { # nonregular f. or unexpected my($what) = -l _ ? 'symlink' : -d _ ? 'directory' : -f _ ? 'file' : 'non-regular file'; my($msg) = "removing unexpected $what $fname"; $msg .= ", it has no corresponding parts object" if !exists $names_to_parts->{$f}; do_log(-1, "WARN: files_to_scan: %s", $msg); if (-d _) { push(@rmdirs, $f) } else { push(@rmfiles, $f) } } elsif (-z _) { # empty file } else { if ($f !~ /^[A-Za-z0-9_.-]+\z/s) { do_log(-1,"WARN: files_to_scan: unexpected/suspicious file name: %s", $f); } push(@$bare_fnames_ref,$f); $bare_fnames{$f} = 1; } } closedir(DIR) or die "Error closing directory $dir: $!"; for my $f (@rmfiles) { my($fname) = $dir . '/' . untaint($f); do_log(5,"files_to_scan: deleting file %s", $fname); unlink($fname) or die "Can't delete $fname: $!"; } undef @rmfiles; for my $d (@rmdirs) { my($dname) = $dir . '/' . untaint($d); do_log(5,"files_to_scan: deleting directory %s", $dname); rmdir_recursively($dname); } undef @rmdirs; # remove entries from %$names_to_parts that have no corresponding files my($fname,$part); while ( ($fname,$part) = each %$names_to_parts ) { next if exists $bare_fnames{$fname}; if (ll(4) && $part->exists) { my($type_short) = $part->type_short; do_log(4,"files_to_scan: info: part %s (%s) no longer present", $fname, (!ref $type_short ? $type_short : join(', ',@$type_short)) ); } delete $names_to_parts->{$fname}; # delete is allowed for the current elem. } ($bare_fnames_ref, $names_to_parts); } 1; __DATA__ # package Amavis::SpamControl; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform c cr ca); import Amavis::Util qw(ll do_log untaint unique_list); import Amavis::Lookup qw(lookup lookup2); import Amavis::rfc2821_2822_Tools qw(make_query_keys qquote_rfc2821_local); } sub new { my($class) = @_; my($self) = bless { scanners_list => [] }, $class; for my $as (@{ca('spam_scanners')}) { if (ref $as && defined $as->[1] && $as->[1] ne '') { my($scanner_name,$module,@args) = @$as; my($scanner_obj); do_log(5, "SpamControl: attempting to load scanner %s, module %s", $scanner_name,$module); { no strict 'subs'; $scanner_obj = $module->new($scanner_name,$module,@args); } if ($scanner_obj) { push(@{$self->{scanners_list}}, [$scanner_obj, @$as]); do_log(2, "SpamControl: scanner %s, module %s", $scanner_name,$module); } else { do_log(5, "SpamControl: no scanner %s, module %s", $scanner_name,$module); } } } $self; } # called at startup, before chroot and before main fork # sub init_pre_chroot { my($self) = @_; for my $as (@{$self->{scanners_list}}) { my($scanner_obj,$scanner_name) = @$as; if ($scanner_obj && $scanner_obj->UNIVERSAL::can('init_pre_chroot')) { $scanner_obj->init_pre_chroot; do_log(1, "SpamControl: init_pre_chroot on %s done", $scanner_name); } } } # called at startup, after chroot and changing UID, but before main fork # sub init_pre_fork { my($self) = @_; for my $as (@{$self->{scanners_list}}) { my($scanner_obj,$scanner_name) = @$as; if ($scanner_obj && $scanner_obj->UNIVERSAL::can('init_pre_fork')) { $scanner_obj->init_pre_fork; do_log(1, "SpamControl: init_pre_fork on %s done", $scanner_name); } } } # called during child process initialization # sub init_child { my($self) = @_; my($failure_msg); for my $as (@{$self->{scanners_list}}) { my($scanner_obj,$scanner_name) = @$as; if ($scanner_obj && $scanner_obj->UNIVERSAL::can('init_child')) { eval { $scanner_obj->init_child; do_log(5, "SpamControl: init_child on %s done", $scanner_name); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1, "init_child on spam scanner %s failed: %s", $scanner_name, $eval_stat); $failure_msg = "init_child $scanner_name failed: $eval_stat" if !defined $failure_msg; }; } } if (defined $failure_msg) { die $failure_msg } } # actual spam checking for every message # sub spam_scan { my($self,$msginfo) = @_; my($failure_msg); for my $as (@{$self->{scanners_list}}) { my($scanner_obj,$scanner_name) = @$as; if ($scanner_obj && $scanner_obj->UNIVERSAL::can('check')) { do_log(5, "SpamControl: calling spam scanner %s", $scanner_name); eval { $scanner_obj->check($msginfo); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1, "checking with spam scanner %s failed: %s", $scanner_name, $eval_stat); $failure_msg = "$scanner_name failed: $eval_stat" if !defined $failure_msg; }; } } if (defined $failure_msg) { die $failure_msg } } # called during child process shutdown # sub rundown_child() { my($self) = @_; for my $as (@{$self->{scanners_list}}) { my($scanner_obj,$scanner_name) = @$as; if ($scanner_obj && $scanner_obj->UNIVERSAL::can('rundown_child')) { eval { $scanner_obj->rundown_child; do_log(5, "SpamControl: rundown_child on %s done", $scanner_name); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1, "rundown_child on spam scanner %s failed: %s", $scanner_name, $eval_stat); }; } } } # check envelope sender and author for white or blacklisting by each recipient; # Saves the result in recip_blacklisted_sender and recip_whitelisted_sender # properties of each recipient object, and updates spam score for each # recipient according to soft-w/b-listing. # sub white_black_list($$$$) { my($msginfo,$sql_wblist,$user_id_sql,$ldap_lookups) = @_; my($fm) = $msginfo->rfc2822_from; my(@rfc2822_from) = !defined($fm) ? () : ref $fm ? @$fm : $fm; my(@senders) = ($msginfo->sender, @rfc2822_from); @senders = unique_list(\@senders); # remove possible duplicates ll(4) && do_log(4,"wbl: checking sender %s", scalar(qquote_rfc2821_local(@senders))); my($any_w,$any_b,$all,$wr,$br); $any_w = 0; $any_b = 0; $all = 1; for my $r (@{$msginfo->per_recip_data}) { # for each recipient next if $r->recip_done; # already dealt with my($wb,$boost); my($found) = 0; my($recip) = $r->recip_addr; my($user_id_ref,$mk_ref); $user_id_ref = $r->user_id; $user_id_ref = [] if !defined $user_id_ref; do_log(5,"wbl: (SQL) recip <%s>, %s matches", $recip, scalar(@$user_id_ref)) if defined $sql_wblist && ll(5); for my $sender (@senders) { for my $ind (0..$#{$user_id_ref}) { # for ALL SQL sets matching the recip my($user_id) = $user_id_ref->[$ind]; my($mkey); ($wb,$mkey) = lookup(0,$sender, Amavis::Lookup::SQLfield->new($sql_wblist,'wb','S',$user_id) ); do_log(4,'wbl: (SQL) recip <%s>, rid=%s, got: "%s"', $recip,$user_id,$wb); if (!defined($wb)) { # NULL field or no match: remains undefined } elsif ($wb =~ /^ *([+-]?\d+(?:\.\d*)?) *\z/) { # numeric my($val) = 0+$1; # penalty points to be added to the score $boost += $val; ll(2) && do_log(2, 'wbl: (SQL) soft-%slisted (%s) sender <%s> => <%s> (rid=%s)', ($val<0?'white':'black'), $val, $sender, $recip, $user_id); $wb = undef; # not hard- white or blacklisting, does not exit loop } elsif ($wb =~ /^[ \000]*\z/) { # neutral, stops the search $found=1; $wb = 0; do_log(5, 'wbl: (SQL) recip <%s> is neutral to sender <%s>', $recip,$sender); } elsif ($wb =~ /^([BbNnFf])[ ]*\z/) { # blacklisted (B,N(o), F(alse)) $found=1; $wb = -1; $any_b++; $br = $recip; $r->recip_blacklisted_sender(1); do_log(5, 'wbl: (SQL) recip <%s> blacklisted sender <%s>', $recip,$sender); } else { # whitelisted (W, Y(es), T(true), or anything else) if ($wb =~ /^([WwYyTt])[ ]*\z/) { do_log(5, 'wbl: (SQL) recip <%s> whitelisted sender <%s>', $recip,$sender); } else { do_log(-1,'wbl: (SQL) recip <%s> whitelisted sender <%s>, '. 'unexpected wb field value: "%s"', $recip,$sender,$wb); } $found=1; $wb = +1; $any_w++; $wr = $recip; $r->recip_whitelisted_sender(1); } last if $found; } if (!$found && defined($ldap_lookups)) { # LDAP queries my($wblist); my($keys_ref,$rhs_ref) = make_query_keys($sender,0,0); my(@keys) = @$keys_ref; unshift(@keys, '<>') if $sender eq ''; # a hack for a null return path $_ = untaint($_) for @keys; # untaint keys $_ = Net::LDAP::Util::escape_filter_value($_) for @keys; do_log(5,'wbl: (LDAP) query keys: %s', join(', ',map("\"$_\"",@keys))); $wblist = lookup(0,$recip,Amavis::Lookup::LDAPattr->new( $ldap_lookups,'amavisBlacklistSender','L-')); for my $key (@keys) { if (grep(lc($_) eq lc($key), @$wblist)) { $found=1; $wb = -1; $br = $recip; $any_b++; $r->recip_blacklisted_sender(1); do_log(5,'wbl: (LDAP) recip <%s> blacklisted sender <%s>', $recip,$sender); } } $wblist = lookup(0,$recip,Amavis::Lookup::LDAPattr->new( $ldap_lookups,'amavisWhitelistSender','L-')); for my $key (@keys) { if (grep(lc($_) eq lc($key), @$wblist)) { $found=1; $wb = +1; $wr = $recip; $any_w++; $r->recip_whitelisted_sender(1); do_log(5,'wbl: (LDAP) recip <%s> whitelisted sender <%s>', $recip,$sender); } } } if (!$found) { # fall back to static lookups if no match # sender can be both white- and blacklisted at the same time my($val); my($r_ref,$mk_ref,@t); # NOTE on the specifics of $per_recip_blacklist_sender_lookup_tables : # the $r_ref below is supposed to be a ref to a single lookup table # for compatibility with pre-2.0 versions of amavisd-new; # Note that this is different from @score_sender_maps, which is # supposed to contain a ref to a _list_ of lookup tables as a result # of the first-level lookup (on the recipient address as a key). # ($r_ref,$mk_ref) = lookup(0,$recip, Amavis::Lookup::Label->new("blacklist_recip<$recip>"), cr('per_recip_blacklist_sender_lookup_tables')); @t = ((defined $r_ref ? $r_ref : ()), @{ca('blacklist_sender_maps')}); $val = lookup2(0,$sender,\@t,Label=>"blacklist_sender<$sender>") if @t; if ($val) { $found=1; $wb = -1; $br = $recip; $any_b++; $r->recip_blacklisted_sender(1); do_log(5,'wbl: recip <%s> blacklisted sender <%s>', $recip,$sender); } # similar for whitelists: ($r_ref,$mk_ref) = lookup(0,$recip, Amavis::Lookup::Label->new("whitelist_recip<$recip>"), cr('per_recip_whitelist_sender_lookup_tables')); @t = ((defined $r_ref ? $r_ref : ()), @{ca('whitelist_sender_maps')}); $val = lookup2(0,$sender,\@t,Label=>"whitelist_sender<$sender>") if @t; if ($val) { $found=1; $wb = +1; $wr = $recip; $any_w++; $r->recip_whitelisted_sender(1); do_log(5,'wbl: recip <%s> whitelisted sender <%s>', $recip,$sender); } } if (!defined($boost)) { # lookup @score_sender_maps if no match with SQL # note the first argument of lookup() is true, requesting ALL matches my($r_ref,$mk_ref) = lookup2(1,$recip, ca('score_sender_maps'), Label=>"score_recip<$recip>"); for my $j (0..$#{$r_ref}) { # for ALL tables matching the recipient my($val,$key) = lookup2(0,$sender,$r_ref->[$j], Label=>"score_sender<$sender>"); if (defined $val && $val != 0) { $boost += $val; ll(2) && do_log(2,'wbl: soft-%slisted (%s) sender <%s> => <%s>, '. 'recip_key="%s"', ($val<0?'white':'black'), $val, $sender, $recip, $mk_ref->[$j]); } } } } # endfor on @senders if ($boost) { # defined and nonzero $r->spam_level( ($r->spam_level || 0) + $boost); my($spam_tests) = 'AM.WBL=' . (0+sprintf("%.3f",$boost)); if (!defined($r->spam_tests)) { $r->spam_tests([ \$spam_tests ]); } else { unshift(@{$r->spam_tests}, \$spam_tests); } } $all = 0 if !$wb; } # endfor on recips if (!ll(2)) { # don't bother preparing a log report which will not be printed } else { my($msg) = ''; if ($all && $any_w && !$any_b) { $msg = "whitelisted" } elsif ($all && $any_b && !$any_w) { $msg = "blacklisted" } elsif ($all) { $msg = "black or whitelisted by all recips" } elsif ($any_b || $any_w) { $msg .= "whitelisted by ".($any_w>1?"$any_w recips, ":"$wr, ") if $any_w; $msg .= "blacklisted by ".($any_b>1?"$any_b recips, ":"$br, ") if $any_b; $msg .= "but not by all,"; } do_log(2,"wbl: %s sender %s", $msg, scalar(qquote_rfc2821_local(@senders))) if $msg ne ''; } ($any_w+$any_b, $all); } 1; __DATA__ # package Amavis::SpamControl::ExtProg; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars :sa c cr ca); import Amavis::Util qw(ll do_log sanitize_str min max minmax prolong_timer get_deadline); import Amavis::ProcControl qw(exit_status_str proc_status_ok kill_proc run_command run_command_consumer); import Amavis::rfc2821_2822_Tools qw(qquote_rfc2821_local); import Amavis::Timing qw(section_time); } use subs @EXPORT_OK; use Errno qw(EIO EINTR EAGAIN ECONNRESET EBADF); use IO::File qw(O_CREAT O_EXCL O_WRONLY); use Time::HiRes (); sub new { my($class, $scanner_name,$module,@args) = @_; my($cmd,$cmdargs,%options) = @args; return if !defined $cmd || $cmd eq ''; bless { scanner_name => $scanner_name, command => $cmd, args => $cmdargs, options => \%options, }, $class; } # pass a mail message to an external (spam checking) program, # extract interesting header fields from the result # sub check { my($self,$msginfo) = @_; my($scanner_name) = $self->{scanner_name}; my($cmd) = $self->{command}; my($cmdargs) = $self->{args}; my($size_limit); my($mbsl) = $self->{options}->{'mail_body_size_limit'}; $mbsl = c('sa_mail_body_size_limit') if !defined $mbsl; if (defined $mbsl) { $size_limit = min(64*1024, $msginfo->orig_header_size) + 1 + min($mbsl, $msginfo->orig_body_size); # don't bother if slightly oversized, it's faster without size checks if ($msginfo->msg_size < $size_limit + 5*1024) { undef $size_limit } } my($prefix) = ''; # fake a local delivery agent by inserting a Return-Path $prefix .= sprintf("Return-Path: %s\n", $msginfo->sender_smtp); $prefix .= sprintf("X-Envelope-To: %s\n", join(",\n ",qquote_rfc2821_local(@{$msginfo->recips}))); my($os_fp) = $msginfo->client_os_fingerprint; $prefix .= sprintf("X-Amavis-OS-Fingerprint: %s\n", sanitize_str($os_fp)) if defined($os_fp) && $os_fp ne ''; my(@av_tests); my($per_recip_data) = $msginfo->per_recip_data; $per_recip_data = [] if !$per_recip_data; for my $r (@$per_recip_data) { my($spam_tests) = $r->spam_tests; if (defined $spam_tests) { push(@av_tests, grep(/^AV\..+=/, split(/,/, join(',',map($$_,@$spam_tests))))); } } $prefix .= sprintf("X-Amavis-AV-Status: %s\n", sanitize_str(join(',',@av_tests))) if @av_tests; $prefix .= sprintf("X-Amavis-PolicyBank: %s\n", c('policy_bank_path')); $prefix .= sprintf("X-Amavis-MessageSize: %d%s\n", $msginfo->msg_size, !defined $size_limit ? '' : ", TRUNCATED to $size_limit"); my($resp_stdout_fh) = IO::File->new; # parent reading side of the pipe my($child_stdout_fh) = IO::File->new; # child stdout writing side of a pipe my($resp_stderr_fh) = IO::File->new; # parent reading side of the pipe my($child_stderr_fh) = IO::File->new; # child stderr writing side of a pipe pipe($resp_stdout_fh,$child_stdout_fh) or die "$scanner_name: Can't create pipe1: $!"; pipe($resp_stderr_fh,$child_stderr_fh) or die "$scanner_name: Can't create pipe2: $!"; binmode($resp_stdout_fh) or die "Can't set pipe1 to binmode: $!"; binmode($resp_stderr_fh) or die "Can't set pipe2 to binmode: $!"; my($proc_fh,$pid) = run_command_consumer('&='.fileno($child_stdout_fh), '&='.fileno($child_stderr_fh), $cmd, @$cmdargs); $child_stdout_fh->close or die "Parent failed to close child side of the pipe1: $!"; $child_stderr_fh->close or die "Parent failed to close child side of the pipe2: $!"; undef $child_stdout_fh; undef $child_stderr_fh; my($remaining_time, $deadline) = get_deadline($scanner_name.'_scan', 0.9, 5); alarm(0); # stop the timer my($proc_fd) = fileno($proc_fh); my($resp_stdout_fd) = fileno($resp_stdout_fh); my($resp_stderr_fd) = fileno($resp_stderr_fh); my($response) = ''; my($response_stderr) = ''; my($response_chopped) = 0; my($child_stat); my($bytes_sent) = 0; my($err_on_child) = 0; my($msg) = $msginfo->mail_text; eval { if (!defined($msg)) { # empty mail } elsif ($msg->isa('MIME::Entity')) { # $msg->print_body($proc_fh); # flushing the pipe? die "$scanner_name: reading from MIME::Entity not implemented"; } else { $msg->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; my($data_source) = $prefix; my($eof_on_response) = 0; my($eof_on_msg) = 0; my($force_eof_on_msg) = 0; my($rout,$wout,$eout); my($rin,$win,$ein); $rin=$win=$ein=''; vec($rin,$resp_stdout_fd,1) = 1; vec($rin,$resp_stderr_fd,1) = 1; for (;;) { vec($win,$proc_fd,1) = 0; vec($win,$proc_fd,1) = 1 if defined $proc_fh && (!$eof_on_msg || $data_source ne ''); $ein = $rin | $win; my($timeout) = max(2, $deadline - Time::HiRes::time); my($nfound,$timeleft) = select($rout=$rin, $wout=$win, $eout=$ein, $timeout); $nfound >= 0 or die "$scanner_name: select failed: $!"; if (vec($rout,$resp_stderr_fd,1)) { my($inbuf) = ''; $! = 0; my($nread) = sysread($resp_stderr_fh,$inbuf,16384); if (!defined($nread)) { if ($!==EAGAIN || $!==EINTR) { Time::HiRes::sleep(0.1); # slow down, just in case } else { do_log(0,"%s: error reading from pipe2: %s", $scanner_name,$!); } } elsif ($nread < 1) { # sysread returns 0 at eof } else { # successful read ll(5) && do_log(5, "rx stderr: %d %s [...]", length($inbuf), substr($inbuf,0,1000)); $response_stderr .= $inbuf if length($response_stderr) < 10000; } } if (vec($rout,$resp_stdout_fd,1)) { my($inbuf) = ''; $! = 0; my($nread) = sysread($resp_stdout_fh,$inbuf,16384); if (!defined($nread)) { if ($!==EAGAIN || $!==EINTR) { Time::HiRes::sleep(0.1); # slow down, just in case } else { $eof_on_response = 1; die "$scanner_name: error reading from pipe1: $!"; } } elsif ($nread < 1) { # sysread returns 0 at eof $eof_on_response = 1; } else { # successful read ll(5) && do_log(5, "rx: %d %s [...]", length($inbuf), substr($inbuf,0,30)); my($response_l) = length($response); if ($response_chopped || $response_l >= 65536) { # ignore the rest of input } else { $response .= $inbuf; my($j) = $response_l <= 1 ? 0 : $response_l - 1; # we only need a mail header from the returned text $response_chopped = 1 if index($response,"\n\n",$j) >= 0; } } } if (vec($wout,$proc_fd,1)) { if ($data_source eq '' && !$eof_on_msg) { my($nread) = $force_eof_on_msg ? 0 : $msg->read($data_source,16384); if (!$nread) { $eof_on_msg = 1; defined $nread or die "$scanner_name: error reading message: $!"; if (defined $proc_fh) { $proc_fh->close or $err_on_child=$! }; undef $proc_fh; do_log(5,"tx: eof"); } if (defined $size_limit) { my($remaining_room) = $size_limit - $bytes_sent; $remaining_room = 0 if $remaining_room < 0; if ($nread > $remaining_room) { $data_source = substr($data_source, 0, $remaining_room); do_log(5,"tx: (size limit) %d -> %d", $nread,$remaining_room); $force_eof_on_msg = 1; } } } if ($data_source ne '' && defined $proc_fh) { ll(5) && do_log(5, "tx: %d %s [...]", length($data_source), substr($data_source,0,30)); # syswrite does a write(2), no need to call $proc_fh->flush my($nwrite) = syswrite($proc_fh, $data_source); if (!defined($nwrite)) { if ($!==EAGAIN || $!==EINTR) { Time::HiRes::sleep(0.1); # slow down, just in case } else { $data_source = ''; $eof_on_msg = 1; # simulate an eof do_log(-1,"%s: error writing to pipe: %s", $scanner_name,$!); $proc_fh->close or $err_on_child=$!; undef $proc_fh; do_log(5,"tx: eof (wr err)"); } } elsif ($nwrite > 0) { # successful write $bytes_sent += $nwrite; if ($nwrite < length($data_source)) { substr($data_source,0,$nwrite) = ''; } else { $data_source = ''; } } } } last if $eof_on_response; if (Time::HiRes::time >= $deadline) { die "$scanner_name: exceeded allowed time\n"; } } } if (defined $proc_fh) { $proc_fh->close or $err_on_child=$! } $child_stat = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"%s failed: %s", $scanner_name,$eval_stat); kill_proc($pid,$scanner_name,1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; }; prolong_timer($scanner_name); # restart timer if (length($response_stderr) > 2000) { $response_stderr = substr($response_stderr,0,2000) . '[...]'; } if (proc_status_ok($child_stat,$err_on_child)) { do_log(2, "%s stderr: %s", $scanner_name,$response_stderr) if $response_stderr ne ''; } else { do_log(-1,"%s stderr: %s", $scanner_name,$response_stderr) if $response_stderr ne ''; die "$scanner_name: error running program $cmd: " . exit_status_str($child_stat,$err_on_child) . "\n"; } my($crm114_score); my($j) = index($response,"\n\n"); # find a header/body delimiter $response = substr($response,0,$j+1) if $j >= 0; # keep just a header if ($cmd =~ /\bcrm/ && $response =~ /^\s*([+-]?\d*(?:\.\d*)?)\s*$/) { $crm114_score = $1; $response = ''; # skip the header parsing loop below } my(@response_lines) = split(/^/m, $response, -1); push(@response_lines, "\n", "\n"); # insure a trailing NL and a separator undef $response; my(%header_field, @header_field_name, $curr_head); # scan mail header section retrieved from an external program on its stdout for my $ln (@response_lines) { # guaranteed to contain header/body separator if ($ln =~ /^[ \t]/) { # folded $curr_head .= $ln; } else { # a new header field, process previous if any if (defined $curr_head) { local($1,$2); if ($curr_head=~/^((?:X-DSPAM|X-CRM114)[^:]*?)[ \t]*:[ \t]*(.*)$/s) { my($hn,$hb) = ($1,$2); my($hnlc) = lc($hn); push(@header_field_name, $hn) if !exists($header_field{$hnlc}); $header_field{$hnlc} = $hb; # keep last } } $curr_head = $ln; last if $ln eq "\n"; } } my($spam_score,$spam_tests); my($score_factor) = $self->{options}->{'score_factor'}; my($dspam_result) = $header_field{lc('X-DSPAM-Result')}; if (defined $dspam_result) { if ($dspam_result =~ /\b(signature|result|probability|confidence)=.*;/) { # combined result, split my(%attribute); for my $attr (split(/;\s*/, $dspam_result)) { local($1,$2); my($n,$v) = ($attr =~ /^([^=]*)=(.*)\z/s) ? ($1,$2) : ('user',$attr); $v =~ s/^"//; $v =~ s/"\z//; $attribute{$n} = $v; } # simulate separate header fields @header_field_name = qw(X-DSPAM-Result X-DSPAM-Class X-DSPAM-Confidence X-DSPAM-Probability X-DSPAM-Signature); for my $hn (@header_field_name) { my($hnlc) = lc($hn); my($name) = $hnlc; $name =~ s/^X-DSPAM-//i; $header_field{$hnlc} = $attribute{$name}; } } $dspam_result = $header_field{lc('X-DSPAM-Result')}; my($dspam_signature) = $header_field{lc('X-DSPAM-Signature')}; $dspam_result = '' if !defined $dspam_result; $dspam_signature = '' if !defined $dspam_signature; chomp($dspam_result); chomp($dspam_signature); $dspam_signature = '' if $dspam_signature eq 'N/A'; $msginfo->supplementary_info('DSPAMRESULT', $dspam_result); $msginfo->supplementary_info('DSPAMSIGNATURE', $dspam_signature); $spam_score = $dspam_result eq 'Spam' ? 10 : -1; # fabricated $score_factor = 1 if !defined $score_factor; $spam_score *= $score_factor; $spam_tests = sprintf("%s.%s=%.3f", $scanner_name, $dspam_result, $spam_score); do_log(2,"%s result: %s, score=%.3f, sig=%s", $scanner_name, $dspam_result, $spam_score, $dspam_signature); } my($crm114_status) = $header_field{lc('X-CRM114-Status')}; if (defined $crm114_score || defined $crm114_status) { local($1,$2); if (!defined $crm114_status) { # presumably using --stats_only # fabricate a Status from score $crm114_status = $crm114_score <= -10 ? uc("spam") : $crm114_score >= +10 ? "GOOD" : "UNSURE"; $header_field{lc('X-CRM114-Status')} = sprintf("%s ( %s )", $crm114_status, $crm114_score); @header_field_name = qw(X-CRM114-Status); } elsif ($crm114_status =~ /^([A-Z]+)\s+\(\s+([-\d\.]+)\s+\)/) { $crm114_status = $1; $crm114_score = $2; } my($crm114_cacheid) = $header_field{lc('X-CRM114-CacheID')}; if (defined $crm114_cacheid && $crm114_cacheid =~ /^sfid-\s*$/i) { delete $header_field{lc('X-CRM114-CacheID')}; $crm114_cacheid = undef; } s/[ \t\r\n]+\z// for ($crm114_status, $crm114_score, $crm114_cacheid); $score_factor = -0.10 if !defined $score_factor; $spam_score = $score_factor * $crm114_score; $spam_tests = sprintf("%s.%s(%s)=%.3f", $scanner_name, $crm114_status, $crm114_score, $spam_score); $msginfo->supplementary_info('CRM114STATUS', sprintf("%s ( %s )", $crm114_status,$crm114_score)); $msginfo->supplementary_info('CRM114SCORE', $crm114_score); $msginfo->supplementary_info('CRM114CACHEID', $crm114_cacheid); do_log(2,"%s result: score=%s (%s), status=%s, cacheid=%s", $scanner_name, $spam_score, $crm114_score, $crm114_status, $crm114_cacheid); } my($hdr_edits) = $msginfo->header_edits; my($use_our_hdrs) = cr('prefer_our_added_header_fields'); my($allowed_hdrs) = cr('allowed_added_header_fields'); my($all_local) = !grep(!$_->recip_is_local, @$per_recip_data); for my $hn (@header_field_name) { my($hnlc) = lc($hn); my($hb) = $header_field{$hnlc}; if (defined $hb) { $hb =~ s/[ \t\r\n]+\z//; # trim trailing whitespace and eol do_log(5,"%s: suppl attr: %s = '%s'", $scanner_name,$hn,$hb); $msginfo->supplementary_info($hn,$hb); # add header fields to passed mail for all recipients if ($all_local && $allowed_hdrs && $allowed_hdrs->{$hnlc} && !($use_our_hdrs && $use_our_hdrs->{$hnlc})) { $hdr_edits->add_header($hn,$hb,2); } } } if (defined $spam_score) { for my $r (@$per_recip_data) { $r->spam_level( ($r->spam_level || 0) + $spam_score ); if (!defined($r->spam_tests)) { $r->spam_tests([ \$spam_tests ]); } else { push(@{$r->spam_tests}, \$spam_tests); } } } section_time($scanner_name); } 1; __DATA__ # package Amavis::SpamControl::SpamdClient; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars :sa c cr ca); import Amavis::Util qw(ll do_log sanitize_str min max minmax); import Amavis::rfc2821_2822_Tools qw(qquote_rfc2821_local); import Amavis::Timing qw(section_time); } use Errno qw(ENOENT EACCES); sub new { my($class, $scanner_name,$module,@args) = @_; my(%options) = @args; bless { scanner_name => $scanner_name, options => \%options }, $class; } # expects spamd started like the following: # spamd -H /var/amavis/home -r /var/amavis/home/spamd.pid -s stderr \ # -u vscan -g vscan -x -P --allow-tell --min-children=2 --max-children=2 sub check { my($self,$msginfo) = @_; my($scanner_name) = $self->{scanner_name}; my($which_section); my($spam_level,$sa_tests,$size_limit); my($mbsl) = $self->{options}->{'mail_body_size_limit'}; $mbsl = c('sa_mail_body_size_limit') if !defined $mbsl; if (defined $mbsl) { $size_limit = min(64*1024, $msginfo->orig_header_size) + 1 + min($mbsl, $msginfo->orig_body_size); # don't bother if slightly oversized, it's faster without size checks undef $size_limit if $msginfo->msg_size < $size_limit + 5*1024; } if (defined $size_limit) { do_log(1,"%s: not wasting time on spamd, ". "message longer than %s bytes: %s+%s", $scanner_name, $mbsl, $msginfo->orig_header_size, $msginfo->orig_body_size); } else { # message not too large, do spam checking my($hdr_edits) = $msginfo->header_edits; # fake a local delivery agent by inserting Return-Path $which_section = 'prepare pseudo header section'; my($hdr_prefix) = ''; $hdr_prefix .= sprintf("Return-Path: %s\n", $msginfo->sender_smtp); $hdr_prefix .= sprintf("X-Envelope-To: %s\n", join(",\n ",qquote_rfc2821_local(@{$msginfo->recips}))); my($os_fp) = $msginfo->client_os_fingerprint; $hdr_prefix .= sprintf("X-Amavis-OS-Fingerprint: %s\n", sanitize_str($os_fp)) if defined($os_fp) && $os_fp ne ''; my(@av_tests); my($per_recip_data) = $msginfo->per_recip_data; $per_recip_data = [] if !$per_recip_data; for my $r (@$per_recip_data) { my($spam_tests) = $r->spam_tests; if (defined $spam_tests) { push(@av_tests, grep(/^AV\..+=/, split(/,/, join(',',map($$_,@$spam_tests))))); } } $hdr_prefix .= sprintf("X-Amavis-AV-Status: %s\n", sanitize_str(join(',',@av_tests))) if @av_tests; $hdr_prefix .= sprintf("X-Amavis-PolicyBank: %s\n", c('policy_bank_path')); my($msg) = $msginfo->mail_text; eval { $which_section = 'spamd_connect'; do_log(3,"connecting to spamd"); my($spamd_handle) = Amavis::IO::RW->new( [ '127.0.0.1:783', '[::1]:783' ], Eol => "\015\012", Timeout => 30); defined $spamd_handle or die "Can't connect to spamd, $@ ($!)"; $spamd_handle->timeout(30); section_time($which_section); $which_section = 'spamd_tx'; do_log(4,"sending to spamd"); $hdr_prefix =~ s{\n}{\015\012}gs; my($msgsize) = length($hdr_prefix); # prepended lines... $msgsize += $msginfo->msg_size; # size as defined by RFC 1870 $msgsize -= $msginfo->skip_bytes; # TODO: adjust for CRLF $spamd_handle->print("SYMBOLS SPAMC/1.3\015\012"); # HEADERS $spamd_handle->print("Content-length: " . $msgsize . "\015\012"); $spamd_handle->print("\015\012"); $spamd_handle->print($hdr_prefix); if (!defined($msg)) { # empty mail } elsif ($msg->isa('MIME::Entity')) { # TODO - cont. length won't match! $msg->print_body($spamd_handle); } else { $msg->seek($msginfo->skip_bytes,0) or die "Can't rewind mail file: $!"; my($nbytes,$buff); while (($nbytes=$msg->read($buff,16384)) > 0) { $buff =~ s{\n}{\015\012}gs; $spamd_handle->print($buff); } defined $nbytes or die "Error reading: $!"; } $spamd_handle->flush; $hdr_prefix = undef; section_time($which_section); $which_section = 'spamd_rx'; do_log(4,"receiving from spamd"); my($version,$resp_code,$resp_msg); my(%attr); local($1,$2,$3); my($ln); my($error); my($first) = 1; while (defined($ln = $spamd_handle->get_response_line)) { do_log(4,"from spamd - resp.hdr: %s", $ln); if ($ln eq "\015\012") { last; } elsif ($first) { $first = 0; $ln =~ s/\015\012\z//; ($version,$resp_code,$resp_msg) = split(/[ \t]+/,$ln,3); } elsif ($ln =~ /^([^:]*?)[ \t]*:[ \t]*(.*)\015\012\z/i) { $attr{lc($1)} = $2; } else { $error = $ln } } if ($first) { do_log(-1,"Empty spamd response") } elsif (defined $error) { do_log(-1,"Error in spamd resp: %s",$error) } elsif ($resp_code !~ /^\d+\z/ || $resp_code != 0) { do_log(-1,"Failure reported by spamd: %s %s %s", $version,$resp_code,$resp_msg); } else { my($reply_len) = 0; while (defined($ln = $spamd_handle->get_response_line)) { do_log(5,"from spamd: %s", $ln); $reply_len += length($ln); $ln =~ s/\015\012\z//; $sa_tests = $ln; } do_log(-1,"Reply from spamd size mismatch: %d %s", $reply_len, $attr{'content-length'} ) if $reply_len != $attr{'content-length'}; } $spamd_handle->close; # terminate the session, ignoring status undef $spamd_handle; $spam_level = $2 if $attr{'spam'} =~ m{(\S+) ; (\S+) / (\S+)}; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(-1,"%s client failed: %s", $scanner_name, $eval_stat); }; section_time($which_section); my($score_factor) = $self->{options}->{'score_factor'}; if (defined $spam_level && defined $score_factor) { $spam_level *= $score_factor; } do_log(2,"%s spamd score=%s, tests=%s", $scanner_name, $spam_level, $sa_tests); for my $r (@$per_recip_data) { $r->spam_level( ($r->spam_level || 0) + $spam_level ); if (!defined($r->spam_tests)) { $r->spam_tests([ \$sa_tests ]); } else { push(@{$r->spam_tests}, \$sa_tests); } } } } 1; __DATA__ # package Mail::SpamAssassin::Logger::Amavislog; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); # let a 'require' understand that this module is already loaded: $INC{'Mail/SpamAssassin/Logger/Amavislog.pm'} = 'amavisd'; import Amavis::Util qw(ll do_log); } sub new { my($class,%args) = @_; my(%llmap) = (error => -1, warn => 0, info => 1, dbg => 3); # $args{debug} is a simple boolean, sets the log level floor to 1 when true if ($args{debug}) { for (keys %llmap) { $llmap{$_} = 1 if $llmap{$_} > 1 } } bless { llmap => \%llmap }, $class; } sub close_log { 1 } sub log_message { my($self, $level,$msg) = @_; my($ll) = $self->{llmap}->{$level}; $ll = 1 if !defined $ll; ll($ll) && do_log($ll, "SA %s: %s", $level,$msg); 1; } 1; package Amavis::SpamControl::SpamAssassin; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); import Amavis::Conf qw(:platform :confvars :sa $daemon_user c cr ca); import Amavis::Util qw(ll do_log sanitize_str prolong_timer add_entropy min max minmax get_deadline); import Amavis::ProcControl qw(exit_status_str proc_status_ok kill_proc run_command run_as_subprocess collect_results collect_results_structured); import Amavis::rfc2821_2822_Tools; import Amavis::Timing qw(section_time); import Amavis::Lookup qw(lookup lookup2); import Amavis::IO::FileHandle; } use subs @EXPORT_OK; use Errno qw(ENOENT EACCES EAGAIN EBADF); use FileHandle; use Mail::SpamAssassin; sub getCommonSAModules { my $self = shift; my(@modules) = qw( Mail::SpamAssassin::Locker Mail::SpamAssassin::Locker::Flock Mail::SpamAssassin::Locker::UnixNFSSafe Mail::SpamAssassin::PersistentAddrList Mail::SpamAssassin::DBBasedAddrList Mail::SpamAssassin::AutoWhitelist Mail::SpamAssassin::BayesStore Mail::SpamAssassin::BayesStore::DBM Mail::SpamAssassin::PerMsgLearner Net::DNS::RR::SOA Net::DNS::RR::NS Net::DNS::RR::MX Net::DNS::RR::A Net::DNS::RR::AAAA Net::DNS::RR::PTR Net::DNS::RR::CNAME Net::DNS::RR::DNAME Net::DNS::RR::TXT Net::DNS::RR::SPF Net::CIDR::Lite Sys::Hostname::Long URI URI::Escape URI::Heuristic URI::QueryParam URI::Split URI::URL URI::WithBase URI::_foreign URI::_generic URI::_ldap URI::_login URI::_query URI::_segment URI::_server URI::_userpass URI::_idna URI::_punycode URI::data URI::ftp URI::gopher URI::http URI::https URI::ldap URI::ldapi URI::ldaps URI::mailto URI::mms URI::news URI::nntp URI::pop URI::rlogin URI::rsync URI::rtsp URI::rtspu URI::sip URI::sips URI::snews URI::ssh URI::telnet URI::tn3270 URI::urn URI::urn::oid URI::file URI::file::Base URI::file::Unix URI::file::Win32 ); # DBD::mysql # Mail::SpamAssassin::BayesStore::SQL # Mail::SpamAssassin::SQLBasedAddrList # ??? ArchiveIterator Reporter Getopt::Long Sys::Syslog lib # Net::Ping @modules; } sub getSA2Modules { qw(Mail::SpamAssassin::UnixLocker Mail::SpamAssassin::BayesStoreDBM Mail::SpamAssassin::SpamCopURI ); } sub getSA31Modules { qw( ); # Mail::SpamAssassin::BayesStore::MySQL # Mail::SpamAssassin::BayesStore::PgSQL } sub getSA32Modules { qw(Mail::SpamAssassin::Bayes Mail::SpamAssassin::Bayes::CombineChi Mail::SpamAssassin::Locales Encode::Detect ); # Mail::SpamAssassin::BayesStore::MySQL # Mail::SpamAssassin::BayesStore::PgSQL # /var/db/spamassassin/compiled/.../Mail/SpamAssassin/CompiledRegexps/body_0.pm } sub getSAPlugins { my($self) = @_; my(@modules); my($sa_version_num) = $self->{version_num}; push(@modules, qw(Hashcash RelayCountry SPF URIDNSBL)) if $sa_version_num>=3; push(@modules, qw(DKIM)) if $sa_version_num >= 3.001002; if ($sa_version_num >= 3.001000) { push(@modules, qw( AWL AccessDB AntiVirus AutoLearnThreshold DCC MIMEHeader Pyzor Razor2 ReplaceTags SpamCop TextCat URIDetail WhiteListSubject)); # 'DomainKeys' plugin fell out of fashion with SA 3.2.0, don't load it } if ($sa_version_num >= 3.002000) { push(@modules, qw( BodyEval DNSEval HTMLEval HeaderEval MIMEEval RelayEval URIEval WLBLEval ASN Bayes BodyRuleBaseExtractor Check HTTPSMismatch OneLineBodyRuleType ImageInfo Rule2XSBody Shortcircuit VBounce)); } if ($sa_version_num >= 3.004000) { push(@modules, qw(AskDNS)); } $_ = 'Mail::SpamAssassin::Plugin::'.$_ for @modules; my(%mod_names) = map(($_,1), @modules); # add supporting modules push(@modules, qw(Razor2::Client::Agent)) if $mod_names{'Mail::SpamAssassin::Plugin::Razor2'}; push(@modules, qw(IP::Country::Fast)) if $mod_names{'Mail::SpamAssassin::Plugin::RelayCountry'}; # push(@modules, # qw(Mail::DomainKeys Mail::DomainKeys::Message Mail::DomainKeys::Policy)) # if $mod_names{'Mail::SpamAssassin::Plugin::DomainKeys'}; push(@modules, qw(Mail::DKIM Mail::DKIM::Verifier)) if $mod_names{'Mail::SpamAssassin::Plugin::DKIM'}; push(@modules, qw(Image::Info Image::Info::GIF Image::Info::JPEG Image::Info::PNG Image::Info::BMP Image::Info::TIFF)) if $mod_names{'Mail::SpamAssassin::Plugin::ImageInfo'}; if ($mod_names{'Mail::SpamAssassin::Plugin::SPF'}) { if ($sa_version_num < 3.002000) { # only the old Mail::SPF::Query was supported push(@modules, qw(Mail::SPF::Query)); } else { # SA 3.2.0 supports both the newer Mail::SPF and the old Mail::SPF::Query # but we won't be loading the Mail::SPF::Query push(@modules, qw( Mail::SPF Mail::SPF::Server Mail::SPF::Request Mail::SPF::Mech Mail::SPF::Mech::A Mail::SPF::Mech::PTR Mail::SPF::Mech::All Mail::SPF::Mech::Exists Mail::SPF::Mech::IP4 Mail::SPF::Mech::IP6 Mail::SPF::Mech::Include Mail::SPF::Mech::MX Mail::SPF::Mod Mail::SPF::Mod::Exp Mail::SPF::Mod::Redirect Mail::SPF::SenderIPAddrMech Mail::SPF::v1::Record Mail::SPF::v2::Record auto::NetAddr::IP::_compV6 auto::NetAddr::IP::short auto::NetAddr::IP::full6 auto::NetAddr::IP::Util::inet_n2dx auto::NetAddr::IP::Util::inet_n2ad auto::NetAddr::IP::Util::inet_any2n auto::NetAddr::IP::Util::ipv6_aton NetAddr::IP NetAddr::IP::Util )); } } if ($mod_names{'Mail::SpamAssassin::Plugin::DomainKeys'} || $mod_names{'Mail::SpamAssassin::Plugin::DKIM'}) { push(@modules, qw( Crypt::OpenSSL::RSA auto::Crypt::OpenSSL::RSA::new_public_key auto::Crypt::OpenSSL::RSA::new_key_from_parameters auto::Crypt::OpenSSL::RSA::get_key_parameters auto::Crypt::OpenSSL::RSA::import_random_seed Digest::SHA Error)); } # HTML/HeadParser.pm # do_log(5, "getSAPlugins %s: %s", $sa_version_num, join(', ',@modules)); @modules; } # invoked by a parent process before forking and chrooting # sub loadSpamAssassinModules { my $self = shift; my($sa_version_num) = $self->{version_num}; my(@modules); # modules to be loaded before chroot takes place push(@modules, $self->getCommonSAModules); if (!defined($sa_version_num)) { die "loadSpamAssassinModules: unknown version of Mail::SpamAssassin"; } elsif ($sa_version_num < 3) { push(@modules, $self->getSA2Modules); } elsif ($sa_version_num >= 3.001 && $sa_version_num < 3.002) { push(@modules, $self->getSA31Modules); } elsif ($sa_version_num >= 3.002) { push(@modules, $self->getSA32Modules); } push(@modules, $self->getSAPlugins); my($missing) = Amavis::Boot::fetch_modules('PRE-COMPILE OPTIONAL MODULES', 0, @modules) if @modules; do_log(2, 'INFO: SA version: %s, %.6f, no optional modules: %s', $self->{version}, $sa_version_num, join(' ',@$missing)) if ref $missing && @$missing; } # invoked by a parent process before forking but after chrooting # sub initializeSpamAssassinLogger { my $self = shift; local($1,$2,$3,$4,$5,$6); # just in case if (!Mail::SpamAssassin::Logger->UNIVERSAL::can('add')) { # old SA? } elsif (!Mail::SpamAssassin::Logger::add(method => 'Amavislog', debug => $sa_debug )) { do_log(-1,"Mail::SpamAssassin::Logger::add failed"); } else { # successfully rigged SpamAssassin with our logger Mail::SpamAssassin::Logger::remove('stderr'); # remove default SA logger unshift(@sa_debug_fac, 'info', !$sa_debug ? () : 'all'); } } # invoked by a parent process before forking but after chrooting # sub new_SpamAssassin_instance { my($self,$running_as_parent) = @_; # pick next available number as an instance name my($sa_instance_name) = sprintf('%s', scalar @{$self->{instances}}); do_log(1, "initializing Mail::SpamAssassin (%s)", $sa_instance_name); my($sa_version_num) = $self->{version_num}; my(@new_sa_debug_fac); for my $fac (@sa_debug_fac) { # handle duplicates and negation: foo,nofoo,x,x my($bfac) = $fac; $bfac =~ s/^none\z/noall/i; $bfac =~ s/^no(?=.)//si; @new_sa_debug_fac = grep(!/^(no)?\Q$bfac\E\z/si, @new_sa_debug_fac); push(@new_sa_debug_fac, $fac); } do_log(2,"SpamAssassin debug facilities: %s", join(',',@sa_debug_fac)); my($sa_args) = { debug => !@sa_debug_fac ? undef : \@sa_debug_fac, save_pattern_hits => grep(lc($_) eq 'all', @sa_debug_fac) ? 1 : 0, dont_copy_prefs => 1, require_rules => 1, stop_at_threshold => 0, need_tags => 'TIMING,LANGUAGES,RELAYCOUNTRY,ASN,ASNCIDR', local_tests_only => $sa_local_tests_only, home_dir_for_helpers => $helpers_home, rules_filename => $sa_configpath, site_rules_filename => $sa_siteconfigpath, # LOCAL_STATE_DIR => '/var/lib', # PREFIX => '/usr/local', # DEF_RULES_DIR => '/usr/local/share/spamassassin', # LOCAL_RULES_DIR => '/usr/local/etc/mail/spamassassin', #see man Mail::SpamAssassin for other options }; if ($sa_version_num < 3.001005 && !defined $sa_args->{LOCAL_STATE_DIR}) { $sa_args->{LOCAL_STATE_DIR} = '/var/lib' } # don't ignore sa-update rules local($1,$2,$3,$4,$5,$6); # avoid Perl bug, $1 gets tainted in compile_now my($spamassassin_obj) = Mail::SpamAssassin->new($sa_args); # $Mail::SpamAssassin::DEBUG->{rbl}=-3; # $Mail::SpamAssassin::DEBUG->{rulesrun}=4+64; if ($running_as_parent) { # load SA config files and rules, try to preload most modules $spamassassin_obj->compile_now; } if (ll(2) && !@{$self->{instances}}) { # created the first/main/only SA instance if ($spamassassin_obj->UNIVERSAL::can('get_loaded_plugins_list')) { my(@plugins) = $spamassassin_obj->get_loaded_plugins_list; do_log(2, "SpamAssassin loaded plugins: %s", join(', ', sort map { my($n) = ref $_; $n =~ s/^Mail::SpamAssassin::Plugin:://; $n } @plugins)); # printf STDOUT ("%s\n", join(", ",@plugins)); # not in use: AccessDB AntiVirus TextCat; ASN BodyRuleBaseExtractor # OneLineBodyRuleType Rule2XSBody Shortcircuit } } # provide a default username my $sa_uname = $spamassassin_obj->{username}; if (!defined $sa_uname || $sa_uname eq '') { $spamassassin_obj->{username} = $sa_uname = $daemon_user } $self->{default_username} = $sa_uname if !defined $self->{default_username}; my($sa_instance) = { instance_name => $sa_instance_name, spamassassin_obj => $spamassassin_obj, loaded_user_name => $sa_uname, loaded_user_config => '', conf_backup => undef, conf_backup_additional => {}, }; # remember some initial settings, like %msa_backup in spamd for (qw(username user_dir userstate_dir learn_to_journal)) { if (exists $spamassassin_obj->{$_}) { $sa_instance->{conf_backup_additional}{$_} = $spamassassin_obj->{$_}; } } push(@{$self->{instances}}, $sa_instance); alarm(0); # seems like SA forgets to clear alarm in some cases umask($self->{saved_umask}); # restore our umask, SA clobbered it section_time('SA new'); $sa_instance; } sub new { my($class, $scanner_name,$module,@args) = @_; my(%options) = @args; my($self) = bless { scanner_name => $scanner_name, options => \%options }, $class; $self->{initialized_stage} = 1; $self->{saved_umask} = umask; my($sa_version) = Mail::SpamAssassin->Version; local($1,$2,$3); my($sa_version_num); # turn '3.1.8-pre1' into 3.001008 $sa_version_num = sprintf("%d.%03d%03d", $1,$2,$3) if $sa_version =~ /^(\d+)\.(\d+)(?:\.(\d+))/; # ignore trailing non-digits $self->{version} = $sa_version; $self->{version_num} = $sa_version_num; $self->{default_username} = undef; $self->{instances} = []; $self; } sub init_pre_chroot { my $self = shift; $self->{initialized_stage} == 1 or die "Wrong initialization sequence: " . $self->{initialized_stage}; $self->loadSpamAssassinModules; $self->{initialized_stage} = 2; } sub init_pre_fork { my $self = shift; $self->{initialized_stage} == 2 or die "Wrong initialization sequence: " . $self->{initialized_stage}; $self->initializeSpamAssassinLogger; $self->new_SpamAssassin_instance(1) for (1 .. max(1,$sa_num_instances)); $self->{initialized_stage} = 3; } sub init_child { my $self = shift; $self->{initialized_stage} == 3 or die "Wrong initialization sequence: " . $self->{initialized_stage}; for my $sa_instance (@{$self->{instances}}) { my $spamassassin_obj = $sa_instance->{spamassassin_obj}; next if !$spamassassin_obj; $spamassassin_obj->call_plugins("spamd_child_init"); umask($self->{saved_umask}); # restore our umask, SA may have clobbered it } $self->{initialized_stage} = 4; } sub rundown_child { my $self = shift; for my $sa_instance (@{$self->{instances}}) { my $spamassassin_obj = $sa_instance->{spamassassin_obj}; next if !$spamassassin_obj; do_log(3,'SA rundown_child (%s)', $sa_instance->{instance_name}); $spamassassin_obj->call_plugins("spamd_child_post_connection_close"); umask($self->{saved_umask}); # restore our umask, SA may have clobbered it } $self->{initialized_stage} = 5; } sub call_spamassassin($$$$) { my($self,$msginfo,$lines,$size_limit) = @_; my(@result); my($mail_obj,$per_msg_status); my($which_section) = 'SA prepare'; my($data_representation) = 'GLOB'; # pass data to SA as ARRAY or a GLOB my($saved_pid) = $$; my($sa_version_num) = $self->{version_num}; my($fh) = $msginfo->mail_text; if ($data_representation eq 'ARRAY') { # read mail into memory, bad choice $which_section = 'SA msg read'; my($ln); my($len) = 0; if (defined $size_limit) { $len += length($_) for @$lines } $fh->seek($msginfo->skip_bytes, 0) or die "Can't rewind mail file: $!"; for ($! = 0; defined($ln=<$fh>); $! = 0) { # header section push(@$lines,$ln); if (defined $size_limit) { $len += length($ln); last if $len > $size_limit } last if $ln eq "\n"; } defined $ln || $!==0 or # returning EBADF at EOF is a perl bug $!==EBADF ? do_log(0,"Error reading mail header section: %s", $!) : die "Error reading mail header section: $!"; if (!defined $size_limit) { for ($! = 0; defined($ln=<$fh>); $! = 0) { push(@$lines,$ln) } # body } else { for ($! = 0; defined($ln=<$fh>); $! = 0) { # body push(@$lines,$ln); $len += length($ln); last if $len > $size_limit; } } defined $ln || $!==0 or # returning EBADF at EOF is a perl bug $!==EBADF ? do_log(1,"Error reading mail body: %s", $!) : die "Error reading mail body: $!"; section_time($which_section); } local(*F); my($eval_stat); $which_section = 'SA prelim'; eval { if ($data_representation eq 'GLOB') { # pass mail as a GLOB to SpamAssassin do_log(2,"truncating a message passed to SA at %d bytes, orig %d", $size_limit, $msginfo->msg_size) if defined $size_limit; # present a virtual file to SA, an original mail file prefixed by @$lines tie(*F,'Amavis::IO::FileHandle'); open(F, $fh,$lines,$size_limit) or die "Can't open SA virtual file: $!"; binmode(F) or die "Can't set binmode on a SA virtual file: $!"; } $which_section = 'SA userconf'; my($sa_default_username) = $self->{default_username}; my($per_recip_data) = $msginfo->per_recip_data; $per_recip_data = [] if !$per_recip_data; my($uconf_maps_ref) = ca('sa_userconf_maps'); my($uname_maps_ref) = ca('sa_username_maps'); $uconf_maps_ref = [] if !$uconf_maps_ref; $uname_maps_ref = [] if !$uname_maps_ref; my(%uconf_filename_available); my(%sa_configs_hash); # collects distinct config names and usernames my($uconf_unsupported) = 0; my($r_ind) = 0; for my $r (@$per_recip_data) { my($uconf,$uname); my($recip_addr) = $r->recip_addr; $uconf = lookup2(0, $recip_addr, $uconf_maps_ref) if @$uconf_maps_ref; $uname = lookup2(0, $recip_addr, $uname_maps_ref) if @$uname_maps_ref; $uconf = '' if !defined $uconf; $uname = $sa_default_username if !defined $uname || $uname eq ''; if ($uconf =~ /^sql:/i) { $uconf = $uname eq $sa_default_username ? '' : 'sql:'.$uname; } if ($sa_version_num < 3.00300 && $uconf ne '') { $uconf = ''; $uconf_unsupported = 1 } if ($uconf eq '') { # ok, no special config required, just using a default } elsif ($uconf =~ /^sql:/) { # assume data is in SQL, possibly an empty set } else { $uconf = "$MYHOME/$uconf" if $uconf !~ m{^/}; if ($uconf_filename_available{$uconf}) { # good, already checked and is available, keep it } elsif (defined $uconf_filename_available{$uconf}) { # defined but false, already checked and failed, use a default config $uconf = ''; } else { # check for existence of a SA user configuration/preferences file my(@stat_list) = stat($uconf); # symlinks-friendly my($errn) = @stat_list ? 0 : 0+$!; my($msg) = $errn == ENOENT ? "does not exist" : $errn ? "is inaccessible: $!" : -d _ ? "is a directory" : !-f _ ? "is not a regular file" : !-r _ ? "is not readable" : undef; if (defined $msg) { do_log(1,'SA user config file "%s" %s, ignoring it', $uconf,$msg); $uconf_filename_available{$uconf} = 0; # defined but false $uconf = ''; # ignoring it, use a default config } else { $uconf_filename_available{$uconf} = 1; } } } # collect lists of recipient indices for each unique config/user pair # the %sa_configs_hash is a two-level hash: on $uconf and $uname my($p) = $sa_configs_hash{$uconf}; if (!$p) { $sa_configs_hash{$uconf} = $p = {} } if (!exists $p->{$uname}) { $p->{$uname} = $r_ind } else { $p->{$uname} .= ',' . $r_ind } $r_ind++; } if ($uconf_unsupported) { do_log(5,'SA user config loading unsupported for SA older than 3.3.0'); } # refresh $sa_instance->{loaded_user_name}, just in case for my $sa_instance (@{$self->{instances}}) { my $spamassassin_obj = $sa_instance->{spamassassin_obj}; next if !$spamassassin_obj; my $sa_uname = $spamassassin_obj->{username}; $sa_instance->{loaded_user_name} = defined $sa_uname ? $sa_uname : ''; } my $sa_instance = $self->{instances}[0]; my($curr_conf) = $sa_instance->{loaded_user_config}; my($curr_user) = $sa_instance->{loaded_user_name}; # switching config files is the most expensive, sort to minimize switching my(@conf_names); # a list of config names for which SA needs to be called; # sorted: current first, baseline second, then the rest push(@conf_names, $curr_conf) if exists $sa_configs_hash{$curr_conf}; push(@conf_names, '') if $curr_conf ne '' && exists $sa_configs_hash{''}; push(@conf_names, grep($_ ne '' && $_ ne $curr_conf, keys %sa_configs_hash)); # call SA checking for each distinct SA userprefs config filename and user for my $conf_user_pair (map { my $c = $_; map([$c,$_], keys %{$sa_configs_hash{$c}}) } @conf_names) { my($uconf,$uname) = @$conf_user_pair; # comma-separated list of recip indices which use this SA config my($rind_list) = $sa_configs_hash{$uconf}{$uname}; do_log(5, "SA user config: \"%s\", username: \"%s\", %s", $uconf, $uname, $rind_list); my $sa_instance; if (@{$self->{instances}} <= 1) { # pick the only choice $sa_instance = $self->{instances}[0]; } else { # choosing a suitably-matching SpamAssassin instance my(@sa_instances_matching_uconf, @sa_instances_matching_both, @sa_instances_available); for my $sa_instance (@{$self->{instances}}) { next if !$sa_instance->{spamassassin_obj}; push(@sa_instances_available, $sa_instance); if ($sa_instance->{loaded_user_config} eq $uconf) { push(@sa_instances_matching_uconf, $sa_instance); if ($sa_instance->{loaded_user_name} eq $uname) { push(@sa_instances_matching_both, $sa_instance); } } } my $fit_descr; if (@sa_instances_matching_both) { # just pick the first $sa_instance = $sa_instances_matching_both[0]; $fit_descr = sprintf('exact fit, %d choices', scalar @sa_instances_matching_both); } elsif (@sa_instances_matching_uconf) { # picking one at random my $j = @sa_instances_matching_uconf <= 1 ? 0 : int(rand(scalar(@sa_instances_matching_uconf))); $sa_instance = $sa_instances_available[$j]; $fit_descr = sprintf('good fit: same config, other user, %d choices', scalar @sa_instances_matching_uconf); } elsif ($uconf eq '') { # the first instance is a good choice for switching to a dflt config $sa_instance = $self->{instances}[0]; $fit_descr = 'need a default config, picking first'; } elsif (@sa_instances_available <= 1) { $sa_instance = $sa_instances_available[0]; $fit_descr = 'different config, picking the only one available'; } elsif (@sa_instances_available == 2) { $sa_instance = $sa_instances_available[1]; $fit_descr = 'different config, picking the second one'; } else { # picking one at random, preferably not the first my $j = 1+int(rand(scalar(@sa_instances_available)-1)); $sa_instance = $sa_instances_available[$j]; $fit_descr = 'different config, picking one at random'; } do_log(2,'SA instance chosen (%s), %s', $sa_instance->{instance_name}, $fit_descr); } my $curr_conf = $sa_instance->{loaded_user_config}; my $curr_user = $sa_instance->{loaded_user_name}; my $spamassassin_obj = $sa_instance->{spamassassin_obj}; if ($curr_conf ne '' && $curr_conf ne $uconf) { # revert SA configuration to its initial state $which_section = 'revert_config'; ref $sa_instance->{conf_backup} or die "panic, no conf_backup available"; for (qw(username user_dir userstate_dir learn_to_journal)) { if (exists $sa_instance->{conf_backup_additional}{$_}) { $spamassassin_obj->{$_} = $sa_instance->{conf_backup_additional}{$_}; } else { delete $spamassassin_obj->{$_}; } } # config leaks fixed in SpamAssassin 3.3.0, SA bug 6205, 6003, 4179 $spamassassin_obj->copy_config($sa_instance->{conf_backup}, undef) or die "copy_config: failed to restore"; $sa_instance->{loaded_user_config} = $curr_conf = ''; do_log(5,"SA user config reverted to a saved copy"); section_time($which_section); } if ($uconf ne '' && $uconf ne $curr_conf) { # load SA user configuration/preferences if (!defined $sa_instance->{conf_backup}) { $which_section = 'save_config'; do_log(5,"saving SA user config"); $sa_instance->{conf_backup} = {}; $spamassassin_obj->copy_config(undef, $sa_instance->{conf_backup}) or die "copy_config: failed to save configuration"; section_time($which_section); } $which_section = 'load_config'; # User preferences include scoring options, scores, whitelists # and blacklists, etc, but do not include rule definitions, # privileged settings, etc. unless allow_user_rules is enabled; # and they never include administrator settings if ($uconf =~ /^sql:/) { $uconf eq 'sql:'.$uname or die "panic: loading SA config mismatch: $uname <-> $uconf"; do_log(5,"loading SA user config from SQL %s", $uname); $spamassassin_obj->load_scoreonly_sql($uname); } else { do_log(5,"loading SA user config file %s", $uconf); $spamassassin_obj->read_scoreonly_config($uconf); } $sa_instance->{loaded_user_config} = $curr_conf = $uconf; section_time($which_section); } if ($uname ne $curr_user) { $which_section = 'SA switch_user'; do_log(5,'switching SA (%s) username "%s" -> "%s"', $sa_instance->{instance_name}, $curr_user, $uname); $spamassassin_obj->signal_user_changed({ username => $uname }); $sa_instance->{loaded_user_name} = $curr_user = $uname; section_time($which_section); } ll(3) && do_log(3, "calling SA parse (%s), SA vers %s, %.6f, ". "data as %s, recips_ind [%s]%s%s", $sa_instance->{instance_name}, $self->{version}, $sa_version_num, $data_representation, $rind_list, ($uconf eq '' ? '' : ", conf: \"$uconf\""), ($uname eq '' ? '' : ", user: \"$uname\"") ); if ($data_representation eq 'GLOB') { seek(F,0,0) or die "Can't rewind a SA virtual file: $!"; } my($data) = $data_representation eq 'ARRAY' ? $lines : \*F; $spamassassin_obj->timer_reset if $spamassassin_obj->UNIVERSAL::can('timer_reset'); $which_section = 'SA parse'; my($remaining_time, $deadline) = get_deadline('SA check', 1, 5); my(%suppl_attrib) = ( 'return_path' => $msginfo->sender_smtp, 'recipients' => [ map(qquote_rfc2821_local($_->recip_addr), @$per_recip_data[split(/,/, $rind_list)]) ], 'originating' => $msginfo->originating ? 1 : 0, 'message_size' => $msginfo->msg_size, !c('enable_dkim_verification') ? () : ('dkim_signatures' => $msginfo->dkim_signatures_all), !defined $deadline ? () : ('master_deadline' => $deadline), 'rule_hits' => [ # known options: rule, area, score, value, ruletype, tflags, descr # { rule=>'AM:TEST1', score=>0.11 }, # { rule=>'TESTTEST', defscore=>0.22, descr=>'my test' }, !defined $size_limit ? () : { rule=>'__TRUNCATED', score=>-0.1, area=>'RAW: ', tflags=>'nice', descr=>"Message size truncated to $size_limit B" }, ], 'amavis_policy_bank_path' => c('policy_bank_path'), ); $mail_obj = $sa_version_num >= 3 ? $spamassassin_obj->parse($data,0,\%suppl_attrib) : Mail::SpamAssassin::NoMailAudit->new(data=>$data, add_From_line=>0); section_time($which_section); $which_section = 'SA check'; if (@conf_names <= 1) { do_log(4,"CALLING SA check (%s)", $sa_instance->{instance_name}); } else { do_log(4,"CALLING SA check (%s) for recips: %s", $sa_instance->{instance_name}, join(", ", @{$suppl_attrib{'recipients'}})); } { local($1,$2,$3,$4,$5,$6); # avoid Perl 5.8.x bug, $1 gets tainted $per_msg_status = $spamassassin_obj->check($mail_obj); } do_log(4,"DONE SA check (%s)", $sa_instance->{instance_name}); section_time($which_section); $which_section = 'SA collect'; my($spam_level,$spam_report,$spam_summary,%supplementary_info); { local($1,$2,$3,$4,$5,$6); # avoid Perl 5.8.x taint bug if ($sa_version_num < 3) { $spam_level = $per_msg_status->get_hits; $supplementary_info{'TESTSSCORES'} = $supplementary_info{'TESTS'} = $per_msg_status->get_names_of_tests_hit; } else { $spam_level = $per_msg_status->get_score; for my $t (qw(TESTS TESTSSCORES ADDEDHEADERHAM ADDEDHEADERSPAM AUTOLEARN AUTOLEARNSCORE SC SCRULE SCTYPE LANGUAGES RELAYCOUNTRY ASN ASNCIDR DCCB DCCR DCCREP DKIMDOMAIN DKIMIDENTITY AWLSIGNERMEAN CRM114STATUS CRM114SCORE CRM114CACHEID)) { $supplementary_info{$t} = $per_msg_status->get_tag($t); } } { # fudge my($crm114_status) = $supplementary_info{'CRM114STATUS'}; my($crm114_score) = $supplementary_info{'CRM114SCORE'}; if (defined $crm114_status && defined $crm114_score) { $supplementary_info{'CRM114STATUS'} = sprintf("%s ( %s )", $crm114_status,$crm114_score); } } $spam_summary = $per_msg_status->get_report; # taints $1 and $2 ! # $spam_summary = $per_msg_status->get_tag('SUMMARY'); $spam_report = $per_msg_status->get_tag('REPORT'); # do the fetching of a TIMING tag last: $supplementary_info{'TIMING'} = $per_msg_status->get_tag('TIMING'); } # section_time($which_section); # don't bother reporting separately, short $which_section = 'SA check finish'; if (defined $per_msg_status) { $per_msg_status->finish; undef $per_msg_status } if (defined $mail_obj) { $mail_obj->finish if $sa_version_num >= 3; undef $mail_obj } # section_time($which_section); # don't bother reporting separately, short # returning the result as a data structure instead of modifying # the $msginfo objects directly is used to make it possible to run # this subroutine as a subprocess; modifications to $msginfo objects # would be lost if done in a context of a spawned process push(@result, { recip_ind_list => $rind_list, user_config => $uconf, spam_level => $spam_level, spam_report => $spam_report, spam_summary => $spam_summary, supplementary_info => \%supplementary_info, }); } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; $which_section = 'SA finish'; if (defined $per_msg_status) # just in case { $per_msg_status->finish; undef $per_msg_status } if (defined $mail_obj) # just in case { $mail_obj->finish if $sa_version_num >= 3; undef $mail_obj } if ($data_representation eq 'GLOB') { close(F) or die "Can't close SA virtual file: $!"; untie(*F); } umask($self->{saved_umask}); # restore our umask, SA may have clobbered it if ($$ != $saved_pid) { eval { do_log(-2,"PANIC, SA checking produced a clone process ". "of [%s], CLONE [%s] SELF-TERMINATING", $saved_pid,$$) }; POSIX::_exit(6); # avoid END and destructor processing } # section_time($which_section); if (defined $eval_stat) { chomp $eval_stat; die $eval_stat } # resignal \@result; } sub check { my($self,$msginfo) = @_; $self->{initialized_stage} == 4 or die "Wrong initialization sequence: " . $self->{initialized_stage}; my($which_section); my($prefix) = ''; my($spam_level,$sa_tests,$spam_report,$spam_summary,$supplementary_info_ref); my($fh) = $msginfo->mail_text; my($hdr_edits) = $msginfo->header_edits; my($score_factor) = $self->{options}->{'score_factor'}; my($mbsl) = $self->{options}->{'mail_body_size_limit'}; $mbsl = c('sa_mail_body_size_limit') if !defined $mbsl; my($size_limit); if (defined $mbsl) { $size_limit = min(64*1024, $msginfo->orig_header_size) + 1 + min($mbsl, $msginfo->orig_body_size); # don't bother if slightly oversized, it's faster without size checks if ($msginfo->msg_size < $size_limit + 5*1024) { undef $size_limit } } # fake a local delivery agent by inserting a Return-Path $prefix .= sprintf("Return-Path: %s\n", $msginfo->sender_smtp); $prefix .= sprintf("X-Envelope-To: %s\n", join(",\n ",qquote_rfc2821_local(@{$msginfo->recips}))); my($os_fp) = $msginfo->client_os_fingerprint; $prefix .= sprintf("X-Amavis-OS-Fingerprint: %s\n", sanitize_str($os_fp)) if defined($os_fp) && $os_fp ne ''; my(@av_tests); for my $r (@{$msginfo->per_recip_data}) { my($spam_tests) = $r->spam_tests; if ($spam_tests) { push(@av_tests, grep(/^AV[.:].+=/, split(/,/, join(',',map($$_,@$spam_tests))))); } } $prefix .= sprintf("X-Amavis-AV-Status: %s\n", sanitize_str(join(',',@av_tests))) if @av_tests; $prefix .= sprintf("X-Amavis-PolicyBank: %s\n", c('policy_bank_path')); $prefix .= sprintf("X-Amavis-MessageSize: %d%s\n", $msginfo->msg_size, !defined $size_limit ? '' : ", TRUNCATED to $size_limit"); for my $hf_name (qw( X-CRM114-Status X-CRM114-CacheID X-CRM114-Notice X-CRM114-Action X-DSPAM-Result X-DSPAM-Class X-DSPAM-Signature X-DSPAM-Processed X-DSPAM-Confidence X-DSPAM-Probability X-DSPAM-User X-DSPAM-Factors)) { my($suppl_attr_val) = $msginfo->supplementary_info($hf_name); if (defined $suppl_attr_val && $suppl_attr_val ne '') { chomp $suppl_attr_val; $prefix .= sprintf("%s: %s\n", $hf_name, sanitize_str($suppl_attr_val)); } } $which_section = 'SA call'; my($proc_fh,$pid); my($eval_stat); my($results_aref); eval { # NOTE ON TIMEOUTS: SpamAssassin may use timer for its own purpose, # disabling it before returning. It seems it only uses timer when # external tests are enabled. local $SIG{ALRM} = sub { my($s) = Carp::longmess("SA TIMED OUT, backtrace:"); # crop at some rather arbitrary limit if (length($s) > 900) { $s = substr($s,0,900-3) . '[...]' } do_log(-1,"%s",$s); }; prolong_timer('spam_scan_sa_pre', 1, 4); # restart the timer # # note: array @lines at this point contains only prepended synthesized # header fields, but may be extended in sub call_spamassassin() by # reading-in the rest of the message; this may or may not happen in # a separate process (called through run_as_subprocess or directly); # lines must each be terminated by a \n character, which must be the # only \n in a line; # my(@lines) = split(/^/m, $prefix, -1); $prefix = undef; if (!$sa_spawned) { $results_aref = call_spamassassin($self,$msginfo,\@lines,$size_limit); } else { ($proc_fh,$pid) = run_as_subprocess(\&call_spamassassin, $self,$msginfo,\@lines,$size_limit); my($results,$child_stat) = collect_results_structured($proc_fh,$pid,'spawned SA',200*1024); $results_aref = $results->[0] if defined $results; } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; section_time($which_section) if $sa_spawned; $which_section = 'SA done'; prolong_timer('spam_scan_sa'); # restart the timer if ($results_aref) { # for each group of recipients using the same SA userconf file for my $h (@$results_aref) { my($rind_list) = $h->{recip_ind_list}; my(@r_list) = @{$msginfo->per_recip_data}[split(/,/,$rind_list)]; my($uconf) = $h->{user_config}; $spam_level = $h->{spam_level}; $spam_report = $h->{spam_report}; $spam_summary = $h->{spam_summary}; $supplementary_info_ref = $h->{supplementary_info}; $supplementary_info_ref = {} if !$supplementary_info_ref; $sa_tests = $supplementary_info_ref->{'TESTSSCORES'}; add_entropy($spam_level,$sa_tests); if (defined $spam_level && defined $score_factor) { $spam_level *= $score_factor; } do_log(3,"spam_scan: score=%s autolearn=%s tests=[%s] recips=%s", $spam_level, $supplementary_info_ref->{'AUTOLEARN'}, $sa_tests, $rind_list); my(%sa_tests_h); if (defined $sa_tests && $sa_tests ne 'none') { for my $t (split(/,[ \t]*/, $sa_tests)) { my($test_name,$score) = split(/=/, $t, 2); $sa_tests_h{$test_name} = $score; } } my($dkim_adsp_suppress) = 0; if (exists $sa_tests_h{'DKIM_ADSP_DISCARD'}) { # must honour ADSP 'discardable', suppress a bounce do_log(2,"spam_scan: dsn_suppress_reason DKIM_ADSP_DISCARD"); $dkim_adsp_suppress = 1; } for my $r (@r_list) { $r->spam_level( ($r->spam_level || 0) + $spam_level ); $r->spam_report($spam_report); $r->spam_summary($spam_summary); if (!defined($r->spam_tests)) { $r->spam_tests([ \$sa_tests ]); } else { # comes last: here we use push, unlike elsewhere which may do unshift push(@{$r->spam_tests}, \$sa_tests); } if ($dkim_adsp_suppress) { $r->dsn_suppress_reason('DKIM_ADSP_DISCARD' . !defined $_ ? '' : ", $_") for $r->dsn_suppress_reason; } } } } if (defined($msginfo->spam_report) || defined($msginfo->spam_summary)) { $spam_report = $msginfo->spam_report . ', ' . $spam_report if $msginfo->spam_report ne ''; $spam_summary = $msginfo->spam_summary . "\n\n" . $spam_summary if $msginfo->spam_summary ne ''; } $msginfo->spam_report($spam_report); $msginfo->spam_summary($spam_summary); for (keys %$supplementary_info_ref) { $msginfo->supplementary_info($_, $supplementary_info_ref->{$_}) } if (defined $eval_stat) { # SA timed out? kill_proc($pid,'a spawned SA',1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; chomp $eval_stat; do_log(-2, "SA failed: %s", $eval_stat); # die "$eval_stat\n" if $eval_stat !~ /timed out\b/; } } 1; __DATA__ # package Amavis::Unpackers; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&init &decompose_part &determine_file_types); import Amavis::Util qw(untaint min max minmax ll do_log snmp_count prolong_timer rmdir_recursively add_entropy); import Amavis::ProcControl qw(exit_status_str proc_status_ok run_command kill_proc collect_results collect_results_structured); import Amavis::Conf qw(:platform :confvars $file c cr ca); import Amavis::Timing qw(section_time); import Amavis::Lookup qw(lookup lookup2); import Amavis::Unpackers::MIME qw(mime_decode); import Amavis::Unpackers::NewFilename qw(consumed_bytes); } use subs @EXPORT_OK; use Errno qw(ENOENT EACCES EINTR EAGAIN); use POSIX qw(SIGALRM); use IO::File qw(O_CREAT O_EXCL O_WRONLY); use Time::HiRes (); use File::Basename qw(basename); use Convert::TNEF; # avoid an exploitable security hole in Convert::UUlib 1.04 and older! use Convert::UUlib 1.05 qw(:constants); # 1.08 or newer is preferred! use Compress::Zlib 1.35; # avoid security vulnerability in <= 1.34 use Archive::Zip 1.14 qw(:CONSTANTS :ERROR_CODES); # recursively descend into a directory $dir containing potentially unsafe # files with unpredictable names, soft links, etc., rename each regular # nonempty file to a directory $outdir giving it a generated name, # and discard all the rest, including the directory $dir. # Return a pair: number of bytes that 'sanitized' files now occupy, # and a number of parts-objects created. # sub flatten_and_tidy_dir($$$;$$); # prototype sub flatten_and_tidy_dir($$$;$$) { my($dir, $outdir, $parent_obj, $item_num_offset, $orig_names) = @_; do_log(4, 'flatten_and_tidy_dir: processing directory "%s"', $dir); my($consumed_bytes) = 0; my($item_num) = 0; my($parent_placement) = $parent_obj->mime_placement; chmod(0750, $dir) or die "Can't change protection of \"$dir\": $!"; local(*DIR); opendir(DIR,$dir) or die "Can't open directory \"$dir\": $!"; # modifying a directory while traversing it can cause surprises, avoid; # avoid slurping the whole directory contents into memory my($f, @rmfiles, @renames, @recurse); while (defined($f = readdir(DIR))) { next if $f eq '.' || $f eq '..'; my($msg); my($fname) = $dir . '/' . $f; my(@stat_list) = lstat($fname); my($errn) = @stat_list ? 0 : 0+$!; if ($errn == ENOENT) { $msg = "does not exist" } elsif ($errn) { $msg = "inaccessible: $!" } if (defined $msg) { die "flatten_and_tidy_dir: \"$fname\" $msg," } add_entropy(@stat_list); my($newpart_obj) = Amavis::Unpackers::Part->new($outdir,$parent_obj); $item_num++; $newpart_obj->mime_placement(sprintf("%s/%d", $parent_placement, $item_num+$item_num_offset) ); # save tainted original member name if available, or a tainted file name my($original_name) = !ref($orig_names) ? undef : $orig_names->{$f}; $newpart_obj->name_declared(defined $original_name ? $original_name : $f); # untaint, but if $dir happens to still be tainted, we want to know and die $fname = $dir . '/' . untaint($f); if (-d _) { $newpart_obj->attributes_add('D'); push(@recurse, $fname); } elsif (-l _) { $newpart_obj->attributes_add('L'); push(@rmfiles, [$fname, 'soft link']); } elsif (!-f _) { $newpart_obj->attributes_add('S'); push(@rmfiles, [$fname, 'nonregular file']); } elsif (-z _) { push(@rmfiles, [$fname, 'empty file']); } else { chmod(0750, $fname) or die "Can't change protection of file \"$fname\": $!"; my($size) = 0 + (-s _); $newpart_obj->size($size); $consumed_bytes += $size; my($newpart) = $newpart_obj->full_name; push(@renames, [$fname, $newpart, $original_name]); } } closedir(DIR) or die "Error closing directory \"$dir\": $!"; my($cnt_u) = scalar(@rmfiles); for my $pair (@rmfiles) { my($fname,$what) = @$pair; do_log(5,'flatten_and_tidy_dir: deleting %s "%s"', $what,$fname); unlink($fname) or die "Can't remove $what \"$fname\": $!"; } undef @rmfiles; my($cnt_r) = scalar(@renames); for my $tuple (@renames) { my($fname,$newpart,$original_name) = @$tuple; ll(5) && do_log(5,'flatten_and_tidy_dir: renaming "%s"%s to %s', $fname, !defined $original_name ? '' : " ($original_name)", $newpart); rename($fname,$newpart) or die "Can't rename \"$fname\" to $newpart: $!"; } undef @renames; for my $fname (@recurse) { do_log(5,'flatten_and_tidy_dir: descending into subdir "%s"', $fname); my($bytes,$cnt) = flatten_and_tidy_dir($fname, $outdir, $parent_obj, $item_num+$item_num_offset, $orig_names); $consumed_bytes += $bytes; $item_num += $cnt; } rmdir($dir) or die "Can't remove directory \"$dir\": $!"; section_time("ren$cnt_r-unl$cnt_u-files$item_num"); ($consumed_bytes, $item_num); } # call 'file(1)' utility for each part, # and associate (save) full and short file content types with each part # sub determine_file_types($$) { my($tempdir, $partslist_ref) = @_; $file ne '' or die "Unix utility file(1) not available, but is needed"; my(@all_part_list) = grep($_->exists, @$partslist_ref); my($initial_num_parts) = scalar(@all_part_list); my($cwd) = "$tempdir/parts"; if (@all_part_list) { chdir($cwd) or die "Can't chdir to $cwd: $!" } my($proc_fh,$pid); my($eval_stat); eval { while (@all_part_list) { my(@part_list,@file_list); # collect reasonably small subset of filenames my($arglist_size) = length($file); # size of a command name itself while (@all_part_list) { # collect as many args as safe, at least one my($nm) = $all_part_list[0]->full_name; local($1); $nm =~ s{^\Q$cwd\E/(.*)\z}{$1}s; # remove cwd from filename # POSIX requires 4 kB as a minimum buffer size for program arguments last if @file_list && $arglist_size + length($nm) + 1 > 4000; push(@part_list, shift(@all_part_list)); # swallow the next one push(@file_list, $nm); $arglist_size += length($nm) + 1; } if (scalar(@file_list) < $initial_num_parts) { do_log(2, "running file(1) on %d (out of %d) files, arglist size %d", scalar(@file_list), $initial_num_parts, $arglist_size); } else { do_log(5, "running file(1) on %d files, arglist size %d", scalar(@file_list), $arglist_size); } ($proc_fh,$pid) = run_command(undef, '&1', $file, @file_list); my($index) = 0; my($ln); for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { do_log(5, "result line from file(1): %s", $ln); chomp($ln); local($1,$2); if ($index > $#file_list) { do_log(-1,"NOTICE: Skipping unexpected output from file(1): %s",$ln); } else { my($part) = $part_list[$index]; # walk through @part_list in sync my($expect) = $file_list[$index]; # walk through @file_list in sync if ($ln !~ /^(\Q$expect\E):[ \t]*(.*)\z/s) { # split file name from type do_log(-1,"NOTICE: Skipping bad output from file(1) ". "at [%d, %s], got: %s", $index,$expect,$ln); } else { my($type_short); my($actual_name) = $1; my($type_long) = $2; $type_short = lookup2(0,$type_long,\@map_full_type_to_short_type_maps); ll(4) && do_log(4, "File-type of %s: %s%s", $part->base_name, $type_long, (!defined $type_short ? '' : !ref $type_short ? "; ($type_short)" : '; (' . join(', ',@$type_short) . ')' ) ); $part->type_long($type_long); $part->type_short($type_short); $part->attributes_add('C') # simpleminded if !ref($type_short) ? $type_short eq 'pgp' # encrypted? : grep($_ eq 'pgp', @$type_short); $index++; } } } defined $ln || $!==0 || $!==EAGAIN or die "Error reading from file(1) utility: $!"; do_log(-1,"unexpected(file): %s",$!) if !defined($ln) && $!==EAGAIN; my($err) = 0; $proc_fh->close or $err = $!; my($child_stat) = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; my(@errmsg); # exit status is 1 when result is 'ERROR: ...', accept it mercifully proc_status_ok($child_stat,$err, 0,1) or push(@errmsg, "failed, ".exit_status_str($child_stat,$err)); if ($index < @part_list) { push(@errmsg, sprintf("parsing failure - missing last %d results", @part_list - $index)); } !@errmsg or die join(", ",@errmsg); # even though exit status 1 is accepted, log a warning nevertheless proc_status_ok($child_stat,$err) or do_log(-1, "file utility failed: %s", exit_status_str($child_stat,$err)); } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; kill_proc($pid,$file,1,$proc_fh,$eval_stat) if defined $pid; }; chdir($tempdir) or die "Can't chdir to $tempdir: $!"; section_time(sprintf('get-file-type%d', $initial_num_parts)); if (defined $eval_stat) { do_log(-2, "file(1) utility (%s) FAILED: %s", $file,$eval_stat); # die "file(1) utility ($file) error: $eval_stat"; } } sub decompose_mail($$) { my($tempdir,$file_generator_object) = @_; my($hold); my(@parts); my($depth) = 1; my($any_undecipherable) = 0; my($which_section) = "parts_decode"; # fetch all not-yet-visited part names, and start a new cycle TIER: while (@parts = @{$file_generator_object->parts_list}) { if ($MAXLEVELS > 0 && $depth > $MAXLEVELS) { $hold = "Maximum decoding depth ($MAXLEVELS) exceeded"; last; } $file_generator_object->parts_list_reset; # new cycle of names # clip to avoid very long log entries my(@chopped_parts) = @parts > 5 ? @parts[0..4] : @parts; ll(4) && do_log(4,"decode_parts: level=%d, #parts=%d : %s", $depth, scalar(@parts), join(', ', (map($_->base_name, @chopped_parts)), (@chopped_parts >= @parts ? () : "...")) ); for my $part (@parts) { # test for existence of all expected files my($fname) = $part->full_name; my($errn) = 0; if ($fname eq '') { $errn = ENOENT } else { my(@stat_list) = lstat($fname); if (@stat_list) { add_entropy(@stat_list) } else { $errn = 0+$! } } if ($errn == ENOENT) { $part->exists(0); # $part->type_short('no-file') if !defined $part->type_short; } elsif ($errn) { die "decompose_mail: inaccessible file $fname: $!"; } elsif (!-f _) { # not a regular file my($what) = -l _ ? 'symlink' : -d _ ? 'directory' : 'non-regular file'; do_log(-1, "WARN: decompose_mail: removing unexpected %s %s", $what,$fname); if (-d _) { rmdir_recursively($fname) } else { unlink($fname) or die "Can't delete $what $fname: $!" } $part->exists(0); $part->type_short(-l _ ? 'symlink' : -d _ ? 'dir' : 'special') if !defined $part->type_short; } elsif (-z _) { # empty file unlink($fname) or die "Can't remove \"$fname\": $!"; $part->exists(0); $part->type_short('empty') if !defined $part->type_short; $part->type_long('empty') if !defined $part->type_long; } else { $part->exists(1); } } determine_file_types($tempdir, \@parts); for my $part (@parts) { if ($part->exists && !defined($hold)) { $hold = decompose_part($part, $tempdir) } $any_undecipherable++ if grep($_ eq 'U', @{ $part->attributes || [] }); } last TIER if defined $hold; $depth++; } section_time($which_section); prolong_timer($which_section); ($hold, $any_undecipherable); } # Decompose one part # sub decompose_part($$) { my($part, $tempdir) = @_; # possible return values from eval: # 0 - truly atomic or unknown or archiver failure; consider atomic # 1 - some archive, successfully unpacked, result replaces original # 2 - probably unpacked, but keep the original (eg self-extracting archive) my($hold); my($eval_stat); my($sts) = 0; my($any_called) = 0; eval { my($type_short) = $part->type_short; my(@ts) = !defined $type_short ? () : !ref $type_short ? ($type_short) : @$type_short; if (@ts) { # when one or more short types are known snmp_count("OpsDecType-".join('.',@ts)); for my $dec_tuple (@{ca('decoders')}) { # first matching decoder wins next if !defined $dec_tuple; my($dec_ts,$code,@args) = @$dec_tuple; if ($code && grep($_ eq $dec_ts, @ts)) { $any_called = 1; $sts = &$code($part,$tempdir,@args); last } } } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; my($ll) = -1; if ($eval_stat =~ /\bExceeded storage quota\b.*\bbytes by/ || $eval_stat =~ /\bMaximum number of files\b.*\bexceeded/) { $hold = $eval_stat; $ll = 1; } do_log($ll,"Decoding of %s (%s) failed, leaving it unpacked: %s", $part->base_name, $part->type_long, $eval_stat); $sts = 2; # keep the original, along with possible decoded files }; if ($any_called) { chdir($tempdir) or die "Can't chdir to $tempdir: $!"; # just in case } if ($sts == 1 && lookup2(0,$part->type_long,\@keep_decoded_original_maps)) { # don't trust this file type or unpacker, # keep both the original and the unpacked file ll(4) && do_log(4,"file type is %s, retain original %s", $part->type_long, $part->base_name); $sts = 2; # keep the original, along with possible decoded files } if ($sts == 1) { ll(5) && do_log(5,"decompose_part: deleting %s", $part->full_name); unlink($part->full_name) or die sprintf("Can't unlink %s: %s", $part->full_name, $!); } ll(4) && do_log(4,"decompose_part: %s - %s", $part->base_name, ['atomic','archive, unpacked','source retained']->[$sts]); section_time('decompose_part') if $any_called; die $eval_stat if $eval_stat =~ /^timed out\b/; # resignal timeout $hold; } # a trivial wrapper around mime_decode() to adjust arguments and result # sub do_mime_decode($$) { my($part, $tempdir) = @_; mime_decode($part,$tempdir,$part); 2; # probably unpacked, but keep the original mail }; # # Uncompression/unarchiving routines # Possible return codes: # 0 - truly atomic or unknown or archiver failure; consider atomic # 1 - some archiver format, successfully unpacked, result replaces original # 2 - probably unpacked, but keep the original (eg self-extracting archive) # if ASCII text, try multiple decoding methods as provided by UUlib # (uuencoded, xxencoded, BinHex, yEnc, Base64, Quoted-Printable) # sub do_ascii($$) { my($part, $tempdir) = @_; ll(4) && do_log(4,"do_ascii: Decoding part %s", $part->base_name); snmp_count('OpsDecByUUlibAttempt'); # prevent uunconc.c/UUDecode() from trying to create temp file in '/' my($old_env_tmpdir) = $ENV{TMPDIR}; $ENV{TMPDIR} = "$tempdir/parts"; my($any_errors) = 0; my($any_decoded) = 0; alarm(0); # stop the timer local($SIG{ALRM}); my($sigset,$action,$oldaction); if ($] < 5.008) { # in old Perl signals could be delivered at any time $SIG{ALRM} = sub { die "timed out\n" }; } elsif ($] < 5.008001) { # Perl 5.8.0 # 5.8.0 does not have POSIX::SigAction::safe but uses safe signals, which # means a runaway uulib can't be aborted; tough luck, upgrade your Perl! $SIG{ALRM} = sub { die "timed out\n" }; # old way, but won't abort } else { # Perl >= 5.8.0 has 'safe signals', and SigAction::safe available # POSIX::sigaction can bypass safe Perl signals on request; # alternatively, use Perl module Sys::SigAction $sigset = POSIX::SigSet->new(SIGALRM); $oldaction = POSIX::SigAction->new; $action = POSIX::SigAction->new(sub { die "timed out\n" }, $sigset, &POSIX::SA_RESETHAND); $action->safe(1); POSIX::sigaction(SIGALRM,$action,$oldaction) or die "Can't set ALRM handler: $!"; do_log(4,"do_ascii: Setting sigaction handler, was %d", $oldaction->safe); } my($eval_stat); eval { # must not go away without calling Convert::UUlib::CleanUp ! my($sts,$count); prolong_timer('do_ascii_pre'); # restart timer $sts = Convert::UUlib::Initialize(); $sts = 0 if !defined $sts; # avoid Use of uninit. value in numeric eq (==) $sts==RET_OK or die "Convert::UUlib::Initialize failed: ". Convert::UUlib::strerror($sts); my($uulib_version) = Convert::UUlib::GetOption(OPT_VERSION); !Convert::UUlib::SetOption(OPT_IGNMODE,1) or die "bad uulib OPT_IGNMODE"; # !Convert::UUlib::SetOption(OPT_DESPERATE,1) or die "bad uulib OPT_DESPERATE"; if (defined $action) { $action->safe(0); # bypass safe Perl signals POSIX::sigaction(SIGALRM,$action) or die "Can't set ALRM handler: $!"; } # may take looong time on malformed messages, allow it to be interrupted ($sts, $count) = Convert::UUlib::LoadFile($part->full_name); if (defined $action) { $action->safe(1); # re-establish safe signal handling POSIX::sigaction(SIGALRM,$action) or die "Can't set ALRM handler: $!"; } if ($sts != RET_OK) { my($errmsg) = Convert::UUlib::strerror($sts) . ": $!"; $errmsg .= ", (???" . Convert::UUlib::strerror(Convert::UUlib::GetOption(OPT_ERRNO))."???)" if $sts == RET_IOERR; die "Convert::UUlib::LoadFile (uulib V$uulib_version) failed: $errmsg"; } ll(4) && do_log(4,"do_ascii: Decoding part %s (%d items), uulib V%s", $part->base_name, $count, $uulib_version); my($uu); my($item_num) = 0; my($parent_placement) = $part->mime_placement; for (my($j) = 0; $uu = Convert::UUlib::GetFileListItem($j); $j++) { $item_num++; ll(4) && do_log(4, "do_ascii(%d): state=0x%02x, enc=%s%s, est.size=%s, name=%s", $j, $uu->state, Convert::UUlib::strencoding($uu->uudet), ($uu->mimetype ne '' ? ", mimetype=" . $uu->mimetype : ''), $uu->size, $uu->filename); if (!($uu->state & FILE_OK)) { $any_errors = 1; do_log(1,"do_ascii: Convert::UUlib info: %s not decodable, %s", $j,$uu->state); } else { my($newpart_obj)=Amavis::Unpackers::Part->new("$tempdir/parts",$part); $newpart_obj->mime_placement("$parent_placement/$item_num"); $newpart_obj->name_declared($uu->filename); my($newpart) = $newpart_obj->full_name; if (defined $action) { $action->safe(0); # bypass safe Perl signals POSIX::sigaction(SIGALRM,$action) or die "Can't set ALRM handlr: $!"; } $! = 0; $sts = $uu->decode($newpart); # decode to file $newpart my($err_decode) = "$!"; if (defined $action) { $action->safe(1); # re-establish safe signal handling POSIX::sigaction(SIGALRM,$action) or die "Can't set ALRM handlr: $!"; } chmod(0750, $newpart) or $! == ENOENT # chmod, don't panic if no file or die "Can't change protection of \"$newpart\": $!"; my($statmsg); my($errn) = lstat($newpart) ? 0 : 0+$!; if ($errn == ENOENT) { $statmsg = "does not exist" } elsif ($errn) { $statmsg = "inaccessible: $!" } elsif ( -l _) { $statmsg = "is a symlink" } elsif ( -d _) { $statmsg = "is a directory" } elsif (!-f _) { $statmsg = "not a regular file" } if (defined $statmsg) { $statmsg = "; file status: $newpart $statmsg" } my($size) = 0 + (-s _); $newpart_obj->size($size); consumed_bytes($size, 'do_ascii'); if ($sts == RET_OK && $errn==0) { $any_decoded = 1; do_log(4,"do_ascii: RET_OK%s", $statmsg) if defined $statmsg; } elsif ($sts == RET_NODATA || $sts == RET_NOEND) { $any_errors = 1; do_log(-1,"do_ascii: Convert::UUlib error: %s%s", Convert::UUlib::strerror($sts), $statmsg); } else { $any_errors = 1; my($errmsg) = Convert::UUlib::strerror($sts) . ":: $err_decode"; $errmsg .= ", " . Convert::UUlib::strerror( Convert::UUlib::GetOption(OPT_ERRNO) ) if $sts == RET_IOERR; die("Convert::UUlib failed: " . $errmsg . $statmsg); } } } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('do_ascii'); # restart timer if (defined $oldaction) { POSIX::sigaction(SIGALRM,$oldaction) or die "Can't restore ALRM handler: $!"; } Convert::UUlib::CleanUp(); snmp_count('OpsDecByUUlib') if $any_decoded; if (defined $old_env_tmpdir) { $ENV{TMPDIR} = $old_env_tmpdir } else { delete $ENV{TMPDIR} } if (defined $eval_stat) { chomp $eval_stat; die "do_ascii: $eval_stat\n" } $any_errors ? 2 : $any_decoded ? 1 : 0; } # use Archive-Zip # sub do_unzip($$;$$) { my($part, $tempdir, $archiver_dummy, $testing_for_sfx) = @_; ll(4) && do_log(4, "Unzipping %s", $part->base_name); # avoid DoS vulnerability in < 2.017, CVE-2009-1391 # Compress::Raw::Zlib->VERSION(2.017); # module not loaded snmp_count('OpsDecByArZipAttempt'); my($zip) = Archive::Zip->new; my(@err_nm) = qw(AZ_OK AZ_STREAM_END AZ_ERROR AZ_FORMAT_ERROR AZ_IO_ERROR); my($retval) = 1; # need to set up a temporary minimal error handler Archive::Zip::setErrorHandler(sub { return 5 }); my($sts) = $zip->read($part->full_name); Archive::Zip::setErrorHandler(sub { die @_ }); my($any_unsupp_compmeth,$any_zero_length); my($encryptedcount,$extractedcount) = (0,0); if ($sts != AZ_OK) { # not a zip? corrupted zip file? other errors? if ($testing_for_sfx && $sts == AZ_FORMAT_ERROR) { # a normal status for executable that is not a self extracting archive do_log(4, "do_unzip: ok, exe is not a zip sfx: %s (%s)", $err_nm[$sts], $sts); } else { do_log(-1, "do_unzip: not a zip: %s (%s)", $err_nm[$sts], $sts); # $part->attributes_add('U'); # perhaps not, it flags as **UNCHECKED** too # # many bounces containing chopped-off zip } $retval = 0; } else { my($item_num) = 0; my($parent_placement) = $part->mime_placement; for my $mem ($zip->members) { my($newpart_obj) = Amavis::Unpackers::Part->new("$tempdir/parts",$part); $item_num++; $newpart_obj->mime_placement("$parent_placement/$item_num"); $newpart_obj->name_declared($mem->fileName); my($compmeth) = $mem->compressionMethod; if ($compmeth!=COMPRESSION_DEFLATED && $compmeth!=COMPRESSION_STORED) { $any_unsupp_compmeth = $compmeth; $newpart_obj->attributes_add('U'); } elsif ($mem->isEncrypted) { $encryptedcount++; $newpart_obj->attributes_add('U','C'); } elsif ($mem->isDirectory) { $newpart_obj->attributes_add('D'); } else { # want to read uncompressed - set to COMPRESSION_STORED my($oldc) = $mem->desiredCompressionMethod(COMPRESSION_STORED); $sts = $mem->rewindData; $sts == AZ_OK or die sprintf("%s: error rew. member data: %s (%s)", $part->base_name, $err_nm[$sts], $sts); my($newpart) = $newpart_obj->full_name; my($outpart) = IO::File->new; # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] $outpart->open($newpart, untaint(O_CREAT|O_EXCL|O_WRONLY), 0640) or die "Can't create file $newpart: $!"; binmode($outpart) or die "Can't set file $newpart to binmode: $!"; my($size) = 0; while ($sts == AZ_OK) { my($buf_ref); ($buf_ref, $sts) = $mem->readChunk; $sts == AZ_OK || $sts == AZ_STREAM_END or die sprintf("%s: error reading member: %s (%s)", $part->base_name, $err_nm[$sts], $sts); my($buf_len) = length($$buf_ref); if ($buf_len > 0) { $size += $buf_len; $outpart->print($$buf_ref) or die "Can't write to $newpart: $!"; consumed_bytes($buf_len, 'do_unzip'); } } $any_zero_length = 1 if $size == 0; $newpart_obj->size($size); $outpart->close or die "Error closing $newpart: $!"; $mem->desiredCompressionMethod($oldc); $mem->endRead; $extractedcount++; } } snmp_count('OpsDecByArZip'); } if ($any_unsupp_compmeth) { $retval = 2; do_log(-1, "do_unzip: %s, unsupported compr. method: %s", $part->base_name, $any_unsupp_compmeth); } elsif ($any_zero_length) { # possible zip vulnerability exploit $retval = 2; do_log(1, "do_unzip: %s, members of zero length, archive retained", $part->base_name); } elsif ($encryptedcount) { $retval = 2; do_log(1, "do_unzip: %s, %d members are encrypted, %s extracted, archive retained", $part->base_name, $encryptedcount, !$extractedcount ? 'none' : $extractedcount); } $retval; } # use external decompressor program from the compress/gzip/bzip2/xz family # sub do_uncompress($$$) { my($part, $tempdir, $decompressor) = @_; ll(4) && do_log(4,"do_uncompress %s by %s", $part->base_name,$decompressor); my($decompressor_name) = basename((split(' ',$decompressor))[0]); snmp_count("OpsDecBy\u${decompressor_name}"); my($newpart_obj) = Amavis::Unpackers::Part->new("$tempdir/parts",$part); $newpart_obj->mime_placement($part->mime_placement."/1"); my($newpart) = $newpart_obj->full_name; my($type_short, $name_declared) = ($part->type_short, $part->name_declared); local($1); my(@rn); # collect recommended file names push(@rn,$1) if $part->type_long =~ /^\S+\s+compressed data, was "(.+)"(\z|, from\b)/; for my $name_d (!ref $name_declared ? ($name_declared) : @$name_declared) { next if $name_d eq ''; my($name) = $name_d; for (!ref $type_short ? ($type_short) : @$type_short) { $_ eq 'F' and $name=~s/\.F\z//; $_ eq 'Z' and $name=~s/\.Z\z// || $name=~s/\.tg?z\z/.tar/; $_ eq 'gz' and $name=~s/\.gz\z// || $name=~s/\.tgz\z/.tar/; $_ eq 'bz' and $name=~s/\.bz\z// || $name=~s/\.tbz\z/.tar/; $_ eq 'bz2' and $name=~s/\.bz2?\z// || $name=~s/\.tbz\z/.tar/; $_ eq 'xz' and $name=~s/\.xz\z// || $name=~s/\.txz\z/.tar/; $_ eq 'lzo' and $name=~s/\.lzo\z//; $_ eq 'rpm' and $name=~s/\.rpm\z/.cpio/; } push(@rn,$name) if !grep($_ eq $name, @rn); } $newpart_obj->name_declared(@rn==1 ? $rn[0] : \@rn) if @rn; my($proc_fh,$pid); my($retval) = 1; prolong_timer('do_uncompress_pre'); # restart timer my($eval_stat); eval { ($proc_fh,$pid) = run_command($part->full_name, '/dev/null', split(' ',$decompressor)); my($rv,$err) = run_command_copy($newpart,$proc_fh,$pid); # may die undef $proc_fh; undef $pid; if (!proc_status_ok($rv,$err)) { # unlink($newpart) or die "Can't unlink $newpart: $!"; my($msg) = sprintf('Error running decompressor %s on %s, %s', $decompressor, $part->base_name, exit_status_str($rv,$err)); # bzip2 and gzip use status 2 as a warning about corrupted file if (proc_status_ok($rv,$err, 2)) {do_log(0,"%s",$msg)} else {die $msg} } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('do_uncompress'); # restart timer if (defined $eval_stat) { $retval = 0; chomp $eval_stat; kill_proc($pid,$decompressor,1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; die "do_uncompress: $eval_stat\n"; # propagate failure } $retval; } # use Compress::Zlib to inflate # sub do_gunzip($$) { my($part, $tempdir) = @_; my($retval) = 0; do_log(4, "Inflating gzip archive %s", $part->base_name); snmp_count('OpsDecByZlib'); my($gz) = Amavis::IO::Zlib->new; $gz->open($part->full_name,'rb') or die("do_gunzip: Can't open gzip file ".$part->full_name.": $!"); my($newpart_obj) = Amavis::Unpackers::Part->new("$tempdir/parts",$part); $newpart_obj->mime_placement($part->mime_placement."/1"); my($newpart) = $newpart_obj->full_name; my($outpart) = IO::File->new; # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] $outpart->open($newpart, untaint(O_CREAT|O_EXCL|O_WRONLY), 0640) or die "Can't create file $newpart: $!"; binmode($outpart) or die "Can't set file $newpart to binmode: $!"; my($nbytes,$buff); my($size) = 0; while (($nbytes=$gz->read($buff,16384)) > 0) { $outpart->print($buff) or die "Can't write to $newpart: $!"; $size += $nbytes; consumed_bytes($nbytes, 'do_gunzip'); } my($err) = defined $nbytes ? 0 : $!; $newpart_obj->size($size); $outpart->close or die "Error closing $newpart: $!"; undef $buff; # release storage my(@rn); # collect recommended file name my($name_declared) = $part->name_declared; for my $name_d (!ref $name_declared ? ($name_declared) : @$name_declared) { next if $name_d eq ''; my($name) = $name_d; $name=~s/\.(gz|Z)\z// || $name=~s/\.tgz\z/.tar/; push(@rn,$name) if !grep($_ eq $name, @rn); } $newpart_obj->name_declared(@rn==1 ? $rn[0] : \@rn) if @rn; if (defined $nbytes && $nbytes==0) { $retval = 1 } # success else { do_log(-1, "do_gunzip: Error reading file %s: %s", $part->full_name,$err); unlink($newpart) or die "Can't unlink $newpart: $!"; $newpart_obj->size(undef); $retval = 0; } $gz->close or die "Error closing gzipped file: $!"; $retval; } # DROPED SUPPORT for Archive::Tar; main drawback of this module is: it either # loads an entire tar into memory (horrors!), or when using extract_archive() # it does not relativize absolute paths (which makes it possible to store # members in any directory writable by uid), and does not provide a way to # capture contents of members with the same name. Use pax program instead! # #use Archive::Tar; #sub do_tar($$) { # my($part, $tempdir) = @_; # snmp_count('OpsDecByArTar'); # # Work around bug in Archive-Tar # my $tar = eval { Archive::Tar->new($part->full_name) }; # if (!defined($tar)) { # chomp $@; # do_log(4, "Faulty archive %s: %s", $part->full_name, $@); # die $@ if $@ =~ /^timed out\b/; # resignal timeout # return 0; # } # do_log(4,"Untarring %s", $part->base_name); # my($item_num) = 0; my($parent_placement) = $part->mime_placement; # my(@list) = $tar->list_files; # for (@list) { # next if m{/\z}; # ignore directories # # this is bad (reads whole file into scalar) # # need some error handling, too # my $data = $tar->get_content($_); # my($newpart_obj) = Amavis::Unpackers::Part->new("$tempdir/parts",$part); # $item_num++; $newpart_obj->mime_placement("$parent_placement/$item_num"); # my($newpart) = $newpart_obj->full_name; # my($outpart) = IO::File->new; # $outpart->open($newpart, O_CREAT|O_EXCL|O_WRONLY, 0640) # or die "Can't create file $newpart: $!"; # binmode($outpart) or die "Can't set file $newpart to binmode: $!"; # $outpart->print($data) or die "Can't write to $newpart: $!"; # $newpart_obj->size(length($data)); # consumed_bytes(length($data), 'do_tar'); # $outpart->close or die "Error closing $newpart: $!"; # } # 1; #} # use external program to expand 7-Zip archives # sub do_7zip($$$;$) { my($part, $tempdir, $archiver, $testing_for_sfx) = @_; ll(4) && do_log(4, "Expanding 7-Zip archive %s", $part->base_name); my($decompressor_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${decompressor_name}Attempt"); my($last_line); my($bytes) = 0; my($mem_cnt) = 0; my($retval) = 1; my($proc_fh,$pid); my($fn) = $part->full_name; prolong_timer('do_7zip_pre'); # restart timer my($eval_stat); eval { ($proc_fh,$pid) = run_command(undef, '&1', $archiver, 'l', '-slt', "-w$tempdir/parts", '--', $fn); my($ln); my($name,$size,$attr); my($entries_cnt) = 0; for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { $last_line = $ln if $ln !~ /^\s*$/; # keep last nonempty line chomp($ln); local($1); if ($ln =~ /^\s*\z/) { if (defined $name || defined $size) { do_log(5,'do_7zip: member: %s "%s", %s bytes', $attr,$name,$size); if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } if (defined $size && $size > 0) { $bytes += $size; $mem_cnt++ } } undef $name; undef $size; undef $attr; } elsif ($ln =~ /^Path = (.*)\z/s) { $name = $1 } elsif ($ln =~ /^Size = ([0-9]+)\z/s) { $size = $1 } elsif ($ln =~ /^Attributes = (.*)\z/s) { $attr = $1 } } defined $ln || $!==0 || $!==EAGAIN or die "Error reading (1): $!"; do_log(-1,"unexpected(do_7zip_1): %s",$!) if !defined($ln) && $!==EAGAIN; if (defined $name || defined $size) { do_log(5,'do_7zip: member: %s "%s", %s bytes', $attr,$name,$size); if (defined $size && $size > 0) { $bytes += $size; $mem_cnt++ } } # consume all remaining output to avoid broken pipe for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { $last_line = $ln if $ln !~ /^\s*$/ } defined $ln || $!==0 || $!==EAGAIN or die "Error reading (2): $!"; do_log(-1,"unexpected(do_7zip_2): %s",$!) if !defined($ln) && $!==EAGAIN; my($err) = 0; $proc_fh->close or $err = $!; my($rv) = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; local($1,$2); if (proc_status_ok($rv,$err,1) && $mem_cnt > 0 && $bytes > 0) { # just warn do_log(4,"do_7zip: warning, %s", exit_status_str($rv,$err)); } elsif (!proc_status_ok($rv,$err)) { die("can't get a list of archive members: " . exit_status_str($rv,$err) ."; ".$last_line); } if ($mem_cnt > 0 || $bytes > 0) { consumed_bytes($bytes, 'do_7zip-pre', 1); # pre-check on estimated size snmp_count("OpsDecBy\u${decompressor_name}"); ($proc_fh,$pid) = run_command(undef, '&1', $archiver, 'x', '-bd', '-y', "-w$tempdir/parts", "-o$tempdir/parts/7zip", '--', $fn); collect_results($proc_fh,$pid,$archiver,16384,[0,1]); undef $proc_fh; undef $pid; my($errn) = lstat("$tempdir/parts/7zip") ? 0 : 0+$!; if ($errn != ENOENT) { my($b) = flatten_and_tidy_dir("$tempdir/parts/7zip", "$tempdir/parts", $part); consumed_bytes($b, 'do_7zip'); } } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('do_7zip'); # restart timer if (defined $eval_stat) { $retval = 0; chomp $eval_stat; kill_proc($pid,$archiver,1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; # if ($testing_for_sfx) { die "do_7zip: $eval_stat" } # else { do_log(-1, "do_7zip: %s", $eval_stat) }; die "do_7zip: $eval_stat\n" # propagate failure } $retval; } # use external program to expand RAR archives # sub do_unrar($$$;$) { my($part, $tempdir, $archiver, $testing_for_sfx) = @_; ll(4) && do_log(4, "Expanding RAR archive %s", $part->base_name); my($decompressor_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${decompressor_name}Attempt"); # unrar exit codes: SUCCESS=0, WARNING=1, FATAL_ERROR=2, CRC_ERROR=3, # LOCK_ERROR=4, WRITE_ERROR=5, OPEN_ERROR=6, USER_ERROR=7, MEMORY_ERROR=8, # CREATE_ERROR=9, USER_BREAK=255 my(@list); my($hypcount) = 0; my($encryptedcount) = 0; my($lcnt) = 0; my($member_name); my($bytes) = 0; my($last_line); my($item_num) = 0; my($parent_placement) = $part->mime_placement; my($retval) = 1; my($fn) = $part->full_name; my($proc_fh,$pid); my(@common_rar_switches) = qw(-c- -p- -idcdp); # -av- prolong_timer('do_unrar_pre'); # restart timer my($eval_stat); eval { ($proc_fh,$pid) = run_command(undef, '&1', $archiver, 'v',@common_rar_switches,'--',$fn); # jump hoops because there is no simple way to just list all the files my($ln); my($entries_cnt) = 0; for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { $last_line = $ln if $ln !~ /^\s*$/; # keep last nonempty line chomp; if ($ln =~ /^unexpected end of archive/) { last; } elsif ($ln =~ /^------/) { $hypcount++; last if $hypcount >= 2; } elsif ($hypcount < 1 && $ln =~ /^Encrypted file:/) { do_log(4,"do_unrar: %s", $ln); $part->attributes_add('U','C'); } elsif ($hypcount == 1) { $lcnt++; local($1,$2,$3); if ($lcnt % 2 == 0) { # information line (every other line) if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } if ($ln !~ /^\s+(\d+)\s+(\d+)\s+(\d+%|-->|<--|<->)/) { do_log($testing_for_sfx ? 4 : -1, "do_unrar: can't parse info line for \"%s\" %s", $member_name,$ln); } elsif (defined $member_name) { do_log(5,'do_unrar: member: "%s", size: %s', $member_name,$1); if ($1 > 0) { $bytes += $1; push(@list, $member_name) } } undef $member_name; } elsif ($ln =~ /^(.)(.*)\z/s) { $member_name = $2; # all but the first character (space or '*') if ($1 eq '*') { # member is encrypted $encryptedcount++; $item_num++; # make a phantom entry - carrying only name and attributes my($newpart_obj) = Amavis::Unpackers::Part->new("$tempdir/parts",$part); $newpart_obj->mime_placement("$parent_placement/$item_num"); $newpart_obj->name_declared($member_name); $newpart_obj->attributes_add('U','C'); undef $member_name; # makes no sense extracting encrypted files } } } } defined $ln || $!==0 || $!==EAGAIN or die "Error reading (1): $!"; do_log(-1,"unexpected(unrar_1): %s",$!) if !defined($ln) && $!==EAGAIN; $ln = undef; # consume all remaining output to avoid broken pipe for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { $last_line = $ln if $ln !~ /^\s*$/ } defined $ln || $!==0 || $!==EAGAIN or die "Error reading (2): $!"; do_log(-1,"unexpected(unrar_2): %s",$!) if !defined($ln) && $!==EAGAIN; my($err) = 0; $proc_fh->close or $err = $!; my($rv) = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; local($1,$2); if (proc_status_ok($rv,$err, 7)) { # USER_ERROR die printf("perhaps this %s does not recognize switches ". "-av- and -idcdp, it is probably too old. Upgrade: %s", $archiver, 'http://www.rarlab.com/'); } elsif (proc_status_ok($rv,$err, 3)) { # CRC_ERROR # NOTE: password protected files in the archive cause CRC_ERROR do_log(4,"do_unrar: CRC_ERROR - undecipherable, %s", exit_status_str($rv,$err)); $part->attributes_add('U'); } elsif (proc_status_ok($rv,$err, 1) && @list && $bytes > 0) { # WARNING, probably still ok do_log(4,"do_unrar: warning, %s", exit_status_str($rv,$err)); } elsif (!proc_status_ok($rv,$err)) { die("can't get a list of archive members: " . exit_status_str($rv,$err) ."; ".$last_line); } elsif (!$bytes && $last_line =~ /^\Q$fn\E is not RAR archive$/) { chomp($last_line); die $last_line; } elsif ($last_line !~ /^\s*(\d+)\s+(\d+)/s) { do_log(-1,"do_unrar: unable to obtain orig total size: %s", $last_line); } else { do_log(4,"do_unrar: summary size: %d, sum of sizes: %d", $2,$bytes) if abs($bytes - $2) > 100; $bytes = $2 if $2 > $bytes; } consumed_bytes($bytes, 'do_unrar-pre', 1); # pre-check on estimated size if (!@list) { do_log(4,"do_unrar: no archive members, or not an archive at all"); if ($testing_for_sfx) { return 0 } else { $part->attributes_add('U') } } else { snmp_count("OpsDecBy\u${decompressor_name}"); # unrar/rar can make a dir by itself, but can't hurt (sparc64 problem?) mkdir("$tempdir/parts/rar", 0750) or die "Can't mkdir $tempdir/parts/rar: $!"; ($proc_fh,$pid) = run_command(undef, '&1', $archiver, qw(x -inul -ver -o- -kb), @common_rar_switches, '--', $fn, "$tempdir/parts/rar/"); collect_results($proc_fh,$pid,$archiver,16384, [0,1,3] ); # one of: SUCCESS, WARNING, CRC undef $proc_fh; undef $pid; my($errn) = lstat("$tempdir/parts/rar") ? 0 : 0+$!; if ($errn != ENOENT) { my($b) = flatten_and_tidy_dir("$tempdir/parts/rar", "$tempdir/parts", $part); consumed_bytes($b, 'do_unrar'); } } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('do_unrar'); # restart timer if ($encryptedcount) { do_log(1, "do_unrar: %s, %d members are encrypted, %s extracted, archive retained", $part->base_name, $encryptedcount, !@list ? 'none' : scalar(@list) ); $retval = 2; } if (defined $eval_stat) { $retval = 0; chomp $eval_stat; kill_proc($pid,$archiver,1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; # if ($testing_for_sfx) { die "do_unrar: $eval_stat" } # else { do_log(-1, "do_unrar: %s", $eval_stat) }; die "do_unrar: $eval_stat\n" # propagate failure } $retval; } # use external program to expand LHA archives # sub do_lha($$$;$) { my($part, $tempdir, $archiver, $testing_for_sfx) = @_; ll(4) && do_log(4, "Expanding LHA archive %s", $part->base_name); my($decompressor_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${decompressor_name}Attempt"); # lha needs extension .exe to understand SFX! # the downside is that in this case it only sees MS files in an archive my($fn) = $part->full_name; symlink($fn, $fn.".exe") or die sprintf("Can't symlink %s %s.exe: %s", $fn, $fn, $!); my(@list); my(@checkerr); my($retval) = 1; my($proc_fh,$pid); prolong_timer('do_lha_pre'); # restart timer my($eval_stat); eval { # ($proc_fh,$pid) = run_command(undef, '&1', $archiver, 'lq', $fn); ($proc_fh,$pid) = run_command(undef, '&1', $archiver, 'lq', $fn.".exe"); my($ln); my($entries_cnt) = 0; for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { chomp($ln); local($1); if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } if ($ln =~ m{/\z}) { # ignore directories } elsif ($ln =~ /^LHa: (Warning|Fatal error): /) { push(@checkerr,$ln) if @checkerr < 3; } elsif ($ln=~m{^(?:\S+\s+\d+/\d+|.{23})(?:\s+\S+){5}\s*(\S.*?)\s*\z}s) { my($name) = $1; $name = $1 if $name =~ m{^(.*) -> (.*)\z}s; # symlink push(@list, $name); } else { do_log(5,"do_lha: skip: %s", $ln) } } defined $ln || $!==0 || $!==EAGAIN or die "Error reading: $!"; do_log(-1,"unexpected(do_lha): %s",$!) if !defined($ln) && $!==EAGAIN; my($err) = 0; $proc_fh->close or $err = $!; my($child_stat) = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; if (!proc_status_ok($child_stat,$err) || @checkerr) { die('(' . join(", ",@checkerr) .') ' .exit_status_str($child_stat,$err)); } elsif (!@list) { $part->attributes_add('U') if !$testing_for_sfx; die "no archive members, or not an archive at all"; } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('do_lha'); # restart timer if (defined $eval_stat) { unlink($fn.".exe") or do_log(-1, "Can't unlink %s.exe: %s", $fn,$!); $retval = 0; chomp $eval_stat; kill_proc($pid,$archiver,1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; # if ($testing_for_sfx) { die "do_lha: $eval_stat" } # else { do_log(-1, "do_lha: %s", $eval_stat) }; die "do_lha: $eval_stat\n"; # propagate failure } else { # preliminary archive traversal done, now extract files snmp_count("OpsDecBy\u${decompressor_name}"); my($rv); eval { # store_mgr may die, make sure we unlink the .exe file $rv = store_mgr($tempdir, $part, \@list, $archiver, 'pq', $fn.".exe"); 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; unlink($fn.".exe") or do_log(-1, "Can't unlink %s.exe: %s", $fn,$!); if (defined $eval_stat) { die "do_lha: $eval_stat\n" } # propagate failure $rv==0 or die exit_status_str($rv); } $retval; } # use external program to expand ARC archives; # works with original arc, or a GPL licensed 'nomarch' # (http://rus.members.beeb.net/nomarch.html) # sub do_arc($$$) { my($part, $tempdir, $archiver) = @_; my($decompressor_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${decompressor_name}"); my($is_nomarch) = $archiver =~ /nomarch/i; ll(4) && do_log(4,"Unarcing %s, using %s", $part->base_name, ($is_nomarch ? "nomarch" : "arc") ); my($cmdargs) = ($is_nomarch ? '-l -U' : 'ln') . ' ' . $part->full_name; my($proc_fh,$pid) = run_command(undef, '/dev/null', $archiver, split(' ',$cmdargs)); my(@list); my($ln); my($entries_cnt) = 0; for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } push(@list,$ln); } defined $ln || $!==0 || $!==EAGAIN or die "Error reading: $!"; do_log(-1,"unexpected(do_arc): %s",$!) if !defined($ln) && $!==EAGAIN; my($err) = 0; $proc_fh->close or $err = $!; my($child_stat) = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; proc_status_ok($child_stat,$err) or do_log(-1, 'do_arc: %s',exit_status_str($child_stat,$err)); #*** no spaces in filenames allowed??? local($1); map { s/^([^ \t\r\n]*).*\z/$1/s } @list; # keep only filenames if (@list) { # store_mgr may die, allow failure to propagate my($rv) = store_mgr($tempdir, $part, \@list, $archiver, ($is_nomarch ? ('-p', '-U') : 'p'), $part->full_name); do_log(-1, 'arc %', exit_status_str($rv)) if $rv; } 1; } # use external program to expand ZOO archives # sub do_zoo($$$) { my($part, $tempdir, $archiver) = @_; my($is_unzoo) = $archiver =~ m{\bunzoo[^/]*\z}i ? 1 : 0; ll(4) && do_log(4,"Expanding ZOO archive %s, using %s", $part->base_name, ($is_unzoo ? "unzoo" : "zoo") ); my($decompressor_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${decompressor_name}"); my(@list); my($separ_count) = 0; my($bytes) = 0; my($ln,$last_line); my($retval) = 1; my($fn) = $part->full_name; my($proc_fh,$pid); symlink($fn, "$fn.zoo") # Zoo needs extension of .zoo! or die sprintf("Can't symlink %s %s.zoo: %s", $fn,$fn,$!); prolong_timer('do_zoo_pre'); # restart timer my($eval_stat); my($entries_cnt) = 0; eval { ($proc_fh,$pid) = run_command(undef, '&1', $archiver, $is_unzoo ? qw(-l) : qw(l), "$fn.zoo"); for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { $last_line = $ln if $ln !~ /^\s*$/; # keep last nonempty line if ($ln =~ /^------/) { $separ_count++ } elsif ($separ_count == 1) { local($1,$2); if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } if ($ln !~ /^\s*(\d+)(?:\s+\S+){6}\s+(?:[0-7]{3,})?\s*(.*)$/) { do_log(3,"do_zoo: can't parse line %s", $ln); } else { do_log(5,'do_zoo: member: "%s", size: %s', $2,$1); if ($1 > 0) { $bytes += $1; push(@list,$2) } } } } defined $ln || $!==0 || $!==EAGAIN or die "Error reading: $!"; do_log(-1,"unexpected(do_zoo): %s",$!) if !defined($ln) && $!==EAGAIN; my($err) = 0; $proc_fh->close or $err = $!; my($rv) = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; local($1); if (!proc_status_ok($rv,$err)) { die("can't get a list of archive members: " . exit_status_str($rv,$err) ."; ".$last_line); } elsif ($last_line !~ /^\s*(\d+)\s+\d+%\s+\d+/s) { do_log(-1,"do_zoo: unable to obtain orig total size: %s", $last_line); } else { do_log(4,"do_zoo: summary size: %d, sum of sizes: %d", $1,$bytes) if abs($bytes - $1) > 100; $bytes = $1 if $1 > $bytes; } consumed_bytes($bytes, 'do_zoo-pre', 1); # pre-check on estimated size $retval = 0 if @list; if (!$is_unzoo) { # unzoo cannot cleanly extract to stdout without prepending a clutter # store_mgr may die my($rv) = store_mgr($tempdir,$part,\@list,$archiver,'xpqqq:',"$fn.zoo"); do_log(-1,"do_zoo (store_mgr) %s", exit_status_str($rv)) if $rv; } else { # this code section can handle zoo and unzoo # but zoo is unsafe in this mode (and so is unzoo, a little less so) my($cwd) = "$tempdir/parts/zoo"; mkdir($cwd, 0750) or die "Can't mkdir $cwd: $!"; chdir($cwd) or die "Can't chdir to $cwd: $!"; # don't use "-j ./" in unzoo, it does not protect from relative paths! # "-j X" is less bad, but: "unzoo: 'X/h/user/01.lis' cannot be created" ($proc_fh,$pid) = run_command(undef, '&1', $archiver, $is_unzoo ? qw(-x -j X) : qw(x), "$fn.zoo", $is_unzoo ? '*;*' : () ); collect_results($proc_fh,$pid,$archiver,16384,[0]); undef $proc_fh; undef $pid; my($b) = flatten_and_tidy_dir("$tempdir/parts/zoo", "$tempdir/parts", $part); consumed_bytes($b, 'do_zoo'); } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('do_zoo'); # restart timer if (defined $eval_stat) { $retval = 0; chomp $eval_stat; kill_proc($pid,$archiver,1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; do_log(-1,"do_zoo: %s", $eval_stat); } chdir($tempdir) or die "Can't chdir to $tempdir: $!"; unlink("$fn.zoo") or die "Can't unlink $fn.zoo: $!"; if (defined $eval_stat) { die "do_zoo: $eval_stat\n" } # propagate failure $retval; } # use external program to expand ARJ archives # sub do_unarj($$$;$) { my($part, $tempdir, $archiver, $testing_for_sfx) = @_; do_log(4, "Expanding ARJ archive %s", $part->base_name); my($decompressor_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${decompressor_name}Attempt"); # options to arj, ignored by unarj # provide some password in -g to turn fatal error into 'bad password' error $ENV{ARJ_SW} = "-i -jo -b5 -2h -jyc -ja1 -gsecret -w$tempdir/parts"; # unarj needs extension of .arj! my($fn) = $part->full_name; symlink($part->full_name, $fn.".arj") or die sprintf("Can't symlink %s %s.arj: %s", $fn, $fn, $!); my($retval) = 1; my($proc_fh,$pid); prolong_timer('do_unarj_pre'); # restart timer my($eval_stat); eval { # obtain total original size of archive members from the index/listing ($proc_fh,$pid) = run_command(undef, '&1', $archiver, 'l', $fn.".arj"); my($last_line); my($ln); for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { $last_line = $ln if $ln !~ /^\s*$/ } defined $ln || $!==0 || $!==EAGAIN or die "Error reading (1): $!"; do_log(-1,"unexpected(do_unarj_1): %s",$!) if !defined($ln) && $!==EAGAIN; my($err) = 0; $proc_fh->close or $err = $!; my($rv) = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; if (!proc_status_ok($rv,$err, 0,1,3)) { # one of: success, warn, CRC err $part->attributes_add('U') if !$testing_for_sfx; die "not an ARJ archive? ".exit_status_str($rv,$err); } elsif ($last_line =~ /^\Q$fn\E.arj is not an ARJ archive$/) { die "last line: $last_line"; } elsif ($last_line !~ /^\s*(\d+)\s*files\s*(\d+)/s) { $part->attributes_add('U') if !$testing_for_sfx; die "unable to obtain orig size of files: $last_line, ". exit_status_str($rv,$err); } else { consumed_bytes($2, 'do_unarj-pre', 1); # pre-check on estimated size } # unarj has very limited extraction options, arj is much better! mkdir("$tempdir/parts/arj",0750) or die "Can't mkdir $tempdir/parts/arj: $!"; chdir("$tempdir/parts/arj") or die "Can't chdir to $tempdir/parts/arj: $!"; snmp_count("OpsDecBy\u${decompressor_name}"); ($proc_fh,$pid) = run_command(undef, '&1', $archiver, 'e', $fn.".arj"); my($encryptedcount,$skippedcount) = (0,0); my($entries_cnt) = 0; for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } $encryptedcount++ if $ln =~ /^(Extracting.*\bBad file data or bad password|File is password encrypted, Skipped)\b/s; $skippedcount++ if $ln =~ /(\bexists|^File is password encrypted|^Unsupported .*), Skipped\b/s; } defined $ln || $!==0 || $!==EAGAIN or die "Error reading (2): $!"; do_log(-1,"unexpected(do_unarj_2): %s",$!) if !defined($ln) && $!==EAGAIN; $err = 0; $proc_fh->close or $err = $!; $rv = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; chdir($tempdir) or die "Can't chdir to $tempdir: $!"; if (proc_status_ok($rv,$err, 0,1)) {} # success, warn elsif (proc_status_ok($rv,$err, 3)) # CRC err { $part->attributes_add('U') if !$testing_for_sfx } else { do_log(0, "unarj: error extracting: %s",exit_status_str($rv,$err)) } # add attributes to the parent object, because we didn't remember names # of its scrambled members $part->attributes_add('U') if $skippedcount; $part->attributes_add('C') if $encryptedcount; my($errn) = lstat("$tempdir/parts/arj") ? 0 : 0+$!; if ($errn != ENOENT) { my($b) = flatten_and_tidy_dir("$tempdir/parts/arj", "$tempdir/parts",$part); consumed_bytes($b, 'do_unarj'); snmp_count("OpsDecBy\u${decompressor_name}"); } proc_status_ok($rv,$err, 0,1,3) # one of: success, warn, CRC err or die "unarj: can't extract archive members: ". exit_status_str($rv,$err); if ($encryptedcount || $skippedcount) { do_log(1, "do_unarj: %s, %d members are encrypted, %d skipped, archive retained", $part->base_name, $encryptedcount, $skippedcount); $retval = 2; } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('do_unarj'); # restart timer unlink($fn.".arj") or die "Can't unlink $fn.arj: $!"; if (defined $eval_stat) { $retval = 0; chomp $eval_stat; kill_proc($pid,$archiver,1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; # if ($testing_for_sfx) { die "do_unarj: $eval_stat" } # else { do_log(-1, "do_unarj: %s", $eval_stat) }; die "do_unarj: $eval_stat\n" # propagate failure } $retval; } # use external program to expand TNEF archives # sub do_tnef_ext($$$) { my($part, $tempdir, $archiver) = @_; do_log(4, "Extracting from TNEF encapsulation (ext) %s", $part->base_name); my($archiver_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${archiver_name}"); mkdir("$tempdir/parts/tnef",0750) or die "Can't mkdir $tempdir/parts/tnef: $!"; my($retval) = 1; my($proc_fh,$pid); prolong_timer('do_tnef_ext_pre'); # restart timer my($rem_quota) = max(10*1024, untaint(consumed_bytes(0,'do_tnef_ext'))); my($eval_stat); eval { ($proc_fh,$pid) = run_command(undef, '&1', $archiver, '--number-backups', '-x', "$rem_quota", '-C', "$tempdir/parts/tnef", '-f', $part->full_name); collect_results($proc_fh,$pid,$archiver,16384,[0]); undef $proc_fh; undef $pid; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('do_tnef_ext'); # restart timer if (defined $eval_stat) { $retval = 0; chomp $eval_stat; do_log(-1, "tnef_ext: %s", $eval_stat); } my($b) = flatten_and_tidy_dir("$tempdir/parts/tnef","$tempdir/parts",$part); if ($b > 0) { do_log(4, "tnef_ext extracted %d bytes from a tnef container", $b); consumed_bytes($b, 'do_tnef_ext'); } if (defined $eval_stat) { die "do_tnef_ext: $eval_stat\n" } # propagate $retval; } # use Convert-TNEF # sub do_tnef($$) { my($part, $tempdir) = @_; do_log(4, "Extracting from TNEF encapsulation (int) %s", $part->base_name); snmp_count('OpsDecByTnef'); my($tnef) = Convert::TNEF->read_in($part->full_name, {output_dir=>"$tempdir/parts", buffer_size=>16384, ignore_checksum=>1}); defined $tnef or die "Convert::TNEF failed: ".$Convert::TNEF::errstr; my($item_num) = 0; my($parent_placement) = $part->mime_placement; for my $a ($tnef->message, $tnef->attachments) { for my $attr_name ('AttachData','Attachment') { my($dh) = $a->datahandle($attr_name); if (defined $dh) { my($newpart_obj)= Amavis::Unpackers::Part->new("$tempdir/parts",$part); $item_num++; $newpart_obj->mime_placement("$parent_placement/$item_num"); $newpart_obj->name_declared([$a->name, $a->longname]); my($newpart) = $newpart_obj->full_name; my($outpart) = IO::File->new; # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] $outpart->open($newpart, untaint(O_CREAT|O_EXCL|O_WRONLY), 0640) or die "Can't create file $newpart: $!"; binmode($outpart) or die "Can't set file $newpart to binmode: $!"; my($filepath) = $dh->path; my($size) = 0; if (defined $filepath) { my($io,$nbytes,$buff); $dh->binmode(1); $io = $dh->open("r") or die "Can't open MIME::Body handle: $!"; while (($nbytes=$io->read($buff,16384)) > 0) { $outpart->print($buff) or die "Can't write to $newpart: $!"; $size += $nbytes; consumed_bytes($nbytes, 'do_tnef_1'); } defined $nbytes or die "Error reading from MIME::Body handle: $!"; $io->close or die "Error closing MIME::Body handle: $!"; undef $buff; # release storage } else { my($buff) = $dh->as_string; my($nbytes) = length($buff); $outpart->print($buff) or die "Can't write to $newpart: $!"; $size += $nbytes; consumed_bytes($nbytes, 'do_tnef_2'); } $newpart_obj->size($size); $outpart->close or die "Error closing $newpart: $!"; } } } $tnef->purge if defined $tnef; 1; } # The pax and cpio utilities usually support the following archive formats: # cpio, bcpio, sv4cpio, sv4crc, tar (old tar), ustar (POSIX.2 tar). # The utilities from http://heirloom.sourceforge.net/ support # several other tar/cpio variants such as SCO, Sun, DEC, Cray, SGI # sub do_pax_cpio($$$) { my($part, $tempdir, $archiver) = @_; my($archiver_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${archiver_name}"); ll(4) && do_log(4,"Expanding archive %s, using %s", $part->base_name,$archiver_name); my($is_pax) = $archiver_name =~ /^cpio/i ? 0 : 1; do_log(-1,"WARN: Using %s instead of pax can be a security ". "risk; please add: \$pax='pax'; to amavisd.conf and check that ". "the pax(1) utility is available on the system!", $archiver_name) if !$is_pax; my(@cmdargs) = $is_pax ? qw(-v) : qw(-i -t -v); my($proc_fh,$pid) = run_command($part->full_name, '/dev/null', $archiver, @cmdargs); my($bytes) = 0; local($1,$2); local($_); my($entries_cnt) = 0; for ($! = 0; defined($_=$proc_fh->getline); $! = 0) { chomp; next if /^\d+ blocks\z/; last if /^(cpio|pax): (.*bytes read|End of archive volume)/; if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } if (!/^ (?: \S+\s+ ){4} (\d+) \s+ (.+) \z/xs) { do_log(-1,"do_pax_cpio: can't parse toc line: %s", $_); } else { my($size,$mem) = ($1,$2); if ($mem =~ /^( (?: \s* \S+ ){3} (?: \s+ \d{4}, )? ) \s+ (.+)\z/xs) { $mem = $2; # strip away time and date } elsif ($mem =~ /^\S \s+ (.+)\z/xs) { # -rwxr-xr-x 1 1121 users 3135 C errorReport.sh $mem = $1; # strip away a letter in place of a date (?) } $mem = $1 if $is_pax && $mem =~ /^(.*) =[=>] (.*)\z/; # hard or soft link do_log(5,'do_pax_cpio: size: %5s, member: "%s"', $size,$mem); $bytes += $size if $size > 0; } } defined $_ || $!==0 || $!==EAGAIN or die "Error reading (1): $!"; do_log(-1,"unexpected(pax_cpio_1): %s",$!) if !defined($_) && $!==EAGAIN; # consume remaining output to avoid broken pipe collect_results($proc_fh,$pid,'do_pax_cpio/1',16384,[0]); undef $proc_fh; undef $pid; consumed_bytes($bytes, 'do_pax_cpio/pre', 1); # pre-check on estimated size mkdir("$tempdir/parts/arch", 0750) or die "Can't mkdir $tempdir/parts/arch: $!"; my($name_clash) = 0; my(%orig_names); # maps filenames to archive member names when possible prolong_timer('do_pax_cpio_pre'); # restart timer my($eval_stat); eval { chdir("$tempdir/parts/arch") or die "Can't chdir to $tempdir/parts/arch: $!"; my(@cmdargs) = $is_pax ? qw(-r -k -p am -s /[^A-Za-z0-9_]/-/gp) : qw(-i -d --no-absolute-filenames --no-preserve-owner); ($proc_fh,$pid) = run_command($part->full_name, '&1', $archiver, @cmdargs); my($output) = ''; my($ln); my($entries_cnt) = 0; for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { chomp($ln); if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } if (!$is_pax || $ln !~ /^(.*) >> (\S*)\z/) { $output .= $ln."\n" } else { # parse output from pax -s///p my($member_name,$file_name) = ($1,$2); if (!exists $orig_names{$file_name}) { $orig_names{$file_name} = $member_name; } else { do_log(0,'do_pax_cpio: member "%s" is hidden by a '. 'previous archive member "%s", file: %s', $member_name, $orig_names{$file_name}, $file_name); undef $orig_names{$file_name}; # cause it to exist but undefined $name_clash = 1; } } } defined $ln || $!==0 || $!==EAGAIN or die "Error reading (2): $!"; do_log(-1,"unexpected(pax_cpio_2): %s",$!) if !defined($ln) && $!==EAGAIN; my($err) = 0; $proc_fh->close or $err = $!; my($child_stat) = defined $pid && waitpid($pid,0) > 0 ? $? : undef; undef $proc_fh; undef $pid; chomp($output); proc_status_ok($child_stat,$err) or die(exit_status_str($child_stat,$err).' '.$output); 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('do_pax_cpio'); # restart timer chdir($tempdir) or die "Can't chdir to $tempdir: $!"; my($b) = flatten_and_tidy_dir("$tempdir/parts/arch", "$tempdir/parts", $part, 0, \%orig_names); consumed_bytes($b, 'do_pax_cpio'); if (defined $eval_stat) { chomp $eval_stat; kill_proc($pid,$archiver,1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; die "do_pax_cpio: $eval_stat\n"; # propagate failure } $name_clash ? 2 : 1; } # command line unpacker from stuffit.com for Linux # decodes Macintosh StuffIt archives and others # (but it appears the Linux version is buggy and a security risk, not to use!) # sub do_unstuff($$$) { my($part, $tempdir, $archiver) = @_; my($archiver_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${archiver_name}"); do_log(4,"Expanding archive %s, using %s", $part->base_name,$archiver_name); mkdir("$tempdir/parts/unstuff", 0750) or die "Can't mkdir $tempdir/parts/unstuff: $!"; my($proc_fh,$pid) = run_command(undef, '&1', $archiver, # '-q', "-d=$tempdir/parts/unstuff", $part->full_name); collect_results($proc_fh,$pid,$archiver,16384,[0]); undef $proc_fh; undef $pid; my($b) = flatten_and_tidy_dir("$tempdir/parts/unstuff", "$tempdir/parts", $part); consumed_bytes($b, 'do_unstuff'); 1; } # ar is a standard Unix binary archiver, also used by Debian packages # sub do_ar($$$) { my($part, $tempdir, $archiver) = @_; ll(4) && do_log(4,"Expanding Unix ar archive %s", $part->full_name); my($archiver_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${archiver_name}"); my($proc_fh,$pid) = run_command(undef, '/dev/null', $archiver, 'tv', $part->full_name); my($ln); my($bytes) = 0; local($1,$2,$3); my($entries_cnt) = 0; for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { chomp($ln); if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } if ($ln !~ /^(?:\S+\s+){2}(\d+)\s+((?:\S+\s+){3}\S+)\s+(.*)\z/) { do_log(-1,"do_ar: can't parse contents listing line: %s", $ln); } else { do_log(5,"do_ar: member: \"%s\", size: %s", $3,$1); $bytes += $1 if $1 > 0; } } defined $ln || $!==0 || $!==EAGAIN or die "Error reading: $!"; do_log(-1,"unexpected(do_ar): %s",$!) if !defined($ln) && $!==EAGAIN; # consume remaining output to avoid broken pipe collect_results($proc_fh,$pid,'ar-1',16384,[0]); undef $proc_fh; undef $pid; consumed_bytes($bytes, 'do_ar-pre', 1); # pre-check on estimated size mkdir("$tempdir/parts/ar", 0750) or die "Can't mkdir $tempdir/parts/ar: $!"; chdir("$tempdir/parts/ar") or die "Can't chdir to $tempdir/parts/ar: $!"; ($proc_fh,$pid) = run_command(undef, '&1', $archiver, 'x', $part->full_name); collect_results($proc_fh,$pid,'ar-2',16384,[0]); undef $proc_fh; undef $pid; chdir($tempdir) or die "Can't chdir to $tempdir: $!"; my($b) = flatten_and_tidy_dir("$tempdir/parts/ar","$tempdir/parts",$part); consumed_bytes($b, 'do_ar'); 1; } sub do_cabextract($$$) { my($part, $tempdir, $archiver) = @_; do_log(4, "Expanding cab archive %s", $part->base_name); my($archiver_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${archiver_name}"); local($_,$1,$2); my($bytes) = 0; my($ln); my($entries_cnt) = 0; my($proc_fh,$pid) = run_command(undef, '/dev/null', $archiver, '-l', $part->full_name); for ($! = 0; defined($ln=$proc_fh->getline); $! = 0) { chomp($ln); next if $ln =~ /^(File size|----|Viewing cabinet:|\z)/; if ($entries_cnt++, $MAXFILES && $entries_cnt > $MAXFILES) { die "Maximum number of files ($MAXFILES) exceeded" } if ($ln !~ /^\s* (\d+) \s* \| [^|]* \| \s (.*) \z/x) { do_log(-1, "do_cabextract: can't parse toc line: %s", $ln); } else { do_log(5, 'do_cabextract: member: "%s", size: %s', $2,$1); $bytes += $1 if $1 > 0; } } defined $ln || $!==0 || $!==EAGAIN or die "Error reading: $!"; do_log(-1,"unexpected(cabextract): %s",$!) if !defined($ln) && $!==EAGAIN; # consume remaining output to avoid broken pipe (just in case) collect_results($proc_fh,$pid,'cabextract-1',16384,[0]); undef $proc_fh; undef $pid; mkdir("$tempdir/parts/cab",0750) or die "Can't mkdir $tempdir/parts/cab: $!"; ($proc_fh,$pid) = run_command(undef, '/dev/null', $archiver, '-q', '-d', "$tempdir/parts/cab", $part->full_name); collect_results($proc_fh,$pid,'cabextract-2',16384,[0]); undef $proc_fh; undef $pid; my($b) = flatten_and_tidy_dir("$tempdir/parts/cab", "$tempdir/parts", $part); consumed_bytes($b, 'do_cabextract'); 1; } sub do_ole($$$) { my($part, $tempdir, $archiver) = @_; do_log(4,"Expanding MS OLE document %s", $part->base_name); my($archiver_name) = basename((split(' ',$archiver))[0]); snmp_count("OpsDecBy\u${archiver_name}"); mkdir("$tempdir/parts/ole",0750) or die "Can't mkdir $tempdir/parts/ole: $!"; my($proc_fh,$pid) = run_command(undef, '&1', $archiver, '-v', '-i', $part->full_name, '-d',"$tempdir/parts/ole"); collect_results($proc_fh,$pid,$archiver,16384,[0]); undef $proc_fh; undef $pid; my($b) = flatten_and_tidy_dir("$tempdir/parts/ole", "$tempdir/parts", $part); if ($b > 0) { do_log(4, "ripOLE extracted %d bytes from an OLE document", $b); consumed_bytes($b, 'do_ole'); } 2; # always keep the original OLE document } # Check for self-extracting archives. Note that we do not depend on # file magic here since it's not reliable. Instead we will try each # archiver. # sub do_executable($$@) { my($part, $tempdir, $unrar, $lha, $unarj) = @_; ll(4) && do_log(4,"Check whether %s is a self-extracting archive", $part->base_name); # ZIP? return 2 if eval { do_unzip($part,$tempdir,undef,1) }; chomp $@; do_log(3, "do_executable: not a ZIP sfx, ignoring: %s", $@) if $@ ne ''; # RAR? return 2 if defined $unrar && eval { do_unrar($part,$tempdir,$unrar,1) }; chomp $@; do_log(3, "do_executable: not a RAR sfx, ignoring: %s", $@) if $@ ne ''; # # LHA? not safe, tends to crash # return 2 if defined $lha && eval { do_lha($part,$tempdir,$lha,1) }; # chomp $@; # do_log(3, "do_executable: not a LHA sfx, ignoring: %s", $@) if $@ ne ''; # ARJ? return 2 if defined $unarj && eval { do_unarj($part,$tempdir,$unarj,1) }; chomp $@; do_log(3, "do_executable: not an ARJ sfx, ignoring: %s", $@) if $@ ne ''; return 0; } # my($k,$v,$fn); # while (($k,$v) = each(%::)) { # local(*e)=$v; $fn=fileno(\*e); # printf STDOUT ("%-10s %-10s %s\n",$k,$v,$fn) if defined $fn; # } # Given a file handle (typically opened pipe to a subprocess, as returned # by run_command), copy from it to a specified output file in binary mode. # sub run_command_copy($$$) { my($outfile, $ifh, $pid) = @_; my($ofh) = IO::File->new; # O_WRONLY etc. can become tainted in Perl5.8.9 [perlbug #62502] $ofh->open($outfile, untaint(O_CREAT|O_EXCL|O_WRONLY), 0640) # calls sysopen or die "Can't create file $outfile: $!"; binmode($ofh) or die "Can't set file $outfile to binmode: $!"; binmode($ifh) or die "Can't set binmode on pipe: $!"; my($eval_stat); my($rv,$rerr); $rerr = 0; eval { my($nread,$nwrite,$tosend,$offset,$inbuf); for (;;) { $nread = sysread($ifh, $inbuf, 16384); if (!defined($nread)) { if ($!==EAGAIN || $!==EINTR) { Time::HiRes::sleep(0.1) } # just in case else { die "Error reading: $!" } } elsif ($nread < 1) { # sysread returns 0 at eof last; } else { consumed_bytes($nread, 'run_command_copy'); $tosend = $nread; $offset = 0; while ($tosend > 0) { # handle partial writes $nwrite = syswrite($ofh, $inbuf, $tosend, $offset); if (!defined($nwrite)) { if ($!==EAGAIN || $!==EINTR) { Time::HiRes::sleep(0.1) }#justincase else { die "Error writing to $outfile: $!" } } elsif ($nwrite < 1) { Time::HiRes::sleep(0.1); # just in case } else { $tosend -= $nwrite; $offset += $nwrite; } } } } $ifh->close or $rerr = $!; $rv = defined $pid && waitpid($pid,0) > 0 ? $? : undef; $ofh->close or die "Error closing $outfile: $!"; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; # remember error, close socket ignoring status $rerr = $!; $ifh->close; $rv = defined $pid && waitpid($pid,0) > 0 ? $? : undef; do_log(-1, "run_command_copy: %s", $eval_stat); $ofh->close or do_log(-1, "Error closing %s: %s", $outfile,$!); }; if (defined $eval_stat) { die "run_cc: $eval_stat\n" } # propagate failure ($rv,$rerr); # return subprocess termination status and reading/close errno } # extract listed files from archive and store each in a new file # sub store_mgr($$$@) { my($tempdir, $parent_obj, $list, $archiver, @args) = @_; my($item_num) = 0; my($parent_placement) = $parent_obj->mime_placement; my($retval) = 0; my($proc_fh,$pid); prolong_timer('store_mgr_pre'); # restart timer my($eval_stat); eval { for my $f (@$list) { next if $f =~ m{/\z}; # ignore directories my($newpart_obj) = Amavis::Unpackers::Part->new("$tempdir/parts",$parent_obj); $item_num++; $newpart_obj->mime_placement("$parent_placement/$item_num"); $newpart_obj->name_declared($f); # store tainted name my($newpart) = $newpart_obj->full_name; ll(5) && do_log(5,'store_mgr: extracting "%s" to file %s using %s', $f, $newpart, $archiver); if ($f =~ m{^\.?[A-Za-z0-9_][A-Za-z0-9/._=~-]*\z}) { #presumably safe arg } else { # this is not too bad, as run_command does not use shell do_log(1, 'store_mgr: NOTICE: suspicious file name "%s"', $f); } ($proc_fh,$pid) = run_command(undef, '/dev/null', $archiver, @args, untaint($f)); my($rv,$err) = run_command_copy($newpart,$proc_fh,$pid); # may die my($ll) = proc_status_ok($rv,$err) ? 5 : 1; ll($ll) && do_log($ll,"store_mgr: extracted by %s, %s", $archiver, exit_status_str($rv,$err)); $retval = $rv if $retval == 0 && $rv != 0; } 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!" }; prolong_timer('store_mgr'); # restart timer if (defined $eval_stat) { $retval = 0; chomp $eval_stat; kill_proc($pid,$archiver,1,$proc_fh,$eval_stat) if defined $pid; undef $proc_fh; undef $pid; die "store_mgr: $eval_stat\n"; # propagate failure } $retval; # return the first nonzero status (if any), or 0 } 1; __DATA__ # package Amavis::DKIM::CustomSigner; sub new { my($class,%params) = @_; bless { %params }, $class; } sub sign_digest { my($self_key, $digest_alg_name, $digest) = @_; my($code) = $self_key->{CustomSigner}; &$code($digest_alg_name, $digest, %$self_key); } 1; package Amavis::DKIM; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&dkim_key_postprocess &generate_authentication_results &dkim_make_signatures &adjust_score_by_signer_reputation &collect_some_dkim_info); import Amavis::Conf qw(:platform c cr ca $myproduct_name %dkim_signing_keys_by_domain @dkim_signing_keys_list @dkim_signing_keys_storage); import Amavis::Util qw(min max minmax untaint ll do_log unique_list get_deadline proto_encode proto_decode); import Amavis::rfc2821_2822_Tools qw(split_address quote_rfc2821_local qquote_rfc2821_local format_time_interval); import Amavis::Timing qw(section_time); import Amavis::Lookup qw(lookup lookup2); } use subs @EXPORT_OK; use IO::File (); use Crypt::OpenSSL::RSA (); use MIME::Base64; use Mail::DKIM::Verifier 0.31; use Mail::DKIM::Signer 0.31; use Mail::DKIM::TextWrap; use Mail::DKIM::Signature; use Mail::DKIM::DkSignature; # Convert private keys (as strings in PEM format) into RSA objects # and do some pre-processing on @dkim_signing_keys_list entries # (may run unprivileged) # sub dkim_key_postprocess() { # convert private keys (as strings in PEM format) into RSA objects for my $ks (@dkim_signing_keys_storage) { my($pkcs1,$dev,$inode,$fname) = @$ks; if (ref($pkcs1) && UNIVERSAL::isa($pkcs1,'Crypt::OpenSSL::RSA')) { # it is already a Crypt::OpenSSL::RSA object } else { # assume a string is a private key in PEM format, convert it to RSA obj $ks->[0] = Crypt::OpenSSL::RSA->new_private_key($pkcs1); } } for my $ent (@dkim_signing_keys_list) { my($domain) = $ent->{domain}; $dkim_signing_keys_by_domain{$domain} = [] if !$dkim_signing_keys_by_domain{$domain}; } my($any_wild); my($j) = 0; for my $ent (@dkim_signing_keys_list) { $ent->{v} = 'DKIM1' if !defined $ent->{v}; # provide a default if (defined $ent->{n}) { # encode n as qp-section (RFC 4871, RFC 2047) $ent->{n} =~ s{([\000-\037\177=;"])}{sprintf('=%02X',ord($1))}egs; } my($domain) = $ent->{domain}; if (ref($domain) eq 'Regexp') { $ent->{domain_re} = $domain; $any_wild = sprintf("key#%d, %s", $j+1, $domain) if !defined $any_wild; } elsif ($domain =~ /\*/) { # wildcarded signing domain in a key declaration, evil, asks for trouble! # support wildcards in signing domain for compatibility with dkim_milter my($regexp) = $domain; $regexp =~ s/\*{2,}/*/gs; # collapse successive wildcards # '*' is a wildcard, quote the rest $regexp =~ s{ ([@#/.^$|*+?(){}\[\]\\]) }{$1 eq '*' ? '.*' : '\\'.$1}gex; $regexp = '^' . $regexp . '\\z'; # implicit anchors $regexp =~ s/^\^\.\*//s; # remove leading anchor if redundant $regexp =~ s/\.\*\\z\z//s; # remove trailing anchor if redundant $regexp = '(?:)' if $regexp eq ''; # just in case, non-empty regexp # presence of {'domain_re'} entry lets get_dkim_key use this regexp # instead of a direct string comparision with {'domain'} $ent->{domain_re} = qr{$regexp}; # compiled regexp object $any_wild = sprintf("key#%d, %s", $j+1, $domain) if !defined $any_wild; } # %dkim_signing_keys_by_domain entries contain lists of indices into # the @dkim_signing_keys_list of all potentially applicable signing keys. # This hash (keyed by domain name) avoids linear searching for signing # keys for all fully-specified domains in @dkim_signing_keys_list. # Wildcarded entries must still be looked up sequentially at run-time # to preserve the declared order and the 'first match wins' paradigm. # Such entries are only supported for compatibility with dkim_milter # and are evil because amavisd has no quick way of verifying that DNS RR # really exists, so signatures generated by amavisd can fail when not all # possible DNS resource records exist for wildcarded signing domains. # if (!defined($ent->{domain_re})) { # no regexp, just plain match on domain push(@{$dkim_signing_keys_by_domain{$domain}}, $j); } else { # a wildcard in a signing domain, compatibility with dkim_milter # wildcarded signing domain potentially matches any _by_domain entry for my $d (keys %dkim_signing_keys_by_domain) { push(@{$dkim_signing_keys_by_domain{$d}}, $j); } # the '*' entry collects only wildcarded signing keys $dkim_signing_keys_by_domain{'*'} = [] if !$dkim_signing_keys_by_domain{'*'}; push(@{$dkim_signing_keys_by_domain{'*'}}, $j); } $j++; } do_log(0,"dkim: wildcard in signing domain (%s), may produce unverifiable ". "signatures with no published public key, avoid!", $any_wild) if $any_wild; } # Fetch a private DKIM signing key for a given signing domain, with its # resource-record (RR) constraints compatible with proposed signature options. # The first such key is returned as a hash; if no key is found an empty hash # is returned. When a selector (s) is given it must match the selector of # a key; when algorithm (a) is given, the key type and a hash algorithm must # match the desired use too; the service type (s) must be 'email' or '*'; # when identity (i) is given it must match the granularity (g) of a key; # # sign.opts. key options # ---------- ----------- # d => domain # s => selector # a => k, h(list) # i => g, t=s # sub get_dkim_key(@) { @_ % 2 == 0 or die "get_dkim_key: a list of pairs is expected as query opts"; my(%options) = @_; # signature options (v, a, c, d, h, i, l, q, s, t, x, z), # of which d is required, while s, a and t are optional but taken into # account in searching for a compatible key - the rest are ignored my(%key_options); my($domain) = $options{d}; my($selector) = $options{s}; defined $domain && $domain ne '' or die "get_dkim_key: domain is required, but tag 'd' is missing"; $domain = lc($domain); $selector = lc($selector) if defined $selector; my(@indices) = $dkim_signing_keys_by_domain{$domain} ? @{$dkim_signing_keys_by_domain{$domain}} : $dkim_signing_keys_by_domain{'*'} ? @{$dkim_signing_keys_by_domain{'*'}} : (); if (@indices) { $selector = $selector eq '' ? undef : lc($selector) if defined $selector; local($1,$2); my($keytype,$hashalg) = defined $options{a} && $options{a} =~ /^([a-z0-9]+)-(.*)\z/is ? ($1,$2) : ('rsa',undef); my($identity_localpart,$identity_domain) = !defined($options{i}) ? () : split_address($options{i}); $identity_localpart = '' if !defined $identity_localpart; $identity_domain = '' if !defined $identity_domain; # find the first key (associated with a domain) with compatible options for my $j (@indices) { my($ent) = $dkim_signing_keys_list[$j]; next unless defined $ent->{domain_re} ? $domain =~ $ent->{domain_re} : $domain eq $ent->{domain}; next if defined $selector && $ent->{selector} ne $selector; next if $keytype ne (exists $ent->{k} ? $ent->{k} : 'rsa'); next if exists $ent->{s} && !(grep($_ eq '*' || $_ eq 'email', split(/:/, $ent->{s})) ); next if defined $hashalg && exists $ent->{'h'} && !(grep($_ eq $hashalg, split(/:/, $ent->{'h'})) ); if (defined($options{i})) { if (lc($identity_domain) eq $domain) { # ok } elsif (exists $ent->{t} && (grep($_ eq 's', split(/:/,$ent->{t})))) { next; # no subdomains allowed } if (!exists($ent->{g}) || $ent->{g} eq '*') { # ok } elsif ($ent->{g} =~ /^ ([^*]*) \* (.*) \z/xs) { next if $identity_localpart !~ /^ \Q$1\E .* \Q$2\E \z/xs; } else { next if $identity_localpart ne $ent->{g}; } } %key_options = %$ent; last; # found a suitable match } } if (defined $key_options{key_storage_ind}) { # obtain actual key from @dkim_signing_keys_storage ($key_options{key}) = @{$dkim_signing_keys_storage[$key_options{key_storage_ind}]}; } %key_options; } # send a query to a signing service, collect its response and parse it; # the protocol is much like the AM.PDP protocol, except that attributes # are different # sub query_signing_service($$) { my($server, $query) = @_; my($remaining_time, $deadline) = get_deadline('query_signing_service'); my($sock) = Amavis::IO::RW->new($server, Eol => "\015\012", Timeout => 10); $sock or die "Error connecting to a signing server $server: $!"; my($req_id) = sprintf("%08x", rand(0x7fffffff)); my($req_id_attr) = proto_encode('request_id', $req_id); $sock->print(join('', map($_."\015\012", (@$query, $req_id_attr, '')))) or die "Error sending a query to a signing server"; ll(5) && do_log(5, "dkim: query_signing_service, query: %s", join('; ', @$query, $req_id_attr)); $sock->flush or die "Error flushing signing server session"; # collect a reply $sock->timeout(max(2, $deadline - Time::HiRes::time)); my(%attr,$ln); local($1,$2); while (defined($ln = $sock->get_response_line)) { last if $ln eq "\015\012"; # end of a response block if ($ln =~ /^ ([^=\000\012]*?) = ([^\012]*?) \015\012 \z/xsi) { $attr{proto_decode($1)} = proto_decode($2); } } $sock->close or die "Error closing session to a signing server $server: $!"; ll(5) && do_log(5, "dkim: query_signing_service, got: %s", join('; ', map($_.'='.$attr{$_}, keys %attr))); $attr{request_id} eq $req_id or die "Answer id '$attr{request_id}' from $server ". "does not match the query id '$req_id'"; \%attr; } # send candidate originator addresses and signature options to a signing # service and let it choose a selector 's' and a domain 'd', thus uniquely # identifying a signing key # sub let_signing_service_choose($$$$) { my($server, $msginfo, $sender_search_list_ref, $sig_opt_prelim) = @_; my(@query) = ( proto_encode('request', 'choose_key'), proto_encode('log_id', $msginfo->log_id), ); # provide some additional information potentially useful in decision-making if ($sig_opt_prelim) { for my $opt (sort keys %$sig_opt_prelim) { push(@query, proto_encode('sig.'.$opt, $sig_opt_prelim->{$opt})); } } push(@query, proto_encode('sender', $msginfo->sender_smtp)); for my $r (@{$msginfo->per_recip_data}) { push(@query, proto_encode('recip', $r->recip_addr_smtp)); } for my $pair (!$sender_search_list_ref ? () : @$sender_search_list_ref) { my($addr,$addr_src) = @$pair; push(@query, proto_encode('candidate', $addr_src, qquote_rfc2821_local($addr))); } my($attr); eval { $attr = query_signing_service($server,\@query); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(0, "query_signing_service failed: %s", $eval_stat); }; my(%sig_options, $chosen_addr_src, $chosen_addr); if ($attr) { for my $opt (keys %$attr) { if ($opt =~ /^sig\.(.+)\z/) { $sig_options{$1} = $attr->{$opt} if !exists($sig_options{$1}); } } if (defined $attr->{chosen_candidate}) { ($chosen_addr_src, $chosen_addr) = split(' ', $attr->{chosen_candidate}, 2); } } (!$attr ? undef : \%sig_options, $chosen_addr_src, $chosen_addr); } # a CustomSigner callback routine passed to Mail::DKIM in place of a key; # the routine will be called by Mail::DKIM::Algorithm::*rsa_sha* routines # instead of calling their own Mail::DKIM::PrivateKey::sign_digest() # sub remote_signer { my($digest_alg_name, $digest, %args) = @_; # $digest: header digest (binary), ready for signing, # e.g. $algorithm->{header_digest}->digest my($server) = $args{Server}; # our own info passed back to us my($msginfo) = $args{MsgInfo}; # our own info passed back to us my(@query) = ( proto_encode('request', 'sign'), proto_encode('digest_alg', $digest_alg_name), proto_encode('digest', encode_base64($digest,'')), proto_encode('s', $args{Selector}), proto_encode('d', $args{Domain}), proto_encode('log_id', $msginfo->log_id), ); my($attr, $b, $reason); eval { $attr = query_signing_service($server, \@query); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; $reason = $eval_stat; }; if ($attr) { $b = $attr->{b}; $reason = $attr->{reason} } if (!defined($b) || $b eq '') { $reason = 'no signature from a signing server' if !defined $reason; # die "Can't sign, $reason, query: " . join('; ',@query) . "\n"; do_log(0, "dkim: can't sign, %s, query: %s", $reason, join('; ',@query)); return ''; # Mail::DKIM::Algorithm::rsa_sha256 doesn't like undef } else { return decode_base64($b); # resulting signature } } # prepare requested DKIM signatures for a provided message, # returning them as a list of Mail::DKIM::Signature objects # sub dkim_make_signatures($$;$) { my($msginfo,$initial_submission,$callback) = @_; my(@signatures); # resulting signature objects my(%sig_options); # signature options and constraints for choosing a key my(%key_options); # options associated with a signing key my(@tried_domains); # used for logging a failure my($chosen_addr,$chosen_addr_src); my($do_sign) = 0; my($fm) = $msginfo->rfc2822_from; # authors my(@rfc2822_from) = !defined($fm) ? () : ref $fm ? @$fm : $fm; my($allowed_hdrs) = cr('allowed_added_header_fields'); my($from_str) = join(', ', qquote_rfc2821_local(@rfc2822_from)); # logging if (length($from_str) > 100) { $from_str = substr($from_str,0,100).'[...]' } if (!$allowed_hdrs || !$allowed_hdrs->{lc('DKIM-Signature')}) { do_log(5, "dkim: inserting a DKIM-Signature header field disabled"); } elsif (!$msginfo->originating) { do_log(5, "dkim: not signing mail which is not originating from our site"); } elsif ($msginfo->is_in_contents_category(CC_VIRUS)) { do_log(2, "dkim: not signing infected mail (from inside), From: %s", $from_str); } elsif ($msginfo->is_in_contents_category(CC_SPAM)) { # it is prudent not to sign outgoing spam, otherwise an attacker may be # able to replay a signed message, re-sending it to other recipients # in bulk directly from botnets do_log(2, "dkim: not signing spam (from inside), From: %s", $from_str); } elsif ($msginfo->is_in_contents_category(CC_SPAMMY)) { do_log(2, "dkim: not signing suspected spam (from inside), From: %s", $from_str); } else { # Choose a signing key based on the first match on the following # addresses (in this order): 2822.From, followed by 2822.Resent-From and # 2822.Resent-Sender address pairs traversed top-down by resent blocks, # followed by 2822.Sender and 2821.mail_from. We choose to look up # a From first, as it generates an author domain signature, but the # search order on remaining entries is admittedly unusual. # Btw, dkim-milter uses the following search order: # Resent-Sender, Resent-From, Sender, From. # Only a signature based on 2822.From is considered an author domain # signature, others are just third-party signatures and have no more # merit than any other third-party signature according to RFC 4871. # my($rf) = $msginfo->rfc2822_resent_from; my($rs) = $msginfo->rfc2822_resent_sender; my(@rfc2822_resent_from, @rfc2822_resent_sender); @rfc2822_resent_from = @$rf if defined $rf; @rfc2822_resent_sender = @$rs if defined $rs; my(@search_list); # collects candidate addresses for choosing a signing key # author addresses go first (typically exactly one, but possibly more) push(@search_list, map([$_,'From'], @rfc2822_from)); # merge Resent-From and Resent-Sender addresses by resent blocks, top-down; # a merge is simplified by the fact that there is an equal number of # resent blocks in @rfc2822_resent_from and @rfc2822_resent_sender lists while (@rfc2822_resent_from || @rfc2822_resent_sender) { # for each resent block while (@rfc2822_resent_from) { my($addr) = shift(@rfc2822_resent_from); last if !defined $addr; # undef delimits resent blocks push(@search_list, [$addr, 'Resent-From']); } while (@rfc2822_resent_sender) { my($addr) = shift(@rfc2822_resent_sender); last if !defined $addr; # undef delimits resent blocks push(@search_list, [$addr, 'Resent-Sender']); } } push(@search_list, [$msginfo->rfc2822_sender, 'Sender']); push(@search_list, [$msginfo->sender, 'mail_from']); { # remove duplicates and empty addresses my(%addr_seen); @search_list = grep { my($a,$src) = @$_; defined $a && $a ne '' && !$addr_seen{$a}++ } @search_list; } ll(2) && do_log(2, "dkim: candidate originators: %s", join(", ", map($_->[1].':'.qquote_rfc2821_local($_->[0]), @search_list))); # dkim_signwith_sd() may provide a ref to a pair [selector,domain] - if # available (e.g. by a custom hook), it will force signing with a private # key associated with this selector and domain, otherwise we fall back # to consulting an external service if available, or else we use our # built-in algorithm for choosing a selector & domain and their associated # signing key # my($sd_pair) = $msginfo->dkim_signwith_sd; if (ref($sd_pair) eq 'ARRAY') { my($s,$d) = @$sd_pair; if (defined $s && $s ne '' && defined $d && $d ne '') { do_log(5, "dkim: dkim_signwith_sd presets d=%s, s=%s", $d,$s); $sig_options{s} = $s; $sig_options{d} = $d; } } my($dkim_signing_service) = c('dkim_signing_service'); if (defined $dkim_signing_service && $dkim_signing_service ne '') { # try the signing service: it should provide an 's' and 'd' if it has # a suitable signing key available, and/or may supply signing options, # overriding the defaults set so far my($sig_opt_ref); ($sig_opt_ref, $chosen_addr_src, $chosen_addr) = let_signing_service_choose($dkim_signing_service, $msginfo, \@search_list, undef); if ($sig_opt_ref) { # merge returned signature options with ours while (my($k,$v) = each(%$sig_opt_ref)) { $sig_options{$k} = $v if defined $v } } } my($sobm) = ca('dkim_signature_options_bysender_maps'); # last resort: fall back to our local configuration settings for my $pair (@search_list) { my($addr,$addr_src) = @$pair; my($addr_localpart,$addr_domain) = split_address($addr); $addr_domain = lc($addr_domain); # fetch a list of hashes from all entries matching the address my($dkim_options_ref,$mk_ref); ($dkim_options_ref,$mk_ref) = lookup2(1,$addr,$sobm) if $sobm && @$sobm; $dkim_options_ref = [] if !defined $dkim_options_ref; # signature options (parenthesized options are set automatically): # (v), a, (b), (bh), c, d, (h), i, (l), q, s, (t), x, (z) # place a catchall default at the end of the list of options; push(@$dkim_options_ref, { c => 'relaxed/simple', a => 'rsa-sha256' }); # start each iteration with the same set of options collected so far my(%tmp_sig_options) = %sig_options; # traverse list of hashes from specific to general, first match wins for my $opts_hash_ref (@$dkim_options_ref) { next if ref $opts_hash_ref ne 'HASH'; # just in case while (my($k,$v) = each(%$opts_hash_ref)) { # for each entry in a hash $tmp_sig_options{$k} = $v if !exists $tmp_sig_options{$k}; } } # a default for a signing domain is a domain of each tried address if (!exists($tmp_sig_options{d})) { my($d) = $addr_domain; $d =~ s/^\@//; $tmp_sig_options{d} = $d } push(@tried_domains, $tmp_sig_options{d}); ll(5) && do_log(5, "dkim: signature options for %s(%s): %s", $addr, $addr_src, join('; ', map($_.'='.$tmp_sig_options{$_}, keys %tmp_sig_options))); # find a private key associated with a signing domain and selector, # and meeting constraints %key_options = get_dkim_key(%tmp_sig_options) if defined $tmp_sig_options{d} && $tmp_sig_options{d} ne ''; # my(@domain_path); # host.sub.example.com sub.example.com example.com com # $addr_domain =~ s/^\@//; $addr_domain =~ s/\.\z//; # if ($addr_domain !~ /\[/) { # don't split address literals # for (my $d=$addr_domain; $d ne ''; $d =~ s/^[^.]*(?:\.|\z)//s) # { push(@domain_path,$d) } # } # for my $d (@domain_path) { # $tmp_sig_options{d} = $d; # %key_options = get_dkim_key(%tmp_sig_options); # last if defined $key_options{key}; # } my($key) = $key_options{key}; if (defined $key && $key ne '') { # found; copy the key and its options $tmp_sig_options{key} = $key; $tmp_sig_options{s} = $key_options{selector}; $chosen_addr = $addr; $chosen_addr_src = $addr_src; # merge the just collected signature options into the final set while (my($k,$v) = each(%tmp_sig_options)) { $sig_options{$k} = $v if defined $v } last; } } # provide defaults for 'c' and 'a' tags if missing $sig_options{c} = 'relaxed/simple' if !exists $sig_options{c}; $sig_options{a} = 'rsa-sha256' if !exists $sig_options{a}; # prepare for a second stage of using an external signing service: # when we do have a 's' and 'd', thus uniquely identifying a signing key, # but do not have a key ourselves, we'll provide a callback routine # in place of a key object so that Mail::DKIM will call it at the time # of signing, and our routine will consult a remote signing service # if (!defined $sig_options{key} && defined $dkim_signing_service && $dkim_signing_service ne '' && defined $sig_options{d} && $sig_options{d} ne '' && defined $sig_options{s} && $sig_options{s} ne '') { my($s) = $sig_options{s}; my($d) = $sig_options{d}; # let Mail::DKIM use our custom code for signing (pref. 0.38 or later) $key_options{key} = Amavis::DKIM::CustomSigner->new( CustomSigner => \&remote_signer, MsgInfo => $msginfo, Selector => $s, Domain => $d, Server => $dkim_signing_service); $key_options{selector} = $s; $key_options{domain} = $d; $sig_options{key} = $key_options{key}; } if (!defined $sig_options{d} || $sig_options{d} eq '') { do_log(2, "dkim: not signing, empty signing domain, From: %s",$from_str); } elsif (!defined $sig_options{key} || $sig_options{key} eq '') { do_log(2, "dkim: not signing, no applicable private key for domains %s,". " s=%s, From: %s", join(", ",@tried_domains), $sig_options{s}, $from_str); } else { # copy key's options to signature options for convenience for (keys %key_options) { $sig_options{'KEY.'.$_} = $key_options{$_} if /^[ghknst]\z/ } $sig_options{'KEY.key_ind'} = $key_options{key_ind}; # check matching of identity to a signing domain or provide a default; # presence of a t=s flag in a public key RR prohibits subdomains in i my($key_allows_subdomains) = grep($_ eq 's', split(/:/,$sig_options{'KEY.t'})) ? 0 : 1; if (defined $sig_options{i}) { # explicitly given, possibly empty # have mercy: provide a leading '@' if missing $sig_options{i} = '@'.$sig_options{i} if $sig_options{i} ne '' && $sig_options{i} !~ /\@/; } elsif (!$key_allows_subdomains) { # we have no other choice but to keep it at its default @d } else { # the public key record permits subdomains # provide default for i in a form of a sender's domain local($1); if ($chosen_addr =~ /\@([^\@]*)\z/) { my($identity_domain) = lc($1); if ($identity_domain =~ /.\.\Q$sig_options{d}\E\z/si) { $sig_options{i} = '@'.$identity_domain; do_log(5, "dkim: identity defaults to %s", $sig_options{i}); } } } if (!defined $sig_options{i} || $sig_options{i} eq '') { $do_sign = 1; # just sign, don't bother with i } else { # check if the requested i is compatible with d local($1); my($identity_domain) = $sig_options{i} =~ /\@([^\@]*)\z/ ? $1 : ''; if (!$key_allows_subdomains && lc($identity_domain) ne lc($sig_options{d})) { do_log(2, "dkim: not signing, identity domain %s not the same as ". "a signing domain %s, flags t=%s, From: %s", $sig_options{i}, $sig_options{d}, $sig_options{'KEY.t'}, $from_str); } elsif ($key_allows_subdomains && $identity_domain !~ /(^|\.)\Q$sig_options{d}\E\z/i) { do_log(2, "dkim: not signing, identity %s not a subdomain of %s, ". "From: %s", $sig_options{i}, $sig_options{d}, $from_str); } else { $do_sign = 1; } } } } if ($do_sign) { # avoid adding same signature on multiple passes through MTA my($sigs_ref) = $msginfo->dkim_signatures_valid; for my $sig (!defined($sigs_ref) ? () : @$sigs_ref) { if ( lc($sig_options{d}) eq lc($sig->domain) && (!defined $sig_options{i} || $sig_options{i} eq $sig->identity) ) { do_log(2, "dkim: not signing, already signed by domain %s, ". "From: %s", $sig_options{d}, $from_str); $do_sign = 0; } } } if ($do_sign) { # relative expiration time $sig_options{x} = $msginfo->rx_time + $sig_options{ttl} if defined $sig_options{ttl} && $sig_options{ttl} > 0; # remove redundant options with RFC 4871 -default values for my $k (keys %sig_options) { delete $sig_options{$k} if !defined $k } delete $sig_options{i} if lc($sig_options{i}) eq '@'.lc($sig_options{d}); delete $sig_options{c} if $sig_options{c} eq 'simple/simple' || $sig_options{c} eq 'simple'; delete $sig_options{q} if $sig_options{q} eq 'dns/txt'; if (ref $callback eq 'CODE') { &$callback($msginfo,\%sig_options) } if (ll(2)) { my($opts) = join(', ', map($_ eq 'key' ? () : ($_.'=>'.$sig_options{$_}), sort keys %sig_options)); do_log(2,"dkim: signing (%s), From: %s (%s:%s), %s", grep(/\@\Q$sig_options{d}\E\z/si, @rfc2822_from) ? 'author' : '3rd-party', $from_str, $chosen_addr_src, qquote_rfc2821_local($chosen_addr), $opts); } my($key) = $sig_options{key}; if (UNIVERSAL::isa($key,'Crypt::OpenSSL::RSA')) { # my($pkcs1) = $key->get_private_key_string; # most compact # $pkcs1 =~ s/^---.*?---(?:\r?\n|\z)//gm; $pkcs1 =~ tr/\r\n//d; # $key = Mail::DKIM::PrivateKey->load(Data => $pkcs1); $key = Mail::DKIM::PrivateKey->load(Cork => $key); # avail since 0.31 } elsif (ref $key) { # already a Mail::DKIM::PrivateKey or Amavis::DKIM::CustomSigner object } else { $key = Mail::DKIM::PrivateKey->load(File => $key); # read from a file } # Sendmail milter interface does not provide a just-generated Received # header field to milters. Milters therefore need to fabricate a pseudo # Received header field in order to provide client IP address to a filter. # Unfortunately it is not posible to reliably fabricate a header field # which will exactly match the later-inserted one, so we must not sign # it to avoid a likely possibility of a signature being invalidated. my($conn) = $msginfo->conn_obj; my($appl_proto) = !$conn ? undef : $conn->appl_proto; my($skip_topmost_received) = defined($appl_proto) && ($appl_proto eq 'AM.PDP' || $appl_proto eq 'AM.CL'); my($policyfn) = sub { my($dkim) = shift; my($signed_header_fields_ref) = cr('signed_header_fields') || {}; my($hfn) = $dkim->{header_field_names}; my(@field_names_to_be_signed); # # when $signed_header_fields_ref->{$nm} is greater than 1 it indicates # that one surplus occurrence of a header filed name in an 'h' tag # should be inserted, consequently prohibiting further instances of # such header field to be added to a message header section without # breaking a signature; useful for example for a From and Subject # if ($hfn) { my(%hfn_cnt); $hfn_cnt{lc $_}++ for @$hfn; for (@$hfn) { my($nm) = lc($_); push(@field_names_to_be_signed, $nm); $hfn_cnt{$nm}--; if (!$hfn_cnt{$nm} && $signed_header_fields_ref->{$nm} > 1) { # causes signing one additional null occurrence of a header field push(@field_names_to_be_signed, $nm); } } } @field_names_to_be_signed = grep($signed_header_fields_ref->{$_}, @field_names_to_be_signed); if ($skip_topmost_received) { # don't sign topmost Received header field for my $j (0..$#field_names_to_be_signed) { if (lc($field_names_to_be_signed[$j]) eq 'received') { splice(@field_names_to_be_signed,$j,1); last } } } my $expiration; if (defined $sig_options{x}) { $expiration = $sig_options{x}; my $j = int($expiration); $expiration = $expiration > $j ? $j+1 : $j; # ceiling } $dkim->add_signature( Mail::DKIM::Signature->new( Selector => $sig_options{s}, Domain => $sig_options{d}, Timestamp => int($msginfo->rx_time), # floor Headers => join(':', reverse @field_names_to_be_signed), Key => $key, !defined $sig_options{c} ? () : (Method => $sig_options{c}), !defined $sig_options{a} ? () : (Algorithm => $sig_options{a}), !defined $sig_options{q} ? () : (Query => $sig_options{q}), !defined $sig_options{i} ? () : (Identity => $sig_options{i}), !defined $expiration ? () : (Expiration => $expiration), # ceiling )); undef; }; # end sub my($dkim_wrapper); eval { my($dkim_signer) = Mail::DKIM::Signer->new(Policy => $policyfn); $dkim_signer or die "Could not create a Mail::DKIM::Signer object\n"; # # NOTE: dkim wrapper will strip bare CR before signing, which suits # forwarding by SMTP which does the same; with other forwarding methods # such as a pipe or milter, bare CRs in a message may break signatures # # feeding mail to a DKIM signer $dkim_wrapper = Amavis::Out::SMTP->new_dkim_wrapper($dkim_signer,1); my($msg) = $msginfo->mail_text; # a file handle or a MIME::Entity object if (defined($msg) && !$msg->isa('MIME::Entity')) { $msg->seek($msginfo->skip_bytes, 0) or die "dkim_sign_mail: Can't rewind mail file: $!"; } my($hdr_edits) = $msginfo->header_edits; $hdr_edits = Amavis::Out::EditHeader->new if !$hdr_edits; my($received_cnt) = $hdr_edits->write_header($msg,$dkim_wrapper,!$initial_submission); if (!defined($msg)) { # empty mail } elsif ($msg->isa('MIME::Entity')) { $msg->print_body($dkim_wrapper); } else { my($nbytes,$buff); while (($nbytes=$msg->read($buff,16384)) > 0) { $dkim_wrapper->print($buff) or die "Can't write to dkim signer: $!"; } defined $nbytes or die "Error reading: $!"; } $dkim_wrapper->close or die "Can't close dkim wrapper: $!"; undef $dkim_wrapper; $dkim_signer->CLOSE or die "Can't close dkim signer: $!"; @signatures = $dkim_signer->signatures; undef $dkim_signer; 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(0, "dkim: signing error: %s", $eval_stat); }; if (defined $dkim_wrapper) { $dkim_wrapper->close } # ignoring status section_time('fwd-data-dkim'); } # signatures must have all the required tags: d, s, b, bh; check to make sure # if (ll(5)) { do_log(5, "dkim: %s", $_->as_string) for @signatures } my(@sane_signatures); for my $s (@signatures) { my(@missing); for my $pair ( ['d', $s->domain], ['s', $s->selector], ['b', $s->data], ['bh', $s->body_hash] ) { my($tag,$val) = @$pair; push(@missing,$tag) if !defined($val) || $val eq ''; } if (!@missing) { push(@sane_signatures, $s); # remember just the last one (typically the only one) $msginfo->dkim_signwith_sd( [$s->selector, $s->domain] ); } else { do_log(2, "dkim: signature is missing tag %s, skipping: %s", join(',',@missing), $s->as_string); } } @sane_signatures; } # prepare Authentication-Results header fields according to RFC 5451 # and RFC 6008 # sub generate_authentication_results($;$$) { my($msginfo,$allow_none,$sigs_ref) = @_; $sigs_ref = $msginfo->dkim_signatures_all if @_ < 3; my($authservid) = c('myauthservid'); $authservid = c('myhostname') if !defined $authservid || $authservid eq ''; # note that RFC 5451 declares A-R header field as structured, which is why # we are inserting a \n into top-level locations suitable for folding, # and let sub hdr() choose suitable folding points my(@results, %all_b, %all_b_valid, %all_b_8); my($sig_cnt_dk, $sig_cnt_dkim, $result_str) = (0, 0, ''); for my $sig (!$sigs_ref ? () : @$sigs_ref) { # first pass my($sig_result, $details, $str); $sig_result = $sig->result; if (defined $sig_result) { $sig_result = lc $sig_result; } else { ($sig_result, $details) = ('pass', 'just generated, assumed good'); $sig->result($sig_result, $details); } my $valid = $sig_result eq 'pass'; if ($valid) { my $expiration_time = $sig->expiration; if (defined $expiration_time && $expiration_time =~ /^\d{1,12}\z/ && $msginfo->rx_time > $expiration_time) { ($sig_result, $details) = ('fail', 'good, but expired'); $sig->result($sig_result, $details); $valid = 0; } } if ($sig->isa('Mail::DKIM::DkSignature')) { $sig_cnt_dk++ } else { $sig_cnt_dkim++ }; my $b = $sig->data; if (defined $b) { $b =~ tr/ \t\n//d; # remove FWS, just in case $all_b_8{substr($b,0,8)}++; $all_b{$b}++; $all_b_valid{$b}++ if $valid; } } # RFC 5451 result: none, pass, fail, policy, neutral, temperror, permerror # Mail::DKIM result: pass, fail, invalid, temperror, none for my $sig (!$sigs_ref ? () : @$sigs_ref) { # second pass my $result_val; # RFC 5451 result value my $sig_result = lc $sig->result; my $details = $sig->result_detail; my $valid = $sig_result eq 'pass'; if ($valid) { $result_val = 'pass'; } else { # map a Mail::DKIM::Signature result into an RFC 5451 result value $result_val = $sig_result eq 'temperror' ? 'temperror' : $sig_result eq 'fail' ? 'fail' : $sig_result eq 'invalid' ? 'neutral' : 'permerror'; } my $d = $sig->domain; $d = lc $d if defined $d; my $str = ''; my $add_header_b; # RFC 6008, should we add a header.b for this signature? my $key_size; eval { my $pk = $sig->get_public_key; $key_size = $pk->cork->size * 8 if $pk && $pk->cork; }; if ($sig->isa('Mail::DKIM::DkSignature')) { $add_header_b = 1 if $sig_cnt_dk > 1; my($rfc2822_sender) = $msginfo->rfc2822_sender; my($fm) = $msginfo->rfc2822_from; my(@rfc2822_from) = !defined($fm) ? () : ref $fm ? @$fm : $fm; my $id = defined $d ? '@'.$d : ''; $str .= ";\n domainkeys=" . $result_val; $str .= sprintf(' (%d-bit key)', $key_size) if defined $key_size; if (defined $details && $details ne '' && lc $details ne lc $result_val){ local($1); # turn it into an RFC 2045 quoted-string $details =~ s{([\000-\037\177"\\])}{\\$1}gs; # RFC 5322 qtext $str .= "\n reason=\"$details\""; } if (@rfc2822_from && $rfc2822_from[0] =~ /(\@[^\@]*)\z/s && lc($1) eq $id) { $str .= "\n header.from=" . join(',', map(quote_rfc2821_local($_), @rfc2822_from)); } if (defined($rfc2822_sender) && $rfc2822_sender =~ /(\@[^\@]*)\z/s && lc($1) eq $id) { $str .= "\n header.sender=" . quote_rfc2821_local($rfc2822_sender); } } else { # a DKIM signature $add_header_b = 1 if $sig_cnt_dkim > 1; $str .= ";\n dkim=" . $result_val; $str .= sprintf(' (%d-bit key)', $key_size) if defined $key_size; if (defined $details && $details ne '' && lc $details ne lc $result_val){ local($1); # turn it into an RFC 2045 quoted-string $details =~ s{([\000-\037\177"\\])}{\\$1}gs; # RFC 5322 qtext $str .= "\n reason=\"$details\""; } } $str .= "\n header.d=" . $d if defined $d; my $b = $sig->data; if (defined $b && $add_header_b) { # RFC 6008: The value associated with this item in the header field # MUST be at least the first eight characters of the digital signature # (the "b=" tag from a DKIM-Signature) for which a result is being # relayed, and MUST be long enough to be unique among the results # being reported. $b =~ tr/ \t\n//d; # remove FWS, just in case if ($b !~ m{^ [A-Za-z0-9+/]+ =* \z}xs) { # ensure base64 syntax do_log(2, "generate_AR: bad signature tag b=%s", $b); } elsif ($all_b{$b} > 1 && $all_b_valid{$b} && !$valid) { # exact duplicates: do not report invalid ones if at least one is valid # RFC 6008 section 6.2.: a cautious implementation could discard # the false negative in that instance. do_log(2, "generate_AR: not reporting bad duplicates: %s", $b); $str = ''; # ditch the report for this signature } elsif ($all_b_8{$b} > $all_b{$b}) { do_log(2, "generate_AR: not reporting b for collisions: %s", $b); } else { $str .= "\n header.b=" . substr($b,0,8); } } $result_str .= $str; } # just provide a single A-R with all results combined push(@results, $result_str) if $result_str ne ''; push(@results, ";\n dkim=none") if !@results && $allow_none; $_ = sprintf("%s (%s)%s", $authservid, $myproduct_name, $_) for @results; @results; # none, one, or more A-R header field bodies } # adjust spam score for each recipient so that the final spam score # will be shifted towards a fixed score assigned to a signing domain (its # 'reputation', as obtained through @signer_reputation_maps); the formula is: # adjusted_spam_score = f*reputation + (1-f)*spam_score; 0 <= f <= 1 # which has the same semantics as auto_whitelist_factor in SpamAssassin AWL # sub adjust_score_by_signer_reputation($) { my($msginfo) = @_; my($reputation_factor) = c('reputation_factor'); $reputation_factor = 0 if $reputation_factor < 0; $reputation_factor = 1 if $reputation_factor > 1; my($sigs_ref) = $msginfo->dkim_signatures_valid; if (defined $reputation_factor && $reputation_factor > 0 && defined $sigs_ref && @$sigs_ref > 0) { my($best_reputation_signer,$best_reputation_score); my($srm) = ca('signer_reputation_maps'); # walk through all valid signatures, find best (smallest) reputation value for my $sig (@$sigs_ref) { my($sdid) = lc($sig->domain); my($val,$key) = lookup2(0, '@'.$sdid, $srm); if (defined $val && (!defined $best_reputation_score || $val < $best_reputation_score)) { $best_reputation_signer = $sdid; $best_reputation_score = $val; } } if (defined $best_reputation_score) { my($ll) = 2; # initial log level for my $r (@{$msginfo->per_recip_data}) { my($spam_level) = $r->spam_level; next if !defined $spam_level; my($new_level) = $reputation_factor * $best_reputation_score + (1-$reputation_factor) * $spam_level; $r->spam_level($new_level); my($spam_tests) = 'AM.DKIM_REPUT=' . (0+sprintf("%.3f", $new_level-$spam_level)); if (!defined($r->spam_tests)) { $r->spam_tests([ \$spam_tests ]); } else { unshift(@{$r->spam_tests}, \$spam_tests); } ll($ll) && do_log($ll, "dkim: score %.3f adjusted to %.3f due to reputation ". "(%s) of a signer domain %s", $spam_level, $new_level, $best_reputation_score, $best_reputation_signer); $ll = 5; # reduce log clutter after the first recipient } } } } # check if we have a valid author domain signature and do # other DKIM pre-processing; called from collect_some_dkim() # sub collect_some_dkim_info($) { my($msginfo) = @_; my($rfc2822_sender) = $msginfo->rfc2822_sender; my(@rfc2822_from) = $msginfo->rfc2822_from; # now that we have a parsed From, check if we have a valid # author domain signature and do other DKIM pre-processing my(@bank_names, %bank_names, %bn_auth_already_queried); my($atpbm) = ca('author_to_policy_bank_maps'); my(@signatures_valid); my($sigs_ref) = $msginfo->dkim_signatures_all; my($sig_ind) = 0; # index of a signature in a signature array for my $sig (!defined($sigs_ref) ? () : @$sigs_ref) { # for each signature my($valid) = lc($sig->result) eq 'pass'; my($timestamp_age, $creation_time, $expiration_time); if (!$sig->isa('Mail::DKIM::DkSignature')) { $creation_time = $sig->timestamp; # method only implemented for DKIM sig $timestamp_age = $msginfo->rx_time - $creation_time if defined $creation_time && $creation_time =~ /^\d{1,12}\z/; } $expiration_time = $sig->expiration; my($expired) = defined $expiration_time && $expiration_time =~ /^\d{1,12}\z/ && ($msginfo->rx_time > $expiration_time || ( defined $creation_time && $creation_time =~ /^\d{1,12}\z/ && $creation_time > $expiration_time ) ); my($sdid) = lc($sig->domain); # See if a signature matches address in any of the sender/author fields. # In the absence of an explicit Sender header field, the first author # acts as the 'agent responsible for the transmission of the message'. my(@addr_list) = ($msginfo->sender, defined $rfc2822_sender ? $rfc2822_sender : $rfc2822_from[0], @rfc2822_from); for my $addr (@addr_list) { next if !defined $addr; local($1); my($domain); $domain = lc($1) if $addr =~ /\@([^\@]*)\z/s; # turn addresses in @addr_list into booleans, representing match outcome $addr = defined($domain) && $domain eq $sdid ? 1 : 0; } # label which header fields are covered by each signature; # doesn't work for old DomainKeys signatures where h may be missing # and where recurring header fields may only be listed once my(@signed_header_field_names) = map(lc($_), $sig->headerlist); # 'h' tag { my(%field_counts); $field_counts{$_}++ for @signed_header_field_names; for (my $j=-1; ; $j--) { # walk through header fields, bottom-up my($f_ind,$f_name,$fld) = $msginfo->get_header_field(undef,$j); last if !defined $f_ind; # reached the top if ($field_counts{$f_name} > 0) { # header field is covered by this sig $msginfo->header_field_signed_by($f_ind,$sig_ind); # store sig index $field_counts{$f_name}--; } } } if ($valid && !$expired) { push(@signatures_valid, $sig); my($sig_domain) = $sig->domain; $sig_domain = '?' if !$sig_domain; # make sure it is true as a boolean # # note that only the author domain signature (based on RFC 2822.From) # is a valid concept in ADSP; we are also using the same rules to match # against RFC 2822.Sender and envelope sender address, but results are # only of informational/curiosity interest and deeper significance # must not be attributed to dkim_envsender_sig and dkim_sender_sig! # $msginfo->dkim_envsender_sig($sig_domain) if $addr_list[0]; $msginfo->dkim_sender_sig($sig_domain) if $addr_list[1]; $msginfo->dkim_author_sig($sig_domain) if grep($_, @addr_list[2..$#addr_list]); # SDID matches addr $msginfo->dkim_thirdparty_sig($sig_domain) if !$msginfo->dkim_author_sig; if (@$atpbm) { # any author to policy bank name mappings? for my $j (0..$#rfc2822_from) { # for each author (usually only one) my($key) = $rfc2822_from[$j]; # query key: as-is author address for author domain signatures, and # author address with '/@signer-domain' appended for 3rd party sign. # e.g.: 'user@example.com', 'user@sub.example.com/@example.org' for my $opt ( ($addr_list[$j+2] ? '' : ()), '/@'.lc($sig->domain) ) { next if $bn_auth_already_queried{$key.$opt}; my($result,$matchingkey) = lookup2(0,$key,$atpbm, Label=>'AuthToPB', $opt eq '' ? () : (AppendStr=>$opt)); $bn_auth_already_queried{$key.$opt} = 1; if ($result) { if ($result eq '1') { # a handy usability trick to supply a hardwired policy bank # name when acl-style lookup table is used, which can only # return a boolean (undef, 0, or 1) $result = 'AUTHOR_APPROVED'; } # $result is a list of policy banks as a comma-separated string my(@pbn); # collect list of newly encountered policy bank names for (map { s/^[ \t]+//; s/[ \t]+\z//; $_ } split(/,/,$result)) { next if $_ eq '' || $bank_names{$_}; push(@pbn,$_); $bank_names{$_} = 1; } if (@pbn) { push(@bank_names,@pbn); ll(2) && do_log(2, "dkim: policy bank %s by %s", join(',',@pbn), $matchingkey); } } } } } } if (ll(5)) { my($pubkey,$eval_stat); # Mail::DKIM >=0.31 caches result; it can die with "not available" eval { $pubkey = $sig->get_public_key; 1; } or do { $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; }; if (defined $eval_stat) { do_log(5, "dkim: public key s=%s d=%s, error: %s", $sig->selector, $sig->domain, $eval_stat); } elsif (!$pubkey) { do_log(5, "dkim: no public key s=%s d=%s",$sig->selector,$sig->domain); } else { do_log(5, "dkim: public key s=%s d=%s%s", $sig->selector, $sig->domain, join('',map { my $v = $pubkey->get_tag($_); defined $v ? " $_=$v" : '' } qw(v g h k t s))); } } ll(2) && do_log(2, "dkim: %s%s%s %s signature by d=%s, From: %s, ". "a=%s, c=%s, s=%s, i=%s%s%s%s", $valid ? 'VALID' : 'FAILED', $expired ? ', EXPIRED' : '', $timestamp_age >= -1 ? '' : ', IN_FUTURE:('.format_time_interval(-$timestamp_age).')', join('+', (map($_ ? 'Author' : (), @addr_list[2..$#addr_list])), $addr_list[1] ? 'Sender' : (), $addr_list[0] ? 'MailFrom' : (), !grep($_, @addr_list) ? 'third-party' : ()), $sig->domain, join(", ", qquote_rfc2821_local(@rfc2822_from)), $sig->algorithm, scalar($sig->canonicalization), $sig->selector, $sig->identity, !$msginfo->originating ? '' : ', ORIG [' . $msginfo->client_addr . ']:' . $msginfo->client_port, !defined($msginfo->is_mlist) ? '' : ", m.list(".$msginfo->is_mlist.")", $valid ? '' : ', '.$sig->result_detail, ); $sig_ind++; } if (@bank_names) { # ignore nonexisting bank names @bank_names = grep(defined $Amavis::policy_bank{$_}, unique_list(\@bank_names)); if (@bank_names) { Amavis::load_policy_bank($_,$msginfo) for @bank_names; $msginfo->originating(c('originating')); # may have changed } } $msginfo->dkim_signatures_valid(\@signatures_valid) if @signatures_valid; # if (ll(5) && $sig_ind > 0) { # # show which header fields are covered by which signature # for (my $j=0; ; $j++) { # my($f_ind,$f_name,$fld) = $msginfo->get_header_field(undef,$j); # last if !defined $f_ind; # my(@sig_ind) = $msginfo->header_field_signed_by($f_ind); # do_log(5, "dkim: %-5s %s.", !@sig_ind ? '' : '['.join(',',@sig_ind).']', # substr($fld,0,54)); # } # } } 1; __DATA__ # package Amavis::Tools; use strict; use re 'taint'; use warnings; use warnings FATAL => qw(utf8 void); no warnings 'uninitialized'; BEGIN { require Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); $VERSION = '2.303'; @ISA = qw(Exporter); @EXPORT_OK = qw(&show_or_test_dkim_public_keys &generate_dkim_private_key &convert_dkim_keys_file); import Amavis::Conf qw(:platform c cr ca @dkim_signing_keys_list @dkim_signing_keys_storage); import Amavis::Util qw(untaint ll do_log); import Amavis::rfc2821_2822_Tools qw(rfc2822_timestamp); } use subs @EXPORT_OK; use Errno qw(ENOENT EACCES); use IO::File qw(O_RDONLY O_WRONLY O_RDWR O_APPEND O_CREAT O_EXCL); use Crypt::OpenSSL::RSA (); # Prints DNS TXT resource records for corresponding DKIM private keys (as # previously declared by calls to dkim_key) in a format directly suitable # for inclusion in DNS zone files. If an argument is provided the result is # restricted to listed domains only, otherwise RR for all domains are shown. # Note that a domain may have more than one RR: one RR for each selector. # # When a search argument is provided (even if '.'), the printed list is # sorted according to reversed domain labels (e.g. com.example.sub.host), # entries with the same domain are kept in original order. When there are # no search arguments, the original order is retained. # sub show_or_test_dkim_public_keys($$) { my($cmd,$args) = @_; my(@seek_domains) = @$args; # when list is empty all domains are implied my(@sort_list) = map { my($d) = lc($dkim_signing_keys_list[$_]->{domain}); my($d_re) = $dkim_signing_keys_list[$_]->{domain_re}; [$_, $d, $d_re, join('.',reverse split(/\./,$d,-1))] } 0 .. $#dkim_signing_keys_list; if (@seek_domains) { # sort only when there are any search arguments present @sort_list = sort {$a->[3] cmp $b->[3] || $a->[0] <=> $b->[0]} @sort_list; } my($any) = 0; for my $e (@sort_list) { my($j,$domain,$domain_re) = @$e; local($1); next if @seek_domains && !grep { defined $domain_re ? lc($_) =~ /$domain_re/ : /^\.(.*)\z/s ? $domain eq lc($1) || $domain =~ /(?:\.|\z)\Q$1\E\z/si : $domain eq lc($_) } @seek_domains; $any++; my($key_opts) = $dkim_signing_keys_list[$j]; if ($cmd eq 'testkeys' || $cmd eq 'testkey') { test_dkim_key(%$key_opts); } else { my($key_storage_ind) = $key_opts->{key_storage_ind}; my($key,$dev,$inode,$fname) = @{ $dkim_signing_keys_storage[$key_storage_ind] }; my(@pub) = split(/\r?\n/, $key->get_public_key_x509_string); @pub = grep(!/^---.*?---\z/ && !/^[ \t]*\z/, @pub); my(@tags) = map($_.'='.$key_opts->{$_}, grep(defined $key_opts->{$_}, qw(v g h k s t n))); printf("; key#%d, domain %s, %s\n", $key_opts->{key_ind} + 1, $domain, $fname) if defined $fname; printf("; CANNOT DECLARE A WILDCARDED LABEL IN DNS, ". "AVOID OR EDIT MANUALLY!\n") if defined $key_opts->{domain_re}; printf("%s._domainkey.%s.\t%s TXT (%s)\n\n", $key_opts->{selector}, $domain, '3600', join('', map("\n" . ' "' . $_ . '"', join('; ',@tags,'p='), @pub)) ); } } if (!@dkim_signing_keys_list) { printf("No DKIM private keys declared in a config file.\n"); } elsif (!$any) { printf("No DKIM private keys match the selection list.\n"); } } sub test_dkim_key(@) { my(%key_options) = @_; my($now) = Time::HiRes::time; my($key_storage_ind) = $key_options{key_storage_ind}; my($key,$dev,$inode,$fname) = @{ $dkim_signing_keys_storage[$key_storage_ind] }; if (UNIVERSAL::isa($key,'Crypt::OpenSSL::RSA')) { $key = Mail::DKIM::PrivateKey->load(Cork => $key); # avail since 0.31 # my($pkcs1) = $key->get_private_key_string; # most compact # $pkcs1 =~ s/^---.*?---(?:\r?\n|\z)//gm; $pkcs1 =~ tr/\r\n//d; # $key = Mail::DKIM::PrivateKey->load(Data => $pkcs1); } my($policyfn) = sub { my($dkim) = shift; $dkim->add_signature( Mail::DKIM::Signature->new( Selector => $key_options{selector}, Domain => $key_options{domain}, Method => 'simple/simple', Algorithm => 'rsa-sha256', Timestamp => int($now), Expiration => int($now)+24*3600, Key => $key, )); undef; }; my($msg) = sprintf( "From: test\@%s\nMessage-ID: <123\@%s>\nDate: %s\nSubject: test\n\ntest\n", $key_options{domain}, $key_options{domain}, rfc2822_timestamp($now)); $msg =~ s{\n}{\015\012}gs; my(@gen_signatures, @read_signatures); eval { my($dkim_signer) = Mail::DKIM::Signer->new(Policy => $policyfn); $dkim_signer or die "Could not create a Mail::DKIM::Signer object"; $dkim_signer->PRINT($msg) or die "Can't write to dkim: $!"; $dkim_signer->CLOSE or die "Can't close dkim signer: $!"; @gen_signatures = $dkim_signer->signatures; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; print STDERR "dkim signing failed: $eval_stat\n"; }; $msg = $_->as_string . "\015\012" . $msg for @gen_signatures; eval { my($dkim_verifier) = Mail::DKIM::Verifier->new; $dkim_verifier or die "Could not create a Mail::DKIM::Verifier object"; $dkim_verifier->PRINT($msg) or die "Can't write to dkim: $!"; $dkim_verifier->CLOSE or die "Can't close dkim_verifier: $!"; @read_signatures = $dkim_verifier->signatures; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; print STDERR "dkim verification failed: $eval_stat\n"; }; # printf("%s\n", $fname) if defined $fname; printf("TESTING#%d: %-33s => %s\n", $key_options{key_ind} + 1, $_->selector . '._domainkey.' . $_->domain, $_->result_detail) for @read_signatures; } sub generate_dkim_private_key(@) { my($fname,$nbits) = @_; my($fh); eval { $nbits = 1024 if !defined($nbits) || $nbits eq ''; $nbits =~ /^\d+\z/ or die "Number of bits in a key must be numeric\n"; $nbits >= 512 or die "Number of bits too small (suggested 768..1536)\n"; $nbits <= 4096 or die "Number of bits too large (suggested 768..1536)\n"; defined $fname && $fname ne '' or die "File name for a key not provided\n"; $fh = IO::File->new; $fh->open(untaint($fname), O_CREAT|O_EXCL|O_RDWR, 0600) or die "Can't create file \"$fname\": $!\n"; my($rsa) = Crypt::OpenSSL::RSA->generate_key($nbits); $fh->print($rsa->get_private_key_string) or die "Error writing key to a file \"$fname\": $!\n"; $fh->close or die "Can't close file \"$fname\": $!\n"; undef $fh; printf STDERR ("Private RSA key successfully written to file \"%s\" ". "(%d bits, PEM format) \n", $fname,$nbits); 1; } or do { my $eval_stat = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; $fh->close if defined $fh; # ignoring status die "genrsa: $eval_stat\n"; } } # Reads a dkim-filter -compatible key specifications. From the dkim-filter # man page: The keyfile should contain a set of lines of the form # sender-pattern:signing-domain:keypath where sender-pattern is a pattern # to match against message senders (with a special character "*" interpreted # as "zero or more characters"), signing-domain is the domain to announce as # the signing domain when generating signatures (or a '*', implying author's # domain), and keypath is a path to the PEM-formatted private key to be used # for signing messages which match the sender-pattern. The selector used in # the signature will be the filename portion of keypath. A line starting # with "/" is interpreted as a root directory for keys, meaning the keypath # values after that line in the file are taken relative to that path. If a # file referenced by keypath cannot be opened, the filter will try again by # appending ".pem" and then ".private". '#'-delimited comments and blank # lines are ignored. # sub convert_dkim_keys_file($) { my($keysfile) = @_; my($inp) = IO::File->new; $inp->open($keysfile,'<') or die "dkim_key_file: Can't open file $keysfile for reading: $!"; my($basedir,@options,@opt_re,%domain_selectors); my($rn) = 0; my($ln); for ($! = 0; defined($ln=$inp->getline); $! = 0) { chomp($ln); $rn++; local($1); my($selector,$key_fn); if ($ln =~ /^ \s* (?: \# | \z)/xs) { # skip empty and all-comment lines } elsif ($ln =~ m{^/}) { $basedir = $ln; $basedir .= '/' if $basedir !~ m{/\z}; } else { my($sender_pattern,$signing_domain,$keypath) = map { s/^\s+//; s/\s+\z//; $_ } split(/:/,$ln,3); defined $sender_pattern && $sender_pattern ne '' or die "Error in $keysfile, empty sender pattern, line $rn: $ln\n"; defined $keypath && $keypath ne '' || $signing_domain eq '' or die "Error in $keysfile, empty file name field, line $rn: $ln\n"; $keypath = $basedir . $keypath if defined $basedir && $keypath !~ m{^/}; for my $ext ('', '.pem', '.private') { my($errn) = stat($keypath.$ext) ? 0 : 0+$!; if ($errn != ENOENT) { $key_fn = $keypath.$ext; last } } defined $key_fn or die "File $keypath does not exist, $keysfile line $rn: $ln\n"; $selector = lc($1) if $keypath =~ m{ (?: ^ | / ) ( [^/]+? ) (?: \.pem | \.private )? \z }xs; # must convert sender pattern to unquoted form to match actual addresses my($sender_domain); if ($sender_pattern eq '*' || $sender_pattern eq '*@*') { $sender_pattern = $sender_domain = '*'; } else { my($sender_localpart); ($sender_localpart, $sender_domain) = Amavis::rfc2821_2822_Tools::split_address( Amavis::rfc2821_2822_Tools::unquote_rfc2821_local($sender_pattern)); $sender_domain =~ s/^\@//; $sender_domain = lc($sender_domain); $sender_pattern = $sender_localpart . '@' . $sender_domain; } if ($signing_domain eq '*') { $signing_domain = $sender_domain } $signing_domain = lc($signing_domain); if ($signing_domain ne '' && !$domain_selectors{$signing_domain}{$selector}) { # dkim_key($signing_domain,$selector,$key_fn); # declare a signing key printf("dkim_key(%-18s %-12s '%s');\n", "'".$signing_domain."',", "'".$selector."',", $key_fn); $domain_selectors{$signing_domain}{$selector} = 1; } if ($signing_domain eq $sender_domain) { $signing_domain = '*' } push(@options, [$sender_pattern, $signing_domain, $selector]); } } defined $ln || $!==0 or die "Error reading from $keysfile: $!"; $inp->close or die "Error closing $keysfile: $!"; # # prepare by_sender signature options lookup table when non-default # signing is required (e.g. third-party signatures) # my($in_options) = 0; for my $opt (@options) { my($sender_pattern, $signing_domain, $selector) = @$opt; if ($signing_domain eq '*') { # implies author domain signature, no need for special options } else { $sender_pattern =~ s/\*{2,}/*/gs; # collapse successive wildcards $sender_pattern =~ # '*' is a wildcard, quote the rest s{ ([@#/.^$|*+?(){}\[\]\\]) }{ $1 eq '*' ? '.*' : '\\'.$1 }gex; $sender_pattern = '^' . $sender_pattern . '\\z'; # implicit anchors # remove trailing first, leading next, preferring /^.*\z/ -> /^/, not /\z/ $sender_pattern =~ s/\.\*\\z\z//s; # remove trailing anchor if redundant $sender_pattern =~ s/^\^\.\*//s; # remove leading anchor if redundant $sender_pattern = '(?:)' if $sender_pattern eq ''; # just in case $signing_domain = undef if $signing_domain eq ''; $selector = undef if $selector eq ''; # case insensitive matching for compatibility with dkim-milter push(@opt_re, [ qr/$sender_pattern/is => ( !defined($signing_domain) || keys(%{$domain_selectors{$signing_domain}})==1 ? { d => $signing_domain } : { d => $signing_domain, s => $selector } ) ]); if (!$in_options) { printf("\n%s\n", '@dkim_signature_options_bysender_maps = (new_RE('); $in_options = 1; } printf(" [ %-30s => { d=>%s%s} ],\n", 'qr/' . $sender_pattern . '/is', !defined($signing_domain) ? 'undef' : "'".$signing_domain."'", !defined($signing_domain) || keys %{$domain_selectors{$signing_domain}} == 1 ? '' : !defined($selector) ? ', s=>undef' : ", s=>'".$selector."'"); } } printf("%s\n", '));') if $in_options; # use Data::Dump (); Data::Dump::dump(@opt_re); # unshift(@dkim_signature_options_bysender_maps, # Amavis::Lookup::RE->new(@opt_re)) if @opt_re; } 1; __DATA__ # # ============================================================================= # This text section governs how a main per-message amavisd-new log entry (at # log level 0) is formed (config variable $log_short_templ). Empty disables it. [?%#D|#|Passed # [? [:ccat|major] |# OTHER|CLEAN|MTA-BLOCKED|OVERSIZED|BAD-HEADER-[:ccat|minor]|SPAMMY|SPAM|\ UNCHECKED|BANNED (%F)|INFECTED (%V)] {[:actions_performed]}# , [? %p ||%p ][?%a||[?%l||LOCAL ][:client_addr_port] ][?%e||\[%e\] ]%s -> [%D|,]# [? %q ||, quarantine: %q]# [? %Q ||, Queue-ID: %Q]# [? %m ||, Message-ID: %m]# [? %r ||, Resent-Message-ID: %r]# [? %i ||, mail_id: %i]# , Hits: [:SCORE]# , size: %z# [? [:partition_tag] ||, pt: [:partition_tag]]# [~[:remote_mta_smtp_response]|["^$"]||[", queued_as: "]]\ [remote_mta_smtp_response|[~%x|["queued as ([0-9A-Za-z]+)$"]|["%1"]|["%0"]]|/]# #[? [:header_field|Subject]||, Subject: [:dquote|[:mime2utf8|[:header_field|Subject]|100|1]]]# #[? [:header_field|From]||, From: [:uquote|[:mime2utf8|[:header_field|From]|100|1]]]# #[? %#T ||, Tests: \[[%T|,]\]]# [? [:dkim|sig_sd] ||, dkim_sd=[:dkim|sig_sd]]# [? [:dkim|newsig_sd] ||, dkim_new=[:dkim|newsig_sd]]# , %y ms# ] [?%#O|#|Blocked # [? [:ccat|major|blocking] |# OTHER|CLEAN|MTA-BLOCKED|OVERSIZED|BAD-HEADER-[:ccat|minor]|SPAMMY|SPAM|\ UNCHECKED|BANNED (%F)|INFECTED (%V)] {[:actions_performed]}# , [? %p ||%p ][?%a||[?%l||LOCAL ][:client_addr_port] ][?%e||\[%e\] ]%s -> [%O|,]# [? %q ||, quarantine: %q]# [? %Q ||, Queue-ID: %Q]# [? %m ||, Message-ID: %m]# [? %r ||, Resent-Message-ID: %r]# [? %i ||, mail_id: %i]# , Hits: [:SCORE]# , size: %z# [? [:partition_tag] ||, pt: [:partition_tag]]# #[? [:header_field|Subject]||, Subject: [:dquote|[:mime2utf8|[:header_field|Subject]|100|1]]]# #[? [:header_field|From]||, From: [:uquote|[:mime2utf8|[:header_field|From]|100|1]]]# #[? %#T ||, Tests: \[[%T|,]\]]# [? [:dkim|sig_sd] ||, dkim_sd=[:dkim|sig_sd]]# [? [:dkim|newsig_sd] ||, dkim_new=[:dkim|newsig_sd]]# , %y ms# ] __DATA__ # # ============================================================================= # This text section governs how a verbose per-message amavisd-new log entry # is formed (config variable $log_verbose_templ). An empty text will prevent # a verbose log entry, multiline text will produce multiple log entries, one # for each nonempty line. Syntax is explained in the README.customize file. [?%#D|#|Passed # [? [:ccat|major] |# OTHER|CLEAN|MTA-BLOCKED|OVERSIZED|BAD-HEADER-[:ccat|minor]|SPAMMY|SPAM|\ UNCHECKED|BANNED (%F)|INFECTED (%V)] {[:actions_performed]}# , [? %p ||%p ][?%a||[?%l||LOCAL ][:client_addr_port] ][?%e||\[%e\] ]%s -> [%D|,]# [? [:tls_in] ||, tls: [:tls_in]]# [? %q ||, quarantine: %q]# [? %Q ||, Queue-ID: %Q]# [? %m ||, Message-ID: %m]# [? %r ||, Resent-Message-ID: %r]# [? %i ||, mail_id: %i]# , Hits: [:SCORE]# , size: %z# [? [:partition_tag] ||, pt: [:partition_tag]]# [~[:remote_mta_smtp_response]|["^$"]||[", queued_as: "]]\ [remote_mta_smtp_response|[~%x|["queued as ([0-9A-Za-z]+)$"]|["%1"]|["%0"]]|/]# [? [:header_field|Subject]||, Subject: [:dquote|[:mime2utf8|[:header_field|Subject]|100|1]]]# [? [:header_field|From]||, From: [:uquote|[:mime2utf8|[:header_field|From]|100|1]]]# [? [:dkim|author] || (dkim:AUTHOR)]# [? [:useragent|name] ||, [:useragent|name]: [:uquote|[:useragent|body]]]# , helo=[:client_helo]# [? %#T ||, Tests: \[[%T|,]\]]# [? [:banning_rule_key] ||, b.key=[:banning_rule_key]]# [? [:banning_rule_comment] ||, b.com=[:banning_rule_comment]]# [? [:banning_rule_rhs] ||, b.rhs=[:banning_rule_rhs]]# [? [:banned_parts_as_attr] ||, b.parts=[:banned_parts_as_attr]]# [:supplementary_info|SCTYPE|, shortcircuit=%%s]# [:supplementary_info|AUTOLEARN|, autolearn=%%s]# [:supplementary_info|AUTOLEARNSCORE|, autolearnscore=%%s]# [? [:supplementary_info|LANGUAGES] ||, languages=[:uquote|[:supplementary_info|LANGUAGES]]]# [? [:supplementary_info|RELAYCOUNTRY] ||, relaycountry=[:uquote|[:supplementary_info|RELAYCOUNTRY]]]# [? [:supplementary_info|ASN] ||, asn=[:uquote|[:supplementary_info|ASN] [:supplementary_info|ASNCIDR]]]# #[? [:supplementary_info|DCCB] ||, dcc=[:supplementary_info|DCCB]:[:uquote|[:supplementary_info|DCCR]]]# #[? [:supplementary_info|DCCREP] ||, dcc_rep=[:supplementary_info|DCCREP]]# #[:supplementary_info|AWLSIGNERMEAN|, signer_avg=%%s]# #[? [:dkim|domain] ||, dkim_d=[:dkim|domain]]# [? [:dkim|identity] ||, dkim_i=[:dkim|identity]]# [? [:dkim|sig_sd] ||, dkim_sd=[:dkim|sig_sd]]# [? [:dkim|newsig_sd] ||, dkim_new=[:dkim|newsig_sd]]# [? [:rusage|ru_maxrss] ||, rss=[:rusage|ru_maxrss]]# , %y ms# ] [?%#O|#|Blocked # [? [:ccat|major|blocking] |# OTHER|CLEAN|MTA-BLOCKED|OVERSIZED|BAD-HEADER-[:ccat|minor]|SPAMMY|SPAM|\ UNCHECKED|BANNED (%F)|INFECTED (%V)] {[:actions_performed]}# , [? %p ||%p ][?%a||[?%l||LOCAL ][:client_addr_port] ][?%e||\[%e\] ]%s -> [%O|,]# [? [:tls_in] ||, tls: [:tls_in]]# [? %q ||, quarantine: %q]# [? %Q ||, Queue-ID: %Q]# [? %m ||, Message-ID: %m]# [? %r ||, Resent-Message-ID: %r]# [? %i ||, mail_id: %i]# , Hits: [:SCORE]# , size: %z# [? [:partition_tag] ||, pt: [:partition_tag]]# [? [:header_field|Subject]||, Subject: [:dquote|[:mime2utf8|[:header_field|Subject]|100|1]]]# [? [:header_field|From]||, From: [:uquote|[:mime2utf8|[:header_field|From]|100|1]]]# [? [:dkim|author] || (dkim:AUTHOR)]# [? [:useragent|name] ||, [:useragent|name]: [:uquote|[:useragent|body]]]# , helo=[:client_helo]# [? %#T ||, Tests: \[[%T|,]\]]# [? [:banning_rule_key] ||, b.key=[:banning_rule_key]]# [? [:banning_rule_comment] ||, b.com=[:banning_rule_comment]]# [? [:banning_rule_rhs] ||, b.rhs=[:banning_rule_rhs]]# [? [:banned_parts_as_attr] ||, b.parts=[:banned_parts_as_attr]]# [:supplementary_info|SCTYPE|, shortcircuit=%%s]# [:supplementary_info|AUTOLEARN|, autolearn=%%s]# [:supplementary_info|AUTOLEARNSCORE|, autolearnscore=%%s]# [? [:supplementary_info|LANGUAGES] ||, languages=[:uquote|[:supplementary_info|LANGUAGES]]]# [? [:supplementary_info|RELAYCOUNTRY] ||, relaycountry=[:uquote|[:supplementary_info|RELAYCOUNTRY]]]# [? [:supplementary_info|ASN] ||, asn=[:uquote|[:supplementary_info|ASN] [:supplementary_info|ASNCIDR]]]# #[? [:supplementary_info|DCCB] ||, dcc=[:supplementary_info|DCCB]:[:uquote|[:supplementary_info|DCCR]]]# #[? [:supplementary_info|DCCREP] ||, dcc_rep=[:supplementary_info|DCCREP]]# #[:supplementary_info|AWLSIGNERMEAN|, signer_avg=%%s]# #[? [:dkim|domain] ||, dkim_d=[:dkim|domain]]# [? [:dkim|identity] ||, dkim_i=[:dkim|identity]]# [? [:dkim|sig_sd] ||, dkim_sd=[:dkim|sig_sd]]# [? [:dkim|newsig_sd] ||, dkim_new=[:dkim|newsig_sd]]# [? [:rusage|ru_maxrss] ||, rss=[:rusage|ru_maxrss]]# , %y ms# ] __DATA__ # # ============================================================================= # This text section governs how a main per-recipient amavisd-new log entry # is formed (config variable $log_recip_templ). An empty text will prevent a # log entry, multi-line text will produce multiple log entries, one for each # nonempty line. Macro %. might be useful, it counts recipients starting # from 1. Syntax is explained in the README.customize file. # Long header fields will be automatically wrapped by the program. # [?%#D|#|Passed # #([:ccat|name|main]) # [? [:ccat|major] |OTHER|CLEAN|MTA-BLOCKED|OVERSIZED|BAD-HEADER|SPAMMY|SPAM|\ UNCHECKED|BANNED (%F)|INFECTED (%V)]# , %s -> [%D|,], Hits: %c# , tag=[:tag_level], tag2=[:tag2_level], kill=[:kill_level]# [~[:remote_mta_smtp_response]|["^$"]||\ ["queued as ([0-9A-Za-z]+)"]|[", queued_as: %1"]|[", fwd: %0"]]# , %0/%1/%2/%k# ] [?%#O|#|Blocked # #([:ccat|name|blocking]) # [? [:ccat|major|blocking] |# OTHER|CLEAN|MTA-BLOCKED|OVERSIZED|BAD-HEADER|SPAMMY|SPAM|\ UNCHECKED|BANNED (%F)|INFECTED (%V)]# , %s -> [%O|,], Hits: %c# , tag=[:tag_level], tag2=[:tag2_level], kill=[:kill_level]# , %0/%1/%2/%k# ] __DATA__ # # ============================================================================= # This is a template for (neutral: non-virus, non-spam, non-banned) # DELIVERY STATUS NOTIFICATIONS to sender. # For syntax and customization instructions see README.customize. # The From, To and Date header fields will be provided automatically. # Long header fields will be automatically wrapped by the program. # Subject: [?%#D|Undeliverable mail|Delivery status notification]\ [? [:ccat|major] |||, MTA-BLOCKED\ |, OVERSIZED message\ |, invalid header section[=explain_badh|1]\ [?[:ccat|minor]||: bad MIME|: unencoded 8-bit character\ |: improper use of control char|: all-whitespace header line\ |: header line longer than 998 characters|: header field syntax error\ |: missing required header field|: duplicate header field|]\ |, UNSOLICITED BULK EMAIL apparently from you\ |, UNSOLICITED BULK EMAIL apparently from you\ |, contents UNCHECKED\ |, BANNED contents type (%F)\ |, VIRUS in message apparently from you (%V)\ ] Message-ID: [? %#D |#|Your message WAS SUCCESSFULLY RELAYED to:[\n %D] [~[:dsn_notify]|["\\bSUCCESS\\b"]|\ and you explicitly requested a delivery status notification on success.\n]\ ] [? %#N |#|The message WAS NOT relayed to:[\n %N] ] [:wrap|78|||This [?%#D|nondelivery|delivery] report was \ generated by the program amavisd-new at host %h. \ Our internal reference code for your message is %n/%i] # ccat_min 0: other, 1: bad MIME, 2: 8-bit char, 3: NUL/CR, # 4: empty, 5: long, 6: syntax, 7: missing, 8: multiple [? [:explain_badh] ||[? [:ccat|minor] |INVALID HEADER |INVALID HEADER: BAD MIME HEADER SECTION OR BAD MIME STRUCTURE |INVALID HEADER: INVALID 8-BIT CHARACTERS IN HEADER SECTION |INVALID HEADER: INVALID CONTROL CHARACTERS IN HEADER SECTION |INVALID HEADER: FOLDED HEADER FIELD LINE MADE UP ENTIRELY OF WHITESPACE |INVALID HEADER: HEADER LINE LONGER THAN RFC 5322 LIMIT OF 998 CHARACTERS |INVALID HEADER: HEADER FIELD SYNTAX ERROR |INVALID HEADER: MISSING REQUIRED HEADER FIELD |INVALID HEADER: DUPLICATE HEADER FIELD |INVALID HEADER ] [[:wrap|78| | |%X]\n] ]\ # [:wrap|78|| |Return-Path: %s[?[:dkim|envsender]|| (OK)]] [:wrap|78|| |From: [:header_field|From|100][?[:dkim|author]|| (dkim:AUTHOR)]] [? [:header_field|Sender]|#|\ [:wrap|78|| |Sender: [:header_field|Sender|100]\ [?[:dkim|sender]|| (dkim:SENDER)]]] [? %m |#|[:wrap|78|| |Message-ID: %m]] [? %r |#|[:wrap|78|| |Resent-Message-ID: %r]] [? %#X|#|[? [:useragent] |#|[:wrap|78|| |[:useragent]]]] [? %j |#|[:wrap|78|| |Subject: [:header_field|Subject|100]]] # ccat_min 0: other, 1: bad MIME, 2: 8-bit char, 3: NUL/CR, # 4: empty, 5: long, 6: syntax, 7: missing, 8: multiple [? [:explain_badh] ||[? [:ccat|minor] |# 0: other |# 1: bad MIME |# 2: 8-bit char WHAT IS AN INVALID CHARACTER IN A MAIL HEADER SECTION? The RFC 5322 document specifies rules for forming internet messages. It does not allow the use of characters with codes above 127 to be used directly (non-encoded) in a mail header section. If such characters (e.g. with diacritics) from ISO Latin or other alphabets need to be included in a header section, these characters need to be properly encoded according to RFC 2047. Such encoding is often done transparently by mail reader (MUA), but if automatic encoding is not available (e.g. by some older MUA) it is a user's responsibility to avoid using such characters in a header section, or to encode them manually. Typically the offending header fields in this category are 'Subject', 'Organization', and comment fields or display names in e-mail addresses of 'From', 'To' or 'Cc'. Sometimes such invalid header fields are inserted automatically by some MUA, MTA, content filter, or other mail handling service. If this is the case, such service needs to be fixed or properly configured. Typically the offending header fields in this category are 'Date', 'Received', 'X-Mailer', 'X-Priority', 'X-Scanned', etc. If you don't know how to fix or avoid the problem, please report it to _your_ postmaster or system manager. # [~[:useragent]|^X-Mailer:\\s*Microsoft Outlook Express 6\\.00|[" If using Microsoft Outlook Express as your MUA, make sure its settings under: Tools -> Options -> Send -> Mail Sending Format -> Plain & HTML are: "MIME format" MUST BE selected, and "Allow 8-bit characters in headers" MUST NOT be enabled! "]]# |# 3: NUL/CR IMPROPER USE OF CONTROL CHARACTER IN A MESSAGE HEADER SECTION The RFC 5322 document specifies rules for forming internet messages. It does not allow the use of control characters NUL and bare CR to be used directly in a mail header section. |# 4: empty IMPROPERLY FOLDED HEADER FIELD LINE MADE UP ENTIRELY OF WHITESPACE The RFC 5322 document specifies rules for forming internet messages. In section '3.2.2. Folding white space and comments' it explicitly prohibits folding of header fields in such a way that any line of a folded header field is made up entirely of white-space characters (control characters SP and HTAB) and nothing else. |# 5: long HEADER LINE LONGER THAN RFC 5322 LIMIT OF 998 CHARACTERS The RFC 5322 document specifies rules for forming internet messages. Section '2.1.1. Line Length Limits' prohibits each line of a header section to be more than 998 characters in length (excluding the CRLF). |# 6: syntax |# 7: missing MISSING REQUIRED HEADER FIELD The RFC 5322 document specifies rules for forming internet messages. Section '3.6. Field Definitions' specifies that certain header fields are required (origination date field and the "From:" originator field). |# 8: multiple DUPLICATE HEADER FIELD The RFC 5322 document specifies rules for forming internet messages. Section '3.6. Field Definitions' specifies that certain header fields must not occur more than once in a message header section. |# other ]]# __DATA__ # # ============================================================================= # This is a template for VIRUS/BANNED SENDER NOTIFICATIONS. # For syntax and customization instructions see README.customize. # The From, To and Date header fields will be provided automatically. # Long header fields will be automatically wrapped by the program. # Subject: [? [:ccat|major] |Clean message from you\ |Clean message from you\ |Clean message from you (MTA blocked)\ |OVERSIZED message from you\ |BAD-HEADER in message from you\ |Spam claiming to be from you\ |Spam claiming to be from you\ |A message with UNCHECKED contents from you\ |BANNED contents from you (%F)\ |VIRUS in message apparently from you (%V)\ ] [? %m |#|In-Reply-To: %m] Message-ID: [? [:ccat|major] |Clean|Clean|MTA-BLOCKED|OVERSIZED|INVALID HEADER|\ Spammy|Spam|UNCHECKED contents|BANNED CONTENTS ALERT|VIRUS ALERT] Our content checker found [? %#V |#|[:wrap|78| | |[? %#V |viruses|virus|viruses]: %V]] [? %#F |#|[:wrap|78| | |banned [? %#F |names|name|names]: %F]] [? %#X |#|[[:wrap|78| | |%X]\n]] in email presumably from you %s to the following [? %#R |recipients|recipient|recipients]:[ -> %R] Our internal reference code for your message is %n/%i [? %a |#|[:wrap|78|| |First upstream SMTP client IP address: \[%a\] %g]] [? %e |#|[:wrap|78|| |According to a 'Received:' trace,\ the message apparently originated at: \[%e\], %t]] [:wrap|78|| |Return-Path: %s[?[:dkim|envsender]|| (OK)]] [:wrap|78|| |From: [:header_field|From|100][?[:dkim|author]|| (dkim:AUTHOR)]] [? [:header_field|Sender]|#|\ [:wrap|78|| |Sender: [:header_field|Sender|100]\ [?[:dkim|sender]|| (dkim:SENDER)]]] [? %m |#|[:wrap|78|| |Message-ID: %m]] [? %r |#|[:wrap|78|| |Resent-Message-ID: %r]] [? %j |#|[:wrap|78|| |Subject: [:header_field|Subject|100]]] [? %#D |Delivery of the email was stopped! ]# [? %#V ||Please check your system for viruses, or ask your system administrator to do so. ]# [? %#V |[? %#F ||# The message [?%#D|has been blocked|triggered this warning] because it contains a component (as a MIME part or nested within) with declared name or MIME type or contents type violating our access policy. To transfer contents that may be considered risky or unwanted by site policies, or simply too large for mailing, please consider publishing your content on the web, and only sending an URL of the document to the recipient. Depending on the recipient and sender site policies, with a little effort it might still be possible to send any contents (including viruses) using one of the following methods: - encrypted using pgp, gpg or other encryption methods; - wrapped in a password-protected or scrambled container or archive (e.g.: zip -e, arj -g, arc g, rar -p, or other methods) Note that if the contents is not intended to be secret, the encryption key or password may be included in the same message for recipient's convenience. We are sorry for inconvenience if the contents was not malicious. The purpose of these restrictions is to cut the most common propagation methods used by viruses and other malware. These often exploit automatic mechanisms and security holes in more popular mail readers (Microsoft mail readers and browsers are a common target). By requiring an explicit and decisive action from the recipient to decode mail, the danger of automatic malware propagation is largely reduced. # # Details of our mail restrictions policy are available at ... ]]# __DATA__ # # ============================================================================= # This is a template for non-spam (e.g. VIRUS,...) ADMINISTRATOR NOTIFICATIONS. # For syntax and customization instructions see README.customize. # Long header fields will be automatically wrapped by the program. # From: %f Date: %d Subject: [? [:ccat|major] |Clean mail|Clean mail|MTA-blocked mail|\ OVERSIZED mail|INVALID HEADER in mail|Spammy|Spam|UNCHECKED contents in mail|\ BANNED contents (%F) in mail|VIRUS (%V) in mail]\ FROM [?%l||LOCAL ][?%a||[:client_addr_port] ]%s To: [? %#T |undisclosed-recipients:;|[%T|, ]] [? %#C |#|Cc: [%C|, ]] Message-ID: [? %#V |No viruses were found. |A virus was found: %V |Two viruses were found:\n %V |%#V viruses were found:\n %V ] [? %#F |#|[:wrap|78|| |Banned [?%#F|names|name|names]: %F]] [? %#X |#|Bad header:[\n[:wrap|78| | |%X]]] [? %#W |#\ |Scanner detecting a virus: %W |Scanners detecting a virus: %W ] Content type: [:ccat|name|main]# [? [:ccat|is_blocked_by_nonmain] ||, blocked for [:ccat|name]] Internal reference code for the message is %n/%i [? %a |#|[:wrap|78|| |First upstream SMTP client IP address: \[%a\] %g]] [? %e |#|[:wrap|78|| |According to a 'Received:' trace,\ the message apparently originated at: \[%e\], %t]] [:wrap|78|| |Return-Path: %s[?[:dkim|envsender]|| (OK)]] [:wrap|78|| |From: [:header_field|From][?[:dkim|author]|| (dkim:AUTHOR)]] [? [:header_field|Sender]|#|\ [:wrap|78|| |Sender: [:header_field|Sender]\ [?[:dkim|sender]|| (dkim:SENDER)]]] [? %m |#|[:wrap|78|| |Message-ID: %m]] [? %r |#|[:wrap|78|| |Resent-Message-ID: %r]] [? %j |#|[:wrap|78|| |Subject: %j]] [? %q |Not quarantined.|The message has been quarantined as: %q] [? %#S |Notification to sender will not be mailed. ]# [? %#D |#|The message WILL BE relayed to:[\n%D] ] [? %#N |#|The message WAS NOT relayed to:[\n%N] ] [? %#V |#|[? %#v |#|Virus scanner output:[\n %v] ]] __DATA__ # # ============================================================================= # This is a template for VIRUS/BANNED/BAD-HEADER RECIPIENTS NOTIFICATIONS. # For syntax and customization instructions see README.customize. # Long header fields will be automatically wrapped by the program. # From: %f Date: %d Subject: [? [:ccat|major] |Clean mail|Clean mail|MTA-blocked mail|\ OVERSIZED mail|INVALID HEADER in mail|Spammy|Spam|UNCHECKED contents in mail|\ BANNED contents (%F) in mail|VIRUS (%V) in mail] TO YOU from %s [? [:header_field|To] |To: undisclosed-recipients:;|To: [:header_field|To]] [? [:header_field|Cc] |#|Cc: [:header_field|Cc]] Message-ID: [? %#V |[? %#F ||BANNED CONTENTS ALERT]|VIRUS ALERT] Our content checker found [? %#V |#|[:wrap|78| | |[?%#V|viruses|virus|viruses]: %V]] [? %#F |#|[:wrap|78| | |banned [?%#F|names|name|names]: %F]] [? %#X |#|[[:wrap|78| | |%X]\n]] in an email to you [? %#V |from:|from probably faked sender:] %o [? %#V |#|claiming to be: %s] Content type: [:ccat|name|main]# [? [:ccat|is_blocked_by_nonmain] ||, blocked for [:ccat|name]] Our internal reference code for your message is %n/%i [? %a |#|[:wrap|78|| |First upstream SMTP client IP address: \[%a\] %g]] [? %e |#|[:wrap|78|| |According to a 'Received:' trace,\ the message apparently originated at: \[%e\], %t]] [:wrap|78|| |Return-Path: %s[?[:dkim|envsender]|| (OK)]] [:wrap|78|| |From: [:header_field|From][?[:dkim|author]|| (dkim:AUTHOR)]] [? [:header_field|Sender]|#|\ [:wrap|78|| |Sender: [:header_field|Sender]\ [?[:dkim|sender]|| (dkim:SENDER)]]] [? %m |#|[:wrap|78|| |Message-ID: %m]] [? %r |#|[:wrap|78|| |Resent-Message-ID: %r]] [? [:useragent] |#|[:wrap|78|| |[:useragent]]] [? %j |#|[:wrap|78|| |Subject: %j]] [? %q |Not quarantined.|The message has been quarantined as: %q] Please contact your system administrator for details. __DATA__ # # ============================================================================= # This is a template for spam SENDER NOTIFICATIONS. # For syntax and customization instructions see README.customize. # The From, To and Date header fields will be provided automatically. # Long header fields will be automatically wrapped by the program. # Subject: Considered UNSOLICITED BULK EMAIL, apparently from you [? %m |#|In-Reply-To: %m] Message-ID: A message from %s[ to: %R] was considered unsolicited bulk e-mail (UBE). Our internal reference code for your message is %n/%i The message carried your return address, so it was either a genuine mail from you, or a sender address was faked and your e-mail address abused by third party, in which case we apologize for undesired notification. We do try to minimize backscatter for more prominent cases of UBE and for infected mail, but for less obvious cases some balance between losing genuine mail and sending undesired backscatter is sought, and there can be some collateral damage on either side. [? %a |#|[:wrap|78|| |First upstream SMTP client IP address: \[%a\] %g]] [? %e |#|[:wrap|78|| |According to a 'Received:' trace,\ the message apparently originated at: \[%e\], %t]] [:wrap|78|| |Return-Path: %s[?[:dkim|envsender]|| (OK)]] [:wrap|78|| |From: [:header_field|From|100][?[:dkim|author]|| (dkim:AUTHOR)]] [? [:header_field|Sender]|#|\ [:wrap|78|| |Sender: [:header_field|Sender|100]\ [?[:dkim|sender]|| (dkim:SENDER)]]] [? %m |#|[:wrap|78|| |Message-ID: %m]] [? %r |#|[:wrap|78|| |Resent-Message-ID: %r]] # [? [:useragent] |#|[:wrap|78|| |[:useragent]]] [? %j |#|[:wrap|78|| |Subject: [:header_field|Subject|100]]] [? %#X |#|\n[[:wrap|78|| |%X]\n]] [? %#D |Delivery of the email was stopped! ]# # # Spam scanner report: # [%A # ]\ __DATA__ # # ============================================================================= # This is a template for spam ADMINISTRATOR NOTIFICATIONS. # For syntax and customization instructions see README.customize. # Long header fields will be automatically wrapped by the program. # From: %f Date: %d Subject: Spam FROM [?%l||LOCAL ][?%a||[:client_addr_port] ]%s To: [? %#T |undisclosed-recipients:;|[%T|, ]] [? %#C |#|Cc: [%C|, ]] Message-ID: Content type: [:ccat|name|main]# [? [:ccat|is_blocked_by_nonmain] ||, blocked for [:ccat|name]] Internal reference code for the message is %n/%i [? %a |#|[:wrap|78|| |First upstream SMTP client IP address: \[%a\] %g]] [? %e |#|[:wrap|78|| |According to a 'Received:' trace,\ the message apparently originated at: \[%e\], %t]] [:wrap|78|| |Return-Path: %s[?[:dkim|envsender]|| (OK)]] [:wrap|78|| |From: [:header_field|From][?[:dkim|author]|| (dkim:AUTHOR)]] [? [:header_field|Sender]|#|\ [:wrap|78|| |Sender: [:header_field|Sender]\ [?[:dkim|sender]|| (dkim:SENDER)]]] [? %m |#|[:wrap|78|| |Message-ID: %m]] [? %r |#|[:wrap|78|| |Resent-Message-ID: %r]] [? [:useragent] |#|[:wrap|78|| |[:useragent]]] [? %j |#|[:wrap|78|| |Subject: %j]] [? %q |Not quarantined.|The message has been quarantined as: %q] [? %#D |#|The message WILL BE relayed to:[\n%D] ] [? %#N |#|The message WAS NOT relayed to:[\n%N] ] Spam scanner report: [%A ]\ __DATA__ # # ============================================================================= # This is a template for the plain text part of a RELEASE FROM A QUARANTINE, # applicable if a chosen release format is 'attach' (not 'resend'). # From: %f Date: %d Subject: \[released message\] %j To: [? %#T |undisclosed-recipients:;|[%T|, ]] [? %#C |#|Cc: [%C|, ]] Message-ID: Please find attached a message which was held in a quarantine, and has now been released. [:wrap|78|| |Return-Path: %s[?[:dkim|envsender]|| (OK)]] [:wrap|78|| |From: [:header_field|From][?[:dkim|author]|| (dkim:AUTHOR)]] [? [:header_field|Sender]|#|\ [:wrap|78|| |Sender: [:header_field|Sender]\ [?[:dkim|sender]|| (dkim:SENDER)]]] # [? %m |#|[:wrap|78|| |Message-ID: %m]] # [? %r |#|[:wrap|78|| |Resent-Message-ID: %r]] # [? [:useragent] |#|[:wrap|78|| |[:useragent]]] [? %j |#|[:wrap|78|| |Subject: %j]] Our internal reference code for the message is %n/%i # [~[:report_format]|["^attach$"]|["[? [:attachment_password] |#| Contents of the attached mail message may pose a threat to your computer or could be a social engineering deception, so it should be handled cautiously. To prevent undesired automatic opening, the attached original mail message has been wrapped in a password-protected ZIP archive. Here is the password that allows opening of the attached archive: [:attachment_password] Note that the attachment is not strongly encrypted and the password is not a strong secret (being displayed in this non-encrypted text), so this attachment is not suitable for guarding a secret contents. The sole purpose of this password protection it to prevent undesired accidental or automatic opening of a message, either by some filtering software, a virus scanner, or by a mail reader. ]"]|]# __DATA__ # # ============================================================================= # This is a template for the plain text part of a problem/feedback report, # with either the original message included in-line, or attached, # or the message is structured as a FEEDBACK REPORT NOTIFICATIONS format. # See RFC 5965 - "An Extensible Format for Email Feedback Reports". # From: %f Date: %d Subject: Fw: %j To: [? %#T |undisclosed-recipients:;|[%T|, ]] [? %#C |#|Cc: [%C|, ]] Message-ID: #Auto-Submitted: auto-generated This is an e-mail [:feedback_type] report for a message \ [? %a |\nreceived on %d,|received from\nIP address [:client_addr_port] on %d,] [:wrap|78|| |Return-Path: %s] [:wrap|78|| |From: [:header_field|From][?[:dkim|author]|| (dkim:AUTHOR)]] [? [:header_field|Sender]|#|[:wrap|78|| |Sender: [:header_field|Sender]]] [? %m |#|[:wrap|78|| |Message-ID: %m]] [? %r |#|[:wrap|78|| |Resent-Message-ID: %r]] [? %j |#|[:wrap|78|| |Subject: [:header_field|Subject|100]]] [?[:dkim|author]|#| A first-party DKIM or DomainKeys signature is valid, d=[:dkim|author].] Reporting-MTA: %h Our internal reference code for the message is %n/%i [~[:report_format]|["^(arf|attach|dsn)$"]|["\ A complete original message is attached. [~[:report_format]|["^arf$"]|\ For more information on the ARF format please see RFC 5965. ]"]|["\ A complete original message in its pristine form follows: "]]# __DATA__ # # ============================================================================= # This is a template for the plain text part of an auto response (e.g. # vacation, out-of-office), see RFC 3834. # From: %f Date: %d To: [? %#T |undisclosed-recipients:;|[%T|, ]] [? %#C |#|Cc: [%C|, ]] Reply-To: postmaster@%h Message-ID: Auto-Submitted: auto-replied [:wrap|76||\t|Subject: Auto: autoresponse to: %s] [? %m |#|In-Reply-To: %m] Precedence: junk This is an auto-response to a message \ [? %a |\nreceived on %d,|received from\nIP address \[%a\] on %d,] envelope sender: %s (author) From: [:rfc2822_from] [? %j |#|[:wrap|78|| |Subject: %j]] [?[:dkim|author]|#| A first-party DKIM or DomainKeys signature is valid, d=[:dkim|author].] amavisd-new-2.7.1/amavisd-new-qmqpqq.patch000640 000621 000620 00000027014 11747106160 020244 0ustar00markcmi000000 000000 --- amavisd.ori 2012-04-29 02:31:15.655242548 +0200 +++ amavisd 2012-04-29 02:33:01.036242047 +0200 @@ -102,4 +102,5 @@ # Amavis::In::SMTP #( Amavis::In::Courier ) +# Amavis::In::QMQPqq # Amavis::Out::SMTP::Protocol # Amavis::Out::SMTP::Session @@ -4041,4 +4042,5 @@ $myproduct_name, $conn->socket_port eq '' ? 'unix socket' : "port ".$conn->socket_port); + # must not use proto name QMQPqq in 'with' $s .= "\n with $smtp_proto" if $smtp_proto=~/^(ES|S|L)MTPS?A?\z/i; #RFC 3848 $s .= "\n id $id" if defined $id && $id ne ''; @@ -10004,4 +10006,5 @@ $extra_code_sql_lookup $extra_code_ldap $extra_code_in_ampdp $extra_code_in_smtp $extra_code_in_courier + $extra_code_in_qmqpqq $extra_code_out_smtp $extra_code_out_pipe $extra_code_out_bsmtp $extra_code_out_local $extra_code_p0f @@ -10031,4 +10034,5 @@ # Amavis::In::AMPDP, Amavis::In::SMTP and In::Courier objects use vars qw($ampdp_in_obj $smtp_in_obj $courier_in_obj); +use vars qw($qmqpqq_in_obj); # Amavis::In::QMQPqq object use vars qw($sql_dataset_conn_lookups); # Amavis::Out::SQL::Connection object @@ -10735,4 +10739,5 @@ do_log(0,"SMTP-in proto code %s loaded", $extra_code_in_smtp ?'':" NOT"); do_log(0,"Courier proto code %s loaded", $extra_code_in_courier ?'':" NOT"); + do_log(0,"QMQPqq-in proto code %s loaded", $extra_code_in_qmqpqq ?'':" NOT"); do_log(0,"SMTP-out proto code %s loaded", $extra_code_out_smtp ?'':" NOT"); do_log(0,"Pipe-out proto code %s loaded", $extra_code_out_pipe ?'':" NOT"); @@ -11349,5 +11354,9 @@ die "unavailable support for protocol: $suggested_protocol"; } elsif ($suggested_protocol eq 'QMQPqq') { - die "unavailable support for protocol: $suggested_protocol"; + if (!$extra_code_in_qmqpqq) { + die "incoming TCP connection, but dynamic QMQPqq code not loaded"; + } + $qmqpqq_in_obj = Amavis::In::QMQPqq->new if !$qmqpqq_in_obj; + $qmqpqq_in_obj->process_qmqpqq_request($sock,$conn,\&check_mail); } elsif ($suggested_protocol eq 'TCP-LOOKUP') { #postfix maps, experimental process_tcp_lookup_request($sock, $conn); @@ -11445,4 +11454,5 @@ do_log(5,"child_finish_hook: invoking DESTROY methods"); undef $smtp_in_obj; undef $ampdp_in_obj; undef $courier_in_obj; + undef $qmqpqq_in_obj; undef $sql_storage; undef $sql_wblist; undef $sql_lookups; undef $sql_dataset_conn_lookups; undef $sql_dataset_conn_storage; @@ -11455,4 +11465,5 @@ # do_log(5,"at the END handler: invoking DESTROY methods"); undef $smtp_in_obj; undef $ampdp_in_obj; undef $courier_in_obj; + undef $qmqpqq_in_obj; undef $sql_storage; undef $sql_wblist; undef $sql_lookups; undef $sql_dataset_conn_lookups; undef $sql_dataset_conn_storage; @@ -15487,4 +15498,5 @@ $extra_code_sql_lookup, $extra_code_ldap, $extra_code_in_ampdp, $extra_code_in_smtp, $extra_code_in_courier, + $extra_code_in_qmqpqq, $extra_code_out_smtp, $extra_code_out_pipe, $extra_code_out_bsmtp, $extra_code_out_local, $extra_code_p0f, @@ -15807,5 +15819,11 @@ undef $extra_code_in_courier; } - if ($needed_protocols_in{'QMQPqq'}) { die "In::QMQPqq code not available" } + if ($needed_protocols_in{'QMQPqq'}) { + eval $extra_code_in_qmqpqq or die "Problem in the In::QMQPqq code: $@"; + # release memory occupied by the source code + undef $extra_code_in_qmqpqq; $extra_code_in_qmqpqq = 1; + } else { + undef $extra_code_in_qmqpqq; + } } @@ -19292,4 +19310,276 @@ __DATA__ # +package Amavis::In::QMQPqq; +use strict; +# use re 'taint'; # (is this module ready for this yet?) + +BEGIN { + use Exporter (); + use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); + $VERSION = '1.18'; + @ISA = qw(Exporter); +} +use POSIX qw(strftime); +use Errno qw(ENOENT); + +BEGIN { + import Amavis::Conf qw(:platform :confvars :dynamic_confvars c cr ca); + import Amavis::Util qw(ll do_log am_id new_am_id prolong_timer + debug_oneshot sanitize_str rmdir_recursively); + import Amavis::Lookup qw(lookup); + import Amavis::Timing qw(section_time); + import Amavis::rfc2821_2822_Tools; + import Amavis::TempDir; + import Amavis::In::Message; + import Amavis::In::Connection; +} + +sub new($) { + my($class) = @_; + my($self) = bless {}, $class; + $self->{bytesleft} = undef; # bytes left for whole package + $self->{len} = undef; # set by getlen() method + $self->{sock} = undef; # connected socket + $self->{proto} = undef; # protocol + $self->{tempdir} = Amavis::TempDir->new; # TempDir object + $self->{session_closed_normally} = undef; # closed properly? (waited for K/Z/D) + $self; +} + +sub DESTROY { + my($self) = shift; + eval { do_log(5,"Amavis::In::QMQPqq DESTROY called, sock=%s, normal=%s", + $self->{sock}, $self->{session_closed_normally}) }; + eval { + if (ref($self->{sock}) && ! $self->{session_closed_normally}) { + $self->qmqpqq_resp("Z","Service shutting down, closing channel"); + } + }; + if ($@ ne '') + { my($eval_stat) = $@; eval { do_log(1,"QMQPqq shutdown: %s",$eval_stat) } } +} + +# get byte, die if no bytes left +sub getbyte($) { +my($self) = shift; +if(!$self->{bytesleft}--) { + die("No bytes left"); + } +if(defined($_ = $self->{sock}->getc)) { + return($_); + } +die("EOF on socket"); +} + +sub getlen($) { +my($self) = shift; +my($ch,$len); + +for(;;) { + $ch = $self->getbyte; + if($ch eq ':') { + return($self->{len} = $len); + } + if($ch !~ /^\d$/) { + die("Char '$ch' is not a number while determining length"); + } + $len .= $ch; + } +} + +sub getcomma($) { +my($self) = shift; +if($self->getbyte ne ',') { + die("Comma expected, found '$_'"); + } +} + +sub getnetstring($$) { +my($self) = shift; +($self->{sock}->read($_[0],$self->getlen) == $self->{len}) || + die("EOF on socket"); +$self->{bytesleft} -= $self->{len}; +$self->getcomma; +} + +# Accept a QMQPqq connect +# and call content checking for the message received +# +sub process_qmqpqq_request($$$$) { +my($self,$sock,$conn,$check_mail) = @_; +# $sock: connected socket from Net::Server +# $conn: information about client connection +# $check_mail: subroutine ref to be called with file handle + +$self->{proto} = "QMQPqq"; +$self->{session_closed_normally} = 0; # closed properly? +$self->{sock} = $sock; # store $sock info for getbyte() method +$self->{bytesleft} = 20; # initial bytesleft value, there should + # NEVER EVER be longer email than 10^20 (approximately) + # bytes but increase if needed ;) +$self->{len} = undef; + +my($msginfo); +my($sender,@recips); +my($len); + +new_am_id(undef, $Amavis::child_invocation_count, undef); +Amavis::Timing::init(); + +$conn->smtp_proto("QMQPqq"); # the name of the method is too specific +my($eval_stat); +eval { + # get length of whole package + $self->{bytesleft} = $self->getlen; + + # get length of 'email' + $len = $self->getlen; + section_time('initial length determination'); + + # prepare tempdir + Amavis::check_mail_begin_task(); + $self->{tempdir}->prepare; + $self->{tempdir}->prepare_file; + + $msginfo = Amavis::In::Message->new; + $msginfo->rx_time(time); + $msginfo->delivery_method(c('forward_method')); + + # get 'email' + $self->{tempdir}->empty(0); + my $size = 16384; + while(($len > 0) && ($sock->read($_,($len >= $size ? $size : $size = $len)) == $size)) { + (print {$self->{tempdir}->fh} $_) || + die("Can't write to mail file: $!"); + $len -= $size; + } + if($len > 0) { + die("EOF on socket"); + } + $self->{tempdir}->fh->flush || die("Can't flush mail file: $!"); + $self->{tempdir}->fh->seek(0,1) || die("Can't seek on file: $!"); + $self->{bytesleft} -= $self->{len}; + section_time('email receiving'); + # comma has to follow + $self->getcomma; + + # get sender (presumably in unquoted form, really???) + $self->getnetstring($sender); + section_time('sender receiving'); + + # get recips (presumably in unquoted form, really???) + my $i = 0; + while($self->{bytesleft}) { + $self->getnetstring($recips[$i++]); + } + section_time('recips receiving'); + + # final comma has to follow + $self->{bytesleft} = 1; + $self->getcomma; + + $msginfo->sender($sender); + $msginfo->sender_smtp(qquote_rfc2821_local($sender)); + $msginfo->recips(\@recips); + + do_log(1, sprintf("%s:%s:%s %s: %s -> %s Received: %s", + $self->{proto},$conn->socket_ip eq $inet_socket_bind ? + '' : '['.$conn->socket_ip.']', + $conn->socket_port, $self->{tempdir_pers}, + $msginfo->sender_smtp, + join(',', map { $_->recip_addr_smtp } + @{$msginfo->per_recip_data}), + join(' ', + ($msginfo->msg_size eq '' ? () + : 'SIZE='.$msginfo->msg_size), + ($msginfo->body_type eq '' ? () + : 'BODY='.$msginfo->body_type), + received_line($conn,$msginfo,am_id(),0) ) + )); + + $msginfo->mail_tempdir($self->{tempdir}->path); + $msginfo->mail_text_fn($self->{tempdir}->path . '/email.txt'); + $msginfo->mail_text($self->{tempdir}->fh); + + my($smtp_resp,$exit_code,$preserve_evidence) = + &$check_mail($conn,$msginfo,0); + + if ($preserve_evidence) { $self->{tempdir}->preserve(1) } + + if ($smtp_resp !~ /^4/ && + grep { !$_->recip_done } @{$msginfo->per_recip_data}) { + die("TROUBLE/MISCONFIG: not all recipients done, ". + "\$forward_method is \"$forward_method\""); + } + + if ($smtp_resp !~ /^4/ && + grep { !$_->recip_done } @{$msginfo->per_recip_data}) { + if ($msginfo->delivery_method eq '') { + do_log(2,"not all recipients done, forward_method is empty"); + } + else { + die "TROUBLE: (MISCONFIG) not all recipients done, " . + "forward_method is: " . $msginfo->delivery_method; + } + } + + # all ok + if($smtp_resp =~ /^2/) { + $self->qmqpqq_resp("K",$smtp_resp); + } + # permanent reject + elsif($smtp_resp =~ /^5/) { + $self->qmqpqq_resp("D",$smtp_resp); + } + # temporary reject (or other error if !~ /^4/) + else { + $self->qmqpqq_resp("Z",$smtp_resp); + } + 1; +} or do { + $eval_stat = $@ ne '' ? $@ : "errno=$!"; +}; + +$self->{tempdir}->clean; +alarm(0); do_log(4,"timer stopped after QMQPqq eval"); + +if($eval_stat ne '') { + chomp $eval_stat; + do_log(0,"QMQPqq: NOTICE: $eval_stat"); + $self->qmqpqq_resp("Z","Service shutting down, $eval_stat"); + } +# report elapsed times by section for each transaction +do_log(2, "%s", Amavis::Timing::report()); + +$self->{session_closed_normally} = 1; +# closes connection after child_finish_hook +} + +# sends a QMQPqq response consisting of K/D/Z code and an optional message; +# slow down evil clients by delaying response on permanent errors +sub qmqpqq_resp($$$;$$) { +my($self,$code,$resp,$penalize,$line) = @_; +if($code !~ /^(K|Z|D)$/) { + die("Internal error(2): bad QMQPqq response code: '$code'"); + } +if($penalize) { + do_log(0,"QMQPqq: $resp; PENALIZE: $line"); + sleep 5; + section_time('QMQPqq penalty wait'); + } +$resp = sanitize_str($resp,1); +do_log(4,"QMQPqq> $resp"); +$self->{sock}->print($self->netstring($code . $resp)); +} + +sub netstring($$) { +my($self,$string) = @_; +return(sprintf("%d:%s,",length($string),$string)); +} + +1; + +__DATA__ +# package Amavis::Out::SMTP::Protocol; use strict; --- amavisd.conf.ori 2012-04-29 02:31:32.418239154 +0200 +++ amavisd.conf 2012-04-29 02:33:01.037242947 +0200 @@ -55,6 +55,6 @@ # option(s) -p overrides $inet_socket_port and $unix_socketname -$inet_socket_port = 10024; # listen on this local TCP port(s) -# $inet_socket_port = [10024,10026]; # listen on multiple TCP ports +$protocol = 'QMQPqq'; # suggested protocol to use on all input sockets +$inet_socket_port = 10628; # accept connections on this local TCP port(s) $policy_bank{'MYNETS'} = { # mail originating from @mynetworks amavisd-new-2.7.1/amavisd-nanny000751 000621 000620 00000027251 11220735171 016164 0ustar00markcmi000000 000000 #!/usr/bin/perl -T #------------------------------------------------------------------------------ # This is amavisd-nanny, a program to show the status # and keep an eye on the health of child processes in amavisd-new. # # Author: Mark Martinec # Copyright (C) 2004-2009 Mark Martinec, All Rights Reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of the author, nor the name of the "Jozef Stefan" # Institute, nor the names of contributors may be used to endorse or # promote products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER # OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # #(the license above is the new BSD license, and pertains to this program only) # # Patches and problem reports are welcome. # The latest version of this program is available at: # http://www.ijs.si/software/amavisd/ #------------------------------------------------------------------------------ use strict; use re 'taint'; use warnings; no warnings 'uninitialized'; use Errno qw(ESRCH ENOENT); use POSIX qw(strftime); use Time::HiRes (); use BerkeleyDB; use vars qw($VERSION); $VERSION = 1.400; my($idlettl) = 3*60*60; # idle children are sent a SIGTERM # after this many seconds my($activettl) = 10*60; # stuck active children are sent a SIGTERM # after this many seconds my($dbfile) = 'nanny.db'; my($db_home) = # DB databases directory defined $ENV{'AMAVISD_DB_HOME'} ? $ENV{'AMAVISD_DB_HOME'} : '/var/amavis/db'; my($wakeuptime) = 2; # -w, sleep time in seconds, may be fractional my($repeatcount); # -c, repeat count (when defined) sub fmt_age($$$) { my($t,$state_bar,$idling) = @_; $t = int($t); my($char) = $idling ? '.' : '='; my($bar_l) = $idling ? $t : length($state_bar); my($bar) = substr( ($char x 9 . ':') x 3 . $char x 5, 0,$bar_l); if (!$idling) { $state_bar = substr($state_bar,0,length($bar)-2) . substr($state_bar,-1,1) . '>' if length($state_bar) > length($bar); for my $j (0 .. length($bar)-1) { substr($bar,$j,1) = substr($state_bar,$j,1) if substr($bar,$j,1) eq '=' && substr($state_bar,$j,1) ne ' '; } } my($s) = $t % 60; $t = int($t/60); my($m) = $t % 60; $t = int($t/60); my($h) = $t % 24; $t = int($t/24); my($d) = $t; my($str) = sprintf("%d:%02d:%02d", $h,$m,$s); $str = (!$d ? " " : sprintf("%dd",$d)) . $str; $str . ' ' . $bar; }; # main program starts here my($normal_termination) = 0; $SIG{INT} = sub { die "\n" }; # do the END code block when interrupted while (@ARGV) { my($opt) = shift @ARGV; my($val) = shift @ARGV; if ($opt eq '-w' && $val =~ /^\+?\d+(?:\.\d*)?\z/) { $wakeuptime = $val } elsif ($opt eq '-c' && $val =~ /^[+-]?\d+\z/) { $repeatcount = $val } else { print <<'EOD'; States legend: A accepted a connection b begin with a protocol for accepting a request m 'MAIL FROM' smtp command started a new transaction in the same session d transferring data from MTA to amavisd = content checking just started G generating and verifying unique mail_id D decoding of mail parts V virus scanning S spam scanning P pen pals database lookup and updates r preparing results Q quarantining and preparing/sending notifications F forwarding mail to MTA . content checking just finished sp space indicates idle (elapsed bar is showing dots) EOD die "Usage: $0 [-c ] [-w ]\n"; } } print <<'EOD'; process-id task-id elapsed in elapsed-bar (dots indicate idle) or state idle or busy EOD my(%waittime); # associative array on pid my($env,$db,$old_db_inode,@dbstat,$cursor); my(%proc_last_timestamp, %proc_state_bars); for (;;) { last if defined $repeatcount && $repeatcount <= 0; @dbstat = stat("$db_home/$dbfile"); my($errn) = @dbstat ? 0 : 0+$!; $errn==0 || $errn==ENOENT or die "stat $db_home/$dbfile: $!"; if (defined $db && $old_db_inode != $dbstat[1]) { $db->db_close==0 or die "BDB db_close error: $BerkeleyDB::Error $!"; undef $db; printf STDERR ("Reopening nanny database %s/%s\n", $db_home,$dbfile); } if (!defined $db && $errn==0) { $old_db_inode = $dbstat[1]; $env = BerkeleyDB::Env->new( -Home => $db_home, -Flags => DB_INIT_CDB | DB_INIT_MPOOL, -ErrFile => \*STDOUT, -Verbose => 1); defined $env or die "BDB no env: $BerkeleyDB::Error $!"; $db = BerkeleyDB::Hash->new(-Filename => $dbfile, -Env => $env); defined $db or die "BDB no dbN 1: $BerkeleyDB::Error $!"; } $| = 0; my(%proc_timestamp, %proc_state, %proc_task_id); my($stat,$key,$val); my($now); my($eval_stat,$interrupt); $interrupt = ''; if (!defined $db) { printf STDERR ("No nanny database %s/%s; waiting...\n", $db_home,$dbfile); } else { $repeatcount-- if defined $repeatcount && $repeatcount > 0; print "\n"; my($h1) = sub { $interrupt = $_[0] }; local(@SIG{qw(INT HUP TERM TSTP QUIT ALRM USR1 USR2)}) = ($h1) x 8; eval { $cursor = $db->db_cursor; # obtain read lock defined $cursor or die "db_cursor error: $BerkeleyDB::Error"; $now = Time::HiRes::time; local($1,$2); my($now_utc_iso8601) = strftime("%Y%m%dT%H%M%S",gmtime(int($now))); while ( ($stat=$cursor->c_get($key,$val,DB_NEXT)) == 0 ) { if ($val !~ /^(\d+(?:\.\d*)?) (.*?) *\z/s) { print STDERR "Bad db entry: $key, $val\n"; } else { $proc_timestamp{$key} = $1; my($task_id) = $2; $proc_state{$key} = $1 if $task_id =~ s/^([^0-9])//; $proc_task_id{$key} = $task_id; if (!exists $proc_state_bars{$key}) { # new process appeared $proc_last_timestamp{$key} = $proc_timestamp{$key}; $proc_state_bars{$key} = ''; } } } $stat==DB_NOTFOUND or die "c_get: $BerkeleyDB::Error $!"; $cursor->c_close==0 or die "c_close error: $BerkeleyDB::Error"; $cursor = undef; }; my($eval_stat) = $@; if (defined $db) { $cursor->c_close if defined $cursor; # unlock, ignoring status $cursor = undef; } } if ($interrupt ne '') { kill($interrupt,$$) } # resignal, ignoring status elsif ($eval_stat ne '') { chomp($eval_stat); die "BDB $eval_stat\n" } for my $key (keys(%proc_state_bars)) { # remove old entries if (!exists($proc_timestamp{$key})) { delete $proc_timestamp{$key}; delete $proc_task_id{$key}; delete $proc_state_bars{$key}; } } my(@to_be_removed,@killed); for my $pid (sort {$a<=>$b} keys %proc_timestamp) { $proc_state{$pid} = ' ' if $proc_state{$pid} eq ''; my($idling) = $proc_task_id{$pid} eq '' && $proc_state{$pid} =~ /^[. ]?\z/s; my($age) = $now - $proc_timestamp{$pid}; if ($idling) { $proc_state_bars{$pid} = '' } else { $proc_state_bars{$pid} = '' if $proc_timestamp{$pid} ne $proc_last_timestamp{$pid}; my($len) = int($age+0.5); $len = 1 if $len < 1; my($str) = $proc_state_bars{$pid}; if ($len > length($str)) { # replicate last character to desired size my($ch) = $str eq '' ? '=' : substr($str,-1,1); $str .= $ch x ($len - length($str)); } substr($str,$len-1,1) = $proc_state{$pid}; $proc_state_bars{$pid} = $str; } $proc_last_timestamp{$pid} = $proc_timestamp{$pid}; my($ttl) = $idling ? $idlettl : $activettl; my($n) = kill(0,$pid); # test if the process is still there if ($n == 0 && $! != ESRCH) { die "Can't check the process $pid: $!"; } elsif ($n == 0) { # ESRCH means there is no such process printf STDERR ("PID %s: %-11s went away %s\n", $pid, $proc_task_id{$pid} || $proc_state{$pid}, fmt_age($age, $proc_state_bars{$pid}, $idling) ); push(@to_be_removed, $pid); } elsif ($age <= $ttl) { # all ok printf STDERR ("PID %s: %-11s %s\n", $pid, $proc_task_id{$pid} || $proc_state{$pid}, fmt_age($age, $proc_state_bars{$pid}, $idling) ); } else { # send a SIGTERM printf STDERR ("PID %s: %-11s terminated %s\n", $pid, $proc_task_id{$pid} || $proc_state{$pid}, fmt_age($age, $proc_state_bars{$pid}, $idling) ); if (kill('TERM',$pid) || $! == ESRCH) { push(@killed,$pid) } else { warn "Can't SIGTERM $pid: $!" } } } if (@to_be_removed) { local($SIG{'INT'}) = 'IGNORE'; $cursor = $db->db_cursor(DB_WRITECURSOR); # obtain a write lock defined $cursor or die "BDB db_cursor error: $BerkeleyDB::Error"; for my $key (@to_be_removed) { my($val); my($stat) = $cursor->c_get($key,$val,DB_SET); $stat==0 || $stat==DB_NOTFOUND or die "BDB c_get: $BerkeleyDB::Error, $!."; if ($stat==0) { # remove existing entry $cursor->c_del==0 or die "BDB c_del: $BerkeleyDB::Error, $!."; } } $cursor->c_close==0 or die "BDB c_close error: $BerkeleyDB::Error"; $cursor = undef; } my($delay) = 1; # seconds while (@killed) { Time::HiRes::sleep($delay); $delay = 2; for my $pid (@killed) { $waittime{$pid}++; printf STDERR ("PID %s: sending SIGKILL in %d s\n", $pid, 30-$waittime{$pid}); if ($waittime{$pid} > 30) { # send a SIGKILL kill('KILL',$pid) or $! == ESRCH or warn "Can't SIGKILL $pid: $!"; $waittime{$pid} = 0; } elsif (kill(0,$pid)) { # process is still there } elsif ($! != ESRCH) { # problem? warn "Can't check process $pid: $!"; } else { # no longer around printf STDERR ("PID %s: %-11s successfully killed\n", $pid); delete($waittime{$pid}); $pid = undef; } } @killed = grep {defined} @killed; printf STDERR ("Waiting for the process to terminate: %s\n", join(', ',@killed)) if @killed; } $| = 1; last if defined $repeatcount && $repeatcount <= 0; Time::HiRes::sleep($wakeuptime) if $wakeuptime > 0; } # forever $normal_termination = 1; END { if (defined $db) { $cursor->c_close if defined $cursor; # ignoring status $db->db_close==0 or die "BDB db_close error: $BerkeleyDB::Error $!"; } print STDERR "exited\n" if !$normal_termination; } amavisd-new-2.7.1/test-messages/000751 000621 000620 00000000000 11124443211 016240 5ustar00markcmi000000 000000 amavisd-new-2.7.1/p0f-analyzer.pl000751 000621 000620 00000025720 11260363236 016343 0ustar00markcmi000000 000000 #!/usr/bin/perl -T #------------------------------------------------------------------------------ # This is p0f-analyzer.pl, a program to continuously read log reports from p0f # utility, keep results in cache for a couple of minutes, and answer queries # over UDP from some program (like amavisd-new) about collected data. # # Author: Mark Martinec # Copyright (C) 2006 Mark Martinec, All Rights Reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of the author, nor the name of the "Jozef Stefan" # Institute, nor the names of contributors may be used to endorse or # promote products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER # OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # #(the license above is the new BSD license, and pertains to this program only) # # Patches and problem reports are welcome. # The latest version of this program is available at: # http://www.ijs.si/software/amavisd/ #------------------------------------------------------------------------------ use strict; use re 'taint'; use Errno qw(EAGAIN EINTR); use Socket; use vars qw($VERSION); $VERSION = '1.400'; # Example usage: # p0f -i bge0 -l 'tcp dst port 25' 2>&1 | p0f-analyzer.pl 2345 # # In the p0f filter expression above specify an IP address of this host where # your MTA is listening for incoming mail, in place of host.example.com above. # Match the UDP port number (like 2345 above) with the port to which a client # will be sending queries ($os_fingerprint_method in amavisd.conf). # argument should be a free UDP port where queries will be accepted on $ARGV[0] =~ /^[0-9]+\z/ or die <<'EOD'; Specify a valid UDP port as an argument. Usage: p0f-analyzer.pl Example usage: p0f -l 'tcp dst port 25' 2>&1 | p0f-analyzer.pl 2345 EOD my($port) = untaint($ARGV[0]); # my($bind_addr) = '0.0.0.0'; # bind to all IPv4 interfaces my($bind_addr) = '127.0.0.1'; # bind just to a loopback interface my(@inet_acl) = qw( 127.0.0.1 ); # list of IP addresses from which queries # will be accepted, others are ignored my($retention_time) = 10*60; # time to keep collected information in cache my($debug) = 0; # nonzero enables log messages to STDERR do_log(1, "p0f-analyzer version %s starting", $VERSION); do_log(1, "listening on UDP port %s, allowed queries from: %s", $port, join(", ",@inet_acl)); socket(S, PF_INET, SOCK_DGRAM, getprotobyname('udp')) or die "socket: $!"; my($packed_addr); $packed_addr = inet_aton($bind_addr) or die "inet_aton: bad IP address [$bind_addr]: $!"; bind(S, sockaddr_in($port,$packed_addr)) or die "binding to [$bind_addr] failed: $!"; my($fn_sock) = fileno(S); my($fn_stdin) = fileno(STDIN); my($rin,$rout); $rin = ''; vec($rin,$fn_sock,1) = 1; vec($rin,$fn_stdin,1) = 1; my(%src); my($cnt_since_cleanup) = 0; binmode(STDIN) or die "Can't set STDIN binmode: $!"; for (;;) { my($nfound,$timeleft) = select($rout=$rin, undef, undef, undef); my($now) = time; if (vec($rout,$fn_sock,1)) { my($port,$iaddr,$paddr,$clientaddr); my($inbuf); $paddr = recv(S,$inbuf,64,0); if (!defined($paddr)) { if ($!==EAGAIN || $!==EINTR) { # false alarm, nothing can be read } else { die "recv: $!"; } } else { ($port,$iaddr) = sockaddr_in($paddr) if $paddr ne ''; $clientaddr = inet_ntoa($iaddr) if defined $iaddr; if (!defined($clientaddr)) { do_log(1, "query from unknown client"); } elsif (!grep {$_ eq $clientaddr} @inet_acl) { do_log(1, "query from non-approved client: %s:%s",$clientaddr,$port); } elsif ($port < 1024 || $port == 2049 || $port > 65535) { do_log(1, "query from questionable port: %s:%s", $clientaddr,$port); } elsif ($inbuf !~ /^([^ ]+) (.*)$/s) { do_log(1, "invalid query syntax from %s:%s", $clientaddr,$port); } else { my($query,$nonce) = ($1,$2); my($src_ip,$src_port); if ($query =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\z/s) { $src_ip = $query; $src_port = 0; # old style query } elsif ($query =~ /^ \[ ([^\]]*) \] (?: : (\d{1,5}) )? \z/xs) { $src_ip = $1; $src_port = $2; if ($src_ip =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\z/) { # IPv4 } elsif ($src_ip =~ /^ (?: (?: IPv6: )? 0{0,4} (?: : 0{0,4} ){1,4} : FFFF : )? ( \d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} )\z/xsi) { $src_ip = $1; # IPv4-mapped IPv6 address, alternative dec, form } elsif ($src_ip =~ /^ (?: IPv6: )? [0-9a-f]{0,4} (?: : [0-9a-f]{0,4} ){2,7} \z/xsi) { $src_ip =~ s/^IPv6://i; } elsif ($src_ip =~ /^ (?: IPv6: )? [0-9a-f]{0,4} (?: : [0-9a-f]{0,4} ){2,5} : \d{1,3} (?: \. \d{1,3} ){3} \z/xsi) { $src_ip =~ s/^IPv6://i; } else { undef $src_ip } } $src_port = 0 if !defined($src_port); if (length($nonce) > 1024) { do_log(1, "invalid query from %s:%s, nonce too long: %d chrs", $clientaddr,$port,length($nonce)); } elsif ($nonce !~ /^([\040-\177].*)\z/s) { do_log(1, "invalid query from %s:%s, forbidden char in nonce", $clientaddr,$port); } elsif (!defined($src_ip) || $src_port > 65535) { do_log(1, "invalid query from %s:%s, bad IP address or port: %s", $clientaddr,$port,$query); } else { do_log(1, "query from %s:%s: %s", $clientaddr,$port,$inbuf); my($resp) = ''; if (exists($src{$src_ip})) { if ($src_port > 0) { # source port known, must match exactly $resp = $src{"[$src_ip]:$src_port"}{d} if exists $src{"[$src_ip]:$src_port"}; } else { # source port not known, find the closest match for my $e (@{$src{$src_ip}}) { if ($resp eq '') { $resp = $e->{d} } elsif ($e->{d} eq $resp) {} else { # keep the longest common string head my($j); my($resp_l) = length($resp); for ($j=0; $j<$resp_l; $j++) { last if substr($e->{d},$j,1) ne substr($resp,$j,1) } if ($j < $resp_l) { # do_log(1, "TRUNCATED to %d: %s %s => /%s/", # $j, $resp, $e->{d}, substr($resp,0,$j)); $resp = substr($resp,0,$j); } } last; } } } $resp = $query.' '.$nonce.' '.$resp; do_log(1, "response to %s:%s: %s", $clientaddr,$port,$resp); defined(send(S, $resp."\015\012", 0, $paddr)) or die "send: $!"; } } } } if (vec($rout,$fn_stdin,1)) { $cnt_since_cleanup++; my($line); $! = 0; my($nbytes) = sysread(STDIN,$line,1024); if (!defined($nbytes)) { if ($!==EAGAIN || $!==EINTR) { # false alarm, nothing can be read } else { die "Read: $!"; } } elsif ($nbytes < 1) { # sysread returns 0 at eof last; # eof } else { chomp($line); local($1,$2,$3,$4,$5,$6); $line =~ /^(\d+\.\d+\.\d+\.\d+):(\d+)[ -]*(.*) \ ->\ (\d+\.\d+\.\d+\.\d+):(\d+)\s*(.*)$/x or next; my($src_ip,$src_port,$src_t,$dst_ip,$dst_port,$src_d) = ($1,$2,$3,$4,$5,$6); my($descr) = "$src_t, $src_d"; my($entry) = { t=>$now, p=>$src_port, c=>1, d=>$descr }; $src{"[$src_ip]:$src_port"} = $entry; if (!exists($src{$src_ip})) { do_log(2, "first: [%s]:%s %.70s", $src_ip,$src_port,$descr); $src{$src_ip} = [ $entry ]; } else { my($found) = 0; for my $e (@{$src{$src_ip}}) { if ($e->{d} eq $descr) { $e->{c}++; $e->{p} = '*'; $e->{t} = $now, $found = 1; do_log(2, "deja-vu: [%s]:%s, cnt=%d %.70s", $src_ip,$src_port,$e->{c},$descr); last; } } if (!$found) { push(@{$src{$src_ip}}, $entry); do_log(2, "stored: [%s]:%d %.70s", $src_ip,$src_port,$descr); } } } if ($cnt_since_cleanup > 50) { for my $k (keys %src) { if (ref($src{$k}) ne 'ARRAY') { if ($src{$k}{t} + $retention_time < $now) { do_log(2, "EXPIRED: %s, age = %d s", $k, $now - $src{$k}{t}); delete $src{$k}; } } else { my(@kept) = grep { $_->{t} + $retention_time >= $now } @{$src{$k}}; if (!@kept) { do_log(2, "EXPIRED: %s, age = %d s", $k, $now - $src{$k}[0]{t}); delete $src{$k}; } elsif (@kept != @{$src{$k}}) { do_log(2, "SHRUNK: %s, %d -> %d", $k, scalar(@{$src{$k}}), scalar(@kept)); @{$src{$k}} = @kept; } } } $cnt_since_cleanup = 0; } } } do_log(1, "normal termination"); exit 0; # Return untainted copy of a string (argument can be a string or a string ref) sub untaint($) { no re 'taint'; my($str); if (defined($_[0])) { local($1); # avoid Perl taint bug: tainted global $1 propagates taintedness $str = $1 if (ref($_[0]) ? ${$_[0]} : $_[0]) =~ /^(.*)\z/s; } $str; } # write log entry sub do_log($$;@) { my($level,$errmsg,@args) = @_; if ($level <= $debug) { $errmsg = sprintf($errmsg,@args) if @args; print STDERR $errmsg,"\n"; } 1; } amavisd-new-2.7.1/AMAVIS-MIB.txt000640 000621 000620 00000324222 11517621601 015616 0ustar00markcmi000000 000000 AMAVIS-MIB DEFINITIONS ::= BEGIN IMPORTS MODULE-IDENTITY, OBJECT-TYPE, NOTIFICATION-TYPE, Counter32, Counter64, Gauge32, Integer32, Unsigned32, TimeTicks, enterprises, mib-2 FROM SNMPv2-SMI TEXTUAL-CONVENTION, DisplayString, TruthValue, TimeInterval, TimeStamp FROM SNMPv2-TC MODULE-COMPLIANCE, OBJECT-GROUP, NOTIFICATION-GROUP FROM SNMPv2-CONF; amavis MODULE-IDENTITY LAST-UPDATED "201101250000Z" ORGANIZATION "Institut Jozef Stefan" CONTACT-INFO "Mark Martinec Institut Jozef Stefan Jamova 39 SI-1000 Ljubljana Slovenia Mark.Martinec@ijs.si" DESCRIPTION "The MIB module describing amavisd-new statistics counters and gauges." REVISION "200906010000Z" DESCRIPTION "The original version of this MIB." REVISION "201011150000Z" DESCRIPTION "Added variables userCounter1 .. userCounter10 and userGauge1 .. userGauge10 ." REVISION "201101140000Z" DESCRIPTION "Add notes indicating that mail checks results cache is no longer used since version 2.7.0 of amavisd-new." REVISION "201101250000Z" DESCRIPTION "Added variables inMsgsStatus* at 1.19 - 1.26, combining a final mail processing status with mail flow direction." ::= { enterprises 15312 2 1 } -- 1.3.6.1.4.1.15312 enterprises . Jozef Stefan Institute -- 1.3.6.1.4.1.15312.2 amavisd-new -- 1.3.6.1.4.1.15312.2.1 amavisd-new SNMP -- 1.3.6.1.4.1.15312.2.1.1 amavisd-new SNMP Statistics -- 1.3.6.1.4.1.15312.2.1.2 amavisd-new Process status -- 1.3.6.1.4.1.15312.2.2 amavisd-new LDAP Elements amavisStats OBJECT IDENTIFIER ::= { amavis 1 } amavisProc OBJECT IDENTIFIER ::= { amavis 2 } amavisMta OBJECT IDENTIFIER ::= { amavis 3 } -- =================== -- amavisStats section -- =================== sysDescr OBJECT-TYPE SYNTAX DisplayString MAX-ACCESS read-only STATUS current DESCRIPTION "A textual string containing information about the running amavisd program." ::= { amavisStats 1 1 } sysObjectID OBJECT-TYPE SYNTAX OBJECT IDENTIFIER MAX-ACCESS read-only STATUS current DESCRIPTION "The vendor's authoritative identification of the network management subsystem contained in the entity. This value provides an easy and unambiguous means for determining `what kind of box' is being managed." ::= { amavisStats 1 2 } sysUpTime OBJECT-TYPE SYNTAX TimeTicks MAX-ACCESS read-only STATUS current DESCRIPTION "The time (in hundredths of a second) since the amavisd program was last (re)started. Useful to distinguish counter wraparounds from program restarts." ::= { amavisStats 1 3 } sysContact OBJECT-TYPE SYNTAX DisplayString MAX-ACCESS read-only STATUS current DESCRIPTION "The textual identification of the contact person for this amavisd instance, together with information on how to contact this person. Corresponds to a configuration variable $snmp_contact ." ::= { amavisStats 1 4 } sysName OBJECT-TYPE SYNTAX DisplayString MAX-ACCESS read-only STATUS current DESCRIPTION "A fully qualified name of the host (or a domain name of a virtual instance). Corresponds to a configuration variable $myhostname ." ::= { amavisStats 1 5 } sysLocation OBJECT-TYPE SYNTAX DisplayString MAX-ACCESS read-only STATUS current DESCRIPTION "The physical location of this host or amavisd instance. Corresponds to a configuration variable $snmp_location ." ::= { amavisStats 1 6 } sysServices OBJECT-TYPE SYNTAX Integer32 MAX-ACCESS read-only STATUS current DESCRIPTION "A value which indicates the set of services that this entity may potentially offer. Always 64 (=applications)." ::= { amavisStats 1 7 } -- orig local -- 0 0 InMsgsOpenRelay -- 0 1 InMsgsInbound -- 0 x (non-originating: inbound or open relay) -- 1 0 InMsgsOutbound -- 1 1 InMsgsInternal -- 1 x InMsgsOriginating (outbound or internal) -- x 0 (departing: outbound or open relay) -- x 1 (local: inbound or internal) -- x x InMsgs -- orig=x local=x inMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd since its start, regardless of their origin or destination." ::= { amavisStats 2 1 } -- orig=0 local=1 inMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off) and having at least one recipient local. On a properly configured system (not an open relay), all recipients of a non-originating message should be local." ::= { amavisStats 2 2 } -- orig=1 local=0 inMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on) and having at least one recipient nonlocal." ::= { amavisStats 2 3 } -- orig=1 local=1 inMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on) and having at least one recipient local." ::= { amavisStats 2 4 } -- orig=1 local=x inMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on). An originating message with multiple recipients when some of them are local and some nonlocal is counted as both internal and outbound. When there are no such messages with mixed recipients, the following relation holds: inMsgsOriginating = inMsgsInternal + inMsgsOutbound ." ::= { amavisStats 2 5 } -- orig=0 local=0 inMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off) and having at least one recipient nonlocal. On a properly configured system this count should be zero at all times. If the count is nonzero, check settings @local_domains_maps, @mynetworks and setting of an 'originating' flag in policy banks." ::= { amavisStats 2 6 } inMsgsStatusAccepted OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd and accepted with a success status code (D_PASS, 2xx), but their forwarding remained the responsibility of an MTA (null delivery_method, typical for a milter setup). Same value as inMsgsStatusDiscardedAll." ::= { amavisStats 2 7 } inMsgsStatusRelayed OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd and accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd. Its value is a sum of inMsgsStatusRelayedUntaggedAll and inMsgsStatusRelayedTaggedAll." ::= { amavisStats 2 8 } inMsgsStatusDiscarded OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received but discarded by amavisd with a success status code (2xx), not forwarded, and no delivery status notification was sent. Effectively a mail message was lost. It is a result of a D_DISCARD setting for all recipients of a message. Note that quarantining uses independent settings, so a message may or may not have been quarantined. Same value as inMsgsStatusDiscardedAll." ::= { amavisStats 2 9 } inMsgsStatusNoBounce OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received but discarded by amavisd with a success status code (2xx), not forwarded, and no delivery status notification was sent. Effectively a mail message was lost. It is a result of a D_BOUNCE setting for all recipients of a message but sending a DSN was suppressed (e.g. high spam score, infected). In other words, D_BOUNCE was converted into a D_DISCARD for this message. Note that quarantining uses independent settings, so a message may or may not have been quarantined. Same value as inMsgsStatusNoBounceAll." ::= { amavisStats 2 10 } inMsgsStatusBounced OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd with a success status code (2xx), but not forwarded, and a delivery status notification was sent to a sender of a message informing him of a reason for blocking a message. It is a result of a D_BOUNCE setting for all recipients of a message. Note that quarantining uses independent settings, so a message may or may not have been quarantined. Same value as inMsgsStatusBouncedAll." ::= { amavisStats 2 11 } inMsgsStatusRejected OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages rejected by amavisd with a reject status code (5xx), not forwarded by amavisd, and no delivery status notification was sent. A duty of notifying the sender of a reject is delegated back to the sending MTA. Note that quarantining uses independent settings, so a message may or may not have been quarantined. Same value as inMsgsStatusRejectedAll." ::= { amavisStats 2 12 } inMsgsStatusTempFailed OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages whose reception resulted in a temporary failure status code (4xx) being send by amavisd back to MTA. A message was not forwarded by amavisd, and no delivery status notification was sent. A message stayed in an MTA queue for future attempts at delivery. Same value as inMsgsStatusTempFailedAll." ::= { amavisStats 2 13 } inMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in all mail messages received by amavisd. Size is calculated according to a definition in RFC 1870." ::= { amavisStats 3 1 } inMsgsSizeInbound OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in all mail messages received by amavisd arriving from outside and having at least one recipient local." ::= { amavisStats 3 2 } inMsgsSizeOutbound OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in all mail messages received by amavisd arriving from inside and having at least one recipient nonlocal." ::= { amavisStats 3 3 } inMsgsSizeInternal OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in all mail messages received by amavisd arriving from inside and having at least one recipient local." ::= { amavisStats 3 4 } inMsgsSizeOriginating OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in all mail messages received by amavisd arriving from inside." ::= { amavisStats 3 5 } inMsgsSizeOpenRelay OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in all mail messages received by amavisd arriving from outside and having at least one nonlocal recipient. Should be zero." ::= { amavisStats 3 6 } inMsgsRecips OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of recipients in all mail messages received by amavisd." ::= { amavisStats 4 1 } inMsgsRecipsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of local recipients in all mail messages received by amavisd arriving from outside." ::= { amavisStats 4 2 } inMsgsRecipsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of nonlocal recipients in all mail messages received by amavisd arriving from inside." ::= { amavisStats 4 3 } inMsgsRecipsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of local recipients in all mail messages received by amavisd arriving from inside." ::= { amavisStats 4 4 } inMsgsRecipsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of recipients in all mail messages received by amavisd arriving from inside (= inMsgsRecipsInternal + inMsgsRecipsOutbound)." ::= { amavisStats 4 5 } inMsgsRecipsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of nonlocal recipients in all mail messages received by amavisd arriving from outside. Should be zero." ::= { amavisStats 4 6 } inMsgsRecipsLocal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of local recipients in all mail messages received by amavisd (= inMsgsRecipsInternal + inMsgsRecipsInbound)." ::= { amavisStats 4 7 } inMsgsBounce OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of bounce mail messages received by amavisd, which include delivery status notifications, disposition notifications, feedback reports (ARF), auto replies, and some mailing list notifications." ::= { amavisStats 5 1 } inMsgsBounceNullRPath OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd having a null envelope sender address (i.e. a null return path). Typically these are delivery status notifications." ::= { amavisStats 5 2 } inMsgsBounceKilled OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of bounce messages received by amavisd and blocked by a bounce killer feature in amavisd, These are bounce messages which carry a message header section of an original message to which they are referring, but it can be demonstrated that the original message did not originate from our site." ::= { amavisStats 5 3 } inMsgsBounceUnverifiable OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of bounce messages received by amavisd and NOT blocked by a bounce killer feature because it was not possible to determine that an original message originated from our site, either because the bounce message did not include a header section of an original mail, or was in some nonstandard form and unparsable." ::= { amavisStats 5 4 } inMsgsBounceRescuedByDomain OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of bounce messages received by amavisd and NOT blocked by a bounce killer feature because a header section of an original mail as included in a bounce message indicated that it is likely the message originated from our site, so this was most likely a legitimate bounce message and not a backscatter message." ::= { amavisStats 5 5 } inMsgsBounceRescuedByOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of bounce messages received by amavisd and NOT blocked by a bounce killer feature because the bounce message was received from an internal network or from an authenticated mail submitter." ::= { amavisStats 5 6 } inMsgsBounceRescuedByPenPals OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of bounce messages received by amavisd and NOT blocked by a bounce killer feature because the bounce message referred to a Message-ID or has a sender/recipient pair matching previous correspondence as shown by evidence in a local SQL logging database." ::= { amavisStats 5 7 } outMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd. This includes passed (relayed) mail messages, notifications, and quarantined messages." ::= { amavisStats 6 1 } outMsgsRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages relayed (forwarded, passed) by amavisd." ::= { amavisStats 6 2 } outMsgsSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages originated by amavisd. This includes quarantined messages, delivery status notifications, notifications to administrator or recipient, as well as other notifications potentially generated by custom hooks, such as feedback reports (ARF) or auto-replies." ::= { amavisStats 6 3 } outMsgsSubmitQuar OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages sent to a quarantine." ::= { amavisStats 6 4 } outMsgsSubmitDsn OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of delivery status notification messages sent to the envelope sender e-mail address." ::= { amavisStats 6 5 } outMsgsSubmitNotif OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of notification messages sent to administrator or to a recipient or to some other configured e-mail address (but not to a mail sender)." ::= { amavisStats 6 6 } outMsgsSubmitAV OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of messages (probes) sent to an external virus scanners, typically SMTP-based." ::= { amavisStats 6 7 } outMsgsSubmitArf OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of abuse reports or feedback notification messages originated by amavisd. Currently these are sent by some custom hook code." ::= { amavisStats 6 8 } outMsgsProtoLocal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages written by amavisd to a local file system. Typically these are quarantine messages when using a local file-based quarantine." ::= { amavisStats 6 9 } outMsgsProtoLocalRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages written by amavisd to a local file system relayed by amavisd. This number should be zero in usual setups." ::= { amavisStats 6 10 } outMsgsProtoLocalSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages written by amavisd to a local file system and generated by amavisd, such as notifications. This number should be zero in usual setups." ::= { amavisStats 6 11 } outMsgsProtoSMTP OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd to an MTA using a SMTP protocol. This includes relayed mail messages as well as messages generated by amavisd, such as notifications." ::= { amavisStats 6 12 } outMsgsProtoSMTPRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of relayed (fordarded, passed) mail messages transmitted by amavisd to an MTA using a SMTP protocol." ::= { amavisStats 6 13 } outMsgsProtoSMTPSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd to an MTA using a SMTP protocol and generated by amavisd, such as notifications." ::= { amavisStats 6 14 } outMsgsProtoLMTP OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd to an MTA using a LMTP protocol. This includes relayed mail messages as well as messages generated by amavisd, such as notifications." ::= { amavisStats 6 15 } outMsgsProtoLMTPRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of relayed (forwarded, passed) mail messages transmitted by amavisd to an MTA using a LMTP protocol." ::= { amavisStats 6 16 } outMsgsProtoLMTPSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd to an MTA using a LMTP protocol and generated by amavisd, such as notifications." ::= { amavisStats 6 17 } outMsgsProtoBSMTP OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd to an MTA using a BSMTP protocol. This includes relayed mail messages as well as messages generated by amavisd, such as notifications." ::= { amavisStats 6 18 } outMsgsProtoBSMTPRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of relayed (forwarded, passed) mail messages transmitted by amavisd to an MTA using a BSMTP protocol." ::= { amavisStats 6 19 } outMsgsProtoBSMTPSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd to an MTA using a BSMTP protocol and generated by amavisd, such as notifications." ::= { amavisStats 6 20 } outMsgsProtoPipe OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd to an MTA using a pipe to an external program. This includes relayed mail messages as well as messages generated by amavisd, such as notifications." ::= { amavisStats 6 21 } outMsgsProtoPipeRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of relayed (forwarded, passed) mail messages transmitted by amavisd to an MTA using a pipe to an external program." ::= { amavisStats 6 22 } outMsgsProtoPipeSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd to an MTA using a pipe to an external program and generated by amavisd, such as notifications." ::= { amavisStats 6 23 } outMsgsProtoSQL OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages stored by amavisd to an SQL database. This includes relayed mail messages as well as messages generated by amavisd such as notifications, but typically corresponds to quarantined messages only." ::= { amavisStats 6 24 } outMsgsProtoSQLRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of relayed mail messages stored by amavisd to an SQL database. Typically the number is always zero in common setups." ::= { amavisStats 6 25 } outMsgsProtoSQLSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages stored by amavisd to an SQL database and generated by amavisd, such as notifications. Typically the number is always zero in common setups." ::= { amavisStats 6 26 } outMsgsDelivers OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages successfully transmitted through any delivery method, such as SMTP, pipe, local file system." ::= { amavisStats 6 27 } outMsgsAttemptFails OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of temporarily unsuccessful delivery attempts through any delivery method, such as SMTP, pipe, local file system." ::= { amavisStats 6 28 } outMsgsRejects OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of delivery attempts rejected by a receiving agent or media, using any delivery method, such as SMTP, pipe, local file system." ::= { amavisStats 6 29 } outMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages transmitted by amavisd through any delivery method, including relayed (forwarded, passed) and quarantined messages, as well as notifications and delivery status reports originated by amavisd. The number may not be exact, as it currently does not reflect modifications by amavisd such as added or edited header fields." ::= { amavisStats 7 1 } outMsgsSizeRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages relayed (passed) by amavisd through any delivery method. The number may not be exact, as it currently does not reflect modifications by amavisd such as added or edited header fields." ::= { amavisStats 7 2 } outMsgsSizeSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages originated by amavisd and sent through any delivery method. These include quarantined messages, as well as notifications and delivery status reports originated by amavisd. The number may not be exact, as it currently does not reflect modifications by amavisd such as added or edited header fields." ::= { amavisStats 7 3 } outMsgsSizeSubmitQuar OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages sent to a quarantine through any delivery method." ::= { amavisStats 7 4 } outMsgsSizeSubmitDsn OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in delivery status notifications originated by amavisd." ::= { amavisStats 7 5 } outMsgsSizeSubmitNotif OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in notification messages originated by amavisd and sent to administrator or to a recipient (but not to a mail sender)." ::= { amavisStats 7 6 } outMsgsSizeSubmitAV OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages sent an external virus scanner for testing. These currently only accounts for SMTP-based virus scanners." ::= { amavisStats 7 7 } outMsgsSizeSubmitArf OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in abuse reports or feedback notification messages originated by amavisd. Currently these are sent by some custom hooks." ::= { amavisStats 7 8 } outMsgsSizeProtoLocal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages stored by amavisd to a local file system, typically corresponding to quarantined messages. The number may not be exact, as it currently does not reflect message modifications by amavisd." ::= { amavisStats 7 9 } outMsgsSizeProtoLocalRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages written by amavisd to a local file system relayed by amavisd. This number should be zero in usual setups." ::= { amavisStats 7 10 } outMsgsSizeProtoLocalSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages written by amavisd to a local file system and generated by amavisd, typically corresponding to quarantined messages." ::= { amavisStats 7 11 } outMsgsSizeProtoSMTP OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages transmitted by amavisd to an MTA using SMTP protocol. This includes relayed (passed) messages, as well as notifications generated by amavisd and messages quarantined to an e-mail address when using this method. The number may not be exact, as it currently does not reflect message modifications by amavisd." ::= { amavisStats 7 12 } outMsgsSizeProtoSMTPRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of relayed (fordarded, passed) mail messages transmitted by amavisd to an MTA using a SMTP protocol." ::= { amavisStats 7 13 } outMsgsSizeProtoSMTPSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages transmitted by amavisd to an MTA using a SMTP protocol and generated by amavisd, such as notifications." ::= { amavisStats 7 14 } outMsgsSizeProtoLMTP OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages transmitted by amavisd to an MTA using LMTP protocol. This includes relayed (passed) messages, as well as notifications generated by amavisd and messages quarantined to an e-mail address when using this method. The number may not be exact, as it currently does not reflect message modifications by amavisd." ::= { amavisStats 7 15 } outMsgsSizeProtoLMTPRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in relayed (forwarded, passed) mail messages transmitted by amavisd to an MTA using a LMTP protocol." ::= { amavisStats 7 16 } outMsgsSizeProtoLMTPSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in mail messages transmitted by amavisd to an MTA using a LMTP protocol and generated by amavisd, such as notifications." ::= { amavisStats 7 17 } outMsgsSizeProtoBSMTP OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages transmitted by amavisd to an MTA using BSMTP protocol. This includes relayed (passed) messages, as well as notifications generated by amavisd and messages quarantined through BSMTP. The number may not be exact, as it currently does not account for message modifications by amavisd." ::= { amavisStats 7 18 } outMsgsSizeProtoBSMTPRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in relayed (forwarded, passed) mail messages transmitted by amavisd to an MTA using a BSMTP protocol." ::= { amavisStats 7 19 } outMsgsSizeProtoBSMTPSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in mail messages transmitted by amavisd to an MTA using a BSMTP protocol and generated by amavisd, such as quarantined messages and notifications." ::= { amavisStats 7 20 } outMsgsSizeProtoPipe OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages transmitted by amavisd through a pipe to an external program. This includes relayed (passed) messages, as well as notifications generated by amavisd and messages quarantined through a pipe. The number may not be exact, as it currently does not reflect message modifications by amavisd." ::= { amavisStats 7 21 } outMsgsSizeProtoPipeRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in relayed (forwarded, passed) mail messages transmitted by amavisd to an MTA using a pipe to an external program." ::= { amavisStats 7 22 } outMsgsSizeProtoPipeSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in mail messages transmitted by amavisd to an MTA using a pipe to an external program and generated by amavisd, such as quarantined messages and notifications." ::= { amavisStats 7 23 } outMsgsSizeProtoSQL OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages stored to an SQL database by amavisd. This includes relayed (passed) messages, as well as notifications generated by amavisd and quarantined message, but typically corresponds to quarantined messages only. The number may not be exact, as it currently does not reflect message modifications by amavisd." ::= { amavisStats 7 24 } outMsgsSizeProtoSQLRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in relayed mail messages stored by amavisd to an SQL database. Typically the number is always zero in common setups." ::= { amavisStats 7 25 } outMsgsSizeProtoSQLSubmit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in mail messages stored by amavisd to an SQL database and generated by amavisd, such as quarantined messages and notifications." ::= { amavisStats 7 26 } quarMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of quarantined message." ::= { amavisStats 8 1 } quarArchMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of messages quarantined to an archival quarantine." ::= { amavisStats 8 2 } quarCleanMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean-contents messages quarantined." ::= { amavisStats 8 3 } quarMtaFailedMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean-contents messages quarantined because of a failure to feed a message back to an MTA. Should be zero at all times, quarantining is not supposed to be enabled for MTA failures." ::= { amavisStats 8 4 } quarOversizedMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean-contents messages quarantined because their size exceeded a configurable limit and quarantining is enabled for oversized messages." ::= { amavisStats 8 5 } quarBadHdrMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of messages quarantined because they have a bad header section and quarantining is enabled for such messages." ::= { amavisStats 8 6 } quarSpammyMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of spammy messages quarantined (above tag2 level) and quarantining is enabled for such messages." ::= { amavisStats 8 7 } quarSpamMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of spam messages quarantined (above kill level) and quarantining is enabled for such messages." ::= { amavisStats 8 8 } quarUncheckedMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of messages quarantined because they could not be checked for infection (e.g. corrupted or encrypted contents, mail bombs) and quarantining is enabled for such messages." ::= { amavisStats 8 9 } quarBannedMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of messages with banned contents quarantined." ::= { amavisStats 8 10 } quarVirusMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of infected messages quarantined." ::= { amavisStats 8 11 } quarAttemptTempFails OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of times quarantining was unsuccessful due to a temporary error reported by a delivery mechanism." ::= { amavisStats 8 12 } quarAttemptFails OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of times quarantining was unsuccessful due to a permanent error reported by a delivery mechanism." ::= { amavisStats 8 13 } quarMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in quarantined message." ::= { amavisStats 9 1 } quarArchMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages quarantined to an archival quarantine." ::= { amavisStats 9 2 } quarCleanMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in quarantine clean-contents messages." ::= { amavisStats 9 3 } quarMtaFailedMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in clean-contents messages quarantined because of a failure to feed a message back to an MTA." ::= { amavisStats 9 4 } quarOversizedMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in clean-contents messages quarantined because their size exceeded a configurable limit." ::= { amavisStats 9 5 } quarBadHdrMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages quarantined because they had a bad header section." ::= { amavisStats 9 6 } quarSpammyMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in quarantined spammy messages (above tag2 level)." ::= { amavisStats 9 7 } quarSpamMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in quarantined spam messages (above kill level)." ::= { amavisStats 9 8 } quarUncheckedMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in messages quarantined because they could not be checked for infection (e.g. corrupted or encrypted contents, mail bombs)." ::= { amavisStats 9 9 } quarBannedMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in quarantined messages with banned contents." ::= { amavisStats 9 10 } quarVirusMsgsSize OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of octets in quarantined infected messages." ::= { amavisStats 9 11 } contentCleanMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents mail messages received by amavisd." ::= { amavisStats 10 1 1 } contentCleanMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of incoming clean contents messages received by amavisd." ::= { amavisStats 10 1 2 } contentCleanMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of outgoing clean contents messages received by amavisd." ::= { amavisStats 10 1 3 } contentCleanMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of internal-to-internal clean contents messages received by amavisd." ::= { amavisStats 10 1 4 } contentCleanMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of originating clean contents messages received by amavisd." ::= { amavisStats 10 1 5 } contentCleanMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents messages received by amavisd from outside and having nonlocal recipients. Should be zero." ::= { amavisStats 10 1 6 } contentMtaFailedMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents mail messages received by amavisd which could not be delivered back to MTA, resulting in a temporary failure on the receiving side." ::= { amavisStats 10 2 1 } contentMtaFailedMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents incoming mail messages received by amavisd which could not be delivered back to MTA, resulting in a temporary failure on the receiving side." ::= { amavisStats 10 2 2 } contentMtaFailedMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents outgoing mail messages received by amavisd which could not be delivered back to MTA, resulting in a temporary failure on the receiving side." ::= { amavisStats 10 2 3 } contentMtaFailedMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents internal-to-internal mail messages received by amavisd which could not be delivered back to MTA, resulting in a temporary failure on the receiving side." ::= { amavisStats 10 2 4 } contentMtaFailedMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents originating mail messages received by amavisd which could not be delivered back to MTA, resulting in a temporary failure on the receiving side." ::= { amavisStats 10 2 5 } contentMtaFailedMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents mail messages received from outside with nonlocal recipients, which could not be delivered back to MTA, resulting in a temporary failure on the receiving side. Should be zero." ::= { amavisStats 10 2 6 } contentOversizedMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents mail messages received by amavisd which exceeded a configured size limit." ::= { amavisStats 10 3 1 } contentOversizedMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents incoming mail messages received by amavisd which exceeded a configured size limit." ::= { amavisStats 10 3 2 } contentOversizedMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents outgoing mail messages received by amavisd which exceeded a configured size limit." ::= { amavisStats 10 3 3 } contentOversizedMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents internal-to-internal mail messages received by amavisd which exceeded a configured size limit." ::= { amavisStats 10 3 4 } contentOversizedMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents originating mail messages received by amavisd which exceeded a configured size limit." ::= { amavisStats 10 3 5 } contentOversizedMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of clean contents mail messages received from outside with nonlocal recipients, which exceeded a configured size limit. Should be zero." ::= { amavisStats 10 3 6 } contentBadHdrMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages with a bad header section received by amavisd." ::= { amavisStats 10 4 1 } contentBadHdrMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of incoming mail messages with a bad header section received by amavisd." ::= { amavisStats 10 4 2 } contentBadHdrMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of outgoing mail messages with a bad header section received by amavisd." ::= { amavisStats 10 4 3 } contentBadHdrMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of internal-to-internal mail messages with a bad header section received by amavisd." ::= { amavisStats 10 4 4 } contentBadHdrMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of originating mail messages with a bad header section received by amavisd." ::= { amavisStats 10 4 5 } contentBadHdrMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages with a bad header section received from outside with nonlocal recipients. Should be zero." ::= { amavisStats 10 4 6 } contentSpammyMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of spammy (tag2) mail messages received by amavisd." ::= { amavisStats 10 5 1 } contentSpammyMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of incoming spammy (tag2) mail messages received by amavisd." ::= { amavisStats 10 5 2 } contentSpammyMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of outgoing spammy (tag2) mail messages received by amavisd." ::= { amavisStats 10 5 3 } contentSpammyMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of internal-to-internal spammy (tag2) mail messages received by amavisd." ::= { amavisStats 10 5 4 } contentSpammyMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of originating spammy (tag2) mail messages received by amavisd." ::= { amavisStats 10 5 5 } contentSpammyMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of spammy (tag2) mail messages received from outside with nonlocal recipients. Should be zero." ::= { amavisStats 10 5 6 } contentSpamMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of spam (kill level) mail messages received by amavisd." ::= { amavisStats 10 6 1 } contentSpamMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of incoming spam (kill level) mail messages received by amavisd." ::= { amavisStats 10 6 2 } contentSpamMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of outgoing spam (kill level) mail messages received by amavisd." ::= { amavisStats 10 6 3 } contentSpamMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of internal-to-internal spam (kill level) mail messages received by amavisd." ::= { amavisStats 10 6 4 } contentSpamMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of originating spam (kill level) mail messages received by amavisd." ::= { amavisStats 10 6 5 } contentSpamMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of spam (kill level) mail messages received from outside with nonlocal recipients. Should be zero." ::= { amavisStats 10 6 6 } contentUncheckedMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd which could not be checked for infection (mail bombs)." ::= { amavisStats 10 7 1 } contentUncheckedMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of incoming mail messages received by amavisd which could not be checked for infection (mail bombs)." ::= { amavisStats 10 7 2 } contentUncheckedMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of outgoing mail messages received by amavisd which could not be checked for infection (mail bombs)." ::= { amavisStats 10 7 3 } contentUncheckedMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of internal-to-internal mail messages received by amavisd which could not be checked for infection (mail bombs)." ::= { amavisStats 10 7 4 } contentUncheckedMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of originating mail messages received by amavisd which could not be checked for infection (mail bombs)." ::= { amavisStats 10 7 5 } contentUncheckedMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received from outside with nonlocal recipients which could not be checked for infection (mail bombs)." ::= { amavisStats 10 7 6 } contentBannedMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of banned-contents mail messages received by amavisd." ::= { amavisStats 10 8 1 } contentBannedMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of incoming banned-contents mail messages received by amavisd." ::= { amavisStats 10 8 2 } contentBannedMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of outgoing banned-contents mail messages received by amavisd." ::= { amavisStats 10 8 3 } contentBannedMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of internal-to-internal banned-contents mail messages received by amavisd." ::= { amavisStats 10 8 4 } contentBannedMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of originating banned-contents mail messages received by amavisd." ::= { amavisStats 10 8 5 } contentBannedMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of banned-contents mail messages received from outside with nonlocal recipients. Should be zero." ::= { amavisStats 10 8 6 } contentVirusMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of infected mail messages received by amavisd." ::= { amavisStats 10 9 1 } contentVirusMsgsInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of incoming infected mail messages received by amavisd." ::= { amavisStats 10 9 2 } contentVirusMsgsOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of outgoing infected mail messages received by amavisd." ::= { amavisStats 10 9 3 } contentVirusMsgsInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of internal-to-internal infected mail messages received by amavisd." ::= { amavisStats 10 9 4 } contentVirusMsgsOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of originating infected mail messages received by amavisd." ::= { amavisStats 10 9 5 } contentVirusMsgsOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of infected mail messages received from outside with nonlocal recipients. Should be zero." ::= { amavisStats 10 9 6 } cacheAttempts OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of times a database of cached previous scanning results was consulted. Cache is not used any longer since 2.7.0." ::= { amavisStats 11 1 } cacheMisses OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of times a database of cached previous scanning results was consulted but did not find a message with the same body hash. Cache is not used any longer since 2.7.0." ::= { amavisStats 11 2 } cacheHits OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of times a database of cached previous scanning results was consulted and found previous scanning results for a message with the same body hash. Cache is not used any longer since 2.7.0." ::= { amavisStats 11 3 } cacheHitsVirusCheck OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of times a database of cached previous scanning results was consulted and found previous (positive or negative) virus scanning results for a message with the same body hash. Cache is not used any longer since 2.7.0." ::= { amavisStats 11 4 } cacheHitsVirusMsgs OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of times a database of cached previous scanning results was consulted and found previous positive virus scanning results for a message with the same body hash. Cache is not used any longer since 2.7.0." ::= { amavisStats 11 5 } outConnNew OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of new (SMTP or LMTP) TCP session establishments to an MTA." ::= { amavisStats 11 6 } outConnQuit OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of (SMTP or LMTP) TCP sessions to an MTA closed down by sending a QUIT command." ::= { amavisStats 11 7 } outConnTransact OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of transactions to an MTA. The SMTP or LMTP protocol allows transferring more than one mail message during the same session. Each MAIL FROM command begins a new SMTP or LMTP transaction." ::= { amavisStats 11 8 } outConnReuseFail OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Amavisd tries to reuse open SMTP or LMTP sessions to an MTA to send additional mail messages without having to tear down and re-establish a session. Before attempting to reuse an open session which hasn't been used for some time, amavisd sends a NOOP command to test if the session is still alive or was torn down at the receiving end. The outConnReuseFail corresponds to unsuccessful NOOP attempts, which indicate that a new session must be opened." ::= { amavisStats 11 9 } outConnReuseRecent OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Amavisd tries to reuse open SMTP or LMTP sessions to an MTA to send additional mail messages without having to tear down and re-establish a session. The outConnReuseRecent corresponds to the number of times an already open and recently used session could be reused." ::= { amavisStats 11 10 } outConnReuseRefreshed OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Amavisd tries to reuse open SMTP or LMTP sessions to an MTA to send additional mail messages without having to tear down and re-establish a session. Before attempting to reuse an open session which hasn't been used for some time, amavisd sends a NOOP command to test if the session is still alive or was torn down at the receiving end. The outConnReuseRefreshed corresponds to successful NOOP attempts, which indicate that an existing session can be re-used." ::= { amavisStats 11 11 } opsDec OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of invocations of decoders while parsing mail messages." ::= { amavisStats 12 1 } opsSpamCheck OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of invocations of a spam checker." ::= { amavisStats 12 2 } opsVirusCheck OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of invocations of a virus checker." ::= { amavisStats 12 3 } penPalsAttempts OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database with a goal of finding a previous message corresponding to a message now being processed." ::= { amavisStats 13 1 } penPalsAttemptsRid OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database with a goal of finding a previous message with a reversed pair of a given sender and recipient address, corresponding to a message now being processed." ::= { amavisStats 13 2 } penPalsAttemptsMid OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database with a goal of finding a previous message with a given MessageID and a sender address, corresponding to a message now being processed." ::= { amavisStats 13 3 } penPalsMisses OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database which did not produce any records." ::= { amavisStats 13 4 } penPalsHits OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database which did find a matching previous message." ::= { amavisStats 13 5 } penPalsHitsRid OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database which did find a matching previous message with a reversed pair of a given sender and recipient address corresponding to a message now being processed." ::= { amavisStats 13 6 } penPalsHitsMid OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database which did find a matching previous message with a given MessageID and a sender address corresponding to a message now being processed." ::= { amavisStats 13 7 } penPalsHitsMidRid OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database which did find a matching previous message with a given MessageID, as well as having a reversed pair of a given sender and recipient address, corresponding to a message now being processed." ::= { amavisStats 13 8 } penPalsSavedFromTag2 OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages that would have reached or exceeded tag2 level (CC_SPAMMY) had it not been for the negative score points contributed by a PenPals lookup." ::= { amavisStats 13 9 } penPalsSavedFromTag3 OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages that would have reached or exceeded tag3 level (CC_SPAMMY,1) had it not been for the negative score points contributed by a PenPals lookup." ::= { amavisStats 13 10 } penPalsSavedFromKill OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages that would have reached or exceeded kill level (CC_SPAM) had it not been for the negative score points contributed by a PenPals lookup." ::= { amavisStats 13 11 } sqlAddrSenderAttempts OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database performed in order to find a record with a sender's e-mail address, creating one if missing." ::= { amavisStats 14 1 } sqlAddrSenderMisses OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL operations on a logging database performed in order to create a record with a sender's address which previously did not exist in a database." ::= { amavisStats 14 2 } sqlAddrSenderHits OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database performed, successfully finding an existing a record with a sender's address." ::= { amavisStats 14 3 } sqlAddrRecipAttempts OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database performed in order to find a record with a recipient's e-mail address, creating one if missing." ::= { amavisStats 14 4 } sqlAddrRecipMisses OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL operations on a logging database performed in order to create a record with a recipient's address which previously did not exist in a database." ::= { amavisStats 14 5 } sqlAddrRecipHits OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of SQL queries on a logging database performed, successfully finding an existing a record with a recipient's address." ::= { amavisStats 14 6 } logEntries OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged. Note that a long log message may be wrapped into more than one line. Contrast with 'logLines'." ::= { amavisStats 15 1 } logEntriesEmerg OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at an Emergency log level. Currently not used by amavisd." ::= { amavisStats 15 2 } logEntriesAlert OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at an Alert log level or more severe. Currently not used by amavisd." ::= { amavisStats 15 3 } logEntriesCrit OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a Critical log level or more severe. Corresponds to log level -3." ::= { amavisStats 15 4 } logEntriesErr OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at an Error log level or more severe. Corresponds to log level -2." ::= { amavisStats 15 5 } logEntriesWarning OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a Warning log level or more severe. Corresponds to log level -1." ::= { amavisStats 15 6 } logEntriesNotice OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a Notice log level or more severe. Corresponds to log level 0." ::= { amavisStats 15 7 } logEntriesInfo OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at an Informational log level or more severe. Corresponds to log level 1." ::= { amavisStats 15 8 } logEntriesDebug OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a Debug log level or more severe, which effectively accounts for all log entries, same as 'logEntries'." ::= { amavisStats 15 9 } logEntriesLevel0 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a log level 0 or more severe." ::= { amavisStats 15 10 } logEntriesLevel1 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a log level 1." ::= { amavisStats 15 11 } logEntriesLevel2 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a log level 2." ::= { amavisStats 15 12 } logEntriesLevel3 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a log level 3." ::= { amavisStats 15 13 } logEntriesLevel4 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a log level 4." ::= { amavisStats 15 14 } logEntriesLevel5 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of all messages logged at a log level 5 or less severe." ::= { amavisStats 15 15 } logLines OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of log lines written. Contrast with 'logEntries'." ::= { amavisStats 15 16 } logRetries OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of times a syslog routine needed to retry its attempt at writing a log line. Purely informational and depending on platform. Does NOT indicate a log message loss, but may indicate a level of contention on a syslog service." ::= { amavisStats 15 17 } timeElapsedTotal OBJECT-TYPE SYNTAX TimeInterval MAX-ACCESS read-only STATUS current DESCRIPTION "Time elapsed while processing mail messages, in units of 0.01 s." ::= { amavisStats 16 1 } timeElapsedReceiving OBJECT-TYPE SYNTAX TimeInterval MAX-ACCESS read-only STATUS current DESCRIPTION "Time elapsed while receiving mail messages, in units of 0.01 s." ::= { amavisStats 16 2 } timeElapsedSending OBJECT-TYPE SYNTAX TimeInterval MAX-ACCESS read-only STATUS current DESCRIPTION "Time elapsed while transmitting mail messages, in units of 0.01 s." ::= { amavisStats 16 3 } timeElapsedDecoding OBJECT-TYPE SYNTAX TimeInterval MAX-ACCESS read-only STATUS current DESCRIPTION "Time elapsed while decoding mail messages, in units of 0.01 s." ::= { amavisStats 16 4 } timeElapsedPenPals OBJECT-TYPE SYNTAX TimeInterval MAX-ACCESS read-only STATUS current DESCRIPTION "Time elapsed on SQL logging and querying operations, in units of 0.01 s." ::= { amavisStats 16 5 } timeElapsedVirusCheck OBJECT-TYPE SYNTAX TimeInterval MAX-ACCESS read-only STATUS current DESCRIPTION "Time elapsed while checking mail messages for viruses, in units of 0.01 s." ::= { amavisStats 16 6 } timeElapsedSpamCheck OBJECT-TYPE SYNTAX TimeInterval MAX-ACCESS read-only STATUS current DESCRIPTION "Time elapsed while checking mail messages for spam, in units of 0.01 s." ::= { amavisStats 16 7 } userCounter1 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 1 } userCounter2 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 2 } userCounter3 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 3 } userCounter4 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 4 } userCounter5 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 5 } userCounter6 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 6 } userCounter7 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 7 } userCounter8 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 8 } userCounter9 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 9 } userCounter10 OBJECT-TYPE SYNTAX Counter64 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose counter, its semantics is assignable by a user." ::= { amavisStats 17 10 } userGauge1 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 1 } userGauge2 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 2 } userGauge3 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 3 } userGauge4 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 4 } userGauge5 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 5 } userGauge6 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 6 } userGauge7 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 7 } userGauge8 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 8 } userGauge9 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 9 } userGauge10 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "A general purpose gauge/integer, its semantics is assignable by a user." ::= { amavisStats 18 10 } inMsgsStatusAcceptedAll OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd and accepted with a success status code (D_PASS, 2xx), but their forwarding remained the responsibility of an MTA (null delivery_method, typical for a milter setup). A synonym with inMsgsStatusAccepted." ::= { amavisStats 19 1 } inMsgsStatusAcceptedInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient local, and accepted with a success status code (D_PASS, 2xx), but their forwarding remained the responsibility of an MTA (null delivery_method, typical for a milter setup)." ::= { amavisStats 19 2 } inMsgsStatusAcceptedOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient nonlocal, and accepted with a success status code (D_PASS, 2xx), but their forwarding remained the responsibility of an MTA (null delivery_method, typical for a milter setup)." ::= { amavisStats 19 3 } inMsgsStatusAcceptedInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient local, and accepted with a success status code (D_PASS, 2xx), but their forwarding remained the responsibility of an MTA (null delivery_method, typical for a milter setup)." ::= { amavisStats 19 4 } inMsgsStatusAcceptedOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), and accepted with a success status code (D_PASS, 2xx), but their forwarding remained the responsibility of an MTA (null delivery_method, typical for a milter setup)." ::= { amavisStats 19 5 } inMsgsStatusAcceptedOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient nonlocal, and accepted with a success status code (D_PASS, 2xx), but their forwarding remained the responsibility of an MTA (null delivery_method, typical for a milter setup). On a properly configured system this count should be zero at all times." ::= { amavisStats 19 6 } inMsgsStatusRelayedUntaggedAll OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd and accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd without adding a warning to a Subject or to a header section or appending an address extension to recipients. A sum of inMsgsStatusRelayedUntaggedAll and inMsgsStatusRelayedTaggedAll is represented by a counter inMsgsStatusRelayed." ::= { amavisStats 20 1 } inMsgsStatusRelayedUntaggedInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient local, accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd, while adding a warning to a Subject or to a header section or appending an address extension to recipients." ::= { amavisStats 20 2 } inMsgsStatusRelayedUntaggedOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient nonlocal, accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd, while adding a warning to a Subject or to a header section or appending an address extension to recipients." ::= { amavisStats 20 3 } inMsgsStatusRelayedUntaggedInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient local, accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd." ::= { amavisStats 20 4 } inMsgsStatusRelayedUntaggedOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd." ::= { amavisStats 20 5 } inMsgsStatusRelayedUntaggedOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient nonlocal, accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd." ::= { amavisStats 20 6 } inMsgsStatusRelayedTaggedAll OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd and accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd, while adding a warning to a Subject or to a header section or appending an address extension to recipients. A sum of inMsgsStatusRelayedUntaggedAll and nMsgsStatusRelayedTaggedAll is represented by a counter inMsgsStatusRelayed." ::= { amavisStats 21 1 } inMsgsStatusRelayedTaggedInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient local, accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd, while adding a warning to a Subject or to a header section or appending an address extension to recipients." ::= { amavisStats 21 2 } inMsgsStatusRelayedTaggedOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient nonlocal, accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd, while adding a warning to a Subject or to a header section or appending an address extension to recipients." ::= { amavisStats 21 3 } inMsgsStatusRelayedTaggedInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient local, accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd, while adding a warning to a Subject or to a header section or appending an address extension to recipients." ::= { amavisStats 21 4 } inMsgsStatusRelayedTaggedOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd, while adding a warning to a Subject or to a header section or appending an address extension to recipients." ::= { amavisStats 21 5 } inMsgsStatusRelayedTaggedOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient nonlocal, accepted with a success status code (D_PASS, 2xx), and forwarded by amavisd, while adding a warning to a Subject or to a header section or appending an address extension to recipients." ::= { amavisStats 21 6 } inMsgsStatusDiscardedAll OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received but discarded by amavisd with a success status code (2xx), not forwarded, and no delivery status notification was sent. Effectively a mail message was lost. It is a result of a D_DISCARD setting for all recipients of a message. Note that quarantining uses independent settings, so a message may or may not have been quarantined. A synonym with inMsgsStatusDiscarded." ::= { amavisStats 22 1 } inMsgsStatusDiscardedInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient local, but discarded by amavisd with a success status code (2xx), not forwarded, and no delivery status notification was sent. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 22 2 } inMsgsStatusDiscardedOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient nonlocal, but discarded by amavisd with a success status code (2xx), not forwarded, and no delivery status notification was sent. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 22 3 } inMsgsStatusDiscardedInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient local, but discarded by amavisd with a success status code (2xx), not forwarded, and no delivery status notification was sent. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 22 4 } inMsgsStatusDiscardedOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), but discarded by amavisd with a success status code (2xx), not forwarded, and no delivery status notification was sent. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 22 5 } inMsgsStatusDiscardedOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient nonlocal, but discarded by amavisd with a success status code (2xx), not forwarded, and no delivery status notification was sent. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 22 6 } inMsgsStatusNoBounceAll OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received but discarded by amavisd with a success status code (2xx), not forwarded, and no delivery status notification was sent. Effectively a mail message was lost. It is a result of a D_BOUNCE setting for all recipients of a message but sending a DSN was suppressed (e.g. high spam score, infected). In other words, D_BOUNCE was converted into a D_DISCARD for this message. Note that quarantining uses independent settings, so a message may or may not have been quarantined. A synonym with inMsgsStatusNoBounce." ::= { amavisStats 23 1 } inMsgsStatusNoBounceInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient local, but discarded by amavisd with a success status code (2xx), not forwarded, a delivery status notification was suppressed. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 23 2 } inMsgsStatusNoBounceOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient nonlocal, but discarded by amavisd with a success status code (2xx), not forwarded, a delivery status notification was suppressed. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 23 3 } inMsgsStatusNoBounceInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient local, but discarded by amavisd with a success status code (2xx), not forwarded, a delivery status notification was suppressed. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 23 4 } inMsgsStatusNoBounceOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), but discarded by amavisd with a success status code (2xx), not forwarded, a delivery status notification was suppressed. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 23 5 } inMsgsStatusNoBounceOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient nonlocal, but discarded by amavisd with a success status code (2xx), not forwarded, a delivery status notification was suppressed. Effectively a mail message was lost. A message may or may not have been quarantined." ::= { amavisStats 23 6 } inMsgsStatusBouncedAll OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd with a success status code (2xx), but not forwarded, and a delivery status notification was sent to a sender of a message informing him of a reason for blocking a message. It is a result of a D_BOUNCE setting for all recipients of a message. Note that quarantining uses independent settings, so a message may or may not have been quarantined. A synonym with inMsgsStatusBounced." ::= { amavisStats 24 1 } inMsgsStatusBouncedInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient local, confirmed by a success status code (2xx) but not forwarded, and a delivery status notification was sent to a sender of a message informing him of a reason for non-delivery. A message may or may not have been quarantined." ::= { amavisStats 24 2 } inMsgsStatusBouncedOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient nonlocal, confirmed by a success status code (2xx) but not forwarded, and a delivery status notification was sent to a sender of a message informing him of a reason for non-delivery. A message may or may not have been quarantined." ::= { amavisStats 24 3 } inMsgsStatusBouncedInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient local, confirmed by a success status code (2xx) but not forwarded, and a delivery status notification was sent to a sender of a message informing him of a reason for non-delivery. A message may or may not have been quarantined." ::= { amavisStats 24 4 } inMsgsStatusBouncedOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), confirmed by a success status code (2xx) but not forwarded, and a delivery status notification was sent to a sender of a message informing him of a reason for non-delivery. A message may or may not have been quarantined." ::= { amavisStats 24 5 } inMsgsStatusBouncedOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient nonlocal, confirmed by a success status code (2xx) but not forwarded, and a delivery status notification was sent to a sender of a message informing him of a reason for non-delivery. A message may or may not have been quarantined." ::= { amavisStats 24 6 } inMsgsStatusRejectedAll OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages rejected by amavisd with a reject status code (5xx), not forwarded by amavisd, and no delivery status notification was sent. A duty of notifying the sender of a reject is delegated back to the sending MTA. Note that quarantining uses independent settings, so a message may or may not have been quarantined. A synonym with inMsgsStatusRejected." ::= { amavisStats 25 1 } inMsgsStatusRejectedInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient local, and rejected by amavisd with a reject status code (5xx), not forwarded by amavisd, and no delivery status notification was sent. A message may or may not have been quarantined." ::= { amavisStats 25 2 } inMsgsStatusRejectedOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient nonlocal, and rejected by amavisd with a reject status code (5xx), not forwarded by amavisd, and no delivery status notification was sent. A message may or may not have been quarantined." ::= { amavisStats 25 3 } inMsgsStatusRejectedInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient local, and rejected by amavisd with a reject status code (5xx), not forwarded by amavisd, and no delivery status notification was sent. A message may or may not have been quarantined." ::= { amavisStats 25 4 } inMsgsStatusRejectedOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), and rejected by amavisd with a reject status code (5xx), not forwarded by amavisd, and no delivery status notification was sent. A message may or may not have been quarantined." ::= { amavisStats 25 5 } inMsgsStatusRejectedOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient nonlocal, and rejected by amavisd with a reject status code (5xx), not forwarded by amavisd, and no delivery status notification was sent. A message may or may not have been quarantined." ::= { amavisStats 25 6 } inMsgsStatusTempFailedAll OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages whose reception resulted in a temporary failure status code (4xx) being send by amavisd back to MTA. A message was not forwarded by amavisd, and no delivery status notification was sent. A message stayed in an MTA queue for future attempts at delivery. A synonym with inMsgsStatusTempFailed." ::= { amavisStats 26 1 } inMsgsStatusTempFailedInbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient local, whose reception resulted in a temporary failure status code (4xx). A message was not forwarded by amavisd, and no delivery status notification was sent." ::= { amavisStats 26 2 } inMsgsStatusTempFailedOutbound OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient nonlocal, whose reception resulted in a temporary failure status code (4xx). A message was not forwarded by amavisd, and no delivery status notification was sent." ::= { amavisStats 26 3 } inMsgsStatusTempFailedInternal OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), having at least one recipient local, whose reception resulted in a temporary failure status code (4xx). A message was not forwarded by amavisd, and no delivery status notification was sent." ::= { amavisStats 26 4 } inMsgsStatusTempFailedOriginating OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from inside ('originating' flag is on), whose reception resulted in a temporary failure status code (4xx). A message was not forwarded by amavisd, and no delivery status notification was sent." ::= { amavisStats 26 5 } inMsgsStatusTempFailedOpenRelay OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of mail messages received by amavisd arriving from outside ('originating' flag is off), having at least one recipient nonlocal, whose reception resulted in a temporary failure status code (4xx). A message was not forwarded by amavisd, and no delivery status notification was sent." ::= { amavisStats 26 6 } -- ================== -- amavisProc section -- ================== procGone OBJECT-TYPE SYNTAX Counter32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of registered amavisd child processes that were abnormally terminated (crashed or killed) since the last start of amavisd-snmp-subagent." ::= { amavisProc 1 1 } procAll OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered amavisd child processes." ::= { amavisProc 1 2 } procIdle OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered but idle amavisd child processes." ::= { amavisProc 1 3 } procBusy OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes." ::= { amavisProc 1 4 } procBusyTransfer OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes transferring data from or to an MTA." ::= { amavisProc 1 5 } procBusyDecode OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes decoding a mail message." ::= { amavisProc 1 6 } procBusyVirus OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes running antivirus scanners." ::= { amavisProc 1 7 } procBusySpam OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes checking a mail message for spam." ::= { amavisProc 1 8 } procBusyOther OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes doing some other activity." ::= { amavisProc 1 9 } procBusy0 OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes, same as procBusy." ::= { amavisProc 2 1 } procBusy01s OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 0.1 or more seconds ago." ::= { amavisProc 2 2 } procBusy02s OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 0.2 or more seconds ago." ::= { amavisProc 2 3 } procBusy05s OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 0.5 or more seconds ago." ::= { amavisProc 2 4 } procBusy1s OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 1 or more seconds ago." ::= { amavisProc 2 5 } procBusy2s OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 2 or more seconds ago." ::= { amavisProc 2 6 } procBusy4s OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 4 or more seconds ago." ::= { amavisProc 2 7 } procBusy8s OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 8 or more seconds ago." ::= { amavisProc 2 8 } procBusy15s OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 15 or more seconds ago." ::= { amavisProc 2 9 } procBusy30s OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 30 or more seconds ago." ::= { amavisProc 2 10 } procBusy1m OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 1 or more minutes ago." ::= { amavisProc 2 11 } procBusy2m OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 2 or more minutes ago." ::= { amavisProc 2 12 } procBusy4m OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 4 or more minutes ago." ::= { amavisProc 2 13 } procBusy8m OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 8 or more minutes ago." ::= { amavisProc 2 14 } procBusy15m OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 15 or more minutes ago." ::= { amavisProc 2 15 } procBusy30m OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 30 or more minutes ago." ::= { amavisProc 2 16 } procBusy1h OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 1 or more hours ago." ::= { amavisProc 2 17 } procBusy2h OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 2 or more hours ago." ::= { amavisProc 2 18 } procBusy4h OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 4 or more hours ago." ::= { amavisProc 2 19 } procBusy8h OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 8 or more hours ago." ::= { amavisProc 2 20 } procBusy15h OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 15 or more hours ago." ::= { amavisProc 2 21 } procBusy30h OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of currently registered and busy amavisd child processes that started processing the current task 30 or more hours ago." ::= { amavisProc 2 22 } -- ================= -- amavisMta section -- ================= mtaQueueEntriesMaildrop OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of files under an MTA directory $queue_directory/maildrop." ::= { amavisMta 1 1 } mtaQueueEntriesIncoming OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of files under an MTA directory $queue_directory/incoming" ::= { amavisMta 1 2 } mtaQueueEntriesActive OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of files under an MTA directory $queue_directory/active" ::= { amavisMta 1 3 } mtaQueueEntriesDeferred OBJECT-TYPE SYNTAX Gauge32 MAX-ACCESS read-only STATUS current DESCRIPTION "Number of files under an MTA directory $queue_directory/deferred" ::= { amavisMta 1 4 } END amavisd-new-2.7.1/amavisd-agent000751 000621 000620 00000032101 11216511312 016117 0ustar00markcmi000000 000000 #!/usr/bin/perl -T #------------------------------------------------------------------------------ # This is amavisd-agent, a demo program to display # SNMP-like counters updated by amavisd-new. # # Author: Mark Martinec # Copyright (C) 2004-2009 Mark Martinec, All Rights Reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of the author, nor the name of the "Jozef Stefan" # Institute, nor the names of contributors may be used to endorse or # promote products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER # OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # #(the license above is the new BSD license, and pertains to this program only) # # Patches and problem reports are welcome. # The latest version of this program is available at: # http://www.ijs.si/software/amavisd/ #------------------------------------------------------------------------------ use strict; use re 'taint'; use warnings; no warnings 'uninitialized'; use Errno qw(ENOENT); use Time::HiRes (); use BerkeleyDB; my($dbfile) = 'snmp.db'; my($db_home) = # DB databases directory defined $ENV{'AMAVISD_DB_HOME'} ? $ENV{'AMAVISD_DB_HOME'} : '/var/amavis/db'; my($wakeuptime) = 10; # -w, sleep time in seconds, may be fractional my($repeatcount); # -c, repeat count (when defined) use vars qw($VERSION); $VERSION = 2.700; use vars qw(%values %virus_by_name); use vars qw(%virus_by_os %spam_by_os %ham_by_os); use vars qw(%history $avg_int $uptime); $avg_int = 5*60; # 5 minute interval sub p1($$@) { my($k,$avg,@tot_k) = @_; printf("%-35s %6d %6.0f/h", $k, $values{$k}, $avg*3600); for my $tot_k (@tot_k) { if ($values{$tot_k} <= 0) { printf(" --- %%") } else { printf(" %7.1f %%", 100*$values{$k}/$values{$tot_k}) } print " ($tot_k)"; } print "\n"; } sub p1_size($$@) { my($k,$avg,@tot_k) = @_; my($scale) = 1024*1024; printf("%-35s %6.0fMB %4.0fMB/h", $k, $values{$k}/$scale, $avg*3600/$scale); for my $tot_k (@tot_k) { if ($values{$tot_k} <= 0) { printf(" --- %%") } else { printf(" %5.1f %%", 100*$values{$k}/$values{$tot_k}) } print " ($tot_k)"; } print "\n"; } sub p1_time($$$$) { my($k,$dv,$dcnt,$tot_k) = @_; printf("%-35s %6.0f s %8s s/msg (%s)\n", $k, $values{$k}/1000, $dcnt < 1 ? "---" : sprintf("%7.3f",$dv/1000/$dcnt), $tot_k); } sub p2($$$$) { my($k,$avg,$tot_k,$href) = @_; if ($values{$tot_k} > 0) { printf("%-35s %6d %6.0f/h %6.1f %% (%s)\n", $k, $href->{$k}, $avg*3600, 100*$href->{$k}/$values{$tot_k}, $tot_k); } } sub enqueue($$$$$) { my($name,$now,$val,$msgcnt,$hold_time) = @_; if (ref $history{$name} ne 'ARRAY') { $history{$name} = [] } my($oldest_useful); for my $j (0..$#{$history{$name}}) { if ($history{$name}->[$j][0] + $hold_time >= $now) { $oldest_useful = $j; last } } if (defined $oldest_useful) { @{$history{$name}} = @{$history{$name}}[$oldest_useful..$#{$history{$name}}]; } push(@{$history{$name}}, [$now,$val,$msgcnt]); my($average,$dv,$dt,$dcnt); my($n) = scalar(@{$history{$name}}); my($oldest) = $history{$name}->[0]; my($latest) = $history{$name}->[$n-1]; $dt = $latest->[0] - $oldest->[0]; $dv = $latest->[1] - $oldest->[1]; $dcnt = $latest->[2] - $oldest->[2]; if ($n < 2 || $dt < $hold_time/2) { $dt = $uptime; $dv = $val; $dcnt = $msgcnt; # average since the start time } if ($dt > 0) { $average = $dv/$dt } ($average, $dv, $dt, $dcnt, $n); } sub fmt_ticks($) { my($t) = @_; my($hh)= $t % 100; $t = int($t/100); my($s) = $t % 60; $t = int($t/60); my($m) = $t % 60; $t = int($t/60); my($h) = $t % 24; $t = int($t/24); my($d) = $t; sprintf("%d days, %d:%02d:%02d.%02d", $d,$h,$m,$s,$hh); }; # main program starts here my($normal_termination) = 0; $SIG{INT} = sub { die "\n" }; # do the END code block while (@ARGV) { my($opt) = shift @ARGV; my($val) = shift @ARGV; if ($opt eq '-w' && $val =~ /^\+?\d+(?:\.\d*)?\z/) { $wakeuptime = $val } elsif ($opt eq '-c' && $val =~ /^[+-]?\d+\z/) { $repeatcount = $val } else { die "Usage: $0 [-c ] [-w ]\n" } } my($stat,$key,$val); my($env,$db,$old_db_inode,@dbstat,$cursor); for (;;) { last if defined $repeatcount && $repeatcount <= 0; @dbstat = stat("$db_home/$dbfile"); my($errn) = @dbstat ? 0 : 0+$!; $errn==0 || $errn==ENOENT or die "stat $db_home/$dbfile: $!"; if (defined $db && $old_db_inode != $dbstat[1]) { $db->db_close==0 or die "BDB db_close error: $BerkeleyDB::Error $!"; undef $db; printf STDERR ("Reopening snmp database %s/%s\n", $db_home,$dbfile); } if (!defined $db && $errn==0) { $old_db_inode = $dbstat[1]; $env = BerkeleyDB::Env->new( -Home => $db_home, -Flags => DB_INIT_CDB | DB_INIT_MPOOL, -ErrFile => \*STDOUT, -Verbose => 1); defined $env or die "BDB no env: $BerkeleyDB::Error $!"; $db = BerkeleyDB::Hash->new(-Filename => $dbfile, -Env => $env); defined $db or die "BDB no dbS 1: $BerkeleyDB::Error $!"; } $| = 0; %values = (); %virus_by_name = (); %virus_by_os = (); %spam_by_os = (); %ham_by_os = (); my($now); my($eval_stat,$interrupt); $interrupt = ''; if (!defined $db) { printf STDERR ("No snmp database %s/%s; waiting...\n", $db_home,$dbfile); } else { $repeatcount-- if defined $repeatcount && $repeatcount > 0; print "\n\n"; my($h1) = sub { $interrupt = $_[0] }; local(@SIG{qw(INT HUP TERM TSTP QUIT ALRM USR1 USR2)}) = ($h1) x 8; eval { $cursor = $db->db_cursor; # obtain read lock defined $cursor or die "db_cursor error: $BerkeleyDB::Error"; $now = Time::HiRes::time; while ( ($stat=$cursor->c_get($key,$val,DB_NEXT)) == 0 ) { if ($key =~ /^(virus\.byname\..*)\z/s) { $virus_by_name{$1} = $val } elsif ($key =~ /^(virus\.byOS\..*)\z/s) { $virus_by_os{$1} = $val } elsif ($key =~ /^(ham\.byOS\..*)\z/s) { $ham_by_os{$1} = $val } elsif ($key =~ /^(?:spam|spammy)\.byOS\.(.*)\z/s) { $spam_by_os{"spam.byOS.$1"} = $val } else { $values{$key} = $val } } $stat==DB_NOTFOUND or die "c_get: $BerkeleyDB::Error $!"; $cursor->c_close==0 or die "c_close error: $BerkeleyDB::Error"; $cursor = undef; }; $eval_stat = $@; if (defined $db) { $cursor->c_close if defined $cursor; # unlock, ignoring status $cursor = undef; } } if ($interrupt ne '') { kill($interrupt,$$) } # resignal elsif ($eval_stat ne '') { chomp($eval_stat); die "BDB $eval_stat\n" } for my $k (sort keys %values) { if ($values{$k} =~ /^(?:C32|C64) (.*)\z/) { $values{$k} = $1; } elsif ($k eq 'sysUpTime' && $values{$k} =~ /^INT (.*)\z/) { $uptime = $now - $1; my($ticks) = int($uptime*100); printf("%-15s %s %s (%s)\n", $k,'TimeTicks', $ticks, fmt_ticks($ticks)); delete($values{$k}); } elsif ($values{$k} =~ /^(?:INT|TIM) (.*)\z/) { $values{$k} = $1; } else { printf("%-15s %s\n", $k,$values{$k}); delete($values{$k}); } } my($msgcnt) = $values{'InMsgs'}; for (sort keys %values) { my($avg,$dv,$dt,$dcnt,$n) = enqueue($_, $now, $values{$_}, $msgcnt, $avg_int); if (/^OpsDecTyp/) {} # later elsif (/^CacheHitsVirusMsgs$/) { p1($_,$avg,'ContentVirusMsgs') } elsif (/^CacheHitsBannedMsgs$/) { p1($_,$avg,'ContentBannedMsgs') } elsif (/^CacheHitsSpamMsgs$/) { p1($_,$avg,'ContentSpamMsgs') } elsif (/^Cache/) { p1($_,$avg,'CacheAttempts') } # elsif (/^Content(.*?)Msgs/) { p1($_,$avg,'Content'.$1.'Msgs') } elsif (/^Content(.*?)Msgs(.*)\z/) { p1($_,$avg,'InMsgs'.$2) } elsif (/^Content/) { p1($_,$avg,'InMsgs') } elsif (/^OpsSql/) { p1($_,$avg,'InMsgsRecips') } elsif (/^InMsgsSize/) { p1_size($_,$avg,'InMsgsSize') } elsif (/^InMsgsRecipsLocal\z/) { p1($_,$avg,'InMsgsRecips') } elsif (/^InMsgsRecips(.*)\z/) { p1($_,$avg,'InMsgs'.$1) } elsif (/^InMsgsBounce./) { p1($_,$avg,'InMsgsBounce') } elsif (/^(InMsgs|Ops)/) { p1($_,$avg,'InMsgs') } elsif (/^OutMsgsSize\z/) { p1_size($_,$avg,'InMsgsSize') } elsif (/^OutMsgsSize/) { p1_size($_,$avg,'OutMsgsSize') } elsif (/^OutMsgs\z/) { p1($_,$avg,'InMsgs') } elsif (/^Out/) { p1($_,$avg,'OutMsgs') } elsif (/^QuarMsgsSize\z/) { p1_size($_,$avg,'InMsgsSize') } elsif (/^QuarMsgsSize/) { p1_size($_,$avg,'QuarMsgsSize') } elsif (/^Quar/) { p1($_,$avg,'QuarMsgs') } elsif (/^LogEntries\z/) { p1($_,$avg,'InMsgs') } elsif (/^Log/) { p1($_,$avg,'LogEntries') } elsif (/^GenMailIdRetries/) { p1($_,$avg,'InMsgs') } elsif (/^PenPalsAttempts\z/) { p1($_,$avg,'InMsgsRecipsLocal') } elsif (/^PenPalsHits\z/) { p1($_,$avg,'PenPalsAttempts')} elsif (/^PenPalsHits./) { p1($_,$avg,'PenPalsHits') } elsif (/^PenPals/) { p1($_,$avg,'PenPalsAttempts') } elsif (/^SqlAddrSenderAttempts\z/) { p1($_,$avg,'InMsgs') } elsif (/^SqlAddrSender/) { p1($_,$avg,'SqlAddrSenderAttempts') } elsif (/^SqlAddrRecipAttempts\z/) { p1($_,$avg,'InMsgsRecips') } elsif (/^SqlAddrRecip/) { p1($_,$avg,'SqlAddrRecipAttempts') } elsif (/^banned\.byOS/) { p1($_,$avg,'InMsgs') } elsif (/^TimeElapsed/i) { p1_time($_,$dv,$dcnt,'InMsgs') } else { p1($_,$avg,undef) } } for (sort { $values{$b}<=>$values{$a} } grep {/^OpsDecTyp/} keys %values) { my($avg,$dv,$dt,$dcnt,$n) = enqueue($_, $now, $values{$_}, $msgcnt, $avg_int); p1($_,$avg,'InMsgs'); } for my $href (\%virus_by_name,\%virus_by_os,\%spam_by_os,\%ham_by_os) { for (keys %$href) { $href->{$_} = $1 if $href->{$_} =~ /^(?:C32|C64) (.*)\z/ } } for my $href (\%virus_by_os,\%spam_by_os,\%ham_by_os) { for (keys %$href) { /^[a-zA-Z]+\.byOS\.(.*)\z/; my($os) = $1; $values{"all.byOS.$os"} += $href->{$_}; } } my($separated) = 0; for my $pair ([\%virus_by_name, 'ContentVirusMsgs',], [\%virus_by_os, 'ContentVirusMsgs',], [\%spam_by_os, 'ContentSpamMsgs', ], [\%ham_by_os, 'ContentCleanMsgs' ] ) { my($href,$tot_k) = @$pair; for (sort {$href->{$b} <=> $href->{$a}} keys %$href) { if (!$separated) { print "\n"; $separated = 1 } my($avg,$dv,$dt,$dcnt,$n) = enqueue($_, $now, $href->{$_}, $msgcnt, $avg_int); p2($_,$avg,$tot_k,$href); } } if (0) { # disabled $separated = 0; for my $href (\%virus_by_os, \%spam_by_os, \%ham_by_os) { for (sort {$href->{$b} <=> $href->{$a}} keys %$href) { if (!$separated) { print "\n"; $separated = 1 } my($avg,$dv,$dt,$dcnt,$n) = enqueue($_, $now, $href->{$_}, $msgcnt, $avg_int); /^[a-zA-Z]+\.byOS\.(.*)\z/; my($os) = $1; p2($_,$avg,"all.byOS.$os",$href); } } $separated = 0; for (sort { $values{$b}<=>$values{$a} } grep {/^all\.byOS\./} keys %values) { if (!$separated) { print "\n"; $separated = 1 } my($avg,$dv,$dt,$dcnt,$n) = enqueue($_, $now, $values{$_}, $msgcnt, $avg_int); p1($_,$avg,'InMsgs'); } } $| = 1; last if defined $repeatcount && $repeatcount <= 0; Time::HiRes::sleep($wakeuptime) if $wakeuptime > 0; } # forever $normal_termination = 1; END { if (defined $db) { $cursor->c_close if defined $cursor; # ignoring status $db->db_close==0 or die "BDB db_close error: $BerkeleyDB::Error $!"; } print STDERR "exited\n" if !$normal_termination; } amavisd-new-2.7.1/LICENSE000640 000621 000620 00000043131 07565334703 014506 0ustar00markcmi000000 000000 GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. amavisd-new-2.7.1/LDAP.schema000640 000621 000620 00000106343 11603361705 015376 0ustar00markcmi000000 000000 #-------------------------------------------------------------------------- # LDAP Schema for amavisd-new Jacques Supcik, PhD #----------------------------- IP-Plus Internet Services # Release 1.2.2 Swisscom Enterprise Solutions Ltd # 30 May 2004 3050 Bern - Switzerland #-------------------------------------------------------------------------- # Copyright (c) 2004 Jacques Supcik, Swisscom Enterprise Solutions Ltd. # Permission is granted to copy, distribute and/or modify this document # under the terms of the GNU Free Documentation License, Version 1.2 # or any later version published by the Free Software Foundation; # with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. # A copy of the license is included in the section entitled "GNU # Free Documentation License". #-------------------------------------------------------------------------- # Changes made to LDAP Schema to make it import and play nicely with # Novell NDS - Michael Tracey, SONOPRESS USA, LLC April 07 2005 # ( uncomment each dn:, changetype:, add:, add X-NDS-NAME attribute, replace # "attributetype" by "attributetypes:" and "objectclasse" by "objectclasses:" # (plural,colon), and unwrap each attributetypes: and objectclasses: ) #-------------------------------------------------------------------------- # 1.3.6.1.4.1.15312 Jozef Stefan Institute's OID # 1.3.6.1.4.1.15312.2 amavisd-new # 1.3.6.1.4.1.15312.2.2 amavisd-new LDAP Elements # 1.3.6.1.4.1.15312.2.2.1 AttributeTypes # 1.3.6.1.4.1.15312.2.2.2 ObjectClasses # 1.3.6.1.4.1.15312.2.2.3 Syntax Definitions #-------------------------------------------------------------------------- # Attribute Types #----------------- #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.1 NAME 'amavisVirusLover' DESC 'Virus Lover' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.2 NAME 'amavisBannedFilesLover' DESC 'Banned Files Lover' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.3 NAME 'amavisBypassVirusChecks' DESC 'Bypass Virus Check' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.4 NAME 'amavisBypassSpamChecks' DESC 'Bypass Spam Check' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.5 NAME 'amavisSpamTagLevel' DESC 'Spam Tag Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.6 NAME 'amavisSpamTag2Level' DESC 'Spam Tag2 Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.7 NAME 'amavisSpamKillLevel' DESC 'Spam Kill Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.8 NAME 'amavisSpamModifiesSubj' DESC 'Modifies Subject on spam - no longer in use since 2.7.0' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.9 NAME 'amavisWhitelistSender' DESC 'White List Sender' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.10 NAME 'amavisBlacklistSender' DESC 'Black List Sender' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.11 NAME 'amavisSpamQuarantineTo' DESC 'Spam Quarantine to' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.12 NAME 'amavisSpamLover' DESC 'Spam Lover' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.13 NAME 'amavisBadHeaderLover' DESC 'Bad Header Lover' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.14 NAME 'amavisBypassBannedChecks' DESC 'Bypass Banned Files Check' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.15 NAME 'amavisBypassHeaderChecks' DESC 'Bypass Header Check' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.16 NAME 'amavisVirusQuarantineTo' DESC 'Virus quarantine location' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.17 NAME 'amavisBannedQuarantineTo' DESC 'Banned Files quarantine location' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.18 NAME 'amavisBadHeaderQuarantineTo' DESC 'Bad Header quarantine location' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.19 NAME 'amavisLocal' DESC 'Is user considered local' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.20 NAME 'amavisMessageSizeLimit' DESC 'Message size limit' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.21 NAME 'amavisWarnVirusRecip' DESC 'Notify virus recipients' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.22 NAME 'amavisWarnBannedRecip' DESC 'Notify banned file recipients' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.23 NAME 'amavisWarnBadHeaderRecip' DESC 'Notify bad header recipients' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.24 NAME 'amavisVirusAdmin' DESC 'Virus admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.25 NAME 'amavisNewVirusAdmin' DESC 'New virus admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.26 NAME 'amavisSpamAdmin' DESC 'Spam admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.27 NAME 'amavisBannedAdmin' DESC 'Banned file admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.28 NAME 'amavisBadHeaderAdmin' DESC 'Bad header admin' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.29 NAME 'amavisBannedRuleNames' DESC 'Banned rule names' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.30 NAME 'amavisSpamDsnCutoffLevel' DESC 'Spam DSN Cutoff Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.31 NAME 'amavisSpamQuarantineCutoffLevel' DESC 'Spam Quarantine Cutoff Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.32 NAME 'amavisSpamSubjectTag' DESC 'Spam Subject Tag' EQUALITY caseExactIA5Match SUBSTR caseExactSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.33 NAME 'amavisSpamSubjectTag2' DESC 'Spam Subject Tag2' EQUALITY caseExactIA5Match SUBSTR caseExactSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.34 NAME 'amavisArchiveQuarantineTo' DESC 'Archive quarantine location' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.35 NAME 'amavisAddrExtensionVirus' DESC 'Address Extension for Virus' EQUALITY caseExactIA5Match SUBSTR caseExactSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.36 NAME 'amavisAddrExtensionSpam' DESC 'Address Extension for Spam' EQUALITY caseExactIA5Match SUBSTR caseExactSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.37 NAME 'amavisAddrExtensionBanned' DESC 'Address Extension for Banned' EQUALITY caseExactIA5Match SUBSTR caseExactSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.38 NAME 'amavisAddrExtensionBadHeader' DESC 'Address Extension for Bad Header' EQUALITY caseExactIA5Match SUBSTR caseExactSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) # NEW since 2.7.0: #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.39 NAME 'amavisSpamTag3Level' DESC 'Spam Tag3 Level' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.40 NAME 'amavisSpamSubjectTag3' DESC 'Spam Subject Tag3' EQUALITY caseExactIA5Match SUBSTR caseExactSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.41 NAME 'amavisUncheckedQuarantineTo' DESC 'Virus quarantine location' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.42 NAME 'amavisCleanQuarantineTo' DESC 'Clean quarantine location' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.43 NAME 'amavisUncheckedLover' DESC 'Unchecked Files Lover' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.44 NAME 'amavisForwardMethod' DESC 'Forward / next hop destination' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.45 NAME 'amavisSaUserConf' DESC 'SpamAssassin user preferences configuration filename' EQUALITY caseExactIA5Match SUBSTR caseExactSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) #dn: cn=schema #changetype: modify #add: attributetypes attributetype ( 1.3.6.1.4.1.15312.2.2.1.46 NAME 'amavisSaUserName' DESC 'SpamAssassin username (for Bayes and AWL lookups)' EQUALITY caseExactIA5Match SUBSTR caseExactSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} SINGLE-VALUE ) # Classes #--------- # amavisAccount # This class is an auxiliary class, this mean that the class will be added # to a structural class. Usually, the structural class is the class that # represent the mail account itself (e.g. an inetOrgPerson) #dn: cn=schema #changetype: modify #add: objectclasses objectclass ( 1.3.6.1.4.1.15312.2.2.2.1 NAME 'amavisAccount' AUXILIARY DESC 'Amavisd Account' SUP top MAY ( amavisVirusLover $ amavisBypassVirusChecks $ amavisSpamLover $ amavisBypassSpamChecks $ amavisBannedFilesLover $ amavisBypassBannedChecks $ amavisBadHeaderLover $ amavisBypassHeaderChecks $ amavisSpamTagLevel $ amavisSpamTag2Level $ amavisSpamKillLevel $ amavisWhitelistSender $ amavisBlacklistSender $ amavisSpamQuarantineTo $ amavisVirusQuarantineTo $ amavisBannedQuarantineTo $ amavisBadHeaderQuarantineTo $ amavisArchiveQuarantineTo $ amavisSpamModifiesSubj $ amavisLocal $ amavisMessageSizeLimit $ amavisWarnVirusRecip $ amavisWarnBannedRecip $ amavisWarnBadHeaderRecip $ amavisVirusAdmin $ amavisNewVirusAdmin $ amavisSpamAdmin $ amavisBannedAdmin $ amavisBadHeaderAdmin $ amavisBannedRuleNames $ amavisSpamDsnCutoffLevel $ amavisSpamQuarantineCutoffLevel $ amavisSpamSubjectTag $ amavisSpamSubjectTag2 $ amavisAddrExtensionVirus $ amavisAddrExtensionSpam $ amavisAddrExtensionBanned $ amavisAddrExtensionBadHeader $ amavisSpamTag3Level $ amavisSpamSubjectTag3 $ amavisUncheckedQuarantineTo $ amavisCleanQuarantineTo $ amavisUncheckedLover $ amavisForwardMethod $ amavisSaUserConf $ amavisSaUserName $ cn $ description ) ) #-------------------------------------------------------------------------- # # GNU Free Documentation License # Version 1.2, November 2002 # # # Copyright (C) 2000,2001,2002 Free Software Foundation, Inc. # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Everyone is permitted to copy and distribute verbatim copies # of this license document, but changing it is not allowed. # # # 0. PREAMBLE # # The purpose of this License is to make a manual, textbook, or other # functional and useful document "free" in the sense of freedom: to # assure everyone the effective freedom to copy and redistribute it, # with or without modifying it, either commercially or noncommercially. # Secondarily, this License preserves for the author and publisher a way # to get credit for their work, while not being considered responsible # for modifications made by others. # # This License is a kind of "copyleft", which means that derivative # works of the document must themselves be free in the same sense. It # complements the GNU General Public License, which is a copyleft # license designed for free software. # # We have designed this License in order to use it for manuals for free # software, because free software needs free documentation: a free # program should come with manuals providing the same freedoms that the # software does. But this License is not limited to software manuals; # it can be used for any textual work, regardless of subject matter or # whether it is published as a printed book. We recommend this License # principally for works whose purpose is instruction or reference. # # # 1. APPLICABILITY AND DEFINITIONS # # This License applies to any manual or other work, in any medium, that # contains a notice placed by the copyright holder saying it can be # distributed under the terms of this License. Such a notice grants a # world-wide, royalty-free license, unlimited in duration, to use that # work under the conditions stated herein. The "Document", below, # refers to any such manual or work. Any member of the public is a # licensee, and is addressed as "you". You accept the license if you # copy, modify or distribute the work in a way requiring permission # under copyright law. # # A "Modified Version" of the Document means any work containing the # Document or a portion of it, either copied verbatim, or with # modifications and/or translated into another language. # # A "Secondary Section" is a named appendix or a front-matter section of # the Document that deals exclusively with the relationship of the # publishers or authors of the Document to the Document's overall subject # (or to related matters) and contains nothing that could fall directly # within that overall subject. (Thus, if the Document is in part a # textbook of mathematics, a Secondary Section may not explain any # mathematics.) The relationship could be a matter of historical # connection with the subject or with related matters, or of legal, # commercial, philosophical, ethical or political position regarding # them. # # The "Invariant Sections" are certain Secondary Sections whose titles # are designated, as being those of Invariant Sections, in the notice # that says that the Document is released under this License. If a # section does not fit the above definition of Secondary then it is not # allowed to be designated as Invariant. The Document may contain zero # Invariant Sections. If the Document does not identify any Invariant # Sections then there are none. # # The "Cover Texts" are certain short passages of text that are listed, # as Front-Cover Texts or Back-Cover Texts, in the notice that says that # the Document is released under this License. A Front-Cover Text may # be at most 5 words, and a Back-Cover Text may be at most 25 words. # # A "Transparent" copy of the Document means a machine-readable copy, # represented in a format whose specification is available to the # general public, that is suitable for revising the document # straightforwardly with generic text editors or (for images composed of # pixels) generic paint programs or (for drawings) some widely available # drawing editor, and that is suitable for input to text formatters or # for automatic translation to a variety of formats suitable for input # to text formatters. A copy made in an otherwise Transparent file # format whose markup, or absence of markup, has been arranged to thwart # or discourage subsequent modification by readers is not Transparent. # An image format is not Transparent if used for any substantial amount # of text. A copy that is not "Transparent" is called "Opaque". # # Examples of suitable formats for Transparent copies include plain # ASCII without markup, Texinfo input format, LaTeX input format, SGML # or XML using a publicly available DTD, and standard-conforming simple # HTML, PostScript or PDF designed for human modification. Examples of # transparent image formats include PNG, XCF and JPG. Opaque formats # include proprietary formats that can be read and edited only by # proprietary word processors, SGML or XML for which the DTD and/or # processing tools are not generally available, and the # machine-generated HTML, PostScript or PDF produced by some word # processors for output purposes only. # # The "Title Page" means, for a printed book, the title page itself, # plus such following pages as are needed to hold, legibly, the material # this License requires to appear in the title page. For works in # formats which do not have any title page as such, "Title Page" means # the text near the most prominent appearance of the work's title, # preceding the beginning of the body of the text. # # A section "Entitled XYZ" means a named subunit of the Document whose # title either is precisely XYZ or contains XYZ in parentheses following # text that translates XYZ in another language. (Here XYZ stands for a # specific section name mentioned below, such as "Acknowledgements", # "Dedications", "Endorsements", or "History".) To "Preserve the Title" # of such a section when you modify the Document means that it remains a # section "Entitled XYZ" according to this definition. # # The Document may include Warranty Disclaimers next to the notice which # states that this License applies to the Document. These Warranty # Disclaimers are considered to be included by reference in this # License, but only as regards disclaiming warranties: any other # implication that these Warranty Disclaimers may have is void and has # no effect on the meaning of this License. # # # 2. VERBATIM COPYING # # You may copy and distribute the Document in any medium, either # commercially or noncommercially, provided that this License, the # copyright notices, and the license notice saying this License applies # to the Document are reproduced in all copies, and that you add no other # conditions whatsoever to those of this License. You may not use # technical measures to obstruct or control the reading or further # copying of the copies you make or distribute. However, you may accept # compensation in exchange for copies. If you distribute a large enough # number of copies you must also follow the conditions in section 3. # # You may also lend copies, under the same conditions stated above, and # you may publicly display copies. # # # 3. COPYING IN QUANTITY # # If you publish printed copies (or copies in media that commonly have # printed covers) of the Document, numbering more than 100, and the # Document's license notice requires Cover Texts, you must enclose the # copies in covers that carry, clearly and legibly, all these Cover # Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on # the back cover. Both covers must also clearly and legibly identify # you as the publisher of these copies. The front cover must present # the full title with all words of the title equally prominent and # visible. You may add other material on the covers in addition. # Copying with changes limited to the covers, as long as they preserve # the title of the Document and satisfy these conditions, can be treated # as verbatim copying in other respects. # # If the required texts for either cover are too voluminous to fit # legibly, you should put the first ones listed (as many as fit # reasonably) on the actual cover, and continue the rest onto adjacent # pages. # # If you publish or distribute Opaque copies of the Document numbering # more than 100, you must either include a machine-readable Transparent # copy along with each Opaque copy, or state in or with each Opaque copy # a computer-network location from which the general network-using # public has access to download using public-standard network protocols # a complete Transparent copy of the Document, free of added material. # If you use the latter option, you must take reasonably prudent steps, # when you begin distribution of Opaque copies in quantity, to ensure # that this Transparent copy will remain thus accessible at the stated # location until at least one year after the last time you distribute an # Opaque copy (directly or through your agents or retailers) of that # edition to the public. # # It is requested, but not required, that you contact the authors of the # Document well before redistributing any large number of copies, to give # them a chance to provide you with an updated version of the Document. # # # 4. MODIFICATIONS # # You may copy and distribute a Modified Version of the Document under # the conditions of sections 2 and 3 above, provided that you release # the Modified Version under precisely this License, with the Modified # Version filling the role of the Document, thus licensing distribution # and modification of the Modified Version to whoever possesses a copy # of it. In addition, you must do these things in the Modified Version: # # A. Use in the Title Page (and on the covers, if any) a title distinct # from that of the Document, and from those of previous versions # (which should, if there were any, be listed in the History section # of the Document). You may use the same title as a previous version # if the original publisher of that version gives permission. # B. List on the Title Page, as authors, one or more persons or entities # responsible for authorship of the modifications in the Modified # Version, together with at least five of the principal authors of the # Document (all of its principal authors, if it has fewer than five), # unless they release you from this requirement. # C. State on the Title page the name of the publisher of the # Modified Version, as the publisher. # D. Preserve all the copyright notices of the Document. # E. Add an appropriate copyright notice for your modifications # adjacent to the other copyright notices. # F. Include, immediately after the copyright notices, a license notice # giving the public permission to use the Modified Version under the # terms of this License, in the form shown in the Addendum below. # G. Preserve in that license notice the full lists of Invariant Sections # and required Cover Texts given in the Document's license notice. # H. Include an unaltered copy of this License. # I. Preserve the section Entitled "History", Preserve its Title, and add # to it an item stating at least the title, year, new authors, and # publisher of the Modified Version as given on the Title Page. If # there is no section Entitled "History" in the Document, create one # stating the title, year, authors, and publisher of the Document as # given on its Title Page, then add an item describing the Modified # Version as stated in the previous sentence. # J. Preserve the network location, if any, given in the Document for # public access to a Transparent copy of the Document, and likewise # the network locations given in the Document for previous versions # it was based on. These may be placed in the "History" section. # You may omit a network location for a work that was published at # least four years before the Document itself, or if the original # publisher of the version it refers to gives permission. # K. For any section Entitled "Acknowledgements" or "Dedications", # Preserve the Title of the section, and preserve in the section all # the substance and tone of each of the contributor acknowledgements # and/or dedications given therein. # L. Preserve all the Invariant Sections of the Document, # unaltered in their text and in their titles. Section numbers # or the equivalent are not considered part of the section titles. # M. Delete any section Entitled "Endorsements". Such a section # may not be included in the Modified Version. # N. Do not retitle any existing section to be Entitled "Endorsements" # or to conflict in title with any Invariant Section. # O. Preserve any Warranty Disclaimers. # # If the Modified Version includes new front-matter sections or # appendices that qualify as Secondary Sections and contain no material # copied from the Document, you may at your option designate some or all # of these sections as invariant. To do this, add their titles to the # list of Invariant Sections in the Modified Version's license notice. # These titles must be distinct from any other section titles. # # You may add a section Entitled "Endorsements", provided it contains # nothing but endorsements of your Modified Version by various # parties--for example, statements of peer review or that the text has # been approved by an organization as the authoritative definition of a # standard. # # You may add a passage of up to five words as a Front-Cover Text, and a # passage of up to 25 words as a Back-Cover Text, to the end of the list # of Cover Texts in the Modified Version. Only one passage of # Front-Cover Text and one of Back-Cover Text may be added by (or # through arrangements made by) any one entity. If the Document already # includes a cover text for the same cover, previously added by you or # by arrangement made by the same entity you are acting on behalf of, # you may not add another; but you may replace the old one, on explicit # permission from the previous publisher that added the old one. # # The author(s) and publisher(s) of the Document do not by this License # give permission to use their names for publicity for or to assert or # imply endorsement of any Modified Version. # # # 5. COMBINING DOCUMENTS # # You may combine the Document with other documents released under this # License, under the terms defined in section 4 above for modified # versions, provided that you include in the combination all of the # Invariant Sections of all of the original documents, unmodified, and # list them all as Invariant Sections of your combined work in its # license notice, and that you preserve all their Warranty Disclaimers. # # The combined work need only contain one copy of this License, and # multiple identical Invariant Sections may be replaced with a single # copy. If there are multiple Invariant Sections with the same name but # different contents, make the title of each such section unique by # adding at the end of it, in parentheses, the name of the original # author or publisher of that section if known, or else a unique number. # Make the same adjustment to the section titles in the list of # Invariant Sections in the license notice of the combined work. # # In the combination, you must combine any sections Entitled "History" # in the various original documents, forming one section Entitled # "History"; likewise combine any sections Entitled "Acknowledgements", # and any sections Entitled "Dedications". You must delete all sections # Entitled "Endorsements". # # # 6. COLLECTIONS OF DOCUMENTS # # You may make a collection consisting of the Document and other documents # released under this License, and replace the individual copies of this # License in the various documents with a single copy that is included in # the collection, provided that you follow the rules of this License for # verbatim copying of each of the documents in all other respects. # # You may extract a single document from such a collection, and distribute # it individually under this License, provided you insert a copy of this # License into the extracted document, and follow this License in all # other respects regarding verbatim copying of that document. # # # 7. AGGREGATION WITH INDEPENDENT WORKS # # A compilation of the Document or its derivatives with other separate # and independent documents or works, in or on a volume of a storage or # distribution medium, is called an "aggregate" if the copyright # resulting from the compilation is not used to limit the legal rights # of the compilation's users beyond what the individual works permit. # When the Document is included in an aggregate, this License does not # apply to the other works in the aggregate which are not themselves # derivative works of the Document. # # If the Cover Text requirement of section 3 is applicable to these # copies of the Document, then if the Document is less than one half of # the entire aggregate, the Document's Cover Texts may be placed on # covers that bracket the Document within the aggregate, or the # electronic equivalent of covers if the Document is in electronic form. # Otherwise they must appear on printed covers that bracket the whole # aggregate. # # # 8. TRANSLATION # # Translation is considered a kind of modification, so you may # distribute translations of the Document under the terms of section 4. # Replacing Invariant Sections with translations requires special # permission from their copyright holders, but you may include # translations of some or all Invariant Sections in addition to the # original versions of these Invariant Sections. You may include a # translation of this License, and all the license notices in the # Document, and any Warranty Disclaimers, provided that you also include # the original English version of this License and the original versions # of those notices and disclaimers. In case of a disagreement between # the translation and the original version of this License or a notice # or disclaimer, the original version will prevail. # # If a section in the Document is Entitled "Acknowledgements", # "Dedications", or "History", the requirement (section 4) to Preserve # its Title (section 1) will typically require changing the actual # title. # # # 9. TERMINATION # # You may not copy, modify, sublicense, or distribute the Document except # as expressly provided for under this License. Any other attempt to # copy, modify, sublicense or distribute the Document is void, and will # automatically terminate your rights under this License. However, # parties who have received copies, or rights, from you under this # License will not have their licenses terminated so long as such # parties remain in full compliance. # # # 10. FUTURE REVISIONS OF THIS LICENSE # # The Free Software Foundation may publish new, revised versions # of the GNU Free Documentation License from time to time. Such new # versions will be similar in spirit to the present version, but may # differ in detail to address new problems or concerns. See # http://www.gnu.org/copyleft/. # # Each version of the License is given a distinguishing version number. # If the Document specifies that a particular numbered version of this # License "or any later version" applies to it, you have the option of # following the terms and conditions either of that specified version or # of any later version that has been published (not as a draft) by the # Free Software Foundation. If the Document does not specify a version # number of this License, you may choose any version ever published (not # as a draft) by the Free Software Foundation. amavisd-new-2.7.1/README_FILES/000751 000621 000620 00000000000 11741066716 015353 5ustar00markcmi000000 000000 amavisd-new-2.7.1/amavisd-submit000751 000621 000620 00000017616 11326362632 016355 0ustar00markcmi000000 000000 #!/usr/bin/perl -T #------------------------------------------------------------------------------ # This is amavisd-submit, a simple demonstrational program, taking an email # message on stdin and submiting it to amavisd daemon. It is functionally # much like the old amavis.c helper program, except that it talks a new # AM.PDP protocol with the amavisd daemon. See README.protocol for the # description of AM.PDP protocol. # # Usage: # amavisd-submit sender recip1 recip2 recip3 ... 'AM.PDP' }; # $unix_socketname = '/var/amavis/amavisd.sock'; # # # Author: Mark Martinec # Copyright (C) 2004,2010 Mark Martinec, All Rights Reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of the author, nor the name of the "Jozef Stefan" # Institute, nor the names of contributors may be used to endorse or # promote products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER # OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # #(the license above is the new BSD license, and pertains to this program only) # # Patches and problem reports are welcome. # The latest version of this program is available at: # http://www.ijs.si/software/amavisd/ #------------------------------------------------------------------------------ use warnings; use warnings FATAL => 'utf8'; no warnings 'uninitialized'; use strict; use re 'taint'; use IO::Socket; use IO::File; use File::Temp (); use Time::HiRes (); use vars qw($VERSION); $VERSION = 2.000; use vars qw($log_level $socketname $tempbase); $log_level = 0; $tempbase = '/var/amavis'; # where to create a temp directory with a msg $socketname = '/var/amavis/amavisd.sock'; # $socketname = '127.0.0.1:9998'; sub sanitize_str { my($str, $keep_eol) = @_; my(%map) = ("\r" => '\\r', "\n" => '\\n', "\f" => '\\f', "\t" => '\\t', "\b" => '\\b', "\e" => '\\e', "\\" => '\\\\'); if ($keep_eol) { $str =~ s/([^\012\040-\133\135-\176])/ # and \240-\376 ? exists($map{$1}) ? $map{$1} : sprintf(ord($1)>255 ? '\\x{%04x}' : '\\%03o', ord($1))/eg; } else { $str =~ s/([^\040-\133\135-\176])/ # and \240-\376 ? exists($map{$1}) ? $map{$1} : sprintf(ord($1)>255 ? '\\x{%04x}' : '\\%03o', ord($1))/eg; } $str; } sub ll($) { my($level) = @_; $level <= $log_level; } sub do_log($$;@) { my($level, $errmsg, @args) = @_; $errmsg = sprintf($errmsg,@args) if @args; print STDERR sanitize_str($errmsg),"\n" if $level <= $log_level; } sub proto_decode($) { my($str) = @_; $str =~ s/%([0-9a-fA-F]{2})/pack("C",hex($1))/eg; $str; } sub proto_encode($@) { my($attribute_name,@strings) = @_; local($1); $attribute_name =~ # encode all but alfanumerics, '_' and '-' s/([^0-9a-zA-Z_-])/sprintf("%%%02x",ord($1))/eg; for (@strings) { # encode % and nonprintables s/([^\041-\044\046-\176])/sprintf("%%%02x",ord($1))/eg; } $attribute_name . '=' . join(' ',@strings); } sub ask_amavisd($$) { my($sock,$query_ref) = @_; my(@encoded_query) = map { /^([^=]+)=(.*)\z/s; proto_encode($1,$2) } @$query_ref; do_log(2,"> %s",$_) for @encoded_query; $sock->print( map { $_."\015\012" } (@encoded_query,'') ) or die "Can't write response to socket: $!"; $sock->flush or die "Can't flush on socket: $!"; my(%attr); local($/) = "\015\012"; # set line terminator to CRLF # must not use \r and \n, which may not be \015 and \012 on certain platforms do_log(2,"waiting for response"); while(<$sock>) { last if /^\015\012\z/; # end of response if (/^ ([^=\000\012]*?) (=|:[ \t]*) ([^\012]*?) \015\012 \z/xsi) { my($attr_name) = proto_decode($1); my($attr_val) = proto_decode($3); if (!exists $attr{$attr_name}) { $attr{$attr_name} = [] } push(@{$attr{$attr_name}}, $attr_val); } } if (!defined($_) && $! != 0) { die "read from socket failed: $!" } \%attr; } sub usage(;$) { my($msg) = @_; print STDERR $msg,"\n\n" if $msg ne ''; my($prog) = $0; $prog =~ s{^.*/(?=[^/]+\z)}{}; print STDERR "$prog version $VERSION\n"; die "Usage: \$ $prog sender recip1 recip2 ... < email.msg\n"; } # Main program starts here @ARGV >= 1 or usage("Not enough arguments"); my($sock); my($is_inet) = $socketname=~m{^/} ? 0 : 1; # simpleminded: unix vs. inet sock if ($is_inet) { # inet socket $sock = IO::Socket::INET->new($socketname) or die "Can't connect to INET socket $socketname: $!"; } else { # unix socket $sock = IO::Socket::UNIX->new(Type => SOCK_STREAM) or die "Can't create UNIX socket: $!"; $sock->connect( pack_sockaddr_un($socketname) ) or die "Can't connect to UNIX socket $socketname: $!"; } my($tempdir) = File::Temp::tempdir('amavis-XXXXXXXXXX', DIR => $tempbase); defined $tempdir && $tempdir ne '' or die "Can't create a temporary directory: $!"; chmod(0750,$tempdir) or die "Can't change protection on directory $tempdir: $!"; my($fname) = "$tempdir/email.txt"; # copy message from stdin to a file email.txt in the temporary directory my($fh) = IO::File->new; $fh->open($fname, O_CREAT|O_EXCL|O_RDWR, 0640) or die "Can't create file $fname: $!"; my($nbytes,$buff); while (($nbytes=read(STDIN,$buff,16384)) > 0) { $fh->print($buff) or die "Error writing to $fname: $!"; } defined $nbytes or die "Error reading mail file: $!"; $fh->close or die "Error closing $fname: $!"; close(STDIN) or die "Error closing STDIN: $!"; my(@query) = ( 'request=AM.PDP', "mail_file=$fname", "tempdir=$tempdir", 'tempdir_removed_by=server', 'sender=<' . shift(@ARGV) . '>', (map {"recipient=<$_>"} @ARGV), # 'delivery_care_of=server', # 'protocol_name=ESMTP', # 'helo_name=b.example.com', # 'client_address=10.2.3.4', ); my($attr_ref) = ask_amavisd($sock,\@query); if (ll(2)) { for my $attr_name (keys %$attr_ref) { for my $attr_val (@{$attr_ref->{$attr_name}}) { do_log(2,"< %s=%s", $attr_name,$attr_val); } } } my($setreply,$exit_code); $setreply = $attr_ref->{'setreply'}->[0] if $attr_ref->{'setreply'}; $exit_code = $attr_ref->{'exit_code'}->[0] if $attr_ref->{'exit_code'}; if (defined $setreply && $setreply =~ /^2\d\d/) { # all ok do_log(1,"%s", $setreply); } elsif (!defined($setreply)) { do_log(0,"Error, missing 'setreply' attribute"); } else { do_log(0,"%s", $setreply); } # may do another request here if needed ... $sock->close or die "Error closing socket: $!"; $exit_code = 0 if $exit_code==99; # same thing in this case, both is ok exit 0+$exit_code; amavisd-new-2.7.1/AAAREADME.first000640 000621 000620 00000004016 10507436504 016001 0ustar00markcmi000000 000000 amavisd-new =========== http://www.ijs.si/software/amavisd/ amavisd-new is an interface between message transfer agent (MTA) and one or more content checkers: virus scanners, and/or SpamAssassin ( http://www.spamassassin.org/ ). amavisd-new is a performance-enhanced and feature-enriched version of amavisd (which in turn is a daemonized version of AMaViS or amavis-perl). It is normally positioned at or near a central mailer, not necessarily where user's mailboxes and final delivery takes place. If you are looking for fully per-user configurable and/or low-message-rate solution to be placed at the final stage of mail delivery (e.g. called from procmail), there may be other solutions more appropriate for your needs. amavisd-new benefits from the use of Perl module Net::Server, which offers a fast pre-forked multi-child environment. Several new features are offered, including SMTP-relay capability. This makes it suitable for mail anti-virus and/or anti-spam checking on a busy mail gateways. All the modifications since the original amavisd done by Mark Martinec, with contribution of ideas, reports and patches from amavis-user mailing list community. See RELEASE_NOTES, INSTALL, and files in the README_FILES/* subdirectory, at least the one pertaining to your MTA (Postfix, Sendmail/milter, Exim). If running a dual-MTA setup (any MTA type), the files README.postfix, README.sendmail-dual and README.exim_v4 describe a similar setup and can be used as guidelines. There may be more recent versions of these files at the web site http://www.ijs.si/software/amavisd/, along with FAQ. There is also a more detailed documentation on selected topics (a work in progress), the most recent version is at: http://www.ijs.si/software/amavisd/amavisd-new-docs.html The two programs in the subdirectory helper-progs/ are only needed for interfacing with sendmail (with milter or with the old setup, but not for the README.sendmail-dual setup), or with older Exim (v3). Helper programs are not needed if using Postfix or Exim v4 or dual-MTA setup. amavisd-new-2.7.1/RELEASE_NOTES000640 000621 000620 00002545243 11747115620 015461 0ustar00markcmi000000 000000 April 29, 2012 amavisd-new-2.7.1 release notes BUG FIXES - prevent rmdir() from failing with 'Invalid argument' on Solaris 10 when deleting a temporary directory: current working directory must not be within a directory which is about to be deleted; reported and diagnosed by Maciej Uhlig; - forwarding or quarantining through a 'pipe:' method failed with "Insecure dependency in exec while running with -T switch" when a sendmail command-line option -N was needed; reported by Andreas Schulze; - when multiple sockets are specified (e.g. in $forward_method) as a redundancy/failover mechanism, and SMTP session caching is enabled, a failed forwarding session does not clear a cached session, so all further attempts are stuck with the failed server, instead of picking a different server from the list; discovered by Michael Storz; - on establishing a SMTP session when multiple sockets are specified (e.g. in $forward_method) as a redundancy/failover mechanism, the random choice never picked the last socket in a list; discovered by Michael Storz; - fix defanging by mimedefang, it was failing with perl 5.10 or later due to an unhandled "Insecure dependency in sprintf" while logging the result if the $log_level was 2 or higher, or when debugging was enabled; thanks to Steve Scotter for a problem report; - fix defanging by Anomy::Sanitizer, it was failing with an error message: "mangling by anomy failed: replacement size 0, mail will pass unmodified"; - fix the 'xz' entry in a default @decoders list (in files amavisd.conf, amavisd.conf-default and amavisd); the first two variants ('xzdec' and 'xz') were glued together, so the xz decoder was only available if found under names 'unxz' or 'xzcat'; - provide a workaround for a bug [rt.cpan.org #64642] in a perl module Encode, which gratuitously untaints a string when encoding or decoding it: https://rt.cpan.org/Public/Bug/Display.html?id=64642 (still unfixed in Encode 2.44, perl 5.14.2); A module Scalar::Util is now required, which should not be a compatibility problem, as this module is a Perl core module since perl 5.8.0. - avoid the use of Encode::is_utf8 due to a bug in a perl module Encode as bundled with versions of Perl 5.8.0 to 5.8.8 (fixed in March 2007): Perl bug tracking: #32687: Encode::is_utf8 on tainted UTF8 string returns false https://rt.perl.org/rt3/Public/Bug/Display.html?id=32687 also referenced by #37170: https://rt.perl.org/rt3/Public/Bug/Display.html?id=37170 This is a re-manifestation of the same problem we had back in 2004, with a workaround provided by amavisd-new-2.2.1. Forgot that people are still using Perl 5.8 :) Reported by Peter Dieth; - fix a warning: _WARN: Invalid conversion in sprintf: "%a" - write informational messages during a stop/start/restart to stdout, instead of to stderr, avoiding unnecessary cron job messages; thanks to Cristian Seres, Sandro Janke and John Griffiths; also: https://bugzilla.redhat.com/show_bug.cgi?id=561389 - fix a syntactically incorrect 'Avira SAVAPI' av entry (missing closing bracket) in a sample configuration file amavisd.conf; - minor: get_body_digest incorrectly logged 8-bit body as 8-bit header; - no longer insist on a minimal version 2.22 of a module Digest::MD5, the 'clone' method is no longer needed since amavisd-new-2.7.0; - do not call $parser->max_parts($MAXFILES) with some old versions of MIME::Parser which did not yet provide this method; - pre-load a module File::Glob even with perl 5.8.0, otherwise autowhitelisting in SpamAssasssin may fail with "Insecure dependency"; - documentation: (files README.sql-mysql and README.sql-pg): fixed a field name "policy.unchecked_lover", previously incorrectly specified as "policy.unchecked_lovers_maps"; reported by TimH; - documentation: fixed the two SELECT examples in files README.sql-pg and README.sql-mysql, the field 'select' needs to be qualified with a table name: 'msgrcpt.content' to avoid ambiguity; reported by Gary V; - documentation bug in amavisd.conf-default: 'ESMTP' is not a valid setting for $protocol, just use 'SMTP' instead; reported by Pascal Volk; COMPATIBILITY - commented out the LHA entry in the default @decoders list and in do_executable(). The program seems to be unmaintained, was seen crashing and as such it may pose a security risk; pointed out by Thomas Jarosch; - due to popular demand, bring the 'spam-tag:' log line back to log level 2 (version 2.7.0 dropped it to log level 3) to retain compatibility with some log analyzers. Caveat: 'spam-tag' string is now entirely in lowercase. Suggested by Stefan Jakobs; OTHER - if a message is quarantined to more than one location using different quarantine methods, the SQL field msgs.quar_type indicates only the type of the last one. When archival quarantining is enabled this choice is unfortunate, as the primary quarantine type is more interesting than the permanent archival quarantine type. This is now reversed, the msgs.quar_type field now reflects the first quarantine type. Suggested by Patrick Ben Koetter. - SMTP session caching now no longer re-uses old sessions which are in use for more than a minute since their establishment; suggested by Michael Storz; - having the archive quarantine enabled should not be a sufficient reason to store information to SQL when $sql_store_info_for_all_msgs is off; Suggested by Patrick Ben Koetter. - ClamAV-clamd and ClamAV-clamd-stream av scanners: changed socket name in a sample configuration file amavisd.conf to /var/run/clamav/clamd.sock (previously the socket name was /var/run/clamav/clamd); this makes it compatible with a default socket name under several Linux distributions and under FreeBSD; suggested by Oliver Schinagl; - documentation updates; --------------------------------------------------------------------------- July 1, 2011 amavisd-new-2.7.0 release notes Contents: NEW FEATURES SUMMARY GENERAL COMPATIBILITY WITH 2.6.4 / 2.6.5 / 2.6.6 BUG FIXES SINCE 2.6.6 BUG FIXES SINCE 2.6.5 BUG FIXES SINCE 2.6.4 NEW FEATURES OPTIMIZATIONS OTHER CLEANING NEW FEATURES SUMMARY - significant improvements affecting a pre-queue content filtering setup (time limiting, warm/flying restart, ...) - requires Postfix 2.7.0 and SpamAssassin 3.3.0, or later; - new daemon amavisd-signer makes it possible to sign mail with DKIM signatures without requiring amavisd process to have access to private signing keys; - added support for the Sophos-SSSP, Avira SAVAPI and ClamAV clamd streaming protocols allows amavisd to communicate with these antivirus solutions; - allow specifying multiple (fail-over) back-end mailers for resubmission of messages from amavisd back to MTA; - support for Postfix 2.8.0 XFORWARD IDENT, passes a local message identifier (queue id) downstream to a post-queue content filter and back to Postfix; - speedup in data transfer rate on receiving large mail via SMTP/LMTP sessions by a factor of 3.9 for plain text sessions, and by a factor of 11 for encrypted (TLS) sessions; - recognize and insert header fields as prepared by SpamAssassin 3.3.0 or later through its 'add_header' configuration option; - a new setting allows a forward_method to be chosen based on a message content type and/or recipient address; this may be useful for outgoing mail routing purposes or to implement sender reputation schemes; - per-recipient (or per- policy bank) SpamAssassin configuration files or SQL configuration sets are supported (@sa_userconf_maps), and per-recipient SQL Bayes database usernames (@sa_username_maps); - new macros: client_helo, client_addr, client_port, client_addr_port, mime2utf8, rusage, ADDEDHEADERHAM, ADDEDHEADERSPAM, banned_parts_as_attr, actions_performed, new arguments to macros dkim, header_field, HEADER, YESNO and YESNOCAPS; - @listen_sockets setting offers a unified configuration of listening sockets; it may be configured directly, or the traditional way: the $inet_socket_port, $unix_socketname and $inet_socket_bind just add their entries to the @listen_sockets list; - lists of lookup tables (the @*_maps variables) can now contain explicit SQL and LDAP lookup objects as their elements, instead of (or in addition to) the implied SQL and LDAP lookups; - a new configuration variable @virus_name_to_policy_bank_maps allows loading of policy banks based on a virus name; - a new configuration variable $mail_id_size_bits allows setting the size of randomly generated mail_id and secret_id codes; - a new configuration variable $sql_store_info_for_all_msgs allows storing information on mail messages selectively just for quarantined messages; - added SNMP counters InMsgsStatus* which combine the final mail checking status with a direction of a mail flow; - optional transparent archival quarantine, retaining envelope recipient addresses on delivery to a dedicated SMTP server; GENERAL With a synergy of four solutions, using amavisd-new in a pre-queue filtering setup became a sensible / better behaved solution: - the "smtpd_proxy_options=speed_adjust" Postfix option, available since Postfix 2.7.0 (20091101), improves decoupling between SMTP clients and a content filter in a proxy setup, reducing the number of content filtering processes needed for the same mail load. With this option turned on, a Postfix SMTP server receives entire message before connecting to a before-queue content filter; - a master_deadline option and its API equivalent, available in SpamAssassin since version 3.3.0, allows for time limiting on lengthy rules checking, while still providing results when a time limit is exceeded; this makes it more suitable for time-sensitive setups like a pre-queue filtering setup; - reworked sub-task time limiting in amavisd, along with its counterpart solution in SpamAssassin, makes it better suited to a real-time nature of pre-queue filtering setups where one has no control over how long SMTP clients are willing to wait at the data-end stage; - a re-purposed command line option 'reload' now does a warm restart, keeping sockets available to an MTA client at all times, thus reducing a chance that an MTA would even notice a content filter's warm restart. Provided that required minimal versions of Postfix and SpamAssassin are available, on can try amavisd in a Postfix proxy setup. The $child_timeout setting needs to be radically reduced in this setup, matching the longest time most SMTP clients are willing to wait, and must be less than Postfix is willing to wait (smtpd_proxy_timeout), which by default is 100 s. A sensible value is somewhat less then a minute (e.g. 45 seconds). Even though RFC 5321 (section 4.5.3.2.6) recommends that clients SHOULD be willing to wait for 10 minutes at data-end stage, it is not uncommon that this recommendation is not adhered to. Note that a pre-queue filtering setup (along with its benefits) still has all its drawbacks, like the need for more filtering processes to accommodate mail arrival rate peaks (instead of averages), and much shorter and unpredictable (client-dependent) time limits. The new features of the three products only rise the thresholds where trouble starts, and make the whole setup better behaved. COMPATIBILITY WITH 2.6.4 / 2.6.5 / 2.6.6 - due to popular demand to reduce undesired and unintentional backscatter, defaults for the settings $final_spam_destiny and $final_banned_destiny were changed. Previously they both defaulted to D_BOUNCE, new defaults are: $final_virus_destiny = D_DISCARD; $final_banned_destiny = D_DISCARD; $final_spam_destiny = D_PASS; $final_bad_header_destiny = D_PASS; Please adjust to will. If you have these settings configured explicitly in a configuration file, this change of a default value does not affect you. For a pre-queue content filtering setup (smtp proxy or milter) a suitable value for undesired content is D_REJECT. For a post-queue filtering setup preferred choices are to tag-and-deliver (D_PASS), or to drop (D_DISCARD) and quarantine. It is still possible to use a D_BOUNCE setting, but please limit and monitor your backscatter. Due to a default setting of @viruses_that_fake_sender_maps the backscatter on viruses has been fully suppressed since amavisd-new-20021116 even with a D_BOUNCE. Backscatter on high-scoring spam has been controllable since amavisd-new-20030616-p8 by a family of settings @spam(_crediblefrom)_dsn_cutoff_level(_bysender)_maps, - several ancient configuration settings were removed or deactivated, see section CLEANING below; - a command line option 'reload' has been renamed to 'restart', corresponding to a shutdown followed by a normal (cold) start; while a command line option 'reload' has been re-purposed to function as a warm/flying restart. See below for details. Protection of some files may need to be examined (configurations files and DKIM private keys should be readable for group vscan/amavis and not writable by UID vscan/amavis); - a failure of all virus scanners no longer automatically tempfails the operation, but flags a message with a CC_UNCHECKED contents category (just like a failure of decoders/dearchivers), and allows the usual controls (*_destiny, *_quarantine_*) to be used to choose behaviour. The $virus_scanners_failure_is_fatal=1 reverts to previous behaviour, see below; - a default value of $hdr_encoding and $bdy_encoding has been changed from 'iso-8859-1' to 'UTF-8' which better suits reporting of banned parts; - default encoding for reading text templates from the tail of a file 'amavisd' has been changed to 'utf8', which allows replacing a default text by a non-ascii Unicode template, encoded as UTF-8; - when using SQL for logging/penpals: three fields need to be added to a table msgrcpt: msgrcpt.content, msgrcpt.rseqnum, msgrcpt.is_local, and one to a table msgs: msgs.originating . Semantics of msgrcpt.content is similar to msgs.content, but reflects individual recipient's settings (e.g. when a message is both banned and spam, a recipient with banning tests disabled will see a message as spam, while other recipient of the same message will consider it banned). The added field may also simplify queries by third party applications. The field msgrcpt.rseqnum uniquely identifies/enumerates recipients within each message, typically by assigning them sequential numbers starting with 1. The only purpose of this field is to make it possible to define a primary key for the table msgrcpt, which may be needed for some clustering/partitioning purposes. Amavisd itself does not require a primary key on this table. The field msgrcpt.is_local should be considered a boolean, its value can be: 'Y' ... yes, recipient is local, i.e. matches @local_domains_maps 'N' ... no, recipient does not match @local_domains_maps ' ' ... unknown - this is a default field value; amavisd always sets this field to either 'Y' or 'N'; The field msgs.originating should be considered a boolean, its value can be: 'Y' ... yes, message is originating from inside or from an authenticated roaming sender (the flag $originating was true); 'N' ... no, message is not submitted by our user ($originating was false); ' ' ... unknown - this is a default field value; amavisd always sets this field to either 'Y' or 'N', reflecting the $originating flag; A combination of msgs.originating and msgrcpt.is_local tells a direction a message is traveling: originating is_local N N open relay (probably misconfigured @local_domains_maps or $originating flag not set) N Y inbound message Y N outbound message Y Y internal message (inside or authenticated -> inside) Note that a direction is a per-recipient property, a multi-recipient message can be outbound for some recipients and internal for others at the same time. The following SQL directives can be used to add these new fields: ALTER TABLE msgrcpt ADD rseqnum integer DEFAULT 0 NOT NULL; ALTER TABLE msgrcpt ADD content char(1) DEFAULT ' ' NOT NULL; ALTER TABLE msgrcpt ADD is_local char(1) DEFAULT ' ' NOT NULL; ALTER TABLE msgs ADD originating char(1) DEFAULT ' ' NOT NULL; If a primary key on table msgrcpt is needed for some reason, try something like the following: *MySQL: UPDATE msgrcpt SET rseqnum=1+floor(999999999*rand()) WHERE rseqnum=0; ALTER TABLE msgrcpt ADD PRIMARY KEY (partition_tag,mail_id,rseqnum); *PostgreSQL: UPDATE msgrcpt SET rseqnum=1+floor(999999999*random()) WHERE rseqnum=0; CREATE UNIQUE INDEX msgrcpt_idx_primary ON msgrcpt (partition_tag,mail_id,rseqnum); If keeping a possibly customized copy of %sql_clause in a configuration file, entries 'ins_rcp' and 'upd_msg' will need to be updated accordingly. To facilitate transition from 2.6.6 to 2.7.0, it is possible to configure amavisd 2.7.0 to supply with SELECT and INSERT clauses a subset of parameters as used by 2.6.6. A configuration setting $sql_schema_version controls this backward compatibility. Its default value is 2.007000 . By setting it to a value below 2.007000 (such as 2.006006 or 2.006004) a subset of parameters as was used with a version 2.6.6 or 2.6.4 is selected. SQL clauses in $sql_clause{'upd_msg'} and $sql_clause{'ins_rcp'} need to be adjusted according to a chosen version of actual parameters. Below is an example of a required setting compatible with both amavisd-new 2.6.6 and 2.7.0, which lets amavisd 2.7.0 use an SQL schema of 2.6.6, which lacks the four newly added fields: our($sql_schema_version) if $myversion_id_numeric < 2.007000; $sql_schema_version = 2.006006; $sql_clause{'upd_msg'} = 'UPDATE msgs SET content=?, quar_type=?, quar_loc=?, dsn_sent=?,'. ' spam_level=?, message_id=?, from_addr=?, subject=?, client_addr=?'. ' WHERE partition_tag=? AND mail_id=?'; $sql_clause{'ins_rcp'} = 'INSERT INTO msgrcpt (partition_tag, mail_id, rid,'. ' ds, rs, bl, wl, bspam_level, smtp_resp) VALUES (?,?,?,?,?,?,?,?,?)'; Note that this is only provided to facilitate transition. Please add the new fields on an opportunity, then remove the above settings from your configuration file and restart amavisd. - SQL fields msgs.content and msgrcpt.content used to encode a content type CC_SPAMMY as 's', and CC_MTA as 't'. With default case-insensitive queries on a data type CHAR it was not possible to distinguish between lowercase 's' (= CC_SPAMMY) and uppercase 'S' (= CC_SPAM), so the CC_SPAMMY is now encoded as 'Y', and CC_MTA as 'T' (just in case). Please adjust your management tools if necessary. - please check SQL data types on fields msgs.mail_id, msgs.secret_id, msgrcpt.mail_id and quarantine.mail_id in existing databases, these must be treated case-insensitively - see details further down, please search further down for "must be treated case-insensitively"; - SQL clause $sql_clause{'sel_quar'} no longer uses a coalesce() function (introduced in amavisd-new-2.6.2) which attempted to deal with NULL quarantine.partition_tag or with undefined $partition_tag, when releasing a message from an SQL quarantine - but payed the price of not using an index. If releasing from an SQL quarantine is desired, either ensure there are no (old) records in a table 'quarantine' with a NULL partition_tag (e.g. replace such fields with a 0, and don't leave $partition_tag undefined in amavisd.conf - set it to 0 for example when partitioning is not needed), or assign a former clause to $sql_clause{'sel_quar'} in amavisd.conf : $sql_clause{'sel_quar'} = 'SELECT mail_text FROM quarantine'. ' WHERE coalesce(partition_tag,0)=coalesce(?,0) AND mail_id=?'. ' ORDER BY chunk_ind'; Thanks to Michael Scheidell and Thomas Gelf for pointing out the inefficiency. - if using Petr Rehor's amavisd-milter to call amavisd, please update it to version 1.5.0 (or later), as earlier versions did not accept a new attribute 'log_id', which is now included in a response from amavisd; - a sample configuration file amavisd.conf-sample was removed from the package - it hasn't been seriously updated for years, and it contained lots of aged or distracting information; - old helper programs amavis.c and amavis-milter.c are no longer distributed with the package, along with the entire helper-progs subdirectory. As a milter client please use the more modern 'amavisd-milter' package by Petr Rehor, available at http://sourceforge.net/projects/amavisd-milter/ - old AM.CL protocol is no longer supported; it was provided for compatibility with versions of AMaViS pre-dating amavisd-new, along with its client programs: old helper programs amavis.c and amavis-milter.c. Handling of release requests and milter requests through AM.PDP protocol remains unaffected; - a sample AM.PDP client program for mail submission to amavisd which was previously distributed as 'helper-progs/amavis.pl' has been renamed to 'amavisd-submit' and slightly modernized. It provides partial functional compatibility with a very early AMaViS client program amavis.c . It takes a message on stdin, copies it to a temporary file, passes its name to amavisd daemon using AM.PDP protocol, and based on the response adjusts its exit status value so that an invoking script or program may decide whether to deliver the mail message or not; - mail_id and secret_id are now composed of characters from a character set [ A-Z, a-z, 0-9, -, _ ] instead of [ A-Z, a-z, 0-9, +, - ] (i.e. now uses underline instead of a plus) to conform to RFC 4648 base64url specification, thus making it potentially easier to specify an id in various GUI/API interfaces without a need for quoting a plus. The change is also reflected in a choice of quarantine file names. Compatibility with releasing of old quarantined messages is retained; - relationship between mail_id and secret_id has changed and is now: mail_id = encode_base64(md5(decode_base64(secret_id))) (i.e. md5 is applied to 9 raw bytes of a secret id) while previously it was: mail_id = encode_base64(md5(secret_id)) (i.e. md5 was applied to 12 base64 characters of a secret id) Releasing from a quarantine still understands old relationship and old base64 encoding character set for compatibility, so no problems are expected even when releasing a mix of old and new quarantined messages. The change may potentially affect some third party application. - caching of virus and spam check results based on a mail body hash has been removed. It was very beneficial years ago when virus storms were common and spam was not personalized. Nowadays (2011) the feature barely pays for itself (savings are comparable to additional processing needed), and is incompatible with per-recipient spam checks (as introduced with this version), and incompatible with DKIM verification on locally originating and signed mail being returned from a mailing list. Rather than trying to fit a square peg into a round hole, the feature is now dropped. Associated configuration variables are still declared for compatibility, but have no effect: $enable_global_cache, $virus_check_negative_ttl, $virus_check_positive_ttl, $spam_check_negative_ttl, $spam_check_positive_ttl - a default value for $lock_file is now undefined instead of the former default value "$helpers_home/amavisd.lock"; an undefined value lets Net::Server choose a suitable temporary file (POSIX::tmpnam) for 'flock' serialization on socket accept(); - updated (rarely used) AV entries 'Sophos SAVI', 'Mail::ClamAV' and 'av_smtp' in an incompatible way (they now use ask_daemon interface instead of a dedicated subroutine), please update your AV entries according to the new sample file amavisd.conf; - internal: spam_level() and spam_tests() are no longer properties of a message but are now a property of each recipient, which makes possible per-recipient spam checking settings (e.g. rules, bayes username, ...); - internal: a delivery_method() is no longer a property of a message, but is now a property of each recipient, which makes per-recipient forwarding possible; - internal: a load_policy_bank() takes one additional argument $msginfo, which is passed on to any policy bank's ACTION routine if a policy bank has one; BUG FIXES SINCE 2.6.6 - take a more cautious approach on keeping evidence on an SMTP session transaction state when feeding a message back to MTA. Under certain abnormal circumstances an MTA could respond to end-of-data with a temporary failure but retain an active transaction state while amavisd would assume the transaction was closed, leading to a 'MAIL transaction in progress' failure on the next message using the same cached SMTP session. Now amavisd considers a transaction state to be unknown when there is any doubt and closes a session instead of caching it, unless the transaction is reliably known to be closed. Problem reported by Ralf Hildebrandt. BUG FIXES SINCE 2.6.5 All bug fixes that were developed during a 2.7.0 developement cycle have been backported to the 2.6 branch and released as 2.6.6. They are all documented in release notes of the 2.6.6 release. BUG FIXES SINCE 2.6.4 All bug fixes and some compatibility measures that were developed during a 2.7.0 developement cycle have been backported to the 2.6 branch and released as 2.6.5. They are all documented in release notes of the 2.6.5 release. NEW FEATURES - a command line option 'reload' has been renamed to 'restart', while keeping its semantics: to stop a currently running daemon, and then promote a process to become a new daemon. This makes a complete and independent restart with all its benefits: can start a chrooted daemon, can access config files or DKIM signing keys accessible only to root, can open sockets/ports otherwise restricted to root, can change inet and unix socket locations, their bindings and port numbers. It will also reset SNMP counters in a database, if it is enabled. A downside is that during a restart existing sockets are closed, so until new sockets are re-established an MTA client experiences connection failures, which is particularly disruptive in a pre-queue MTA setup. - a command line option 'reload' has been re-purposed to function as a warm restart: it now sends a HUP signal to a running daemon, then exits. A running daemon upon receiving a HUP signal will clone its sockets, clear their 'close-on-exec' flag, then restart itself through exec(). A reborn daemon inherits open sockets, does a normal startup (loading perl modules and config files), then reassociates inherited sockets with configured inet and Unix socket names, which is why these must not be changed in a configuration file between reloads. Sockets remain open and available to clients during the whole warm-restart period, requests are queued by kernel (queue size is configurable through $listen_queue_size, defaulting to SOMAXCONN, or there may not be any queues at all when an IP stack is using SYN cookies), so apart from a delay in connection establishment, an MTA client will not notice a restart as long as the IP stack is willing to accept new sessions (as controlled by listen queue size or SYN cookies). This makes the reloading method particularly suitable for pre-queue filter setups. A downside is that a HUP-ed daemon has already dropped root privileges during its first start, so it must restart as a nonprivileged user (typically 'vscan' or 'amavis'), which rules out its ability to chroot, and requires that configuration files, DKIM signing keys files, and perl modules must be readable by this GID or UID, otherwise a restart fails and a daemon process no longer exists. Depending on a version of perl and operating system in use, it might be necessary to specify an absolute path to amavisd on the initial start. To debug warm-restart problems it may be useful to first try a warm restart on a non-daemonized process (started manually as: amavisd foreground, or: amavisd debug), so that potential errors on stderr are visible. A sensible protection of configuration files and files with DKIM keys is to set their group ownership to vscan (amavis) and UID ownership to root, and mode to 0640 (u=rw,g=r,o=). A need for non-root accessibility of DKIM signing keys can be avoided by using a new signing service daemon included with this release (see further down: amavisd-signer). One additional feature of a warm reload is that SNMP counters in a database (visible through amavisd-agent or amavisd-snmp-subagent) are not reset to zero, unlike the restart which clears them. - on stop, restart or reload, currently busy child processes are left to complete their current task instead of being abruptly stopped. This minimizes a disruption experienced by MTA. - added a client-side and server-side support for the IDENT attribute of a Postfix XFORWARD smtp command (available since Postfix version 2.8.0). The attribute allows passing of a local message identifier (MTA queue id) downstream from a front-end MTA to a post-queue content filter and back to a back-end MTA. Amavisd makes this information available through an existing macro %Q (which was previously non-empty only in milter setups), and as such the information appears in the log when using a default amavisd log template. This information is also passed back to a re-entry MTA if it announces a support for this attribute (enabled on a back-end smtpd service with an option smtpd_authorized_xforward_hosts), so the log entries are now easier to correlate in a post-queue filtering setup: back-end MTA: postfix/smtpd[72995]: 553261D1CB0: client=localhost[::1], orig_queue_id=2F5971D1CA3, orig_client=... post-queue content filter: amavis[20341]: (20341-15) Passed CLEAN ... Queue-ID: 2F5971D1CA3, queued_as: 553261D1CB0 front-end MTA: postfix/lmtp[73130]: 2F5971D1CA3: ... relay=127.0.0.1[127.0.0.1]:10024, status=sent (250 2.0.0 from MTA(smtp:[::1]:10025): 250 2.0.0 Ok: queued as 553261D1CB0) - support Postfix 2.9 long queue IDs (enable_long_queue_ids=yes) as available since postfix-20110321 by adjusted default values of $log_short_templ and $log_verbose_templ templates; - improved support for pre-queue content filtering setups: reorganized time limiting on processing to obey more strictly a deadline time, which is the sum of $child_timeout and a timestamp at the moment of a reception of a complete message (SMTP data-end time). The deadline time is also passed to SpamAssassin, which since version 3.3.0 supports a 'master_deadline' option and can gracefully terminate its processing on a time limit, while still providing results collected so far. The setting $sa_timeout is now retired: the variable is still declared for backward compatibility, but has no effect. Instead, the time available for spam scanning is automatically determined from $child_timeout, taking into consideration the actual time left till the deadline; - $child_timeout and $smtpd_timeout settings are now dynamic, i.e. can be changed by a policy bank, which makes it possible to support (on different ports) both the pre-queue and post-queue (e.g. fallback) clients by the same amavisd daemon; - a new configuration variable $soft_bounce (also a member of policy banks) turns rejects, bounces and discards into a temporary failure when true; this is potentially useful as a short-term safety net when testing configuration changes on a low-traffic server; - added an AV entry and supporting code for Sophos-SSSP, implementing the client side of the Sophos SSSP protocol, talking to a savdid daemon (a replacement for Sophie) using its native protocol; - added an AV entry and supporting code for AVIRA SAVAPI protocol, implementing the client side of the protocol, talking to a savapi daemon; - added an AV entry for clamdscan which can serve as a useful backup scanner, connecting as client to a remote clamd; the supplied alternative config file should specify the host IP and port number where clamd is running using TCPAddr and TCPSocket options; suggested by Michael Scheidell; - added an AV entry for ClamAV clamd streaming which can serve as a main or backup scanner, connecting as client to a remote clamd. The client side implements clamd zINSTREAM command, batched in a zIDSESSION / zEND group. This approach is comparable to what is implemented in clamdscan and is somewhat less efficient than passing only a directory name to clamd, but has an advantage that the clamd daemon need not have direct access to amavisd temporary files, and may even be running on a remote host. The cost of this flexibility is additional data transfer. Suggested by Michael Scheidell. - lists of lookup tables (the @*_maps variables) can now contain explicit SQL and LDAP lookup objects as their elements, instead of (or in addition to) the implied SQL and LDAP lookups. A new configuration setting $lookup_maps_imply_sql_and_ldap controls whether the SQL and LDAP lookup objects are implicitly prepended to list in @*_maps variables (when true), or not (when false). The default value is 1 for compatibility with previous versions. Regardless of the $lookup_maps_imply_sql_and_ldap setting, the @*_maps lists of lookup tables/objects may now contain explicit lookup objects for arbitrarily named SQL fields and LDAP attributes. This provides more flexibility: the order of lookups is now configurable (previously SQL and LDAP lookup objects were prepended to lists and thus always looked up first), and the names of SQL fields or LDAP attributes can now be specified as arguments to SQL and LDAP lookup objects (previously field and attribute names were hardwired into code). Three shorthand functions are available for creating SQL lookup (query) objects: q_sql_s, q_sql_n, q_sql_b, and three for creating LDAP lookup (query) objects: q_ldap_s, q_ldap_n, q_ldap_b. The _s, _n and _b suffixes imply a data type of the expected result: a string, a numeric value, and a boolean. Due to Perl's forgiveness a string data type can in most cases be used as a number or as a boolean and may be used when data type conversion and value normalization is not necessary or when a data type is not known. Here are some examples: @spam_kill_level_maps = ( { # a hash-type lookup object 'user1@example.com' => 8, '.example.org' => 7.5, }, q_ldap_n('amavisSpamKillLevel'), # an LDAP lookup object q_sql_n('spam_kill_level'), # an SQL lookup object $sa_kill_level_deflt, # a constant-type pseudo-lookup object ); @spam_subject_tag2_maps = ( q_sql_s('subject_tag'), ); In addition to simple scalar arguments (a field or attribute name), these six lookup object-creating functions can take as their argument a listref of field or attribute names, or a hashref where hash entry values are SQL field names (or LDAP attribute names), and hash entry keys are the result data names. Lookups resulting from such lookup objects will return a hashref of key/value pairs instead of a single scalar result. This is currently only useful in the @dkim_signature_options_bysender_maps list of lookups which expects such hash results (sets of data names and their values, i.e. entire records). The listref argument is just a shorthand notation which can be used in place of a hashref when field names (or attribute names) are the same as the desired result data names. The following alternatives are equivalent: q_sql_s( { 'd' => 'd', 's' => 's', 'ttl' => 'ttl' } ) q_sql_s( { d => 'd', s => 's', ttl => 'ttl' } ) # perl shorthand q_sql_s( [ 'd', 's', 'ttl' ] ) q_sql_s( [qw(d s ttl)] ) # perl shorthand Example (artificial, not necessarily useful): @dkim_signature_options_bysender_maps = ( q_sql_s( ['d', 's', 'ttl'] ), q_ldap_s( ['d', 's', 'a'] ), q_ldap_s({ d => 'sdid', s => 'amavisSelector', a => 'amavisDkimAlg' }), { 'postmaster@example.com' => { a => 'rsa-sha1', ttl => 7*24*3600 }, '.' => { a => 'rsa-sha256', ttl => 30*24*3600 }, }, ); - a new configuration variable $sql_store_info_for_all_msgs when turned off requests storing information on mail messages selectively just for quarantined messages. At the same time turning this setting off also disables pen pals lookups. A default value is 1 (true) as before, indicating that information on all messages is to be stored into tables msgs, msgrcpt and maddr when @storage_sql_dsn is enabled, thus ensuring long-term uniqueness of mail_id and proper operation of pen pals lookups; - a new program is included with a package: amavisd-signer. It is a DKIM signing service daemon for amavisd. It uses an AM.PDP protocol lookalike to receive a request from amavisd and provides two services: choosing a signing key, and signing a message digest with a chosen DKIM private key. Amavisd uses this signing service when a $dkim_signing_service setting is defined and nonempty, and $enable_dkim_signing is true. For each mail message meeting the basic requirements for signing (originating, nonspam and not infected), the first request sent to a signing service passes some information about the message (its author, sender, recipients) and expects the service to choose and provide a suitable signing domain and selector (and optional signature options) when a signing key is available and the service considers it appropriate to sign the message. If the response does not provide a signing domain and selector, amavisd falls back to consulting its own settings (a dkim_key() set of signing keys, and @dkim_signature_options_bysender_maps). The second stage of signing occurs when a signing key has been uniquely identified during the first stage, i.e. when its signing domain and a selector have been determined. Amavisd computes a message digest according to DKIM specifications and passes it to the signing service, along with the signing domain and selector name. The signing service computes the signature and returns it as a 'b' attribute (corresponding to a 'b' tag of a DKIM signature), from which amavisd assembles the signature header field and inserts it into a message. The signing service may still choose not to sign at this stage, e.g. when a private key corresponding to the requested signing domain and selector is not available. If a signing service is not available or cannot sign, amavisd falls back to its own configured list of signing keys ( dkim_key() ) for backward compatibility. The main reason for separating the signing act from the main amavisd daemon is to make it possible to do the DKIM signing without letting amavisd have access to private keys - following the minimal 'need-to-know' security principles. For example, amavisd may be started as non-root or restarted from a jail, while the independent amavisd-signer process remains the only process with access to private signing keys (by running as root or under a separate UID or GID or with an access to a crypto device). Additional benefit is that more complex decisions on which signing key to use for which mail message can be delegated to the signing service, which can be customized (through code changes or replacing it altogether) without touching the main amavisd daemon. To let amavisd use a signing service, specify the signing service's IP address and TCP port number in amavisd.conf, e.g.: $dkim_signing_service = '127.0.0.1:20203'; matching the $inet_socket_bind and @listen_sockets settings near the beginning of the 'amavisd-service' file, then start the signing service and restart amavisd daemon. Currently all the settings for amavisd-service are contained in its file, no external configuration file or command line options are available at present. - a constant D_TEMPFAIL has been added to a set of allowed final_*_destiny values; mostly intended for completeness and testing; - a new setting @listen_sockets offers a unified configuration of listening sockets. This list may be configured directly, or the traditional way: the $inet_socket_port and $unix_socketname just add their entries (if any) to the @listen_sockets list, and $inet_socket_bind provides a default binding IP address for inet or inet6 ports. Each socket specification may either be a unix socket path (as in $unix_socketname), or an inet or or inet6 socket specification (as in a binding IP address $inet_socket_bind combined with a port number $inet_socket_port, delimited by a colon, e.g. '127.0.0.1:10024', '[::1]:10024', '10024'. When only a port number is specified without an IP address, the binding address defaults to $inet_socket_bind, which in turn (if left undefined), defaults to all interfaces. An 'unspecified' binding address '0.0.0.0' implies any socket of the inet family (IPv4), while an 'unspecified' address '::' implies any socket of an inet6 (IPv6) family. Depending on the operating system and its settings an inet6 socket may or may not be able to also accept inet connections. To be able to listen on inet6 (IPv6) sockets requires a patch to the Net::Server module 0.99 or earlier. Example: @listen_sockets = ( '10024', "$MYHOME/amavisd-proxy.sock", '0.0.0.0:10010', '127.0.0.1:10012', '10026', '9998', '[::1]:10028' ); - SMTP and LMTP client code now accepts a listref of peer socket specifications, or a single scalar specification as before. This allows for a failover in case some server is down or refuses connections. It also provides a simpleminded load balancing between next-hop (re-entry) MTA servers, as the selection from a list is random. Session caching still works, so if a recently used SMTP/LMTP session is still open, it will be reused, in which case no server randomization takes place for as long as the established session remains open. Typical configuration variables where this feature is available are: $forward_method, $notify_method, $resend_method, $release_method, and $requeue_method, but only when the specified protocol is smtp: or lmtp:, (not pipe:, local:, sql:, bsmtp:). Example: $forward_method = [ 'smtp:[::1]:10025', 'smtp:[127.0.0.1]:10025', 'smtp:*:10025' ]; $notify_method = [ 'smtp:*:*', 'smtp:192.0.2.10:10025' ]; It is assumed that the protocol specification scheme (e.g. 'smtp:') of all entries in a list is the same. Mixing different protocols in the same list of alternatives is not allowed; - when a message is being released from a quarantine as an attachment ( $release_format = 'attach' ), it is now possible to wrap this attached message into a password-protected ZIP archive to prevent accidental or automatic opening of the possibly malicious original message. The chosen password is included in the first plain text MIME part, along with an explanation / instructions for a recipient. Note that the purpose of password scrambling is only to prevent an accidental or automatic opening of an attachment. It is not intended to be a strong mechanism for keeping messages secret. There is no point in providing excessively long / strong passwords. A template for this first plain-text MIME part can be changed as before by assigning a new text to $notify_release_templ (or modifying a default template near the end of the file 'amavisd'). Three new configuration variables are added, all three are also members of policy banks: $attachment_email_name, $attachment_outer_name, and $attachment_password. Their default values are: $attachment_password = ''; # no password and no ZIP wrapping $attachment_email_name = 'msg-%m.eml'; $attachment_outer_name = 'msg-%m.zip'; The $attachment_email_name is a template for forming a name of a file, which is then inserted into a zip archive. This name will be seen as a filename containing an original mail message when a recipient unzips the archive. The supplied string may contain placeholders, the same placeholders are recognized as for filename templates used to control quarantining. For the record, here is a complete list of placeholders currently recognized: %P => $msginfo->partition_tag %b => $msginfo->body_digest %m => $msginfo->mail_id %n => $msginfo->log_id %i => iso8601 timestamp of a message reception time by amavisd %% => % The $attachment_outer_name is a template for forming a name of a ZIP archive which will be attached to a message. This name will end up in a MIME sub-header field of the attachment, and as such will be used as a filename when a recipient saves the attachment (without unzipping it). The supplied string may contain the same placeholders as above. The $attachment_password setting can be: . an empty string, in which case no ZIP wrapping will occur and no passwords are applied; also the settings $attachment_email_name and $attachment_outer_name have (currently) no effect, as the attachment is not a ZIP archive but the original message itself; this is a default setting for compatibility with earlier versions; . a fixed static string, in which case an original message is wrapped in a ZIP archive and the archive is encrypted with this fixed string password; . an undefined value ( $attachment_password = undef ), in which case a 4-digit random password (PIN) is internally generated for each quarantine release, the rest is the same as with a fixed string; . a subroutine reference, in which case the supplied subroutine is called (in a scalar context), passing it a $msginfo object as the only argument; the subroutine is expected to return a password as a string, or die if it cannot do its job; the returned value is then treated as one of the three cases above, i.e. an empty string disables zipping, an undefined value invokes internal PIN generating code, and any other value is taken as a password for encrypting the archive. Example use: $release_format = 'resend'; # choices: plain, resend (default), attach or: $release_format = 'attach'; $attachment_password = ''; # no archive, just plain attachment or: $release_format = 'attach'; $attachment_password = undef; # internally generated 4-digit random PIN or: $release_format = 'attach'; $attachment_password = 'fooBAR'; # fixed password or: $release_format = 'attach'; $attachment_password = sub { my($msginfo) = @_; my $str = qx'pwgen -N 1 -n -B -s 6'; die "pwgen failed, exit status: $?" if $?; die "pwgen returned empty result" if $str eq ''; return $str; }; As the $attachment_password is a member of policy bank, it is possible to configure amavisd to listen to release requests on two TCP ports, for example using one to release unencrypted false-positive spam messages, and the other to release possibly problematic infected messages. - updated amavisd-snmp-subagent and AMAVIS-MIB.txt by adding ten user-specifiable 64-bit counters and ten user-specifiable 32-bit gauges; Counters are placed into OID tree under 1.3.6.1.4.1.15312.2.1.1.17 and named UserCounter1..UserCounter10, whereas gauges are placed under 1.3.6.1.4.1.15312.2.1.1.18 and named UserGauge1..UserGauge10. A custom hook or a policy bank ACTION hook can be used for adjusting their values by calling snmp_count64() routine, e.g.: Amavis::Util::snmp_count64('UserCounter3', 'UserCounter9'); or: Amavis::Util::snmp_count( ['UserCounter4', 1234, 'C64'] ); - updated amavisd-snmp-subagent and AMAVIS-MIB.txt by adding the following counters, all placed under 1.3.6.1.4.1.15312.2.1.1 : .19.1 InMsgsStatusAcceptedAll same value as .2.7 .19.2 InMsgsStatusAcceptedInbound .19.3 InMsgsStatusAcceptedOutbound .19.4 InMsgsStatusAcceptedInternal .19.5 InMsgsStatusAcceptedOriginating .19.6 InMsgsStatusAcceptedOpenRelay .20.1 InMsgsStatusRelayedUntaggedAll (no equivalent) .20.2 InMsgsStatusRelayedUntaggedInbound .20.3 InMsgsStatusRelayedUntaggedOutbound .20.4 InMsgsStatusRelayedUntaggedInternal .20.5 InMsgsStatusRelayedUntaggedOriginating .20.6 InMsgsStatusRelayedUntaggedOpenRelay .21.1 InMsgsStatusRelayedTaggedAll (no equivalent) .21.2 InMsgsStatusRelayedTaggedInbound .21.3 InMsgsStatusRelayedTaggedOutbound .21.4 InMsgsStatusRelayedTaggedInternal .21.5 InMsgsStatusRelayedTaggedOriginating .21.6 InMsgsStatusRelayedTaggedOpenRelay .22.1 InMsgsStatusDiscardedAll same value as .2.9 .22.2 InMsgsStatusDiscardedInbound .22.3 InMsgsStatusDiscardedOutbound .22.4 InMsgsStatusDiscardedInternal .22.5 InMsgsStatusDiscardedOriginating .22.6 InMsgsStatusDiscardedOpenRelay .23.1 InMsgsStatusNoBounceAll same value as .2.10 .23.2 InMsgsStatusNoBounceInbound .23.3 InMsgsStatusNoBounceOutbound .23.4 InMsgsStatusNoBounceInternal .23.5 InMsgsStatusNoBounceOriginating .23.6 InMsgsStatusNoBounceOpenRelay .24.1 InMsgsStatusBouncedAll same value as .2.11 .24.2 InMsgsStatusBouncedInbound .24.3 InMsgsStatusBouncedOutbound .24.4 InMsgsStatusBouncedInternal .24.5 InMsgsStatusBouncedOriginating .24.6 InMsgsStatusBouncedOpenRelay .25.1 InMsgsStatusRejectedAll same value as .2.12 .25.2 InMsgsStatusRejectedInbound .25.3 InMsgsStatusRejectedOutbound .25.4 InMsgsStatusRejectedInternal .25.5 InMsgsStatusRejectedOriginating .25.6 InMsgsStatusRejectedOpenRelay .26.1 InMsgsStatusTempFailedAll same value as .2.13 .26.2 InMsgsStatusTempFailedInbound .26.3 InMsgsStatusTempFailedOutbound .26.4 InMsgsStatusTempFailedInternal .26.5 InMsgsStatusTempFailedOriginating .26.6 InMsgsStatusTempFailedOpenRelay For compatibility, the following counters appear in the MIB at two locations (both locations present the same value): .19.1 or .2.7 InMsgsStatusAcceptedAll (.20.1 - InMsgsStatusRelayedUntaggedAll) (.21.1 - InMsgsStatusRelayedTaggedAll) .22.1 or .2.9 InMsgsStatusDiscardedAll .23.1 or .2.10 InMsgsStatusNoBounceAll .24.1 or .2.11 InMsgsStatusBouncedAll .25.1 or .2.12 InMsgsStatusRejectedAll .26.1 or .2.13 InMsgsStatusTempFailedAll The value of a counter .2.8 InMsgsStatusRelayed is a sum of: .20.1 InMsgsStatusRelayedUntaggedAll .21.1 InMsgsStatusRelayedTaggedAll Their semantics is documented in AMAVIS-MIB.txt . Suggested by Patrick Ben Koetter. - a policy bank may now provide a custom hook as a hash key 'ACTION'. On loading a policy bank whose ACTION key has an associated value being a subroutine reference, the supplied subroutine is called when a policy bank is loaded, before its remaining keys/values are copied to the current setting. The action routine is passed two arguments: a $msginfo (a ref to an object containing all the information about a message being processed), and a policy name being loaded. Note that $msginfo may be undef if a policy bank is loaded early - before a $msginfo object is created, such as with policy banks associated with a port number or with client's IP address; Example use: $policy_bank{'TRUSTED_BOOKSHOPS'} = { bypass_spam_checks_maps => [1], spam_lovers_maps => [1], ACTION => sub { Amavis::Util::do_log(2,'Buying a book?'); Amavis::Util::snmp_count64('UserCounter2'); }, }; @author_to_policy_bank_maps = ({ 'amazon.com' => 'TRUSTED_BOOKSHOPS', 'amazon.co.uk' => 'TRUSTED_BOOKSHOPS', 'amazon.de' => 'TRUSTED_BOOKSHOPS', }); - a new configuration variable @virus_name_to_policy_bank_maps has been introduced. It allows loading of policy banks based on a virus name as reported by virus scanners. Reported names converted to spam by a @virus_name_to_spam_score_maps are no longer treated as virus names and as such are not eligible to @virus_name_to_policy_bank_maps. The @virus_name_to_policy_bank_maps is a list of lookup tables. A lookup key is each virus name as reported by any virus scanner. A result of a lookup is expected to be a string containing a comma-separated list of policy bank names. Nonexistent policy banks are ignored. Duplicate names are merged into a single name. The most suitable lookup mechanisms are a regexp lookup and a hash lookup, as these are able to provide an arbitrary user-specifiable result (unlike a list-based (ACL) lookup, which can only provide a boolean value). Suggested by Patrick Ben Koetter. Example use: @virus_name_to_policy_bank_maps = ( new_RE( # a regexp lookup [ qr'^(W32/MyDoom|W32/Netsky|Mal/BredoZp)' => 'VIRUS,MASS_VIRUS' ], [ qr'\bEICAR\b'i => 'EICAR_TEST' ], ), 'VIRUS', # constant (pseudo)lookup, catchall for any other virus name ); $policy_bank{'VIRUS'} = { ACTION => sub { Amavis::Util::snmp_count('UserCounter1') }, }; $policy_bank{'EICAR_TEST'} = { log_templ => $log_short_templ . ', EICAR test message, not to worry', final_destiny_by_ccat => { CC_VIRUS() => D_BOUNCE }, }; $policy_bank{'MASS_VIRUS'} = { # mute everything using a big hammer final_destiny_by_ccat => { CC_VIRUS() => D_DISCARD }, warnsender_by_ccat => { REPLACE => 1 }, warnrecip_maps_by_ccat => { REPLACE => 1 }, quarantine_method_by_ccat => { REPLACE => 1 }, admin_maps_by_ccat => { REPLACE => 1 }, newvirus_admin_maps => [], log_templ => 'MASS VIRUS DROPPED, ' . $log_templ, ACTION => sub { Amavis::Util::snmp_count('UserCounter2') }, }; - a policy bank may now be loaded based on a path name of a Unix socket receiving a connection. Example use: @listen_sockets = ( "$helpers_home/amavisd.sock1", "$helpers_home/amavisd.sock2", "$helpers_home/amavisd.sock3", ); $interface_policy{"$helpers_home/amavisd.sock1"} = 'UX-S1'; $interface_policy{"$helpers_home/amavisd.sock2"} = 'UX-S2'; $interface_policy{"$helpers_home/amavisd.sock3"} = 'UX-S3'; $policy_bank{'UX-S1'} = { ... }; $policy_bank{'UX-S2'} = { ... }; $policy_bank{'UX-S3'} = { ... }; - transparent archival quarantine is a special case of archive quarantining which retains all recipient addresses unmodified in an envelope of a message directed to a quarantine. It makes sense when $archive_quarantine_method specifies protocols 'smtp:' or 'lmtp:' or 'bsmtp:' and a dedicated server is used which guarantees these quarantined messages will *not* be delivered to recipients in the envelope. Transparent archiving is used when $archive_quarantine_to (actually the @archive_quarantine_to_maps) results in a reserved string '%a' for all recipients. Think of the '%a' as a placeholder in a replacement string, being substituted by a full original recipient address. There may be other substitution placeholders available in the future, equivalent to placeholders %l, %d, etc. in SQL query templates. Example: $archive_quarantine_method = 'smtp:127.0.0.1:7777'; $archive_quarantine_to = '%a'; @archive_quarantine_to_maps = (\$archive_quarantine_to); or: $archive_quarantine_method = ['smtp:[::1]:7777', 'smtp:127.0.0.1:7777']; @archive_quarantine_to_maps = ( { '.example.com' => '%a', '.example.net' => '%a', '.example.org' => 'quarantine@example.org', '.' => undef, } ); The envelope sender address of messages sent to an archival quarantine is still controlled by the $mailfrom_to_quarantine setting as before. When this value is undef (which is a default) the envelope sender address remains unchanged - is the same as in a received message. Any other value replaces the original sender address, so an empty string implies a null return path. When delivering quarantine messages to a dedicated SMTP server it must be ensured that the receiving server will not bounce or reject quarantine messages or deliver them to recipients specified in the SMTP envelope! If a receiving SMTP server announces a DSN capability in its response to EHLO, amavisd will add option NOTIFY=NEVER with each recipient to prevent potential backscatter. This is an additional safeguard to prevent potential backscatter, therefore it is recommended that the receiving quarantine server implements and announces the DSN capability. Specifying an empty string for the $mailfrom_to_quarantine achieves the same effect (a null return path implies NOTIFY=NEVER) thus preventing backscatter, but loses original sender address in the envelope. Suggested by Patrick Ben Koetter. - as a convenience, two pre-defined logging templates are provided: $log_short_templ and $log_verbose_templ. The former is the same as was a default $log_templ in previous versions, the later is quite verbose and provides most of the interesting information about a message. An initial value of $log_templ is taken from the $log_short_templ. To change $log_templ, either assign a new template directly as before, or, as a shorthand if $log_verbose_templ is appropriate, just assign it, e.g.: $log_templ = $log_verbose_templ; - added a configuration variable @debug_recipient_maps. Similarly to @debug_sender_maps, a debug level logging is temporarily turned on for the duration of processing of this message when a recipient address matches a list of lookup tables @debug_recipient_maps; suggested by Patrick Ben Koetter; - internal: a delivery_method() is now a property of a recipient instead of being a property of a message as a whole. This makes per-recipient forwarding method selection possible. When recipients of a multi-recipient message specify different forwarding methods, a message is forwarded in multiple transactions, one for each unique delivery_method() setting; i.e. recipients are clustered into sets with the same delivery_method setting and a message for each subset of recipients if forwarded as one transaction; - finer custom control over the forwarding method is available through a before_send() custom hook which may override the $r->delivery_method(...) for all or just some of the recipients with whatever forwarding method specification is suitable - for example a next-hop server's IP address or its port number can be chosen based on spam score or based on a sender domain or some other characteristics of a message or of a recipient; - a new configuration variable @forward_method_maps (along with making a delivery_method() a property of a recipient instead of being a property of a message) makes per-recipient forwarding method selection possible; suggested by Ralf Hildebrandt; Example: @forward_method_maps = ({ # use lowercase keys with hash-type lookups! 'user@example.com' => 'smtp:[::1]:10025', '.sub1.example.com' => 'smtp:[::1]:10026', '.sub2.example.com' => 'smtp:[::1]:10027', '.example.net' => 'smtp:[127.0.0.1]:10025', '.example.org' => [ 'smtp:[192.0.2.9]:125', 'smtp:[2001:db8::f]:125' ], '.' => $forward_method, }); - a new configuration variable %forward_method_maps_by_ccat allows the forward_method_maps to depend on content type and allows per-recipient specification of a forward method, such as specifying a next hop MTA's IP address and port number. This offers new possibilities to control mail routing for purposes like implementing sender reputation schemes which dynamically choose an SMTP source IP address (typically of outgoing mail) based on the contents of a mail message or based on recipient's e-mail address or domain. This needs to be complemented by a suitable configuration of an MTA, such as Postfix 2.7.0 or later. The default is to use the $forward_method setting, ensuring compatibility. There is no need to specify entries for content types which are not being forwarded (often: CC_VIRUS, CC_BANNED, CC_SPAM). Example use: $forward_method = 'smtp:[127.0.0.1]:10025'; %forward_method_maps_by_ccat = ( CC_BADH.',3', [ 'smtp:*:10027' ], CC_BADH.',4', [ 'smtp:*:10027' ], CC_BADH.',5', [ 'smtp:*:10027' ], CC_BADH.',6', [ 'smtp:*:10027' ], CC_BADH.',8', [ 'smtp:*:10027' ], CC_SPAMMY, [ 'smtp:[192.0.2.22]:10025' ], CC_CATCHALL, sub { ca('forward_method_maps') }, ); - added a global configuration setting $allow_preserving_evidence, defaults to true. Turning it off disables preserving temporary files (as evidence) in case of trouble, which is potentially useful for unattended and unmonitored operation. The setting has no influence on preserving evidence in case of @debug_sender_maps or @debug_recipient_maps triggering, which always preserves evidence; - an entry for CC_UNCHECKED was added to %admin_maps_by_ccat, defaulting to @virus_admin_maps. Hence administrator notifications are also sent for messages which cannot be decoded (e.g. are encrypted or contain a mangled archive) if virus administrator notifications are enabled. To turn off sending administrator notifications for unchecked contents: delete $admin_maps_by_ccat{&CC_UNCHECKED}; - to avoid a need to directly manipulate the *_by_ccat settings, a set of individual configuration variables associated with CC_UNCHECKED contents category was added, with their default values compatible with earlier versions of amavisd: $final_unchecked_destiny = D_PASS; $unchecked_quarantine_method = undef; $unchecked_quarantine_to = 'unchecked-quarantine'; @unchecked_quarantine_to_maps = (\$unchecked_quarantine_to); LDAP attribute: amavisUncheckedQuarantineTo SQL field: unchecked_quarantine_to - a failure of all virus scanners no longer automatically tempfails the operation, but flags a message with a CC_UNCHECKED contents category (just like a failure of decoders/dearchivers), and allows the usual controls (*_destiny, *_quarantine_*) to be used to choose behaviour; for example: $final_unchecked_destiny = D_TEMPFAIL; $unchecked_quarantine_method = 'local:unchecked/%m.gz'; To revert to a previous behaviour where a failure of all virus scanners resulted in a temporary failure, set the $virus_scanners_failure_is_fatal to true, e.g.: $virus_scanners_failure_is_fatal = 1; The setting $virus_scanners_failure_is_fatal is a member of policy banks. - support decompression of a .xz file format and legacy .lzma file formats through XZ Utils ( http://tukaani.org/xz/ ) if an entry with a decoding program is found in the @decoders list; it defaults to finding a program 'xz' or 'xzdec' in the $path ; - added two new functions: iso8601_year_and_week() and iso8601_yearweek() to accompany the existing function iso8601_week(); they all provide a week-of-the-year number (ISO 8601 / EN 28601, 1..53, in local time zone) given a Unix timestamp (seconds since 1970-01-01T00:00Z) as an argument, optionally together with the corresponding year number. The result is an integer or a pair of integers as follows: $w = iso8601_week($unix_time); # e.g. 49 $yw = iso8601_yearweek($unix_time); # e.g. 201049 ($y, $w) = iso8601_year_and_week($unix_time); # e.g. (2010,49) Semantics is equivalent to PostgreSQL extract(week from ...), and to MySQL week(date,3). These functions can be useful for assigning to a $partition_tag (in amavisd.conf), e.g.: $partition_tag = sub { my($msginfo)=@_; iso8601_week($msginfo->rx_time) }; or: $partition_tag = sub { my($msginfo)=@_; iso8601_yearweek($msginfo->rx_time) }; or based on a day of a week for short-term cycling (Mo=1, Tu=2,... Su=7): $partition_tag = sub { my($msginfo)=@_; ((localtime($msginfo->rx_time))[6]+6)%7+1 }; (a note from a future: starting with 2.8.0 the following is equivalent: $partition_tag = sub { my($msginfo)=@_; iso8601_weekday($msginfo->rx_time) }; ) Suggested by Michael Scheidell. - the two placeholders %k and %a in templates for SQL lookup clauses $sql_clause{'sel_policy'} (i.e. $sql_select_policy) and $sql_clause{'sel_wblist'} (i.e. $sql_select_white_black_list) were augmented by four new placeholders: %l, %u, %e, and %d, potentially facilitating forming of more complex SQL queries; suggested by Marco Fretz. The following replacements are made: %a -> exact/unmodified e-mail address (same as the first entry in %k) %l -> full unmodified localpart (all up to, but not including the '@') %u -> lowercased username (a localpart without extension) %e -> lowercased address extension (including a delimiter), if any %d -> lowercased domain (without '@') %k -> a list of lookup strings, as before (see below) For example, given an e-mail address: User+Foo@Sub.Example.COM the placeholders would be substituted by: %a User+Foo@Sub.Example.COM %l User+Foo %u user %e +foo %d sub.example.com %k User+Foo@sub.exAMPLE.COM user+foo@sub.example.com user@sub.example.com user+foo user @sub.example.com @.sub.example.com @.example.com @.com @. - per-recipient (or per- policy bank, or global) SpamAssassin configuration files or SQL configuration sets are now supported (the @sa_userconf_maps setting, a policy.sa_userconf SQL field). A multi-recipient message whose recipients map to different configuration sets will be checked by calling SpamAssassin multiple times, once for each unique SpamAssassin configuration set. A configuration set is either a filename, or a set of SQL records obtained from SpamAssassin's user_scores_dsn SQL database by calling its method load_scoreonly_sql(). A lookup on a list of lookup tables @sa_userconf_maps may return undef or an empty string implying no user preferences file, or may provide a file name (absolute path, or relative to $MYHOME) of a SpamAssassin's 'user preferences' configuration file, or may start with a string 'sql:' which implies loading user preferences from a user_scores_dsn SQL database (as declared in a SpamAssassin's configuration file) for a username provided by a lookup on @sa_username_maps (see further down). SpamAssassin will be requested to load a user preferences configuration through its read_scoreonly_config() or load_scoreonly_sql() method, which otherwise (in spamd) serves to load user's .spamassassin/user_prefs file or SQL preferences when switching users. See SpamAssassin documentation file sql/README for SQL details. SpamAssassin's SQL database is only consulted if user_scores_dsn is declared in a SpamAssassin configuration file, and the @sa_userconf_maps returns a string starting with 'sql:' (case insensitive, the rest of the string is currently ignored). If a username as provided by a lookup on @sa_username_maps equals the username under which amavisd was started, SpamAssassin's SQL preferences for that username will not be loaded - it is assumed that preferences for a default username are empty, i.e. that it uses a default SpamAssassin configuration. Each time that currently loaded configuration needs to be replaced by another or restored to a systemwide default, an initial SpamAssassin configuration is restored through SpamAssassin's copy_config() method. Note that saving an original SpamAssassin configuration, loading a user configuration, and restoring to the original configuration does not come cheap: it can take 200 ms for a load and restore, and 370 ms for the initial saving of the configuration (saving is only done once per child process, and only if needed). Saved configuration can occupy additional 2 MB of virtual memory, so use the feature sparingly. No penalty occurs until a child process does its first loading of a user configuration, so rarely activated or inactive policy banks or per-recipient setting using this feature do not cause any additional processing or occupy additional memory. According to SpamAssassin documentation, a user preferences file or SQL preferences can include scoring options, scores, whitelists and blacklists, etc. If 'allow_user_rules' is enabled (local.cf), then user preferences file can also include rule definitions and privileged settings - but not administrator settings. The feature is only available since SpamAssassin 3.3.0. Example: @sa_userconf_maps = ( { 'user1@example.com' => '/etc/mail/spamassassin/special_user_config', '.example.org' => 'sql:', } ); Based on a suggestion by Alexander Wirt and initially based on his patch; - added a global configuration setting $sa_num_instances with a default value of 1, which is the only sensible setting for sites not using per-recipient SpamAssassin configuration switching (as described in the previous section). The $sa_num_instances controls the number of Mail::SpamAssassin objects (instances) created by a parent amavisd process during a startup. Each SpamAssassin instance does its own initialization (loading of rules and configuration settings) during a program startup and occupies a sizable portion of virtual memory (like 7 MB on a 64-bit platform with SA 3.4 rules). When switching SpamAssassin configurations (@sa_userconf_maps), and given more than one instance of the Mail::SpamAssassin object, amavisd has a choice of picking an instance which may already have loaded a selected user configuration file previously, and thus save some time by not having to store and reload SpamAssassin state again. This may be beneficial for example when a sizable portion of users use a default SpamAssassin configuration, while other users need a per-user or per-domain preferences settings; Note that as of SpamAssassin 3.3.2 some features (like compiled rules) are global and not a property of a SpamAssassin instance object. The problem is tracked in the SpamAssassin project as Bug 6236. Until this is resolved please consider the feature experimental. - per-recipient (or per- policy bank) SpamAssassin SQL database usernames are supported (setting @sa_username_maps, a policy.sa_username SQL field). This makes it possible to implement per-user or per-user-group or per-domain Bayes databases when SpamAssassin is configured to keep its Bayes database on an SQL server. It also makes it possible to load per-recipient SpamAssassin preferences (configurations) from an SQL database (as described in a previous section). Switching between Bayes usernames is cheap compared to switching between SpamAssassin configuration files. A multi-recipient message whose recipients map to different usernames will be checked by SpamAssassin multiple times, once for each unique username; Example: @sa_username_maps = ( { 'user1@example.com' => 'user1', 'user2@example.com' => 'user2', '.example.com' => 'user_ex', } ); - passes a value of $originating flag to SpamAssassin through its suppl_attrib argument in a $spamassassin->parse call; it is expected that this information would be treated by SpamAssassin 3.4.0 similarly to msa_networks; - a new configuration variable $mail_id_size_bits allows setting the size of randomly generated mail_id and secret_id codes which are used to identify a message on releasing it from a quarantine, and are used as a key when logging to SQL (penpals) or storing to quarantine. The variable specifies a length of mail_id in bits, and must be an integral multiple of 24 (i.e. must be divisible by 6 and by 8). The mail_id is represented externally as a base64url-encoded string of $mail_id_size_bits / 6 characters, and internally as a string of $mail_id_size_bits / 8 octets. The default value is 72 bits, as in previous versions. Sensible values are 48, 72 and 96 bits. See entry "introduce a concept of 'mail_id'" in amavisd-new-2.3.0 release notes for probability analysis of collisions. The default size should fit all practical current needs. The size of SQL fields msgs.mail_id, msgs.secret_id, msgrcpt.mail_id and quarantine.mail_id may need increasing to accommodate $mail_id_size_bits/6 characters if using a non-default value of $mail_id_size_bits. See also the next entry regarding a type for these fields. - SQL fields msgs.mail_id, msgs.secret_id, msgrcpt.mail_id and quarantine.mail_id must be treated case-insensitively. A suitable data type for these fields in PostgreSQL is bytea, and varbinary in MySQL (of size 12 or 16 characters). In order not to lose entropy in mail_id, and not to increase a probability of collisions, please check existing database schema and adjust as necessary, either a data type, or chose a case-sensitive collation setting. See README.sql-pg and README.sql-mysql for an ALTER command to change data type of these fields. - added optional SQL and LDAP lookups for @spam_tag3_level_maps (sql field: policy.spam_tag3_level, ldap attribute: amavisSpamTag3Level), and for @spam_subject_tag3_maps (sql field: policy.spam_subject_tag3, ldap attribute: amavisSpamSubjectTag3); suggested by Thomas Johnson; - added the following LDAP settings to $default_ldap: sslversion, clientcert, clientkey, cafile, capath, verify, sasl, sasl_mech, sasl_auth_id, localaddr, scheme, inet6; allows bind authentication with a certificate or SASL and allows connections to an LDAP server over IPv6; based on a patch by Christian Roessner; - added macros ADDEDHEADERHAM and ADDEDHEADERSPAM, which expand to newly generated header fields which SpamAssassin prepared for insertion into a header section, in case the message is eventually declared to be nonspam or spam respectively; this information is available from SpamAssassin since version 3.3.0; - macro 'dkim' now recognizes two additional keywords: 'selector', 'sig_sd', and 'newsig_sd', see README.customize for details; in particular, the 'newsig_sd' allows to add information on newly applied signature to a main log entry, and is now included in a default log template; - added a macro 'mime2utf8' which takes a string as its first argument, and an optional truncation length as the second. The string is decoded as a MIME-Header string (understands Q or B character set encodings like =?iso-8859-2?Q?...?=, =?koi8-r?B?...?=) and is converted to UTF-8, optionally truncated to the specified size at clean UTF-8 boundaries, and returned as a result. Suggested by Bastian. The macro is useful in a logging template or in notification templates to decode Subject or From header fields, e.g.: [? [:header_field|Subject]||,\ Subject: [:dquote|[:mime2utf8|[:header_field|Subject]|100]]]# - added a macro 'client_helo', which provides a client-supplied EHLO/HELO domain name of the original SMTP session. The information is obtained through an XFORWARD extension to an SMTP protocol as provided by Postfix, or through a 'helo_name' attribute in an AM.PDP request. Add something like: , helo=[:client_helo]# to the log template if it needs to be logged. Suggested by Xueron Nee; - added a macro 'client_addr' which is a synonym for macro 'a'; the information is obtained through an XFORWARD extension to an SMTP protocol as provided by Postfix, or through a 'client_address' attribute in an AM.PDP request; if neither of these are available, the client's IP address is parsed from a topmost Received header field; - added a macro 'client_port', yields a TCP source port number of an original SMTP session; the information is obtained through an XFORWARD extension to an SMTP protocol as provided by Postfix, or through a 'client_port' attribute in an AM.PDP request. See RFC 6302; - added a macro 'client_addr_port' which combines a client's IP address and a TCP source port number (if available) of an original SMTP session; it is similar to: \[[:client_addr]\]:[:client_port] or when a port number is not available: \[[:client_addr]\] This macro is now included in a default main log template, so the TCP source port number is logged along its IP address. This information is useful in reporting abuse (e.g. client behind a NAT), troubleshooting, forensics and law enforcement. If this information is not desired, one may assign a customized template to the $log_templ configuration variable. See RFC 6302: Logging Recommendations for Internet-Facing Servers. Suggested by Rok Potočnik. - added a macro 'banned_parts_as_attr' and an associated per-recipient attribute banned_parts_as_attr(); it provides the same information as a macro 'banned_parts' and its associated attribute banned_parts(), but uses a different syntax, possibly facilitating parsing and reporting names or types or location of banned parts. The following example illustrates the difference. A result is a single string in both cases, wrapped here for clarity, and shows a path in a message tree of a banned leaf node: a macro 'banned_parts' can yield: multipart/mixed | application/octet-stream,.rar,Setup1.1.rar | .exe,.exe-ms,setup.exe while a macro 'banned_parts_as_attr' yields: P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/octet-stream,T=rar,N=Setup1.1.rar | P=p007,L=1/2/4,T=exe,T=exe-ms,N=setup.exe for the same banned part in a message. The single-character attribute names are unchanged from previous versions. For documentation, here is a legend: P: part's base name, i.e. a file name in a ./parts/ temporary directory L: part's location (path) in a mail tree (branch enumeration, top-down) M: MIME type as declared in MIME header fields of a message T: short part's content type according to a file(1) utility and mapped through @map_full_type_to_short_type_maps N: declared part names (none, one or more), as declared in MIME header fields or in an archive (tar, zip, ...) A: part's attributes as follows: U=undecodable, C=crypted, D=directory, S=special(device), L=link - macro 'header_field' and its alias 'HEADER' now has an optional third parameter (index), which chooses the header field in case of multiple header fields of the same name; the default (-1) is to return the last (bottommost) occurrence, as before; see README.customize for details; - added a macro 'actions_performed', which expands into a comma-separated list of words: Accepted, Relayed(Untagged), RelayedTagged, Discarded, Rejected, Bounced, NoBounce or TempFailed, followed by a mail flow direction word: Inbound, Internal, Outbound or OpenRelay. For brevity the 'RelayedUntagged' status appears in this list as 'Relayed'. Additionally, the list may include words Quarantined and Archived. For multirecipient messages it is possible that the list includes more than one combination. The purpose of this macro is to augment the bare-bones 'Passed CLEAN' or 'Blocked SPAM' in the main log entry. For this purpose the default log template now includes this macro call. If the additional information is not desired in the log, please assign a customized template to the $log_templ configuration variable. Some examples of the new log entries: Passed CLEAN {RelayedOutbound}, ... Passed CLEAN {RelayedInbound}, ... Passed CLEAN {RelayedInternal,RelayedOutbound}, ... Passed SPAMMY {RelayedTaggedInbound}, ... Blocked SPAM {RejectedInbound,Quarantined}, ... Blocked INFECTED (Mal/BredoZp-B) {DiscardedInbound,Quarantined}, ... Semantics of entries in the 'actions_performed' list corresponds to the newly added SNMP variables 1.3.6.1.4.1.15312.2.1.1.19 - .26 (with the exception that 'RelayedUntagged' counter is abbreviated in this macro as 'Relayed'). Please see their detailed description in a file AMAVIS-MIB.txt . - added a macro 'rusage', which expands to a resource usage entry as provided by a system service getrusage(2); the argument to the macro should be one of the field names in the structure rusage (see getrusage(2) man page), e.g. ru_utime, ru_stime, ru_maxrss, ru_ixrss, ru_idrss, ru_isrss, ru_minflt, ru_majflt, ru_nswap, ru_inblock, ru_oublock, ru_msgsnd, ru_msgrcv, ru_nsignals, ru_nvcsw, ru_nivcsw; the information is only provided if an optional perl module Unix::Getrusage is available; - SpamAssassin-compatible macros 'YESNO' and 'YESNOCAPS' now optionally accept two string arguments, replacing the default strings 'Yes' and 'No' in the result; - settings $enable_dkim_verification and $enable_dkim_signing are now dynamic, i.e. became members of policy banks, thus facilitating selectively enabling or disabling these features on a policy bank basis; - internal: added a message property object dkim_signwith_sd() which allows custom hooks to provide the DKIM signing code with a selector and a domain name preferred for choosing a signing key. If this information is not available the signing code will consult an external signing service if provided ($dkim_signing_service), or else use the built-in algorithm for choosing a signing key. A custom hook may provide the information as follows: $msginfo->dkim_signwith_sd( ['some_selector', 'some_domain'] ); After a successful signing, the dkim_signwith_sd will contain a pair [selector,domain] which was actually chosen for signing; - recognize and insert header fields as prepared by SpamAssassin 3.3.0 or later through its 'add_header' configuration option; some of the standard X-Spam-* header fields are still overruled by equivalent ones generated by amavisd itself, primarily to provide true per-recipient handling; header field names must still be listed in the associative array %allowed_added_header_fields in order to be inserted; overrides are configurable through %prefer_our_added_header_fields, for example: $prefer_our_added_header_fields{lc('X-Spam-Status')} = 0; - added an attribute 'log_id' to server responses in an AM.PDP protocol, allowing the client to match its request with the amavisd daemon logging; - added LDAP attributes: amavisAddrExtensionVirus, amavisAddrExtensionSpam, amavisAddrExtensionBanned, and amavisAddrExtensionBadHeader for consistency with SQL; suggested by Stefan Palme; - added LDAP attributes: amavisSpamTag3Level, amavisSpamSubjectTag3, amavisUncheckedQuarantineTo, amavisCleanQuarantineTo, amavisUncheckedLover, amavisForwardMethod, amavisSaUserConf and amavisSaUserName for consistency with SQL; - added LDAP attribute amavisDisclaimerOptions, along with its corresponding SQL field 'disclaimer_options'. It finds its way to the list of lookup tables @disclaimer_options_bysender_maps, so the replacement of the _OPTIONS_ placeholder in @altermime_args_disclaimer could be made dynamic; suggested by Quanah Gibson-Mount; - for consistencly, added LDAP attribute amavisUncheckedLover, along with its corresponding SQL field 'unchecked_lover' and a statical list of lookup tables @unchecked_lovers_maps, which appears in the %lovers_maps_by_ccat. Previously the CC_UNCHECKED entry of the %lovers_maps_by_ccat was (ab)used, and shared the @virus_lovers_maps value. Suggested by Patrick Ben Koetter; - added a variable $myprogram_name, which defaults to a program name (perl variable $0), but may be modified in a configuration file typically depending on a value of $instance_name. It is used to dynamically change a process name in $0, which shows up in a ps(1) and top(1) output on most Unix systems. Along with $syslog_ident, it offers a handy way to distinguish amavisd instances. - if running other spam scanners besides SpamAssassin through a @spam_scanners mechanism (such as DSPAM or CRM114), make header fields produced by them visible to SpamAssassin too, so that its rules can benefit from additional information. Note that in order for SpamAssassin to be able to see such header fields from other scanners, such scanners must be listed in the @spam_scanners list *before* the 'SpamAssassin' entry. Suggested by Marco. OPTIMIZATIONS - rewritten a code section on receiving SMTP/LMTP DATA, replacing perl line-by-line reads & processing by reading & processing 32 kB chunks of data at a time; as a result, data transfer rate has been increased by a factor of about 3.9 for plain text session, and by a factor of 11 for encrypted (TLS) session. Measured data rates of an SMTP DATA transfer between Postfix and amavisd on a loopback interface: No TLS (no session encryption): . amavisd receiving, old code: 8.3 MiB/s . amavisd receiving, new code: 32.3 MiB/s . amavisd sending: 18 MiB/s With TLS (encrypted session, AES256-SHA): . amavisd receiving, old code: 1.0 MiB/s . amavisd receiving, new code: 11.2 MiB/s . amavisd sending: 4.3 MiB/s - save about 6 MB of virtual memory per amavisd child process by properly deleting some larger data items from variables, known not to be reused; thanks to the insight of Perl Monks (ikegami) in: http://www.perlmonks.org/?node_id=803515 - speed up lookup_ip_acl lookups on larger lists of CIDR network addresses (like @mynetworks and @inet_acl list) by using a radix trie (Patricia Trie) representation; the patricia trie is used when a module Net::Patricia is available and a list contains more than 20 elements; minimal required version of Net::Patricia is 1.015; - avoid entering and exiting a block in most map() and grep() calls saves on opcodes, achieving a small reduction of code size and a tiny speedup; - 'use constant' for CC_* and D_* constants allows perl to inline them; OTHER - provide a workaround for a Perl 5.8.9 bug #62502, where O_WRONLY, O_APPEND and other Fcntl constants can become tainted; this is an application of the same workaround as already applied in 2.6.3, but covers two additional code sections; the bug could manifest itself as a taint problem during opening a pipe to an external mail submission problem. This only affects perl 5.8.9; the 5.8.8 and 5.10.0 are fine. Tracked down and a patch provided by Petr Rehor; - RFC 5617 now defines an "Author Domain Signature" as a valid signature in which the domain name of the DKIM signing entity, i.e., the d= tag in the DKIM-Signature header field, is the same as the domain name in the Author Address. The change came with draft-ietf-dkim-ssp-10; previously the "Author Domain Signature" was based on an 'i' tag (identity). This change is now followed by amavisd-new 2.7.0 in macro 'dkim', in evaluation of the @author_to_policy_bank_maps list, in internal attribute dkim_author_sig(), and reflected in logging. Similarly, the evaluation of @signer_reputation_maps is now based on a signing domain ('d' tag), instead of the 'i' tag as previously. The change only affects signatures where the domain name of a signing identity is a subdomain of a signing domain (not identical to the signing domain), which is rare in current practices. - updated generating and parsing of Authentication-Results headear field according to RFC 5451 and RFC 6008 - previously it followed a draft-kucherawy-sender-auth-header. This header field is now also inserted for new DKIM signatures as just-generated and inserted to a passed internal-to-internal message when it is eligible for signing; suggested by Florian Effenberger. - added a setting $myauthservid, also a member of policy banks, which controls the "authserv-id" token in the Authentication-Results header field, according to RFC 5451. Its default value is $myhostname as before. Its value must comply with the RFC 5451 syntax ("dot-atom"). Having a separate control may facilitate setups where a message is processed by amavisd more than once, e.g. for DKIM signing of a mailing list fanout messages, where the second pass should not remove an Authentication-Results header field from a first pass. - updated ARF notifications to RFC 5965 (An Extensible Format for Email Feedback Reports); the $report_format = 'arf' implementation was based on ARF draft, now it complies with RFC 5965; - tightened some sanity limits on DKIM verification to better handle mail messages with a huge number of signatures; problem reported by Tuomo Soini; - amavisd.conf: added file types ini, lib, ocx, sys, vxd to the commented-out long list of file types in $banned_filename_re, along with a commented-out list of type names for consideration: asd, asf, asx, url, vcs, wmd, wmz; - updated default @virus_name_to_spam_score_maps with new or changed entries: [ qr'^(Heuristics\.)?Phishing\.' => 0.1 ], [ qr'^Doppelstern\.(Scam4|Phishing)' => 0.1 ], [ qr'^ScamNailer\.Phish\.' => 0.1 ], [ qr'^HTML/Bankish' => 0.1 ], thank to Giampaolo Tomassoni for Heuristics.Phishing and HTML/Bankish; - when inserting a subject tag into a Subject header field, remove existing copies of the same string first to avoid subjects like "***UNCHECKED*** Fwd: ***UNCHECKED*** Re: foo bar" ; based on a patch by Thomas Arendsen Hein; - p0f-analyzer.pl: convert an 'IPv4-mapped IPv6 addresses in alternative form' to an IPv4 address, otherwise the p0f-analyzer.pl would ignore such queries, as the p0f daemon did not handle IPv6 until version 3. The 'IPv4-mapped IPv6 addresses' is returned for an IPv4 connection when TCP/IP stack is configured to allow inet6 sockets to accept inet sessions; problem reported by Vytautas Kasparavicius; - suppress generating a non-delivery notification if a SpamAssassin test DKIM_ADSP_DISCARD is hit, honouring RFC 5617; - amavisd.conf: commented-out calls to do_ascii to match defaults in the amavisd program; the uulib code (as invoked by Convert::UUlib) has a history of stability problems, seems it is causing more grief compared to the benefits it brings; - new AV entry for 'Avira for UNIX 3.x', thanks to g0rbi, Thomas Mueller, Steffen Ille, Klaus Fuerstberger and Andreas Schulze; - add three more exception cases to mercifully ignore an EBADF I/O error due to a Perl bug on line-by-line reading; - entries 'SpamAssassin' and 'SpamdClient' in the @spam_scanners list now recognize options 'mail_body_size_limit' and 'score_factor', to match their behaviour with 'DSPAM' and 'CRM114' entries; - dropped a logging level (from -1 to 2) on a warning message: INFO: dot-stuffing error (only one leading dot): ... as Postfix in a pre-queue proxy filtering setup does not do any dot-stuffing sanitation, so garbage in the DATA section as received by a Postfix smtpd service comes unchanged to a proxy filter; reported by Ralf Heidenreich, confirmed by Victor Duchovni and Wietse Venema; - use module File::Temp to create a temporary working directory, instead of using a home-brewed code; as a result, these directory names are now a bit longer; - avoid slurping the whole directory contents into memory when recursively tidying, removing, or checking a temporary directory, when purging old database files on a restart, and when preparing a list of files to be scanned; - collect a couple of random bytes from /dev/urandom (if available) at a start of the main process and at each child process birth (when our entropy pool is rather depleted), then stir our entropy pool and perl's srand() to prevent File::Temp from working with the same pseudorandom sequence in each child process; - reworked fetching random bits from entropy pool and deriving mail id from secret id (after re-reading RFC 4086); much less of the private entropy accumulator is now exposed to observers; added a new function fetch_entropy_bytes(), dropped a function fetch_entropy(); - a macro %S no longer corresponds to sender_contact, which was a relict from times of early viruses; for compatibility with existing templates it is now equivalent to %s, but should no longer be used and might be retired or re-purposed with the next version; default notification templates were adjusted accordingly - please adjust your customized templates if using them; - drop dependency on Digest::SHA1; - README.chroot: document that sa-update needs to update rules in the jail and refresh the text somewhat; thanks to Francois Rolland; CLEANING - retired often misused settings $warnvirussender and $warnspamsender (but kept marginally useful $warnbannedsender, $warnbadhsender, and their parent %warnsender_by_ccat). To bounce or reject viruses and spam use D_REJECT and D_BOUNCE settings for corresponding $final_*_destiny. It is no longer supported to both deliver (D_PASS) a virus or spam message while also sending a notification to sender. Both retired variables are still declared for compatibility with old config files, but their value is ignored. An attempt to set their value to a non-default value produces a warning. - retired a setting $syslog_priority, it was not particularly useful since the introduction of dynamic syslog priorities with amavisd-new-2.0 . The new behaviour is equivalent to a previous $syslog_priority='debug'; The variable is still declared for compatibility with old config files, but its value is ignored. An attempt to set its value to a non-default value produces a warning. - retired a setting $SYSLOG_LEVEL, it was obsoleted by amavisd-new-2.4.0; please use the setting $syslog_facility instead, defaulting to $syslog_facility='mail' . The variable $SYSLOG_LEVEL is still declared for compatibility with old config files, but its value is ignored. An attempt to set its value to a non-default value produces a warning. - renamed $DO_SYSLOG to $do_syslog, and $LOGFILE to $logfile; old names are kept as aliases for compatibility; - retired a setting $relayhost_is_client, it became obsolete with amavisd-new-2.0. Please use a '*' in place of a host IP address and port number when amavisd should pass a checked mail message back to the same host from which the request came, e.g.: $forward_method = 'smtp:*:*'; The variable is still declared for compatibility with old config files, but its value is ignored. An attempt to set its value to a non-default value produces a warning. - retired a setting $sa_timeout, the variable is still declared for backward compatibility, but has no effect. Instead, the time available for spam scanning is automatically determined from $child_timeout, taking into consideration the actual time left till the deadline; An attempt to set its value to a non-default value produces a warning. - retired a setting $sa_spam_report_header, it was obsoleted in amavisd-new-2.4.3 with the introduction of %allowed_added_header_fields. To enable insertion of X-Spam-Report header field, please use instead: $allowed_added_header_fields{lc('X-Spam-Report')} = 1; The variable is still declared for compatibility with old config files, but its value is ignored. An attempt to set its value to a non-default value produces a warning. - retired settings $sa_spam_modifies_subj and @spam_modifies_subj_maps. Disabling insertion of spam tag into a Subject header field can be achieved by turning off the corresponding entries in %subject_tag_maps_by_ccat: undef $subject_tag_maps_by_ccat{CC_SPAM()}; undef $subject_tag_maps_by_ccat{CC_SPAMMY.',1'}; undef $subject_tag_maps_by_ccat{CC_SPAMMY()}; undef $subject_tag_maps_by_ccat{CC_CLEAN.',1'}; or by emptying corresponding lists of lookup tables, e.g.: @spam_subject_tag_maps = (); @spam_subject_tag2_maps = (); @spam_subject_tag3_maps = (); or individually (by-recipient) by specifying suitable lookup tables in @spam_subject_tag_maps / @spam_subject_tag2_maps / @spam_subject_tag3_maps, either statically, or through SQL or LDAP lookups; Both settings are still declared for compatibility with old config files, but their value is ignored. An attempt to set the value of a variable $sa_spam_modifies_subj to a non-default value produces a warning. - retired a setting $insert_received_line, it was obsoleted in amavisd-new-2.4.3 with the introduction of %allowed_added_header_fields. To disable insertion of a Received header field, please use instead: $allowed_added_header_fields{lc('Received')} = 0; The variable is still declared for compatibility with old config files, but its value is ignored. An attempt to set its value to a non-default value produces a warning. - retired a setting $notify_xmailer_header, the X-Mailer header field is not inserted into notifications, as was a default. The variable is still declared for compatibility with old config files, but its value is ignored. An attempt to set its value to a non-default value produces a warning. - retired a setting $sa_auto_whitelist, it became obsolete with amavisd-new-2.1.0 and SpamAssassin 3.0.0 (released in 2004) by a 'use_auto_whitelist 1' option in local.cf . The variable is still declared for compatibility with old config files, but its value is ignored. An attempt to set its value to a non-default value produces a warning. - retired a deprecated macro 'x-mailer', use macros 'header_field' or 'useragent' instead; - removed a constant CC_TEMPFAIL, it was retired with amavisd-new-2.5.0; - renamed a configuration variable $sql_partition_tag to $partition_tag in order to reflect its more general usage outside of SQL; the old name $sql_partition_tag is retained for compatibility and is an alias for $partition_tag; - dropped compatibility of $final_*_destiny settings with old numerical values (used by versions of amavisd older than amavisd-new-20030314); - internal: retire Amavis::In::Message::client_addr_mynets, no longer in use; - internal: retire Amavis::In::Message::sender_contact, no longer in use; - internal: retire Amavis::In::Message::PerRecip::infected, no longer in use; - internal: drop a redundant argument $conn from the following subroutines: make_received_header_field, check_header_validity, defanged_mime_entity, msg_from_quarantine, check_amcl_policy, postfix_policy, add_forwarding_header_edits_common, add_forwarding_header_edits_per_recip, prepare_modified_mail, do_notify_and_quarantine, do_quarantine, save_info_preliminary, save_info_final, mail_dispatch, dispatch_from_quarantine, virus_scan, spam_scan, white_black_list, Amavis::SpamControl::{ExtProg,SpamdClient,SpamAssassin}::check As a compatibility measure, the do_quarantine() may still be called with or without the first argument $conn, its value is now ignored if present. Although the $conn argument is also redundant in calls to custom hooks (as this information is available through $msginfo->conn_obj), these calls are left unchanged for compatibility with existing custom hooks. --------------------------------------------------------------------------- May 18, 2011 amavisd-new-2.6.6 release notes This version is strictly a maintenance release, it incorporates bug fixes backported from 2.7.0-pre* series and/or posted as patches to the mailing list. BUG FIXES - amavisd-release was not sending a 'mail_file' attribute when a quarantined message was a non-compressed file in a single-level directory quarantine (not SQL-based), causing a release failure; reported by Jarno Huuskonen; - quarantining to SQL was sporadically failing, reporting some unrelated random error (like 'not available' or 'OpenSSL error: header too long'); reported by Tonio; - avoid a warning "_WARN: Use of uninitialized value in string eq at ... line 275." when an SQL-based white/black-listing is used; reported by Tonio; - wrap the sql clause SET NAMES 'utf8' so that only a warning at a log level 2 is issued if an SQL server does not understand the command (SQLite, old versions of MySQL) instead of aborting; reported by Roland Holzner; --------------------------------------------------------------------------- April 7, 2011 amavisd-new-2.6.5 release notes This version is strictly a maintenance release, it incorporates bug fixes backported from 2.7.0-pre* series and/or posted as patches to the mailing list. BUG FIXES - when a back-end MTA rejected a message, amavisd would send a non-delivery status notification, but also propagate the reject status back, which is wrong, only one or the other response would be appropriate. A fix also allows choosing either a D_REJECT, D_BOUNCE or D_DISCARD response for such a case, configurable through %final_destiny_by_ccat at a CC_MTA entry, defaulting to D_REJECT; reported by Peer Heinlein; - checking header section syntax could take excessive amounts of time in some degenerate cases of a very long header section, now fixed; - do not bypass spam checking of a bounce message when its referenced domain in Message-ID is non-local but pen pals are disabled; reported by Stefan; - removed some of the guesswork in bounce killer to prevent false positives in certain cases of forwarding a mail message as an attachment, at the expense of passing through some undesired but nonstandard bounces; (also, deal with non-delivery notifications from yahoogroups.com, and fixed one particular case of a false-positive in bounce killer (mixed/multipart with an attached full message, sent through a mailing list); - fixed a 'Zoo archive' entry in the $map_full_type_to_short_type_re list; - fixed a test for $myhostname being a FQDN to allow IDN domains (with a dash); - fixed a REPLACE hack (feature introduced in 2.6.2) on loading a policy bank; - fixed choosing the module IO::Socket::INET in ask_daemon_internal() to avoid versions of IO::Socket::INET6 older than 2.55 (2.56?) failing with "Address family not supported by protocol family" when an IPv4 address with a port number is specified for connections to a virus scanner; based on a patch by Phil Pearl (Lobbes); - do_unzip: avoid testing a version of Compress::Raw::Zlib, the module may not be loaded at all and the test would fail, resulting in inoperative zip unpacking; reported by Tuomo Soini; - when logging or quarantining to SQL, execute a clause: SET NAMES 'utf8' after connecting to a database to ensure the decoded Subject and From header fields are correctly interpreted by an SQL server as UTF-8 encoded strings. It seems the module DBD::mysql does not observe a MySQL setting for 'character_set_client' and needs an explicit SET NAMES. The problem did not affect PostgreSQL. Reported by Zhang Huangbin; - avoid LDAP lookups aborting the scan when a %d placeholder is used in a $default_ldap{base} setting and the resulting base does not exist in an LDAP schema; reported by Zhang Huangbin; - the amavisd-new 2.6.3 relaxed semantics of a number of hard links on a directory in TempDir::prepare(_dir), but left out an equivalent change necessary in TempDir::check, which is now fixed; the change only affects certain file system (like the one used on Mac OS X); - treat an empty PID file or a junk one-liner file the same as a nonexistent PID file; previously an empty PID file (e.g. after an unclean shutdown) would prevent amavisd from starting; problem reported by Michael Scheidell; - changed amavisd-release to only provide a 'quar_type' attribute in its request when it is reasonably sure of its appropriate value, otherwise leave the decision to the amavisd daemon; this solves releasing from a file-based quarantine when compression is not used and all files are at the top directory; reported by Voytek Eymont; - provide a workaround for a [perl #62048] bug affecting versions of perl older than (approx) 5.12.3, when a banning check if using rules in $banned_namepath_re and a lookup_re() could abort with an: Unwarranted "Malformed UTF-8 character" on certain tainted mail part names (with a valid UTF-8 representation); reported by Jakob Curdes; - provide a workaround for logging to syslog using an old version of Unix::Syslog which didn't prepare and keep its own copy of the 'ident' argument on a call to openlog(3); thanks to Bill Landry; OTHER - ensure compatibility with a new version 5.500 of MIME-Tools, which changed the way mime attributes content-disposition.filename and content-type.name are decoded, now properly respecting their declared encodings (character set). As a result, the declared (recommended) file names of MIME parts are now represented as native Perl character strings (Unicode), and as such may also end up in reported names of banned parts. Regular expressions in @banned_filename_maps, $banned_filename_re and $banned_namepath_re may also see these strings as native Perl characters, along with their MIME-encoded form. The change also affects interpretation of names with earlier versions of MIME-Tools, making them behave more like the 5.500. - amavisd.conf: exclude names starting with 'cid:' from matching the double extensions banning rule, avoiding false positives; - a small update to a default @virus_name_to_spam_score_maps; - the 'originating' flag is now passed on to SpamAssassin through its %suppl_attrib argument - potentially useful with current trunk version of SpamAssassin (treats originating mail submission as a MSA submission), and ignored by older versions; - some documentation updates; the RELEASE_NOTES file is now encoded as UTF-8, instead of ISO-8859-1; --------------------------------------------------------------------------- June 25, 2009 amavisd-new-2.6.4 release notes NOTE: When upgrading Perl to version 5.10 or planning to do so, please do not forget to add a missing /m flag to regular expressions in your existing AV entries (if you haven't already done so with a 2.6.3 upgrade), as suggested in an example file amavisd.conf in a package. Perl 5.8 does not mind missing /m flags, but with perl 5.10 the results from a virus scanner may no longer be properly recognized. See the BUG FIXES section in 2.6.3 release notes. COMPATIBILITY WITH 2.6.3 The output of amavisd-agent and contents of a database snmp.db has changed according to the now published MIB. Several new SNMP counters were added, a few retired, and some renamed. If you are parsing the output of amavisd-agent or accessing snmp.db directly, please review AMAVIS-MIB.txt, and perhaps switch to using the new amavisd-snmp-subagent. BUG FIXES - amavisd failed to start when spam scanning was disabled either by @bypass_spam_checks_maps=(1) or by @spam_scanners=(), giving: Can't locate object method "new" via package "Amavis::SpamControl" As a workaround one could use a @spam_scanners=(undef) to disable spam scanning; reported by Steve; - several decoders failed to propagate "Exceeded storage quota" exception, so the protection of AV scanners against mail bombs was ineffective; reported by Jorgen Lundman; - milter usage (AM.PDP): verbatim header edits inserted a header body of "1" instead of the correct string, for example: "Authentication-Results: 1"; - updated AV entry for BitDefender's bdscan to recognize tabs around a colon in its output; contributed by Steve; - fix parsing of a combined result from DSPAM (option --classify), as earlier versions of DSPAM did not include a signature with a combined result line; problem reported by Marijan Vidmar; NEW FEATURES SUMMARY - provide a true SNMP agent and a MIB, facilitating monitoring the health of a content filtering system, its performance and mail characteristics; - a new AV interface to SMTP-based antivirus scanners; - allow customizing SMTP-status response reason text for blocked messages; - prevent inserting fake copies of certain important mail header fields without breaking a DKIM signature; NEW FEATURES - newly supplied with the package is a program amavisd-snmp-subagent, acting as an SNMP AgentX, exporting amavisd statistical counters database (snmp.db) as well as a child process status database (nanny.db) to a SNMP daemon supporting the AgentX protocol (RFC 2741), such a NET-SNMP. It is similar to combined existing utility programs amavisd-agent and amavisd-nanny, but instead of writing results as text to stdout, it exports data to an SNMP server running on a host (same or remote), making them available to SNMP clients (such a Cacti or mrtg) for monitoring or alerting purposes. The amavisd program does not have any additional requirements, but to run amavisd-snmp-subagent the following Perl modules are required: NetSNMP::OID, NetSNMP::ASN, NetSNMP::agent, NetSNMP::default_store. All of these come with a Net-SNMP package (previously known as "ucd-snmp"), home at http://net-snmp.sourceforge.net/, FreeBSD ports: net-mgmt/net-snmp. Also, an snmpd daemon must be running on a host. It can be an snmpd from a Net-SNMP package or some other SNMP server supporting AgentX protocol. When using snmpd from Net-SNMP, just add the following to its snmpd.conf: master agentx agentXSocket tcp:127.0.0.1:705 so that amavisd-snmp-subagent will be allowed to connect to it. The setup was tested with Net-SNMP versions 5.3.2.3, 5.4.2.1 and 5.2.0. If you experience wild numbers served in Counter64 variables on a 64-bit platform, the following patch (at the server side) solves the problem: http://www.mail-archive.com/ net-snmp-users@lists.sourceforge.net/msg19502.html The patch seems to already be incorporated into version 5.3.3 of Net-SNMP, and into 5.5 (but not in 5.4.2, nor in 5.2.0). A MIB module (SNMP Management information base) is provided in a file AMAVIS-MIB.txt. It is not necessary to make it available to an SNMP server, and not even necessary for SNMP clients, but making it available to clients allows them to display data with names of variables, not just their OIDs. A query example with no MIB modules: snmpbulkwalk -v2c -c xxx host.example.com .1.3.6.1.4.1.15312.2.1 A query example when a file AMAVIS-MIB.txt is in a subdirectory ./mibs/ : snmpbulkwalk -m+AMAVIS-MIB -M-mibs -OQ -v2c -c xxx host.example.com amavis The amavisd-snmp-subagent can be started at any time, either before or after amavisd, and either before or after snmpd. It can also be restarted at any time. Also, amavisd can be restarted without having to restart amavisd-snmp-subagent, as it will automatically notice a database change and connect to a new database. Similarly, an snmpd daemon can be restarted at any time and amavisd-snmp-subagent will reconnect to it if necessary. A natural starting order is: snmpd first, then amavisd and then amavisd-snmp-subagent. Restarting amavisd will reset its counters. An SNMP client typically interprets a decremented value of a counter variable as a wraparound, which results in a large spike when graphing data. There are two common solutions to the problem: a reasonable upper limit can be provided to a client, so that a spike will be treated as invalid data and ignored, or else a AMAVIS-MIB::sysUpTime variable can be monitored, and if its value is smaller than on a previous reading, this indicates that counters were reset (i.e. amavisd was restarted) and values of counters should not be treated as wrapped on maxint. Consult your SNMP client documentation. The amavisd-snmp-subagent should have access to databases snmp,db and nanny.db in a $db_home directory (environment variable AMAVISD_DB_HOME, defaults to /var/amavis/db) and have rights to connect to an snmpd daemon. It is safe to run it as root, although perhaps not necessary. For testing purposes start amavisd-snmp-subagent from a command line using a command line option -f to let it stay in foreground, and optionally increase debug level, e.g: amavisd-snmp-subagent -f -d 5 If everything goes well, start it without -f and let it daemonize. Supplying a filename with an option -P tells a daemonized agent to write its PID to that file, and remove the file on shutdown (on receiving a signal TERM or INT): amavisd-snmp-subagent -P /var/run/amavisd-snmp-subagent.pid Some suggested sets of OIDs making up interesting diagrams (e.g. for displaying by Cacti): counters: * inMsgsStatusRelayed, inMsgsStatusDiscarded, inMsgsStatusNoBounce, inMsgsStatusBounced, inMsgsStatusRejected * inMsgs, inMsgsOriginating * inMsgsSize, inMsgsSizeOriginating * inMsgsSize, outMsgsSizeSubmitQuar, outMsgsSizeRelay * inMsgs, outMsgsRelay, outMsgsSubmitQuar, outMsgsSubmitDsn, outMsgsSubmitNotif * contentCleanMsgs, contentCleanMsgsOriginating * inMsgs, contentSpamMsgs, contentBannedMsgs, contentVirusMsgs * contentSpamMsgsOriginating, contentBannedMsgsOriginating, contentVirusMsgsOriginating * timeElapsedTotal, timeElapsedDecoding, timeElapsedVirusCheck, timeElapsedSpamCheck * procGone gauges: * procBusy, procAll * procBusy, procBusyTransfer, procBusyDecode, procBusyVirus, procBusySpam * procBusy0, procBusy1s, procBusy2s, procBusy4s, procBusy8s * procBusy15s, procBusy30s, procBusy1m, procBusy2m, procBusy4m * mtaQueueEntriesIncoming, mtaQueueEntriesActive, mtaQueueEntriesDeferred Note that even frequent or extensive SNMP queries do not burden amavisd processes. The amavisd-snmp-subagent process keeps a cache of current variable values. It queries one or the other berkeley database as needed, i.e. when cached data is stale and there was an actual SNMP query for a variable in one or the other database. When a berkeley database needs to be accessed, all its data is fetched in one quick sweep by using a database cursor with a read lock, so that data is consistent. No more than one database sweep in 4 seconds is performed, and less often when queries are less frequent and preferably batched in groups. If some time has passed since the last SNMP query (more than 4 seconds currently), resulting values are always fresh as collected from a database at the time of an SNMP query. There is one additional experimental feature - experimental in a sense that it may change or be dropped in future versions. If running Postfix on the same host as amavisd-snmp-subagent, a count of files (mail messages) in each of the Postfix queue directories is provided as Gauge32 variables in the MIB under .1.3.6.1.4.1.15312.2.1.3, i.e. under amavisMta subtree. The following SNMP variables are available: mtaQueueEntriesMaildrop, mtaQueueEntriesIncoming, mtaQueueEntriesActive, mtaQueueEntriesDeferred. Although semantically outside the scope of amavisd, it provides a quick insight into health of an MTA, and indirectly into health of amavisd. Data is made available only if a command 'postconf -h queue_directory' is successful at amavisd-snmp-subagent startup time and provides a sensible result. Like with the other two real databases, MTA directories are only scanned if and when actually queried by an SNMP client (again, subject to caching). As a safety measure for times when MTA queue grows huge, there is a time limit for scanning each directory subtree (currently 5 seconds, which is about how much a typical SNMP client is willing to wait for a response). Also, a long scan time automatically increases cache validity time (time-to-live) of that measurement. - a new experimental interface to SMTP-based antivirus scanners is provided; an @av_scanners entry may look like the following: ['av_smtp', \&ask_av_smtp, ['{}', 'smtp:[127.0.0.1]:5525', 'dummy@localhost'], qr/^2/, qr/^5/, qr/FOUND:\s*(.*?)\s*$/m ], The ask_av_smtp mechanism connects to a virus scanner using the specified protocol (typically SMTP or LMTP) on a given IP address and a port number, considering the virus scanner as an ordinary MTA. The full original message is then fed to the scanner (currently ignoring the "{}" argument), using the original envelope sender address and a given address as a single recipient (defaults to 'dummy@localhost'). It is expected that a virus scanner will accept a clean message (2xx) and reject an infected message (status 5xx). An SMTP response is parsed as usual for any output from a virus scanner, typically considering a response starting with 2 as clean, a response starting by 5 as infected, and anything else as a scanner failure. The SMTP-based virus scanner should be configured not to deliver a message. This may be achieved by feeding its SMTP output to a dummy SMTP listener, such as smtp-sink as supplied by a Postfix package. It is not a particularly efficient interfacing mechanism, but some virus scanners do not provide a choice. Prompted by Kevin M. Myer; - a new configuration variable %smtp_reason_by_ccat allows customizing SMTP-status response reason text. The reason strings are subject to macro expansion, so built-in macros are available (README.customize). Multi-line texts are allowed and produce a valid multi-line SMTP response, but use it sparingly, as some nonstandard mailers may not like it. Currently %smtp_reason_by_ccat is only consulted for blocked messages, a passed clean message still uses a hardwired reason text; suggested by Ralf Hildebrandt, based on a patch by Noah Baker; - support inclusion of null header field names in an 'h' tag of a DKIM signature generated by amavisd for specified header field names, thus preventing third parties from prepending additional occurrences of these header fields without breaking a signature; useful for example for protecting a recipient's mail reader or a filter from being tricked by supplying a duplicate From or Subject header fields; the protection is requested by specifying a value larger than 1 in %signed_header_fields, e.g.: $signed_header_fields{'from'} = 2; $signed_header_fields{'subject'} = 2; $signed_header_fields{'message-id'} = 2; $signed_header_fields{'content-type'} = 2; Please restrict values used in %signed_header_fields to 0, 1, or 2, consider other values reserved for future use. By default the following header fields are protected from duplicates by a DKIM signature generated by amavisd: From, Date, Subject, Content-Type. To revert to a classical behaviour, set their value in %signed_header_fields to 1, e.g.: $signed_header_fields{lc($_)} = 1 for qw(From Date Subject Content-Type); - add a config variable @spam_notifyadmin_cutoff_level_maps, which allows suppressing of spam administrator notifications when spam score exceeds a level resulting from a lookup into this list of lookup tables; suggested by Rudy Gevaert; - new configuration variables: $snmp_contact, $snmp_location, empty strings by default; these end up in a MIB as exported by the new amavis SNMP agent; OTHER - failure of a file(1) utility is now only logged (at syslog level LOG_ERR) and no longer treated as a fatal error; suggested by Matija Grabnar; - require a minimal version 2.017 of Compress::Raw::Zlib when unpacking a zip archive (by Archive::Zip) to avoid an archive (e.g. in an infected mail) from causing amavisd process to hang; thanks to Alexander 'Leo' Bergolth for troubleshooting the memory allocation issue in a zip library; - amavisd-nanny: write notes about lost processes to STDERR instead of to STDOUT, making it easier to use it non-interactively, e.g. from cron; suppress printing of "exited" when not interrupted; based on a patch provided by Thomas Gelf; - amavisd-agent: suppress printing of "exited" when not interrupted; - amavisd-agent: updated according to changes in MIB; - reduce a log level of a virus scanner failure from -2 to -1, unless all virus scanners have failed; - added to @virus_name_to_spam_score_maps : /^Structured\.(SSN|CreditCardNumber)\b/ /^Sanesecurity.TestSig_/ /^Email\.Spammail\b/ /^winnow\.(phish|spam)\./ /^INetMsg\.SpamDomain/ - internal: added user_policy_id and user_policy_id to recipient data in object Amavis::In::Message::PerRecip, facilitating storing users.policy_id into msgrcpt.sql_policy_id; suggested by Stefan Palme; - amavisd.conf-sample: updated comment explaining the use of %a and %k expansions in $sql_select_policy; thanks to Max-Julian Pogner; - documentation: change all "author signature" to "author domain signature" according to draft-ietf-dkim-ssp-10; --------------------------------------------------------------------------- April 22, 2009 amavisd-new-2.6.3 release notes NOTE: When upgrading Perl to version 5.10 or planning to do so, please do not forget to add a missing /m flag to regular expressions in your existing AV entries, as suggested in an example file amavisd.conf in a package. Perl 5.8 does not mind missing /m flags, but with perl 5.10 the results from a virus scanner may no longer be properly recognized. See the BUG FIXES section below. COMPATIBILITY WITH 2.6.2 - support for DSPAM has been removed from Amavis::SpamControl::SpamAssassin module, merging DSPAM scores into SpamAssassin and DSPAM autolearning is no longer available. Nevertheless, it is now possible to use DSPAM instead of SpamAssassin, or by adding results from each. See description below for @spam_scanners; - please see a note below about having to explitly re-assign the @client_ipaddr_policy list if the @mynetworks_maps (not the @mynetworks) is changed in amavisd.conf; - there are no other known incompatibilities with 2.6.2; BUG FIXES - when logging to SQL (pen pals), the msgs.message_id field always received a value '1' instead of a Message-Id, thus making pen pals less effective (only matching on sender/recipient pairs worked, not on message threads) and letting some bounces bypass a bounce killer; bug was introduced with version 2.6.2; reported by Michael Scheidell; - timer was not reset after a persistent failure to connect to a daemonized virus scanner, so a subsequent call to a backup scanner only had 10 seconds available before it was aborted, which was often too short for a command line backup scanner like clamscan; reported by Bill Landry; - if a virus scanner interface did not find a name of a virus in the output of a virus scanner (despite noticing infection), the infection was ignored; reported by Thomas Mueller; - added missing /m flags to regular expressions in AV entries (a bug is revealed with Perl 5.10.0; previous versions of Perl happened to work, unintentionally accepting a /m flag if added late during a regexp evaluation); reported by Rafael; - $banned_namepath_re setting only worked globally, but was not usable in policy banks; reported by Danny Richter; - do_uncompress: signal run_command_copy() errors, instead of returning a status, thus allowing decompose_part() to detect 'Exceeded storage quota' or 'Maximum number of files exceeded', and flag mail as CC_UNCHECKED; - if $mailfrom_notify_admin was not specified in a configuration file but defaulted to an e-mail address in $hdrfrom_notify_admin, the following was reported (due to missing angle brackets) on an attempt to submit a notification: (!)SEND via SMTP: virusalert@example.com -> ... 501 5.1.7 Bad sender address syntax (!)FAILED to notify admin: 501 5.1.7 Failed, id=40690-23, from MTA([::1]:10027): 501 5.1.7 Bad sender address syntax Notification was not sent, the rest of the processing was unaffected; reported by Peter Pechnik, Thomas Mueller, and Stefan Förster; - fetch_modules: only suppress the "Can't locate ... in @INC" diagnostics if exactly the requested module is missing, but do show the error if some subordinate module is missing and preventing the requested module to be loaded; - do_unrar: recognize an information line with a '<->'; - fixed a syntax error in LDAP.ldif; by Quanah Gibson-Mount - fixed a bug in SpamdClient; reported by Filip Valder NEW FEATURES - added a configuration variable @client_ipaddr_policy, which maps smtp client's IP address lookup lists to a policy bank name. This allows for loading a policy bank based on a client IP address, and generalizes a formerly hard-wired mapping of @mynetworks_maps into 'MYNETS'. The list is traversed in order, the first matching networks list stops the search and its associated policy name is used. Suggested by Jo Rhett. The default setting retains backward compatibility: @client_ipaddr_policy = map { $_ => 'MYNETS' } @mynetworks_maps; but please keep in mind that this assignment is made during startup before evaluating a config file, so if amavisd.conf changes the @mynetworks_maps list, the assignment to @client_ipaddr_policy needs to be re-evaluated to retain a desired default. This is not necessary when @mynetworks_maps is left untouched but only its component @mynetworks is changed. Example: @client_ipaddr_policy = ( [qw( 0.0.0.0/8 [::] 127.0.0.0/8 [::1] )] => 'LOCALHOST', [qw( !172.16.1.0/24 172.16.0.0/12 192.168.0.0/16 )] => 'PRIVATENETS', [qw( 192.0.2.0/25 192.0.2.129 192.0.2.130 )] => 'PARTNER', \@some_other_networks => 'OTHER', \@mynetworks => 'MYNETS', ); - large messages beyond $sa_mail_body_size_limit are now partially passed to SpamAssassin and other spam scanners for checking: a copy passed to a spam scanner is truncated near or slightly past the indicated limit. Large messages are no longer given an almost free passage through spam checks. Note that message truncation can invalidate a DKIM or DK signature. If using (non-default) SpamAssassin rules to assign score points to mail with no valid signatures from authors which are expected to always provide a valid signature, the message truncation can cause false positives on these rules. As a workaround, to a truncated message passed to spam scanners, amavisd inserts a header field: X-Amavis-MessageSize: mmmmm, TRUNCATED to nnnnn which can be captured by SpamAssassin rules, e.g.: header __TRUNCATED X-Amavis-MessageSize =~ m{\A[^\n]*TRUNCATED}m and used in rules like NOTVALID_EBAY to prevent them from triggering. Starting with version 3.3.0 of SpamAssassin, its DKIM plugin understands the issue and receives undamaged DKIM signature objects directly from amavisd, so the above workaround is not needed. Also, a hit on a __TRUNCATED rule is automatically generated (explicit header rule is not necessary), just in case it might be useful for some purpose. - supports passing an extra argument suppl_attrib to $spamassassin->parse, as recognized by SpamAssassin 3.3.0, passing a set of DKIM signature objects to a SpamAssassin's plugin DKIM, which saves having to do the same signature verification operation again within a plugin, and provides uncrippled signatures to SpamAssassin even when a large message is truncated by amavisd and only partially submitted to spam analysis; - add global variables $sa_configpath and $sa_siteconfigpath (undef by default), which are passed to SpamAssassin as options 'rules_filename' and 'site_rules_filename' during its initialization call; this makes it easier to run multiple instances of amavisd, each with a different SpamAssassin configuration, using the same amavisd configurations file by taking advantage of option -i; suggested by Noah Baker; - report process resource usage at log level 2 by calling getrusage(1) if a perl module Unix::Getrusage is available; - a configuration variable @spam_scanners is added, along with a module Amavis::SpamControl::ExtProg (which is only loaded if needed). This is similar in concept to @av_scanners list, and allows using amavisd with different spam scanners, not just with SpamAssassin. The default setting is backward compatible: @spam_scanners = ( ['SpamAssassin', 'Amavis::SpamControl::SpamAssassin'], ); The first element of each tuple is a scanner name, the second is a module name to be invoked, it must implement a method new(). Remaining arguments are passed to a module as arguments in a call to its new(). The exact syntax and semantics of these arguments is module-specific and may change in future versions as more experience is gained. Currently supported spam scanners are: - SpamAssassin: backward compatible, uses the module Mail::SpamAssassin directly as before; - SpamdClient: a client to spamd, equivalent to a spamc usage; the main reason for existence of this module is to allow amavisd to serve as a test client for exercising spamd; not envisaged for production use; - CRM114: spawns an external program 'crm'. A well trained crm114 system gives good results (even with a global database). An alternative is to use a CRM114 plugin to SpamAssassin, with a benefit of autolearning and combining its results with other rules, but at some processing cost; - DSPAM: spawns an external program 'dspam'; Spam score and test results from all spam scanners are added together, currently it makes most sense to only have one of these entries enabled at a time. A possible (artificial, not particularly useful) configuration with multiple entries is illustrated by the following setting: @spam_scanners = ( ['SpamAssassin', 'Amavis::SpamControl::SpamAssassin' ], ['SpamdClient', 'Amavis::SpamControl::SpamdClient' ], ['CRM114', 'Amavis::SpamControl::ExtProg', 'crm', [ qw(-u /var/amavis/home/.crm114 mailreaver.crm --dontstore --report_only --stats_only --good_threshold=8 --spam_threshold=-8) ], mail_body_size_limit => 64000, score_factor => -0.20, ], ['DSPAM', 'Amavis::SpamControl::ExtProg', $dspam, [ qw(--stdout --classify --deliver=innocent,spam --mode=tum --tokenizer=chained,noise --user), $daemon_user ], # use option --feature instead of --tokenizer with dspam < 3.8.0 mail_body_size_limit => 64000, score_factor => 1, ], ); A module Amavis::SpamControl::ExtProg implements an interface to external spawned programs. These are expected to receive a mail message on their stdin, and produce a result on their stdout (and errors on stderr). The result typically consists of some header fields the spawned spam scanner wishes to report to a caller, but can also be a complete rewritten header section or a complete rewritten mail message. The ExtProg module just collects the information it needs from the output of a scanner and discards the rest (i.e. an external scanner can not rewrite a message), so to avoid unnecessary processing, it is best to configure an external scanner to only return what is needed. Currently some post-processing of CRM114 and DSPAM results is hard-wired into the ExtProg module. Collected header fields are typically inserted into a passed message, subject to a list of allowed header fields in %allowed_added_header_fields. Some important header fields are also added to a quarantined message, but a different mechanism is involved ($msginfo->supplementary_info, the same mechanism as used in obtaining tags from SpamAssassin). The module Amavis::SpamControl::ExtProg expects two required parameters: a path to a program to spawn, and a ref to a list of command line arguments. Following these two arguments there may be options in a form of key/value pairs. Unrecognized options are ignored. Currently the only two options are: mail_body_size_limit ... the ExtProg module only feeds up to about this number of bytes (or slightly more) of a message to a spam scanner; if unspecified or undefined a default limit is $sa_mail_body_size_limit, and if that is undefined, an entire message is passed regardless of its size; score_factor ... a floating point number by which a score produced by a spam scanner is multiplied to yield a final score (with a SpamAssassin semantics, values near or below 0 are ham, values near or above 5 are spam). Note that crm114 uses opposite sign semantics, so a score_factor for this scanner should be negative. The dspam scanner produces hard-wired score -1 (innocent) or 10 (spam), which is then multiplied by score_factor to yield a final score. OTHER - supports a SpamAssassin plugin CRM114, thanks to Jules M, and to Martin Schütte for his CRM114 plugin for SpamAssassin; - updated AV virus scanner entry for ESET Software ESETS Command Line Interface to version 3.0, commenting out entries for old versions 2.7 and 2.71.12; thanks to Hugo Slabbert; - provide a workaround for a Perl 5.8.9 bug #62502, where O_WRONLY, O_APPEND and other Fcntl constants can become tainted; the bug could manifest itself as a taint problem during file-based quarantining, during MIME decoding, decoding archives and decompressing mail parts, and possibly elsewhere. The bug is triggered by a legitimate code in Archive::Zip and affects subsequent operations in amavisd for the lifetime of the child process. This only affects perl 5.8.9; the 5.8.8 and 5.10.0 are fine. - provide a workaround for a Perl I/O bug, where a bounce killer could abort with: "inspect_a_bounce_message failed: Error reading mail header section: Bad file descriptor at /usr/local/sbin/amavisd line 11486" - uncommented the qr'^MAIL$' in @keep_decoded_original_maps (amavisd.conf and amavisd.conf-sample); seems it is becoming increasingly more important for virus scanners to also see the complete undecoded message; suggested by Michael Scheidell and others; - added qr'^Safebrowsing\.' to the @virus_name_to_spam_score_maps list, suggested by Michael Scheidell; - no longer unconditionally pre-loads the following SpamAssassin modules, as not all sites are using SQL-based bayes and AWL database: Mail::SpamAssassin::BayesStore::SQL Mail::SpamAssassin::BayesStore::MySQL Mail::SpamAssassin::BayesStore::PgSQL Mail::SpamAssassin::SQLBasedAddrList These modules will be dynamically loaded if needed by SpamAssassin after daemonization (which will be logged as "extra modules loaded after daemonizing/chrooting:"). If running chrooted, these modules may not be available in a jail, so their loading can be forced by including their names in the @additional_perl_modules list (in amavisd.conf), either as absolute file paths or as module names, e.g.: @additional_perl_modules = qw( Mail::SpamAssassin::BayesStore::SQL Mail::SpamAssassin::BayesStore::MySQL Mail::SpamAssassin::SQLBasedAddrList ); - when inserting a warning into a defanged mail body, crop very long diagnostics 'WARNING: bad headers - ...' to some sane size; - when all virus scanners failed, let a 451 smtp response message only say 'ALL VIRUS SCANNERS FAILED' or 'NO VIRUS SCANNERS AVAILABLE', but leave out a detailed failure reason for each failing scanner, along with line numbers; suggested by David Schweikert; - reduce log level of a test in TempDir::prepare for a number of links left on a directory after purging it, seems like it does not play well with a file system on Mac OS X, producing an occasional warning: TempDir::prepare: directory /var/amavis/tmp/amavis-2009... has 2 subdirectories reported by Matthias Schmidt; - do not add a per-recipient contents category CC_UNCHECKED to recipients which have virus checking bypassed, and subsequently do not insert $undecipherable_subject_tag into their Subject; suggested by Jorgen Lundman; - more precise bypassing of spam checks: bypass spam checks if it is already known that a message will be blocked due to banned contents; do not bypass spam checks for virus lovers, even if a message is infected; - when checking if a valid DKIM signature is an author domain signature, treat address localpart as case sensitive, according to ADSP and RFC 5321; - testkeys command: improved reporting of signing failures; - removed X-Virus-Scanned from a list of DKIM-signed header fields; - turn on the 'require_rules' option when initializing SpamAssassin to match behaviour with spamd and a spamassassin command; - make sure a temporary directory name supplied through AM.PDP protocol is really a directory before deleting it; - files_to_scan: make sure a file encountered during directory traversal is not a symlink before changing its protection (chmod); - do_ascii: chmod a file only if it is a regular file (not a symlink) and is not readable; - do_7zip: remove the -p option with its dummy password, looks like it has been removed and now produces an 'Incorrect command line' error; - log (at log level 2) a list of loaded SpamAssassin plugins during startup; suggested by Giuseppe Ghibò; - convert_keysfile: do not print @dkim_signature_options_bysender_maps assignment when a list of options is empty and thus redundant; - convert_keysfile: make '*' in the first field equivalent to '*@*'; - internal: when deciding whether to skip spam scanning and penpals checks test for is_in_contents_category(CC_VIRUS) instead of @virusnames; - internal: removed unused method infected() from package Amavis::In::Message::PerRecip; - internal: store_mgr() now signals errors instead of returning a status; update callers of store_mgr() accordingly; - amavisd.conf-default: document that a default value of $bounce_killer_score is 0; pointed out by Michael Scheidell; --------------------------------------------------------------------------- December 15, 2008 amavisd-new-2.6.2 release notes MAIN NEW FEATURES SUMMARY - bounce killer: improved detection of nonstandard bounces; - bounces to be killed no longer waste SpamAssassin time; - tool to convert dkim-filter keysfile into amavisd configuration; - compatibility with SpamAssassin 3.3 (CVS head) regained; - rewritten and expanded documentation section on DKIM signing and verification in amavisd-new-docs.html; COMPATIBILITY WITH 2.6.1 - apart from small differences in logging and notifications, the version 2.6.2 is compatible with 2.6.1, with its configuration file and its environment; - virus scanner entries were updated (as described below, most notably by adding a regexp flag m), so be sure to update existing configuration file; updated virus scanner entries can be used with 2.6.1 too; - the %sql_clause default has changed in detail (see below), if its value is overridden in a configuration file the setting may need updating; BUG FIXES - when feeding a message by SMTP back to MTA and MTA rejects a recipient as invalid and an smtp connection cache is enabled, the SMTP protocol can get out of step, rejecting the next message in the same connection with a "503 5.5.1 Error: nested MAIL command"; this only affects (hopefully) rare sites where recipient validation is performed after content filtering instead of before content filtering; reported by Richard Smits; - logging routines reporting warnings failed to include a diagnostics message in a log, instead only a dry '_WARN:' or '_DIE:' with no explanation was logged; a bug was introduced in 2.6.1; reported by Mike Cappella; - amavisd-release: add a 'partition_tag' attribute to a release request if a specified quarantine name ends up in a partition tag string in square brackets; this feature was announced in 2.6.1 release notes, but never made it into a distribution; - amavisd-report failed on reading a message from SQL quarantine: dispatch_from_quarantine failed: read: sql select failed, DBD::Pg::st fetchrow_arrayref failed: no statement executing reported by Achraf Tangui; - while evaluating compiled regular expressions (qr), perl 5.10.0 ignores flag m when present in the final expression but not in the qr itself, causing messages containing multiple viruses not to report any virus names (mail is still considered infected, but list of names is empty). Changed regular expressions in virus entries by appending a /m flag to regular expressions in the 6th element of each entry. According to Perl maintainers this was a bug in 5.8.x and earlier, and the behaviour of perl 5.10.0 is now according to specs; reported by Martin Huber; - envelope sender address for administrator- and recipient notifications ($mailfrom_notify_admin, $mailfrom_notify_spamadmin, $mailfrom_notify_recip, %mailfrom_notify_admin_by_ccat, %mailfrom_notify_recip_by_ccat) was not expanded when their value is left unspecified in a configuration file and defaults to parsing of $hdrfrom_notify_* settings. This leads to MTA rejecting a notification from 'postmaster@${myhostname}' by a '501 5.1.7 Bad sender address syntax'. Reported by Aleksey Chudov, Jonas Jacobsson, Durk Strooisma, and Adam; - remove unintentionally hard-coded SSL certificate and key file locations stored in variables $smtpd_tls_key_file and $smtpd_tls_cert_file, they are now configurable through a configuration file as intended; - a macro 'rfc2822_sender' now returns a Sender address in a quoted form, just like its cousin 'rfc2822_from'; - when stopping or restarting amavisd, check a PID file for being stale _before_ testing whether a process exists, not the other way around; previously an unlucky starting amavisd process could hit a: Can't send SIG 0 to process [nnnn]: Operation not permitted which prevented its startup when a stale PID was reused by an unrelated process; reported by Zhang Huangbin; - error reporting improvement: localize variables $@ and $! in all DESTROY methods, thus preventing these variables from being clobbered behind the scenes (e.g. by calling eval or system routines from DESTROY), which could cause a surprising empty (or unrelated) error message being reported by surrounding eval blocks; - avoid problematic perl constructs open('|-') and open('-|') which fail to catch certain fork errors, or waits indefinitely when resources are tight; just explicitly create a pipe and call fork in subroutines run_command, run_command_consumer and in run_as_subprocess. The change possibly also solves some mystery cases where amavisd would appear to hang when resources are tight (running out of swap space or near a maxprocesses limit) instead of reporting a fork failure. Problem with fork failing without giving a reason for failure reported by Uwe Kiewel; - amavisd.conf-default: definition of %sql_clause default was out of date; reported by Roland; - releasing a non-existent message from an SQL quarantine produced an inappropriate error message about a subsequent failure, instead of reporting a missing record; reported by Rick (rn). Also let SQL treat a NULL in mail_text.partition_tag as 0 by using coalesce() - changed a $sql_clause{'sel_quar'} from: SELECT mail_text FROM quarantine WHERE partition_tag=? AND mail_id=? ORDER BY chunk_ind into: SELECT mail_text FROM quarantine WHERE coalesce(partition_tag,0)=coalesce(?,0) AND mail_id=? ORDER BY chunk_ind to facilitate transition from not having a partition_tag defined (resulting in NULL partition_tag fields in SQL) into using it as a numeric value (e.g. a week-of-the-year number); - modified AV entry for a grisoft.com virus scanner by adding a regexp flag /m to let ^ match at any line beginning of a possibly multi-line response from a virus scanner; problem reported by John Beranek; - recognize any 'ERROR:' result from a file(1) utility - not just an 'ERROR: Corrupted', and do not treat its exit status 1 as fatal, but just log a warning; - protect logging from being recursively re-entered when an error occurs during writing of a log entry; NEW FEATURES - bounce killer: improved parsing of nonstandard bounce messages (from qmail, spamarrest.com and similar) yields more effective protection against third-party bounces, including those without a Message-ID. An analysis of 1000 previously passed bounces showed that 2/3 of those are now recognized and blocked, bringing a bounce killer rate to 94 % of all received bounces (with about 4 % of passed unverifiable bounces not carrying an original mail header, and a tiny trickle of true bounces), while still ensuring that bounces (in response to our genuine outbound mail) and message disposition notifications (MDN, RFC 3798) are still received reliably. As a reminder: bounce killer is enabled by setting $bounce_killer_score to a large value, e.g. 100. This value is added to a final spam score if a message analysis determines this is a bounce to a third-party message, i.e. a backscatter. Spam score of genuine bounces is not affected. If a $bounce_killer_score value is above 20 and we know for certain the bounce will be killed, SpamAssassin scanning is bypassed, saving substantial resources when under a backscatter storm. A pre-requisite for proper operation of a bounce killer is a working SQL logging database (pen pals), or that outbound DSN messages have a Message-ID with a fully qualified domain name matching the @local_domains_maps list of lookup tables. Parts decoding must also not be disabled ($bypass_decode_parts=0), which is a default. Conditions are easily met when all mail from local users is submitted through a domain's official mailer, which goes hand in hand with the requirement for DKIM signing and for other similar anti-spoofing techniques (SPF, whitelisting by IP address in Received trace, ...). The $bounce_killer_score should not be enabled when not all outgoing mail can be identified either by a local domain name in Message-ID or by being registered in pen pals SQL database, otherwise genuine bounces and returning MDN messages will be considered spam. - to facilitate transition of DKIM signing from dkim-milter to amavisd-new, a new command-line tool is available (the extra utility code is not loaded during normal operation), taking a file name as its argument, e.g.: # amavisd convert_keysfile /var/db/dkim/keysfile.txt and writing to stdout a set of lines that may be directly included into amavisd.conf configurations file, matching semantics of a dkim-filter keys file. It can be useful during transition, or for those who prefer to specify signing keys and sender-to-key mappings as a file in a syntax compatible with options -K -k of dkim-filter, and can live with limitations of such syntax. See dkim-filter(8) man page for details on the syntax. The produced output consists of signing key declarations (calls to a procedure dkim_key), where each call normally corresponds to exactly one DNS resource record publishing a corresponding DKIM public key. When necessary output also produces an assignment to a list of lookup tables @dkim_signature_options_bysender_maps, which supplies non-default mappings of sender domains to signing keys, e.g. when third-party signatures are desired. From the dkim-filter man page: The keyfile should contain a set of lines of the form sender-pattern:signing-domain:keypath where sender-pattern is a pattern to match against message senders (with a special character "*" interpreted as "zero or more characters"), signing-domain is the domain to announce as the signing domain when generating signatures (or a '*', implying author's domain), and keypath is a path to the PEM-formatted private key to be used for signing messages which match the sender-pattern. The selector used in the signature will be the filename portion of keypath. A line starting with "/" is interpreted as a root directory for keys, meaning the keypath values after that line in the file are taken relative to that path. If a file referenced by keypath cannot be opened, the filter will try again by appending ".pem" and then ".private". '#'-delimited comments and blank lines are ignored. - DKIM verification now logs a note (at log level 2) when a signature timestamp is in future; - allow expiration time (tag x) to be requested with DKIM signing, it is now supported since Mail::DKIM 0.29; - when determining which DKIM-signing key should be applied or which disclaimer options to apply, consider also addresses in all Resent-Sender header fields. The search order is: From, followed by Resent-From and Resent-Sender address pairs traversed top-down by resent blocks, followed by Sender, and by envelope sender; - amavisd-report no longer lets amavisd strip header fields found in a quarantined message which were previously inserted by amavisd; most of X-Spam-* and X-Amavisd-* header fields are now retained in a reported message; suggested by Achraf Tangui; - support IPv6 when connecting over an INET socket to virus scanners; - support SMTP, LMTP and TCP_LOOKUP protocols also over Unix sockets; - added an LDAP attribute amavisArchiveQuarantineTo to code and to LDAP.schema; a patch was provided by Anand Palaniswamy (back in October 2006); prompted by Quanah Gibson-Mount, both of zimbra.com. - new file in the package: LDAP.ldif, same schema as in LDAP.schema, but in ldif format; contributed by Quanah Gibson-Mount and independently also by Michael Hall; - @remove_existing_spam_headers_maps is now a per-recipient list of lookup tables, so pre-existing X-Spam* header fields may be selectively removed according to preferences of individual recipients or sub-domains, e.g.: @remove_existing_spam_headers_maps = ({ 'user@example.com' => 0, 'user@office.example.net' => 1, '.office.example.net' => 0, '.' => 1, # all the rest }); - added a macro b64encode, which could be used in log template to facilitate log parsing, perhaps by using the following in a $log_templ: [? [:header_field|Subject]||, \ Subject: [:b64encode|[:header_field|Subject|100]]]# Suggested by Rajkumar S; - added a macro HEADER as a synonym for a macro header_field for compatibility with SpamAssassin; - added a configuration variable $logline_maxlen (default value is 980, lower bound is 50), allowing user to customize syslog line wrapping threshold; based on a patch by Charles A. Scheidecker; - when loading a policy bank, most entries from an associative array (hash) being loaded entirely replace entries of the same key in the currently active policy bank, but entries which are references to a hash are normally merged with existing hashes, replacing only specified key/values but leaving remaining (non-existent in a new hash) key/values pairs unchanged. In some rare cases it would be more desirable to entirely replace existing hashes, which so far was not possible. For this purpose a hack was introduced: if a hash in a policy bank being loaded contains a key name 'REPLACE' (uppercase) and its value is true, this hash replaces a current hash, instead of being merged with it. For example, a policy bank 'DITCH' when loaded replaces a hash %final_destiny_by_ccat entirely, leaving only a key CC_CATCHALL there: $policy_bank{'DITCH'} = { final_destiny_by_ccat => { REPLACE=>1, CC_CATCHALL() => D_DISCARD }, }; Without specifying a REPLACE=>1 remaining key/value pairs in a hash %final_destiny_by_ccat would stay unchanged and only the CC_CATCHALL key/value pair would be replaced by a new setting (which may not be desirable): $policy_bank{'DITCH'} = { final_destiny_by_ccat => { REPLACE=>0, CC_CATCHALL() => D_DISCARD }, }; or equivalently: $policy_bank{'DITCH'} = { final_destiny_by_ccat => { CC_CATCHALL() => D_DISCARD }, }; - rewritten and expanded documentation section on DKIM signing and verification in amavisd-new-docs.html; OTHER - tested with perl 5.10.0; - package Amavis::IO::FileHandle now supports a method READ (i.e. invoked by a perl functions sysread through a tied hash), which is needed by SpamAssassin revisions since 2008-09-25 (3.3), bringing a little speedup to transferring a message from amavisd to SpamAssassin, and avoiding a Perl I/O bug (perl bug 39060; SA: bug 5985) on some installations; - updated @virus_name_to_spam_score_maps to recognize new malware name formats used by some popular third-party ClamAV signatures (Sanesecurity, MSRBL, MBL); thanks to Mike Cappella, Gary V, Wijatmoko U. Prayitno, Steve Basford, Luca Gibelli, Bill Landry, Henrik K; - keep only one (unique) copy of each malware/spam name when infection is downgraded to spam through @virus_name_to_spam_score_maps or when reported as a virus; - macro F now only shows the first (if any) banned leaf part name, preceded by comments from a banning rule regexp (if any), instead of a list of multiple banned parts each with its full MIME/archive path. Note that Perl syntax for a comment within a regexp is: (?# ... ) For example, given the following rule... $banned_filename_re = new_RE( qr'^\.(exe-ms|dll)$(?# rule #9)', # banned file(1) types ); ... a macro expansion of macros banning_rule_key, banning_rule_comment, banning_rule_rhs, banned_parts and F will be: banning_rule_key: (?-xism:^\\.(exe-ms|dll)$(?# rule #9)) banning_rule_comment: rule #9 banning_rule_rhs: 1 banned_parts: multipart/mixed | application/octet-stream,.exe,.exe-ms,videos.exe F: rule #9:application/octet-stream,.exe,.exe-ms,videos.exe Likewise an SMTP response (with D_REJECT) would match a macro F as before and would yield: 554 5.7.0 Reject, id=42721-01 - BANNED: rule #9:application/octet\ -stream,.exe,.exe-ms,videos.exe A default administrator and recipient notification (still using a macro F in their template), a main log entry, as well as a DSN and a rejection message, will now be shorter and hopefully less confusing to an end user. A full list of banned part paths (as previously produced by a macro F) is now available under a new name as a macro 'banned_parts' and can be used in custom templates to retain previous behaviour if desired. Suggested by Andreas Schulze and Peer Heinlein; - remove a 'LIMIT 1' from default $sql_clause{'sel_penpals_msgid'} and from $sql_clause{'sel_penpals'} clauses, it is redundant and it happens to make a MySQL 5.1 optimizer choose a slow plan; investigated by Michael Scheidell; - changed a default $sql_select_policy from: SELECT *, users.id FROM users LEFT JOIN policy ON ... into a: SELECT users.*, policy.*, users.id FROM users LEFT JOIN policy ON ... MySQL and PostgreSQL are happy with a 'SELECT *, users.id', but Oracle wants 'SELECT users.*, policy.*, users.id', which is also acceptable to MySQL and PostgreSQL and shouldn't make any difference; problem reported and a solution provided by Chris Bryant; - optimize storage of DKIM signing keys when multiple calls to dkim_key() (from amavisd.conf) specify the same file to be associated with different keys - now only one copy of a private key is kept in memory; - sanitize (strip) bare CR characters in mail before DKIM-signing a message and when forwarding it over SMTP (or LMTP) protocol which prohibits CR characters outside of CRLF pairs. Previously a DKIM signature generated by amavisd on messages with embedded bare CR characters broke when passed back through Postfix (following a principle of garbage-in, garbage-out). This was mainly an issue when a message was incorrectly sanitized or a disclaimer added by an external program such as altermime, which (due to a bug in versions 0.3.10 and older) could inappropriately introduce CR characters into a message. Reported by Patrick Wong; - retain original rfc822 quoting of envelope sender address when forwarding mail, instead of using a sanitized version (de-quoted & re-quoted); - insert autolearn=... information field into an X-Spam-Status header field, similar to how SpamAssassin does it; suggested by Jonathan Skanes; - an SQL field msgs.spam_level now receives a sum of SA score plus a minimum of internally generated score boosts across all message recipients, to facilitate coarse assessment by third party utilities without having to look into msgrcpt records; previously it only reflected a SA score (but field msgrcpt.bspam_level remain unchanged, storing a sum for each individual recipient as before); - insert "AM:BOOST=boost_scores_list" into a list of triggered spam tests to make visible the internally generated per-recipient spam score boosts (like from: pen pals, soft white/black-listing, bounce killer) in the log and in the X-Spam-Status header field. The 'tests' list in X-Spam-Status or in the log (macro %T) can now look like: tests=[AM:BOOST=+1.3+0.51-1.1, BAYES_99=3.6, ...] Multiple summands appear in multi-recipient messages where boost scores differ between recipients (the list is squashed, only unique values are shown). Apparent mismatch in score addition pointed out by John Beranek. - when quarantining and generating notification for administrator and recipients, and the per-recipient contents category differs from a per-message summarized contents category, use the per-recipient contents category for lookups into settings; a desirable side effect is that recipients which are bypassing some tests (like tests for virus or spam) no longer receive a recipient notification for cases they are not interested in; undesired behaviour pointed out by Erin D. Hughes; - drop log level of 'smtp resp to NOOP' and related messages from 2 to 3 to reduce log clutter; reported by kfx; - log a warning during startup when DKIM verification is not enabled (when $enable_dkim_verification is at its default value of undef). To quench down the warning and keep DKIM verification disabled, set the $enable_dkim_verification explicitly to 0; - when mail with banned part is to be passed but defanged, provide a more informative warning in the text part: WARNING: banning rules detected suspect part(s), do not open unless you know what you are doing suggested by Gerald Macinenti; - amavisd-agent: Content*Msgs* now take as a 100% reference the InMsgs$2 counter instead of Content$1Msgs; - when exec in a forked process fails, call POSIX::_exit with exit status 6 (SIGABRT) instead of 8 which has different meanings on different OS; - ensure a BDB cursor is unlocked in put_initial_snmp_data() even in case of errors or signals during writing of the initial sys* set of SNMP variables; - provide a generous but firm 4 MB sanity limit on a header section size to avoid excessive storage requirements while parsing and storing a runaway header section; the limit also protects DKIM signature verifier on huge headers; exceeded limit does not affect other mail checks and forwarding, only access to individual header fields beyond the limit is crippled and DKIM signatures would most likely be invalidated; - do not pre-load module Mail::SPF::Query with versions of SpamAssassin 3.2.0 or later, it has been replaced by Mail::SPF; - internal: modify mail_to_local_mailbox and do_quarantine to better deal with suppressing multiple quarantining to the same mailbox, e.g. when two quarantining methods are active but point to the same file; - internal: brush up I/O modules for consistency: open method should implicitly close a previously open file, print method should print all its arguments, read methods now support reading to a buffer at an offset; - internal: change most calls to lookup() into calls to a newer lookup2() for added flexibility, adding option 'Label' to some calls to facilitate debugging; - internal: passing options to lookup2 (and to other subordinate lookup methods) as a hash instead of a hashref; - internal: renamed Amavis::In::Message::PerRecip methods: banned_keys -> banning_rule_key, banned_rhs -> banning_rule_rhs, added: banning_rule_comment and banning_reason_short; - internal: replace subroutine unique with two: unique_list and unique_ref; - internal: remove dependency on a module IO::Wrap; - internal: many rather cosmetic changes for consistency, updated comments; - README.sql-mysql: add 'ALTER table' suggestions to change CHAR to BINARY and VARCHAR to VARBINARY data types; suggested by Peter Huetmannsberger: --------------------------------------------------------------------------- June 29, 2008 amavisd-new-2.6.1 release notes BUG FIXES - avoid a bounce-killer's false positive when a message is multipart/mixed with an attached message/rfc822 (looking like a qmail or a MSN bounce) and having attached a message with a foreign Message-ID - by restricting the check to messages with an empty sender address or a 'postmaster' or 'MAILER-DAEMON' author address; - privileges were dropped too early when chrooting, causing chroot to fail (a workaround was to specify a jail directory through a command line option -R); reported by Helmut Schneider; - fix unwarranted 'run_av error: Exceeded allowed time' error when using a virus scanned Mail::ClamAV; reported by Chaminda Indrajith; - fix a bug in helper-progs/amavis-milter.c where atoi could be reading from a non-null terminated string which could result in wrong milter return status, or even cause a read-access violation; reported by Shin-ichi Nagamura; - dsn_cutoff_level was ignored if SpamAssassin was not invoked (e.g. on large messages) even if recip_score_boost was nonzero, causing a DSN not to be suppressed for internally generated large score values; reported by Bernd Probst; - add back the 'Ok, id=..., from MTA(...):' prefix to an MTA status responses on forwarded mail when generating own SMTP status response (it was lost in code transition from 2.5.4 to 2.6.0); reported by Thomas Gelf; - replaced '-ErrFile=>*STDOUT' with '-ErrFile=>\*STDOUT' in a call to BerkeleyDB::Env::new in amavisd-nanny and amavisd-agent; seems it was failing in some setups (even though it was in accordance with a BerkeleyDB module documentation); reported by Leo Baltus; - README.sql-mysql: fixed an SQL data type mismatch between maddr.id (used as a foreign key) and msgs.sid & msgrcpt.rid; they all should be of the same type, either integer unsigned or bigint unsigned; a schema as published in README.sql-mysql could not be built because of a conflict in a data type; reported by Leonardo Rodrigues Magalhães and Zhang Huangbin; NEW FEATURES - recognize an additional place-holder %P in a template used to build a file name in file-based quarantining, for example: $spam_quarantine_method = 'local:Week%P/spam/%m.gz'; A %P is replaced by a current partition tag, which makes it easier to better organize a file-based quarantine by including a partition tag (e.g. an ISO week number) in a file name or a file path. For the record, here is a complete list of place-holders currently recognized in filename templates: %P => $msginfo->partition_tag %b => $msginfo->body_digest %m => $msginfo->mail_id %n => $msginfo->log_id %i => iso8601 timestamp of a message reception time by amavisd %% => % The following example organizes spam quarantine into weekly subdirectories: cd /var/virusmails mkdir -p W01/spam W02/spam ... W53/spam (weeks 01..53) chown -R vscan:vscan W01 W02 ... W53 (weeks 01..53) amavisd.conf: $spam_quarantine_method = 'local:W%P/spam/%m.gz'; $sql_partition_tag = sub { my($msginfo)=@_; sprintf("%02d",iso8601_week($msginfo->rx_time)) }; - add a macro %P as a synonym for a macro 'partition_tag', mainly for completeness with the added place-holder %P in a file name template; OTHER - disabled a do_ascii decoder in the default @decoders list: # ['asc', \&Amavis::Unpackers::do_ascii], # ['uue', \&Amavis::Unpackers::do_ascii], # ['hqx', \&Amavis::Unpackers::do_ascii], # ['ync', \&Amavis::Unpackers::do_ascii], The do_ascii is invoking a module Convert::UUlib which in turn calls a troublesome library uulib, which has a history of security problems and on occasion misinterprets a text file as some encoded text, causing false positives (e.g. making it look like an executable); a recent false positive on base64-decoding reported by Jeffrey Arbuckle; a recent DoS (looping in uulib) reported by Thomas Ritterbach; - added a rule into $map_full_type_to_short_type_re to cope with another example of misclassification by a file(1) utility, where a plain text file is considered a DOS executable: [qr/^DOS executable \(COM\)/ => 'asc'], # misclassified? An example was provided by Leonardo Rodrigues Magalhães; - until the issue is better understood, revert the use of 'my_require' and go back to the standard but less informative 'require'; some people were reporting problems with my_require (loading of some Perl modules can fail, apparently depending on a current directory where amavisd is started from); reports by Tuomo Soini, Max Matslofva, Bill Landry; - use the $myproduct_name value in generated Received header field instead of a hard-wired 'amavisd-new'; suggested by Thomas Gelf; - added missing required header fields to some test mail messages in a directory test-messages to quench down a complaint about a bad header; - changed SQL default clauses in %sql_clause (upd_msg, sel_quar, sel_penpals) to always join tables using both the partition_tag and the mail_id fields, previously just the mail_id field was used in a join. The change has no particular effect (and is not really necessary) on existing 2.6.0 databases where a primary key is mail_id (it is just a redundant extra condition), but saves a day when a primary key is a composite: (partition_tag,mail_id), which may be a requirement of an SQL partitioning mechanism. Thanks to Thomas Gelf for his testing of MySQL partitioning, reporting deficiency in amavisd SQL schema (primary keys) which did not meet MySQL requirements for partitioning, and suggestions; - an AM.PDP release request can specify an additional optional attribute: partition_tag=xx where a requester can supply a partition_tag value of a message to be released. This helps to uniquely identify a message in case where an SQL database did not enforce a mail_id field to be unique (as may be necessary with some partitioning schemes). If a partition_tag information is readily available to a requester, it is advised that the attribute is included in a request even if mail_id is known to be unique. This may expedite a search and provide a double check to a validity of a request. For backward compatibility amavisd performs a query on msgs.mail_id for a partition_tag value if it is missing form a request, the query uses an SQL clause in a new entry $sql_clause{'sel_msg'}. If exactly one record matches, then everything is fine, and releasing may proceed. If multiple records with the same mail_id exist the release request is aborted with a message asking user to supply a disambiguating partition_tag=xx attribute; - a quarantine id for an SQL-quarantined message as logged in a main log entry is changed from: quarantine: aX3C4f6btXgX to: quarantine: aX3C4f6btXgX[25] i.e. a partition_tag in brackets is appended to mail_id. Correspondingly the amavisd-release is also changed to be able to parse 'aX3C4f6btXgX[25]', splitting it into mail_id and partition_tag, and providing each as a separate attribute in an AM.PDP release request; - README.sql-mysql: changed SQL datatype VARCHAR into VARBINARY for data fields mail_id, secret_id and quar_loc, and CHAR into BINARY for msgs.content and msgs.quar_type to preserve case sensitivity on string comparison operators; suggested by Thomas Gelf; The same change should eventually be done on README.sql-pg too, but as PostgreSQL is more picky than MySQL on matching a field data type to a supplied data value, the change of a data type would need to be reflected in SQL calls in amavisd. This will have to wait until some future version of amavisd-new, having to undergo more testing than I have available before the 2.6.1 release. Background information on UNIQUE constraint in table SQL msgs Amavisd does not know and need not be aware of what is a primary key or what are UNIQUE constraints in SQL table msgs. When generating a mail_id for a message being processed, amavisd tries to INSERT a record with a randomly generated mail_id into table msgs (using SQL clause in $sql_clause{'ins_msg'}). If the operation fails, another mail_id is generated and attempt repeated, until it eventually succeeds. Thus it depends entirely on SQL's decision whether a particular record is allowed or would break some UNIQUE constraint. So, by only changing a declaration on table msgs (PRIMARY KEY or adding a CONSTRAINT), it changes what keys amavisd will be allowed to insert and what kind of duplicates would be allowed. Classically the msgs.mail_id is a PRIMARY KEY and as such it is unique. This was a requirement for versions of amavisd up to and including 2.6.0. Starting with 2.6.1 the JOINs have been tightened to include a partition_tag besides mail_id in a relation, which makes it possible to loosen a unique requirement on msgs.mail_id and only require a pair (partition_tag,mail_id) to be unique. In other words, this way the mail_id is only needed to be unique within each partition tag value. This change allows a partitioning scheme to meet requirements on MySQL partitioning. For non-partitioned databases the change shouldn't make any difference, and one is free to choose between having mail_id unique across the entire table or just within each partition_tag value. Changing a primary key to (partition_tag,mail_id) brings consequences to quarantining, in particular to releasing from an SQL quarantine, where it no longer suffices to specify mail_id=xxx in AM.PDP request, but may be necessary to specify also a partition_tag=xx to distinguish between SQL-quarantined messages which happen to have the same mail_id. --------------------------------------------------------------------------- April 23, 2008 amavisd-new-2.6.0 release notes MAIN NEW FEATURES SUMMARY - integrated DKIM signing and verification; see sections A QUICK START TO DKIM VERIFICATION and A QUICK START TO DKIM SIGNING by the end of this release note; - loading of policy banks based on valid DKIM-signed author's address can be used for reliable whitelisting, for bypassing banned checks, etc. - bounce killer feature: uses a pen pals SQL lookup to check inbound DSN; - SQL logging and quarantining tables have a new field 'partition_tag'; - captures SpamAssassin logging, more flexibility specifying SA log areas; - collects and logs SpamAssassin timing breakdown report (requires SA 3.3); - releasing from a quarantine can push a released message to an attachment; - new experimental code for abuse reporting using formats: ARF/attach/plain; - TLS support on the SMTP client and server side; - connection caching by an SMTP client; - amavisd-nanny and amavisd-agent now re-open a database on amavisd restarts; - amavisd-nanny and amavisd-agent new command line option: -c count; - updated p0f-analyzer.pl to support source port number in queries; - amavisd can send queries either to p0f-analyzer.pl or directly to p0f; COMPATIBILITY WITH 2.5.4 - when using SQL for logging (e.g. for a pen pals feature) or for quarantining, SQL tables tables maddr, msgs, msgrcpt and quarantine need to be extended by a new field 'partition_tag'; see below for details; - when SQL logging (pen pals) or SQL lookups are used, one can choose a binary or a character data type for fields users.email, mailaddr.email, and maddr.email; now may be a good opportunity to change a data type to binary (string of bytes); see below for details; - when using SQL for logging, a default for $sql_clause{'upd_msg'} has changed, so if a configuration file replaces this SQL clause by a non-default setting, it needs to be updated; - perl module Mail::DKIM is now required when DKIM verification or signing is enabled or when spam checking by SpamAssassin is used and a DKIM plugin is enabled; a required version of this module is 0.31 (or later); - because privileges are now dropped sooner, pid and lock files as generated by Net::Server can no longer be located in a directory which is not writable by UID under which amavisd is running (e.g. /var/run). A location of these files is controlled by $pid_file and $lock_file settings, and by default are placed in $MYHOME, which still satisfies the new requirement; - white and blacklisting now takes into account both the SMTP envelope sender address, as well as the author address from a header section (address(es) in a 'From:' header field). Note that whitelisting based only on a sender-specified address is mostly useless nowadays. For a reliable whitelisting see @author_to_policy_bank_maps below, as well as a set of whitelisting possibilities in SpamAssassin (based on DKIM, SPF, or on Received header fields); - if using custom hooks, some of the internal functions have changed, in particular the semantics of a method orig_header_fields - use new functions get_header_field() or get_header_field_body() instead; see updated sample code amavisd-custom.conf, and see entries labeled 'internal' below; - a configuration variable $append_header_fields_to_bottom is now obsolete; the variable is still declared for compatibility with old configuration files, but its value is ignored: new header fields are always prepended, i.e. added to the top of a header section; - semantics of a command line option 'debug-sa' has changed due to a merge of SpamAssassin logging with a mainstream amavisd logging mechanism. A command 'amavisd debug-sa' is now equivalent to 'amavisd -d all' with an implied redirection of all logging to stderr. Previously it only rerouted SpamAssassin logging to stderr but did not affect normal amavisd logging, which still followed the usual $DO_SYSLOG and $LOGFILE settings. Also, a SpamAssassin log level 'info' is now turned on by default (as was previously achievable by a command line option '-d info'), and shows merged with a normal amavisd logging at level 1 or higher. The following table shows mapping of SpamAssassin log levels to amavisd log levels, and for completeness also shows mapping of amavisd log levels to syslog priorities (which has not changed since previous version): SA amavisd syslog ----- ------- ----------- -3 LOG_CRIT -2 LOG_ERR error -1 LOG_WARNING warn 0 LOG_NOTICE info 1 LOG_INFO 2 LOG_INFO dbg 3 LOG_DEBUG 4 LOG_DEBUG 5 LOG_DEBUG - an additional requirement for loading a policy bank 'MYUSERS' is that 'originating' flag must be on, which typically means that mail must be coming from internal networks or from authenticated roaming users to be able to load a policy bank 'MYUSERS'; BUG FIXES - run_av: limit the number of filenames given as arguments to a command line scanner to stay within a safe (POSIX) program argument space limit, run a command line scanner multiple times if necessary. This required a larger change in the program (run_av, ask_av) which is why the fix was listed for a long time on a TODO list and not implemented so far. The problem affected command line virus scanners which are unable to traverse a directory by themselves and need a list of filenames as arguments (such as KasperskyLab's aveclient and kavscanner, MkS_Vir mks, and CyberSoft VFind). Actual problem reported by Danny Richter; NEW FEATURES - DKIM signing and verification - see sections below: A QUICK START TO DKIM VERIFICATION and A QUICK START TO DKIM SIGNING. Not to forget upgrading Mail::DKIM to 0.31 (or later) and adding the following to amavisd.conf; $enable_dkim_verification = 1; $enable_dkim_signing = 1; - SQL tables tables maddr, msgs, msgrcpt and quarantine are extended by a new field 'partition_tag'. When amavisd creates new records in these tables, a current value of a configuration variable $sql_partition_tag (or its value from policy banks) is written into 'partition_tag' fields. An undefined value translates to 0. The 'partition_tag' field is usually declared in a schema as an integer, but in principle could be any data type, such as a string. A value of 'partition_tag' field may be used to speed up purging of old records by using partitioned tables (MySQL 5.1 +, PostgreSQL 8.1 +). A sensible value is a week number of a year, or some other slowly changing value, allowing to quickly drop old table partitions without wasting time on deleting individual records. Records in all tables carrying the 'partition_tag' field are self-contained within each value of a field. In other words, foreign keys never reference a record in a subordinate table with a value of a 'partition_tag' field different from the referencing record. Consequently, mail addresses in table maddr are also self-contained within a partition tag, implying that the same mail address may appear in more than one maddr partition (using different 'id's), and that tables msgs and msgrcpt are guaranteed to reference a maddr.id within their own partition tag. Too fine a granularity of partition tags (e.g. changing a value daily) wastes space in table maddr by storing multiple copies of the same mail address. The $sql_partition_tag may be a scalar (usually an integer or a string), or a reference to a subroutine, which will be called with an object of type Amavis::In::Message as argument (giving access to information about a message being processed), and its result will be used as a partition tag value. Possible/typical usage (in amavisd.conf): $sql_partition_tag = sub { my($msginfo)=@_; iso8601_week($msginfo->rx_time) }; yields an ISO 8601 (EN 28601) week number (1..53) corresponding to a mail reception timestamp in a local time zone. This week number definition is equivalent to PostgreSQL extract(week from ...), and MySQL week(date,3). Another possible use of 'partition_tag' field is to let a policy bank set its specific value (a fixed value or a subroutine) for $sql_partition_tag. This would allow for example labeling of SQL records for mail originating from inside with a different partition_tag value, compared to entries for incoming mail, and consequently let them be stored in a separate partition if desired. Amavisd process itself does not use the 'partition_tag' field for its own purposes, all records regardless of their 'partition_tag' value are available for example to pen pals lookups, as before. The field is provided only as a convenience to SQL database maintenance, and can be ignored by smaller sites where current practice of database maintenance is fast enough. If SQL partitioning is not in use (or not intended to be used in a near future), it is more economical to use a fixed value (such as 0, which is a default) for the $sql_partition_tag. Using week numbers as partition tags adds about 50 % to the number of records in table maddr, the exact number depends on retention period and a ratio of regular vs. infrequent mail addresses observed. To convert tables of an existing database, please use ALTER command. Here is a conversion example (MySQL or PostgreSQL, probably others): ALTER TABLE maddr ADD partition_tag integer DEFAULT 0; ALTER TABLE msgs ADD partition_tag integer DEFAULT 0; ALTER TABLE msgrcpt ADD partition_tag integer DEFAULT 0; ALTER TABLE quarantine ADD partition_tag integer DEFAULT 0; As the maddr.email is no longer guaranteed to be unique, but a pair of (maddr.partition_tag, maddr.email) is unique, the constraint and an associated index needs to be changed: => PostgreSQL: ALTER TABLE maddr DROP CONSTRAINT maddr_email_key, ADD CONSTRAINT maddr_email_key UNIQUE (partition_tag,email); => MySQL: ALTER TABLE maddr DROP KEY email, ADD UNIQUE KEY part_email (partition_tag,email); Should a need arise to revert to amavisd-new-2.5.4 while keeping the new partition_tag field, the 'SELECT id FROM maddr ...' may become slow due to dropped index on a field maddr.email, which is replaced by an index on a pair (maddr.partition_tag, maddr.email). The following change to amavisd 2.5.4 solves the problem: @@ -901,2 +901,2 @@ 'sel_adr' => - 'SELECT id FROM maddr WHERE email=?', + 'SELECT id FROM maddr WHERE partition_tag=0 AND email=?', The use of partitioned tables to speed up purging of old records was suggested by Robert Pelletier. - when SQL logging (pen pals) or SQL lookups are used, one can choose a binary or a character data type for fields users.email, mailaddr.email, and maddr.email; now may be a good opportunity to change a data type to binary (string of arbitrary bytes, no character set associated). Background: values of these fields come from SMTP envelope or from a mail header section of processed mail. Even though RFC 2821 and RFC 2822 restrict these addresses to 7-bit ASCII, there is nothing preventing a malicious or misguided sender from supplying any 8-bit byte values. If SQL fields are declared as VARCHAR or CHAR, a character set is associated with data and its rules apply, e.g. control characters may not be permitted, or UTF-8 byte sequences are validated, or a restriction to codes below 128 apply. Depending on strictness of an SQL server on validating data, a violation of character set rules may lead to aborting an SQL operation and failing of mail processing. Even though new standards for e-mail addresses are being negotiated allowing for UTF-8 encoding, an actual e-mail address may still supply arbitrary bytes, which may violate UTF-8 byte sequence rules. A new configuration variable $sql_allow_8bit_address now controls how amavisd passes e-mail addresses to SQL. If a value is true, then it is expected that SQL tables will accept strings of arbitrary bytes for these fields, without associating a character set with data. No data sanitation is done by amavisd. An appropriate SQL data type is 'VARBINARY' or with PostgreSQL a 'BYTEA'. If a value of $sql_allow_8bit_address is false (which is a default for compatibility) then amavisd performs sanitation before passing data to SQL: control characters and characters with codes above 127 are converted to '?', which brings strings within ASCII character set restrictions. A suitable SQL data type is VARCHAR or CHAR. Note that some information is lost in this case. The following clauses can convert pre-2.6.0 tables into the now preferred and more universal form: MySQL: ALTER TABLE users CHANGE email email varbinary(255); ALTER TABLE mailaddr CHANGE email email varbinary(255); ALTER TABLE maddr CHANGE email email varbinary(255); PostgreSQL: ALTER TABLE users ALTER email TYPE bytea USING decode(email,'escape'); ALTER TABLE mailaddr ALTER email TYPE bytea USING decode(email,'escape'); ALTER TABLE maddr ALTER email TYPE bytea USING decode(email,'escape'); If a binary data type is chosen for these three fields, the setting $sql_allow_8bit_address MUST be set to true to let the amavisd program use the appropriate data type in SQL commands, otherwise PostgreSQL will complain with: 'types bytea and character varying cannot be matched' when amavisd tries to execute SQL commands. MySQL is more forgiving and does not complain about a data type mismatch, so one may get away with a mismatch, although it is appropriate to eventually make it right. If a change of a data type of these fields is chosen while using some third-party management interface to SQL data set (e.g. MailZu), make sure the management interface supports the changed data type. This is primarily a concern with PostgreSQL which is more strict in requiring a match between field data types in tables and data in SQL clauses. The need for a change was pointed out by Xavier Romero, reporting that PostgreSQL SQL lookups with pre-2.6.0 versions of amavisd can fail when 8-bit data appears in SMTP envelope addresses: lookup_sql: sql exec: err=7, 22021, DBD::Pg::st execute failed: ERROR: invalid byte sequence for encoding "UTF8" - bounce killer feature: uses a pen pals SQL lookup to check inbound DSN, attempting to match it with a previous outbound message. If a Message-ID found in an attachment of the inbound DSN matches a Message-ID of a message previously sent from our system by a current recipient of the DSN, the DSN message is spared, otherwise it receives $bounce_killer_score spam score points (0 by default, i.e. disabled) and can be blocked as spam (although technically it is just a misdirected bounce, not spam). A received delivery status notifications is parsed looking for attached header section of an original message in an attempt to find a Message-ID. A standard DSN structure (RFC 3462, RFC 3464) is recognized, as well as a few nonstandard but common formats. Other automatic reports and bounces with unknown structure, and no attached header section are ignored for this purpose (are subject to other regular checks). Unfortunately there are still many nonstandard mailers around (12+ years after DSN format standardization) and many ad-hoc filtering solutions which do not supply the essential information. If a Message-ID can be found in an SQL log database matching a previous message sent by a local user (which is now a recipient of a DSN), using a normal pen pals lookup (no extra SQL operations are necessary), or if a domain part of the Message-ID is one of local domains, then the DSN message is considered a genuine bounce, is unaffected by this check and passes normally (subject to other checks). On the other hand, if the attached DSN header does look like a complete original header but it does not meet the above criteria, then it is assumed that the message is a backscatter to a faked address belonging to our local domains, and $bounce_killer_score spam score points are added, so the message can be treated as spam (subject to spam kill level and other spam settings). The only user-configurable setting is $bounce_killer_score (also member of policy banks), its default value is 0. To activate the bounce killer feature, set the $bounce_killer_score to a positive number, e.g. 100. A bounce killer score does not contribute to SpamAssassin's auto-learning. A pre-requisite for proper operation of a bounce killer is a working SQL logging database (pen pals), or at least that all outbound DSN messages have a Message-ID with a domain matching the @local_domains_maps list of lookup tables. The condition is easily met when all mail from local users is submitted through a domain's official mailer, which goes hand in hand with the requirement for DKIM signing and for other similar antispoofing techniques (SPF, whitelisting by IP address in Received trace, ...). A couple of SNMP-like counters are added to facilitate assessing effectiveness of the feature (e.g. viewed by amavisd-agent utility): InMsgsBounce 21310 333/h 9.9 % (InMsgs) InMsgsBounceKilled 19967 312/h 93.7 % (InMsgsBounce) InMsgsBounceRescuedByDomain 7 0/h 0.0 % (InMsgsBounce) InMsgsBounceRescuedByOriginating 242 4/h 1.1 % (InMsgsBounce) InMsgsBounceRescuedByPenPals 67 1/h 0.3 % (InMsgsBounce) InMsgsBounceUnverifiable 1027 16/h 4.8 % (InMsgsBounce) More information on operations can be obtained from a log, search for: inspect_dsn: bounce killed bounce rescued by penpals bounce rescued by domain bounce unverifiable The feature was suggested by Scott F. Crosby. See also http://www.postfix.org/BACKSCATTER_README.html, http://wiki.apache.org/spamassassin/VBounceRuleset and a SpamAssassin man page Mail::SpamAssassin::Plugin::VBounce for additional ideas on fighting joe-jobbed backscatter mail. - a new configuration variable @author_to_policy_bank_maps (also a member of policy banks) is a list of lookup tables (typically only a hash-type lookup table is used), which maps author addresses(es) (each address in a 'From:' header field - typically only one) to one or more policy bank names (a comma-separated list of names). A match can only occur if a valid DKIM author domain signature or a valid DKIM third-party signature is found, so in as much as one can trust the signing domain, loading of arbitrary policy banks can be safe, offering a flexibility of whitelisting against spam (absolute or just contributing score points), bypassing of checks (banned, virus, bad-header), using less restrictive banned rules for certain senders, by-sender routing, turning quarantining/archiving on/off, and other tricks offered by the existing policy bank loading mechanisms. When a message has a valid DKIM (or DomainKeys) author domain signature (i.e. when a 'From:' address matches a signing identity according to DKIM (RFC 4871) or DomainKeys (RFC 4870) rules), a lookup key is an unchanged author address and the usual lookup rules apply (README.lookups - hash lookups). When a valid third-party signature is found, a lookup key (author address) is extended by a '/@' and a lowercased signing domain, as shown in the example below. The semantics is very similar to a whitelist_from_dkim feature in SpamAssassin, but is more flexible as is allows any dynamic amavisd setting to be changed depending on author address, not just skipping of spam checks. A few examples of a SpamAssassin's whitelist_from_dkim (as in local.cf) along with equivalent amavisd @author_to_policy_bank_maps entries follow. To whitelist any From address with a domain example.com when a message has a valid author domain signature (i.e. a signature by the same domain): SA: whitelist_from_dkim *@example.com am: 'example.com' => 'WHITELIST', which is equivalent to a lengthy but redundant: SA: whitelist_from_dkim *@example.com example.com am: 'example.com/@example.com' => 'WHITELIST', Similar to above, but applies to subdomains of example.com carrying a valid author domain signature (i.e. signature BY THE SAME SUBDOMAIN): SA: whitelist_from_dkim *@*.example.com am: '.example.com' => 'WHITELIST', Note that in amavisd hash lookups a '.example.com' implies a parent domain 'example.com' too, while in SpamAssassin and in Postfix maps a parent domain needs its own entry if desired. To whitelist From addresses from subdomains of example.com which carry a valid third-party signature of its parent domain: SA: whitelist_from_dkim *@*.example.com example.com am: '.example.com/@example.com' => 'WHITELIST', To whitelist any From address as long as a message has a valid DKIM or DomainKeys signature by example.com, i.e. a third-party signature. Typical for mailing lists or discussion groups which sign postings. SA: whitelist_from_dkim *@* example.com am: './@example.com' => 'WHITELIST', Here is a complete example that can be included in amavisd.conf: @author_to_policy_bank_maps = ( { # 'friends.example.net' => 'WHITELIST,NOBANNEDCHECK', # 'user1@cust.example.net' => 'WHITELIST,NOBANNEDCHECK', '.ebay.com' => 'WHITELIST', '.ebay.co.uk' => 'WHITELIST', 'members.ebay.co.uk/@ebay.co.uk' => 'WHITELIST', 'ebay.at' => 'WHITELIST', 'ebay.ca' => 'WHITELIST', 'ebay.fr' => 'WHITELIST', 'ebay.de' => 'WHITELIST', 'members.ebay.de/@ebay.de' => 'WHITELIST', '.paypal.co.uk' => 'WHITELIST', '.paypal.com' => 'WHITELIST', # author domain signatures './@paypal.com' => 'WHITELIST', # 3rd-party sign. by paypal.com 'alert.bankofamerica.com' => 'WHITELIST', 'ealerts.bankofamerica.com'=> 'WHITELIST', 'amazon.com' => 'WHITELIST', 'amazon.de' => 'WHITELIST', 'amazon.co.uk' => 'WHITELIST', 'cisco.com' => 'WHITELIST', '.cnn.com' => 'WHITELIST', 'skype.net' => 'WHITELIST', 'welcome.skype.com' => 'WHITELIST', 'cc.yahoo-inc.com' => 'WHITELIST', 'cc.yahoo-inc.com/@yahoo-inc.com' => 'WHITELIST', '.linkedin.com' => 'MILD_WHITELIST', 'google.com' => 'MILD_WHITELIST', 'googlemail.com' => 'MILD_WHITELIST', './@googlegroups.com' => 'MILD_WHITELIST', './@yahoogroups.com' => 'MILD_WHITELIST', './@yahoogroups.co.uk' => 'MILD_WHITELIST', './@yahoogroupes.fr' => 'MILD_WHITELIST', 'yousendit.com' => 'MILD_WHITELIST', 'meetup.com' => 'MILD_WHITELIST', 'dailyhoroscope@astrology.com' => 'MILD_WHITELIST', } ); $policy_bank{'MILD_WHITELIST'} = { score_sender_maps => [ { '.' => [-1.8] } ], }; $policy_bank{'WHITELIST'} = { bypass_spam_checks_maps => [1], spam_lovers_maps => [1], }; $policy_bank{'NOVIRUSCHECK'} = { bypass_decode_parts => 1, bypass_virus_checks_maps => [1], virus_lovers_maps => [1], }; $policy_bank{'NOBANNEDCHECK'} = { bypass_banned_checks_maps => [1], banned_files_lovers_maps => [1], }; - smtp client connection caching is a new feature which allows smtp client code in amavisd to keep an SMTP session to MTA open after forwarding a message or a notification, so that a next mail message that needs to be sent by this child process can avoid re-establishing a session and the initial greeting/EHLO (and TLS) handshake. A current value of a global settings $smtp_connection_cache_enable controls whether a session will be retained after forwarding a message or not. Its default initial value is true. A global setting $smtp_connection_cache_on_demand controls whether amavisd is allowed to dynamically change the $smtp_connection_cache_enable setting according to its estimate of the message frequency. The heuristics is currently very simple: if time interval between a previous task completion by this child process and the arrival of a current message is 5 seconds or less, the $smtp_connection_cache_enable is turned on (which will affect the next message); if the interval is 15 seconds or more, it is turned off. The default value of the $smtp_connection_cache_on_demand is true, thus enabling the adaptive behaviour. On a busy server the connection caching can save some processing time. Savings are substantial if client-side TLS is enabled, otherwise just a few milliseconds are saved. On an idle server the feature may unnecessarily keep sessions to MTA open (until MTA times them out), so one can disable the feature by setting both controls to false (to 0 or undef). To monitor the connection caching effectiveness, some SNMP-like counters were added, so amavisd-agent may display something like: OutConnNew 2764 319/h 98.2 % (OutMsgs) OutConnQuit 2521 291/h 89.5 % (OutMsgs) OutConnReuseFail 7 1/h 0.2 % (OutMsgs) OutConnReuseRecent 21 2/h 0.7 % (OutMsgs) OutConnReuseRefreshed 31 4/h 1.1 % (OutMsgs) OutConnTransact 2816 325/h 100.0 % (OutMsgs) - client-side TLS support is added, i.e. on forwarding a passed mail back to MTA. Currently only encryption is supported, no client certificates are offered. A $tls_security_level_out is a per-policy-bank setting which controls client-side TLS, its value is either undefined (default), or a string: undef ... client-side TLS is disabled (a default setting); 'may' ... TLS is used if MTA offers a STARTTLS capability (RFC 3207), otherwise a plain text SMTP session is established; 'encrypt' TLS is used if MTA offers a STARTTLS capability, otherwise amavisd refuses to forward a message. The client-side TLS imposes some performance penalty on passing a message back to MTA, although it is still reasonably fast: a benchmark indicates a drop in transfer rate by about a factor of 2, from 22 MB/s (no TLS) to 9 MB/s (with TLS). The smtp client connection caching (see previous item) should preferably be left enabled (permanently or opportunistically), as TLS negotiation adds significantly to the initial SMTP handshake time. - server-side TLS support is added, i.e. on accepting mail from MTA. Encryption is supported, server (i.e. amavisd) offers its certificate, but client certificates are not verified. A $tls_security_level_in is a per-policy-bank setting which controls server-side TLS, its value is either undefined (default), or a string: undef ... server-side TLS is disabled, STARTTLS capability is not offered; 'may' ... STARTTLS capability is offered by amavisd, but client is not required to enter TLS, plain text sessions are permitted; 'encrypt' STARTTLS capability is offered and enforced by amavisd, any SMTP command other than STARTTLS, NOOP, EHLO or QUIT is rejected. If $tls_security_level_in is enabled (any value other than undef or 'none'), amavisd offers a certificate to a connecting client requesting TLS, so a path to a certificate and to its private key must be provided through two global settings: $smtpd_tls_cert_file and $smtpd_tls_key_file, e.g.: $smtpd_tls_cert_file = "$MYHOME/cert/amavisd-cert.pem"; $smtpd_tls_key_file = "$MYHOME/cert/amavisd-key.pem"; The private key should be guarded as secret (not world-readable). A self-signed certificate is acceptable by most mailers. Server-side TLS imposes a significant performance penalty on accepting a message from MTA. A benchmark indicates a drop in transfer rate by a factor of 10, from about 10 MB/s (no TLS) to 1 MB/s (using TLS), so it should only be enabled with a good reason or for experimentation. - enhanced a subroutine delivery_status_notification (along with dispatch_from_quarantine and msg_from_quarantine) to produce a message in one of several formats (derived from a message being processed, or from a quarantined message). Its new arguments can be strings as follows: $request_type: dsn, release, requeue, report $msg_format: dsn, arf, attach, plain, resend $feedback_type: abuse, fraud, miscategorized, not-spam, opt-out, opt-out-list, virus, other (according to ARF draft) Per-policy settings $report_format and $release_format control the format of a generated message. Their value can be one of the following strings, although not all combinations make sense: 'arf' .... an abuse report is generated according to draft-shafranovich-feedback-report-04: "An Extensible Format for Email Feedback Reports"; a plain-text part contains text from a template; 'attach'.. generates a report message as plain text according to a template, with an original message attached; 'plain'... generates a simple (flat) mail with an only MIME part containing a text from a template, followed inline by original message (some service providers can't handle abuse reports with attachments, e.g. Yahoo!); 'resend'.. original message is forwarded unchanged, except for an addition of header fields Resent-From, Resent-Sender, Resent-To, Resent-Date and Resent-Message-ID; 'dsn' .... (for internal use) a delivery status notification is generated according to rfc3462, rfc3464 and rfc3461; When a request_type is 'release' or 'requeue', the format of a generated message is governed by a per-policy setting $release_format according to the table above. Only the 'attach', 'plain' and 'resend' values are useful. A default setting is: $release_format = 'resend'; # with alternatives: attach, plain, resend A plain-text part (if used) is taken from a $notify_release_templ template and a sending address is obtained from %hdrfrom_notify_release_by_ccat. When a request_type is 'report', the format of a generated message is governed by a per-policy setting $report_format according to the table above. Only the following settings are useful: arf, attach, plain, resend. A default setting is: $report_format = 'arf'; # alternatives: arf, attach, plain, resend A plain-text part (if used) is obtained from a $notify_report_templ template, and a sending address from %hdrfrom_notify_report_by_ccat. It is possible to automatically generate abuse reports from custom hooks by calling delivery_status_notification() and mail_dispatch(). Extreme care must be taken to only produce legitimate abuse reports (about true fraud and true spam), sent only to parties that are truly responsible for a message being reported. Non-repudiation is a key factor here - trust only header fields covered by a valid DKIM signature, or generated by your own MX MTA (such as an IP address of the last hop), and only report messages received from a network which officially belongs to the party (according to whois). Rate-limiting should be used, and abuse reports on the same abuser should only be sent once in a time interval of several hours. A SQL database can be used to maintain a list of recently reported abusers, thus preventing excessive reports. - introduced a variation of a message release from a quarantine, allowing a releaser to send an abuse report based on a quarantined message. It is implemented by: * enhancing a subroutine delivery_status_notification as described in the previous item; * extending AM.PDP protocol with a 'request=report' attribute which can be used in place of a 'request=release', * enhancing the 'amavisd-release' utility program to allow sending an attribute 'request=release' or 'request=requeue' or 'request=report' based on its program name. By making a soft or hard link named 'amavisd-report' linking to 'amavisd-release', the utility will send a 'request=report' in place of the usual 'request=release', e.g.: # ln -s amavisd-release amavisd-report # ln -s amavisd-release amavisd-requeue $ amavisd-report spam/k/kg2P0rP9Lpu3.gz '' abuse@example.com - releasing from a quarantine can push a released message to an attachment (Content-Type: message/rfc822), with a configurable template for a header section and the plain-text part; select by: $release_format='attach'; suggested by Patrick Ben Koetter; - detect and save a new attribute SOURCE from an XFORWARD smtp command; the value is also accepted as AM.PDP protocol attribute 'client_source'. Possible values are: 'LOCAL', 'REMOTE', or '[UNAVAILABLE]', the information corresponds to 'local_header_rewrite_clients' postfix setting and is not supposed to be used for security decisions according to Postfix documentation (which makes it less interesting for our purposes); - added client and server support for a PORT attribute of an XFORWARD command, allowing MTA to pass a TCP port number of a remote client to a content filter (and back if necessary); the PORT attribute is made available with Postfix version 2.5 (20071004); a source port number is also accepted as an AM.PDP protocol attribute 'client_port'; - updated p0f-analyzer.pl now supports a source port number information in queries while preserving backward compatibility with previous versions of amavisd-new. Version 2.6.0 of amavisd requires a new version of p0f-analyzer.pl (supplied in the 2.6.0 distribution) if operating system fingerprinting is enabled. A source port number information in a query allows p0f-analyzer.pl to locate a matching entry in its cache faster and also more accurately when multiple connections are present from clients behind NAT using the same IP address. The source port number is made available to a content filter since Postfix version 2.5 (20071004); - besides the ability to send queries to p0f-analyzer.pl, amavisd now also supports sending queries directly to a p0f program over a Unix socket using a p0f query protocol. There is a bug in p0f-2.0.8 (and probably in earlier versions) which makes it send back incorrect results at times, i.e. results belonging to some other unrelated session, so a patch to p0f-2.0.8 MUST be applied in order to use a direct querying mechanism - author has been notified. The patch is supplied: p0f-patch. There are currently no advantages (and some disadvantages) in choosing direct queries to p0f, compared to sending queries to p0f-analyzer.pl, so this new method is not currently recommended. Disadvantages are: * p0f uses a linear search over its list of recent sessions (at least as of version 2.0.8), whereas p0f-analyzer.pl uses a fast hash lookup; * p0f keeps a relatively small list of recent sessions which is limited by the number of slots (size can be specified on a command line, but is limited by a linear search time), whereas p0f-analyzer.pl expires old entries according to time since entered and is thus independent of a current mail rate; * a direct p0f query protocol uses packed binary data and its on-the-wire representation may depend on a compiler used, so it may be incompatible with queries sent by amavisd, whereas the p0f-analyzer.pl queries and replies use a more environment-independent textual representation. To let amavisd sent queries directly to p0f, specify a p0f socket path: $os_fingerprint_method = 'p0f:/var/amavis/home/p0f.sock'; and specify an IP address and a port number on which MTA is listening: $os_fingerprint_dst_ip_and_port = '[192.0.2.3]:25'; because p0f requires this information in a query and the information is not made available to a content filter via XFORWARD command (the p0f-analyzer.pl does not need this information). To send queries to p0f-analyzer.pl (traditional and recommended), use: $os_fingerprint_method = 'p0f:127.0.0.1:2345'; as before. The $os_fingerprint_dst_ip_and_port in this case is not needed and is ignored. - usually a sending address in spam messages is faked and it is desirable to suppress most if not all bounces by keeping $sa_dsn_cutoff_level low, but sometimes it may be possible to be more certain of the validity of a sending address, and when such mail is considered spam, it may still be desirable to send a non-delivery notification, knowing that a notification will most likely be addressed to a genuine sender. Two new settings are provided for this purpose: @spam_crediblefrom_dsn_cutoff_level_bysender_maps and @spam_crediblefrom_dsn_cutoff_level_maps (with their default being $sa_crediblefrom_dsn_cutoff_level), complementing the existing @spam_dsn_cutoff_level_bysender_maps and @spam_dsn_cutoff_level_maps. It is expected that $sa_crediblefrom_dsn_cutoff_level would be set somewhat higher than $sa_dsn_cutoff_level, allowing for more bounces to be generated for spam from likely-to-be-genuine senders (possibly false positives). The choice between taking a cutoff value from one or the other pair of settings depends on an attribute $msginfo->sender_credible - when it is true (e.g. some nonempty string) the *spam_crediblefrom_* settings will be used instead of the baseline @spam_dsn_cutoff_level_*maps. An initial value of a sender_credible attribute as provided by amavisd is true if either the 'originating' flag is true (e.g. mail from inside), or if dkim_envsender_sig attribute is true, e.g. a domain of a valid DKIM signature matches envelope sender address, otherwise it is false. A user-provided custom hook code is free to change the value of sender_credible attribute. An exact value does not matter (it is only interpreted as a boolean), but serves for logging purposes. Heuristics may be based on some tests provided by SpamAssassin, on DKIM signatures, on p0f results, on policy banks, etc. Here is one complete example of a custom hook, which turns on the sender_credible attribute based on some criteria. Note that some of the referenced SpamAssassin tests may not yet be available in the last officially released version of SpamAssassin. added to amavisd.conf: include_config_files('/etc/amavisd-custom.conf'); /etc/amavisd-custom.conf : package Amavis::Custom; use strict; sub new { my($class,$conn,$msginfo) = @_; bless {}, $class } sub after_send { my($self,$conn,$msginfo) = @_; if ($msginfo->sender ne '') { my(@cred); local($1); my($tests) = $msginfo->supplementary_info('TESTS'); $tests = '' if !defined($tests) || $tests eq 'none'; push(@cred,'orig') if $msginfo->originating; push(@cred,$1) if $tests =~ /\b(RCVD_IN_DNSWL_HI)\b/; push(@cred,$1) if $tests =~ /\b(RCVD_IN_DNSWL_MED)\b/; push(@cred,$1) if $tests =~ /\b(RP_MATCHES_RCVD)\b/; my($os_fingerprint) = $msginfo->client_os_fingerprint; if ($os_fingerprint !~ /^Windows XP(?![^(]*\b2000 SP)/) { push(@cred,'dkim') if $msginfo->dkim_envsender_sig; push(@cred,$1) if $tests =~ /\b(SPF_PASS)\b/; } $msginfo->sender_credible(join(",",@cred)) if @cred; } } 1; # insure a defined return - a new setting $reputation_factor (also a member of policy banks) with a value between 0 and 1 (default 0.2), controlling an amount of 'bending' of a calculated spam score towards a fixed score assigned to a signer identity (i.e. its 'reputation') through @signer_reputation_maps; the formula is: adjusted_spam_score = f*reputation + (1-f)*spam_score; which has the same semantics as auto_whitelist_factor in SpamAssassin AWL; - a new setting @signer_reputation_maps (also a member of policy banks) may contain a list of lookup tables (typically just one hash lookup table), mapping a signing identity to a score, which is typically a long term average spam score of all messages signed by this signing identity. Based on a lookup result and a formula given above ($reputation_factor), the resulting value (positive or negative) is added to the spam score. Here is an example setting: @signer_reputation_maps = ( { 'ebay.fr' => -10.95, 'ebay.ca' => -9.57, 'ebay.co.uk' => -8.59, 'ebay.com' => -8.03, 'ebay.at' => -3.59, 'ebay.de' => -3.38, 'reply3.ebay.com' => -4.57, 'reply.ebay.com' => -3.20, 'paypal.com' => -6.66, 'intl.paypal.com' => -3.70, 'email.paypal.co.uk' => -0.67, 'alert.bankofamerica.com' => 1.35, 'ucsd.edu' => -7.89, 'izb.knu.ac.kr' => -7.51, 'ijs.si' => -4.44, 'tu-graz.ac.at' => -5.44, 'tugraz.at' => -4.03, 'aitech.ac.jp' => -3.03, 'univie.ac.at' => -2.99, 'uni-bremen.de' => -2.80, 'uu.se' => -2.54, 'univ-tours.fr' => -2.06, 'phys.huji.ac.il' => -2.01, 'cern.ch' => -1.69, 'prime.gushi.org' => -9.85, 'dostech.ca' => -9.31, 'resistor.net' => -9.18, 'kitterman.com' => -9.05, 'schetterer.org' => -9.04, 'hege.li' => -8.96, 'fouter.net' => -8.84, 'inetmsg.com' => -8.81, 'charite.de' => -8.81, 'porcupine.org' => -8.75, 'secnap.net' => -7.99, 'netoyen.net' => -7.58, 'state-of-mind.de' => -6.93, 'gmurray.org.uk' => -5.39, 'mtcc.com' => -4.93, 'messiah.edu' => -4.91, 'consulintel.es' => -2.16, 'delphij.net' => 1.06, 'channing-bete.com' => -9.11, 'megan.vbhcs.org' => -8.64, 'scent-team.com' => -8.32, 'suedfactoring.de' => -8.15, 'sendmail.net' => -5.14, 'cisco.com' => -4.95, 'hermes-softlab.com' => -3.77, 'altn.com' => -2.16, 'amazon.com' => 0.09, 'eurescom.eu' => -0.63, 'skype.net' => -1.50, 'welcome.skype.com' => -0.34, 'newsdesk.world-nuclear-news.org'=> -0.74, 'youtube.com' => 1.11, 'email.innocentive.com' => 1.98, 'update.hallmark.com' => 2.26, 'newsletters.trendmicro.com'=> 3.02, 'mail.communications.sun.com'=> 3.73, 'alerts.hp.com' => 0.51, 'email.greenpeace.org' => 2.04, 'avaaz.org' => 3.51, 'mail.cnn.com' => 4.12, 'm-w.com' => 4.81, 'medcompare.com' => 2.46, 'biocompare.com' => 3.48, 'dentalcompare.com' => 4.58, 'news.biomedcentral.com' => 2.89, 'yahoo.com' => -0.53, 'yahoo.se' => -1.48, 'yahoo.de' => -1.33, 'yahoo.co.uk' => 0.85, 'yahoo.ca' => 1.13, 'yahoo.no' => 1.22, 'yahoo.es' => 2.04, 'yahoo.ie' => 4.48, 'yahoo.fr' => 6.35, 'yahoo.it' => 7.09, 'yahoo.dk' => 7.10, 'yahoo.co.in' => 5.34, 'yahoo.com.cn' => 4.34, 'yahoo.com.ar' => 5.48, 'yahoo.co.jp' => 9.25, 'yahoo.com.au' => 9.30, 'yahoo.com.hk' => 13.97, 'yahoo.cn' => 21.94, 'yahoo-inc.com' => -1.57, 'yahoogroups.com' => -1.76, # 'yahoogroups.co.uk' => 0.10, # 'yahoogroupes.fr' => 5.99, 'yahoogroups.de' => 19.96, 'yahoogrupos.com.br' => 17.41, 'gmail.com' => -3.52, 'googlegroups.com' => -3.10, 'googlemail.com' => 3.42, 'google.com' => 1.47, 'prodigy.net' => -8.91, 'amis.net' => -2.38, 'earthlink.net' => -2.27, 'btinternet.com' => -1.96, 'pacbell.net' => -0.96, 'rogers.com' => 2.05, 'ipost.com' => 2.40, 'incertum.net' => -9.80, 'yousendit.com' => -4.70, 'news.yousendit.com' => 3.38, 'abv.bg' => -4.57, 'uclouvain.be' => -4.48, 'birthdayalarm.com' => -4.32, 'geni.com' => -3.50, 'mail120.subscribermail.com'=> -2.58, 'mail6.subscribermail.com' => -1.97, 'spock.com' => -11.67, 'meetup.com' => -1.94, '123greetings.com' => -1.92, 'rocketmail.com' => -1.91, 'arcamax.com' => -0.81, 'skynet.be' => -0.10, 'news.virtualtourist.com' => 3.58, 'bighip.com' => 10.53, 'investorsinsight.com' => 12.19, 'lspromos2007online.com' => 12.41, 'postmaster-direct.com' => 12.71, 'smsacfriends.com' => 12.88, 'specialtyofficial.com' => 13.03, 'usafisnews.org' => 13.82, 'freelotto.com' => 15.59, 'partnershopping.net' => 19.46, 'muffinlocate.com' => 19.49, 'stallrust.com' => 19.71, 'sugargrowth.com' => 19.75, 'washusual.com' => 20.01, 'poursmock.com' => 20.02, 'overseeinvest.com' => 20.19, 'waveholer.com' => 20.27, 'headattomic.com' => 20.32, 'moneysuffer.com' => 20.84, 'ibnyes.com' => 20.95, 'honorfamous.com' => 20.96, 'trapdull.com' => 20.96, 'solvehigh.com' => 21.57, 'tracerope.com' => 21.68, 'yesalumni.com' => 21.71, 'internetpromotional.net' => 23.16, 'domaingln11track.com' => 23.69, 'news-central-99.com' => 24.00, 'sports-bobble-heads.com' => 24.69, 'dans-fishing-adventure.com'=> 26.39, 'myhottdeals.net' => 26.65, 'nitroda.com' => 28.75, 'spaninns.com' => 29.08, 'pinaycamgirls.net' => 29.24, 'aspsbulletins.com' => 13.06, 'latestsbulletins.com' => 13.35, 'bulletinshops.com' => 13.40, 'academicsbulletins.com' => 13.49, 'tigersbulletins.com' => 14.13, 'nakedsbulletins.com' => 14.17, 'paintedsbulletins.com' => 14.33, 'opensbulletins.com' => 14.58, 'domainsbulletins.com' => 14.61, 'seniorssbulletins.com' => 14.72, 'dodgesbulletins.com' => 15.23, 'mindsbulletins.com' => 15.96, 'researchsbulletins.com' => 15.98, 'salessbulletins.com' => 16.12, 'netsbulletins.com' => 16.83, 'virtualsbulletins.com' => 17.14, 'petbulletins.com' => 17.30, 'citysbulletins.com' => 18.46, 'wearesurveys.net' => 23.36, 'takeoursurveys.net' => 24.50, 'thankyousurveys.net' => 24.62, 'greetingssurveys.info' => 24.63, 'allaboutsurveys.net' => 25.71, 'internetssurvey.info' => 25.98, 'theinternetsurveys.net' => 26.14, 'netsurveysnet.net' => 26.85, 'surveydept.info' => 23.39, 'survysample.info' => 25.82, 'gosurrvey.info' => 26.02, 'surrvytime.info' => 26.41, 'thedailyinfo.info' => 21.96, 'thenetinfo.info' => 23.27, 'nettinfo.info' => 27.41, 'theinfoguide.info' => 29.91, 'alloursamples.info' => 22.71, 'officeofferonline.info' => 24.77, 'cashsuggestion.info' => 26.84, 'insaneofferdeal.info' => 26.87, 'rewardcenterrs.info' => 30.04, 'gifttsgroup.info' => 31.48, 'ourgiftworld.info' => 32.46, 'giftwinninngs.info' => 34.11, 'mygiftwinninngs.net' => 31.12, 'gifttsgroup.net' => 33.05, 'giftwinninngs.net' => 34.95, }); - drop privileges sooner if possible - right after reading config file and before forking; - allow inserting X-Quarantine-ID header field into passed (and quarantined) mail for local recipients only; remote recipients should not be made aware that we may have a copy in a quarantine; reported by Robert Fitzpatrick; - check for multiple occurrences of RFC 2045 and RFC 2183 MIME header fields (in addition to checks on RFC 2822 header fields): MIME-Version Content-Type Content-Transfer-Encoding Content-ID Content-Description Content-Disposition and the RFC 3834 header field: Auto-Submitted - capture SpamAssassin logging and integrate it into the usual amavisd log; suggested by Jeff Moss; - when parsing SpamAssassin log areas/facilities, recognize negations (a name prefixed by 'no') and remove duplicates, last entry wins, e.g. amavisd -d rules,noall,norules,dcc,norules,rules debug-sa is equivalent to: amavisd -d noall,dcc,rules debug-sa - a SpamAssassin debug level 'info' is now implicitly prepend to a list of SpamAssassin facilities; it may be overridden by explicitly negating it, e.g. 'amavisd -d noinfo'; - when a command line argument 'debug-sa' is present, or if $sa_debug is true, a SpamAssassin debug level 'all' is prepend to a list of SpamAssassin facilities, which would bring SpamAssassin log level to 'dbg'; it may be overridden by negating it, e.g. 'amavisd -d noall,plugins,rules debug-sa'; - include milliseconds in a log timestamp when logging is directed to stderr; - create a custom hook object sooner, so that loading a policy bank from a custom hook becomes useful (but not soon enough to influence partition_tag); - added a custom hook after_send, which may suppress DSN, send reports, quarantine, etc; - fetch additional information (tag) 'TIMING' from SpamAssassin, making it available through macro 'supplementary_info' (if a version of SpamAssassin in use provides it - available since 3.3.0); - a SpamAssassin TIMING-SA report (timing breakdown by sections) is now collected and logged at log level 2 when available; the information is available since version 3.3.0 of SpamAssassin (currently in development and available through SVN); - add a global configuration variable $listen_queue_size (undef by default) which is passed as an option 'listen' to Net::Server, which in turn passes it on to listen(2) as a 'backlog' parameter. The Net::Server provides a default value of SOMAXCONN in the absence of a valid integer in $listen_queue_size (e.g. 128, but on Solaris it defaults to 5, which is too small for some purposes). Suggested by David Schweikert. A workaround for a small SOMAXCONN default on Solaris is provided by Net::Server 0.98 (?). - in the absence of an smtp client's IP address (normally received by XFORWARD smtp command from Postfix, or in the 'client_address' attribute of AM.PDP), parse the topmost one or two Received header fields and use the first valid IP address found there; based on a suggestion by Richard Bishop; - new macros (useful in notifications and $log_template): - 'week_iso8601' returns an ISO 8601 week number (between 1 and 53); - 'partition_tag' returns a current value of a $sql_partition_tag variable; - 'dkim' reports various DKIM verification results; - 'report_format' gives one of the: dsn, arf, attach, plain, resend - 'feedback_type' is expected to give one of the ARF (draft) strings: abuse, fraud, miscategorized, not-spam, opt-out, opt-out-list, virus, other - 'rfc2822_from' an e-mail address from a From header field; - 'rfc2822_sender' an e-mail address from a Sender header field, or empty; - 'tls_in' returns TLS ciphers in use by an SMTP session if mail came to amavisd through a TLS-encrypted session, otherwise empty - 'limit' takes two arguments: a string size limit and some string, returning the string from the second argument unchanged if its size is below the limit, or cropped to the size limit, with '[...]' appended; For details see README.customize. - new configuration setting $allow_fixing_long_header_lines, also member of policy banks, defaults to true - provides control over truncation of header section lines longer than 998 characters as limited by RFC 2822. The $allow_fixing_improper_header must also be true for fixing to take place. Previously it was only possible to turn off all header fixes, but not specifically just the long header truncation; - strip X-Spam-* headers and other prepended header fields when releasing a quarantined message; - turn on message part attributes 'C' (crypted) and 'U' (undecipherable) if a MIME Content-Type of that body part is /encrypted; (note: if $defang_undecipherable is turned on, this would push a received PGP/GPG-encrypted MIME top-part into an attachment, just as with other password-protected archives); - fetch additional information (tags) DCCB and DCCR from a SpamAssassin DCC plugin, making them available through a macro 'supplementary_info'; - amavisd-nanny and amavisd-agent now reopen a database if/when the underlying database file is re-created (i.e. when its inode changes), as is the case when amavisd restarts; based on a patch by Rob Foehl; - amavisd-nanny and amavisd-agent got a new command line option: -c count, which can restrict the number of iterations for display; usability deficiency pointed out by John Evans; - amavisd-nanny and amavisd-agent will use a value from an environment variable AMAVISD_DB_HOME (if it exists) as a database home directory, otherwise fall back to a built-in default '/var/amavis/db'; suggested by Leo Baltus; OTHER - added an AV entry for a new version of a ESET File Security (for Linux and FreeBSD) command line scanner (ESET Software ESETS Command Line Interface v 2.71.12); the nod32cli utility has been replaced by esets_cli; update provided by Willi Gruber; - keep time of a message reception (or creation) in rx_time as a floating point value as provided by Time::HiRes::time in order to avoid discrepancy in SNMP-like counters showing elapsed times; suggested by David Schweikert; - store additional attributes in per-message and per-recipient objects, reducing a need for repeated lookups; - to save time the: tag, tag2, tag3 and kill levels are not looked up when a recipient is bypassing spam checks; - no longer reverse-resolve an IP address obtained from a Received header field on infected mail: it does not work for IPv6, it can be stuck for prolonged periods waiting for a response from a non-responsive DNS server, and is only used for logging/notifications (macros %e and %o) on infected mail, so it is not too bad to drop it; - obey the always_bcc_by_ccat even when releasing from a quarantine; thanks to Tomas Horacek; - for consistency with other quarantine methods: store recipient address (e.g. a quarantine mailbox e-mail address) as a quarantine location even if a quarantine method is 'smtp:' or 'lmtp:'; - a quarantined message now receives one additional header field: an X-Envelope-To-Blocked. An X-Envelope-To still holds a complete list of envelope recipients, but the X-Envelope-To-Blocked only lists its subset (in the same order), where only those recipients are listed which did not receive a message (e.g. being blocked by virus/spam/ banning... rules). This facilitates a release of a multi-recipient message from a quarantine in case where some recipients had a message delivered (e.g. spam lovers) and some had it blocked; - a release request now takes its default recipients list from a header field X-Envelope-To-Blocked, no longer from X-Envelope-To. This avoids releasing a message to recipients which have already received it in the first place, e.g. spam lovers. For backward compatibility, if X-Envelope-To-Blocked header field is not found in a quarantined message, the recipients list defaults to X-Envelope-To as before. A release request can still provide its explicit list of recipients to override a default, like before. Loosely based on suggestions by Christer, by Paolo Schiro and others; - when mail is received from a helper program such as a milter, update a true mail size ($msginfo->msg_size) according to size definition in RFC 1870, adjusting for CRLF line terminators; - updated SMTP enhanced status response codes to an AUTH command according to RFC 4954 (SMTP Service Extension for Authentication); - quote_rfc2821_local: within qcontent all non-qtext characters should be represented as quoted-pairs (RFC 2822); in addition to " and \ (which were already handled), this includes NUL, CR, LF, HT, SP and 8-bit characters; - macros T, C and B now bring e-mail addresses in quoted form and in angle brackets, notification templates no longer add angle brackets around %T and %C; - when defanging mail body no longer insert our own Sender header field on a pretense that it helps with DKIM resigning - according to ADSP (ex SSP) the DKIM/ADSP does not care for the Sender header field (unlike a historical DomainKeys); - always provide X-Amavis-PolicyBank header field in a copy of a mail as submitted to SpamAssassin, even if a policy bank path is empty - this allows for simpler SpamAssassin rules to avoid being tricked by a presence of such header field inserted by third parties; - add missing installs of amavisd-nanny and amavisd-agent utilities to amavisd-new.spec; suggested by Eddy Beliveau; - internal: original header section is no longer kept in two copies (in orig_header and in orig_header_fields). Instead, orig_header now holds an array of header *fields* (previously: array of header *lines*), and orig_header_fields now stores a by- header-field-name hash of indices into orig_header (previously it held copies of header field bodies). To facilitate access to individual header fields and allow for top-down and bottom-up search for a n-th occurrence of a header field, two new access methods are provided: get_header_field and get_header_field_body; these are optimized for a quick access to the *last* header field (previously the *first* one was kept easily accessible); - internal: when specific header fields are looked up in an original header section (such as From, To, Subject, Message-ID, ...) now the *last* (the most to the bottom) occurrence of a header field (instead of the first) is used for better compatibility with DKIM which searches for header fields bottom-up; - internal: when a message is received, a current setting of a boolean attribute $originating (global or from current policy banks) is now copied to an Amavis::In::Message object and becomes a property of a message; this allows for other message objects (like notifications, quarantine) to have their own individual setting of this attribute, for example notifications are always flagged as originating and as such are eligible for DKIM signing; - internal: provide an Amavis::MIME::Body::OnOpenFh package which acts as a MIME::Body -type object (read-only) with an underlying representation in an existing (permanently open) temporary mail file, avoiding a need to open it by file name on a separate file handle. It is useful (simpler, faster) when defanging (pushing original mail to an attachment), or when generating notifications or reports which contain an original mail; - internal: new subroutines init_child and rundown_child in the interface module Amavis::SpamControl::SpamAssassin, which call SpamAssassin plugin methods "spamd_child_init" and "spamd_child_post_connection_close"; this is required for correct operation of some SpamAssassin plugins such as the Mail::SpamAssassin::Plugin::DBI; thanks to Michael Scheidell for pointing out that plugin; - internal: renamed a subroutine string_to_mime_entity into build_mime_entity and generalize its functionality to make it useful for generating reports; - internal: generalized subroutine delivery_status_notification which can now also prepare message in a format of a feedback report (ARF) or just as a plain included (inline) original mail; - internal: a subroutine passed to edit_header() may return undef, in which case a header field will be deleted instead of being replaced/edited; - internal: sub lookup_sql incompatible change to the order of arguments; - internal: a small speedup in receiving mail contents over SMTP or LMTP; - internal: abandon package Amavis::Lock, do the four calls to flock directly; - internal: move subroutines dealing with processes from module Amavis::Util into a new module Amavis::ProcControl; - internal: reduce buffer sizes in some copying loops from 64kB back to 32kB, larger buffers do not perform any better; - documentation: amavisd.conf-default: add a list of legacy dynamic configuration variables which can be used in policy banks; thanks to Gary V; - terminology: use a term 'header section' (in comments, log entries and templates) instead of a 'header' to go along with the 2822upd draft; A QUICK START TO DKIM VERIFICATION Starting with 2.6.0, verification of DKIM signatures (and historical DomainKeys signatures) is provided directly by amavisd (not only by a SpamAssassin plugin DKIM). A required version of a perl module Mail::DKIM is 0.31 or later. Signature verification is sufficiently fast so there is no need for concern about extra processing load. To turn on DKIM (and historical DomainKeys) signature verification, please add the following line to amavisd.conf (if not already there): $enable_dkim_verification = 1; Benefits - Whitelisting of banned checks or spam on messages carrying valid DKIM or DomainKeys signatures from trustworthy signers is possible through the @author_to_policy_bank_maps list of lookup tables. The mechanism uses loading of policy banks based on author's e-mail address (addresses in a 'From:' header field) and a signing domain, so a full flexibility of per-policy-bank settings is available. See description of a new configuration variable @author_to_policy_bank_maps earlier in this release notes. - To each message passed to local recipients (matching the @local_domains_maps) amavisd inserts a header field Authentication-Results (according to draft-kucherawy-sender-auth-header, now RFC 5451) for each signature found in a message, reporting a corresponding verification result. These header fields can reliably tell a recipient or his MUA what domains claimed responsibility for a message, or can be used for troubleshooting DKIM signing, verification and tracking of mail transformations. - Can adjust spam score based on some metrics on a signing domain's reputation for valid signatures found in a message. A useful reputation metric is an average long term spam score for past messages signed by a domain, which can currently be provided manually by @signer_reputation_maps in a configuration file (see example earlier in this release notes). A spam score is shifted towards this reputation score by a configurable factor $reputation_factor (value between 0 and 1, default is 0.2) using a formula: adjusted_spam_score = f * reputation + (1-f) * spam_score Semantics of a $reputation_factor is equivalent to auto_whitelist_factor in a SpamAssassin's AWL plugin, which shifts spam score towards a long term spam score average of a sender. - Notifications and bounces show a "(dkim:AUTHOR)" next to a From address, and a "(dkim:SENDER)" next to a Sender address if these header fields were signed and their domain corresponds to a signer's domain identity. - A valid DKIM or DomainKeys signature turns on a 'sender_credible' attribute which serves to choose one of the two DSN cutoff levels, so that delivery status notifications can be restricted to or preferred for likely-to-be-valid sending addresses, and bounces to possibly fake addresses can be minimized. More information on the 'sender_credible' attribute can be found earlier in this release notes. Currently the ADSP (Author Domain Signing Practices, formerly SSP) draft is not implemented, neither by amavisd, nor by SpamAssassin's plugin DKIM. Work on a draft is still in progress, and until it settles and comes into wider use one needs to resort to SpamAssassin rules to block fake mail with no valid signature from domains which are known to be signing all their mail, such as PayPal, eBay, alert.bankofamerica.com, and others. In essence, the ADSP information (usually inferred, or actually published (quite rare today)) from such domains needs to be encoded into SpamAssassin rules. (A note from the future: ADSP is now RFC 5617, SpamAssassin's DKIM plugin does implement ADSP starting with version 3.3.0.) A QUICK START TO DKIM SIGNING 1. Generate one or more keys to be used for signing and enable signing code by adding the following line to amavisd.conf (if not already there): $enable_dkim_signing = 1; # loads DKIM signing code Signing keys must be made available to amavisd, each private key in a separate file in PEM format. Customarily such keys would be generated and kept in a dedicated directory such as /var/db/dkim or /var/lib/dkim, preferably owned by root. Private keys can be generated by a 'openssl genrsa' command (see RFC 4871 Appendix C), or by an amavisd equivalent. Commonly one key per signing domain or one key per signing host is used, but other choices are possible. If such keys were already prepared for some other DKIM-signing solution, they can be reused by amavisd. # amavisd genrsa /var/db/dkim/a.key.pem # amavisd genrsa /var/db/dkim/b.key.pem 786 # amavisd genrsa /var/db/dkim/sel-example-com.key.pem # amavisd genrsa /var/db/dkim/g-guest-ex-com.key.pem # amavisd genrsa /var/db/dkim/notif-mail.key.pem 512 Amavisd already ensures the generated files are only readable by owner, but a manual procedure may require explicitly setting file permissions. Private keys must be protected from unauthorized access, only the signing software such as amavisd should have read access. On a fresh program start the amavisd daemon loads these files before dropping privileges, so if amavisd is started as root these key files are readable regardless of their file access permission. Note however that starting with version 2.7.0 amavisd is also capable of a warm reload, where the daemon restarts with existing UID, so it does not have elevated privileges to read files with private keys. Therefore it is recommended that the file is owned by a group under which amavisd runs and UID root, and the file access permission allows read access for a group, e.g.: -rw-r----- 1 root vscan ... /var/db/dkim/xxx.key.pem 2. Add commands to amavisd.conf to load private keys, associate them with signing domains and selectors, and describe constraints (tags) to be published with public keys. For example: # Load all available private keys and supply their public key RR constraints. # Arguments are a domain, a selector, a key (a file name of a private key in # PEM format), followed by optional attributes/constraints (tags, represented # here as Perl hash key/value pairs) which are allowed by RFC 4871 in a public # key resource record (v, g, h, k, n, s, t), of which only g, h, k, s and t # are considered to be constraints limiting the choice of a signing key. # A command 'amavisd showkeys' can be used for displaying corresponding # public keys in a format directly suitable for inclusion into DNS zone files. # # signing domain selector private key options # ------------- -------- ---------------------- ---------- dkim_key('example.org', 'abc', '/var/db/dkim/a.key.pem'); dkim_key('example.org', 'yyy', '/var/db/dkim/b.key.pem', t=>'s'); dkim_key('example.org', 'zzz', '/var/db/dkim/b.key.pem', h=>'sha256'); dkim_key('example.com', 'sel-2008', '/var/db/dkim/sel-example-com.key.pem', t=>'s:y', g=>'*', k=>'rsa', h=>'sha256:sha1', s=>'email', n=>'testing; 1, 2'); dkim_key('guest.example.com', 'g', '/var/db/dkim/g-guest-ex-com.key.pem'); dkim_key('mail.example.com', 'notif', '/var/db/dkim/notif-mail.key.pem'); A selector paired with a domain name uniquely identifies a key, both for a signer as well as for a recipient. There may be multiple keys for each domain as long as each one has its own selector. A selector along with a domain name will be used by a receiving mailer in assembling a DNS query (selector._domainkey.signingdomain) to fetch a public key from a signing domain's DNS server when verifying signature validity. A selector paired with a domain name will also be used by a signing amavisd when choosing a key applicable to signing, meeting constraints on its public key (tags, RFC 4871 section 3.6) as given by optional arguments. Optional arguments serve as site documentation, may help amavisd choose between multiple choices (ruling out keys with incompatible tags), and supply additional information for step 3. For a list of options (tags) see RFC 4871 section 3.6. Amavisd does not check the syntax of tag values, except for performing qp-section encoding of a tag 'n'. Note the Perl syntax of key/value pairs, e.g. t => 's:y' will end up as "t=s:y", and n => 'testing; 1, 2' will end up encoded as "n=testing=3B 1, 2". 3. Prepare and publish public keys. Public keys can be extracted from generated key files (which contain both a private and a public key). To publish public keys they need to be edited into a format suitable for inclusion in a DNS server's zone file for each signing domain, either by following a procedure in RFC 4871 Appendix C, or if step 2 was completed, by asking amavisd to do so: # amavisd showkeys or more selectively, e.g.: # amavisd showkeys .org example.com This step is not needed if public keys were already prepared and published earlier for some other DKIM-signing solution. 4. Edit zone files in master DNS server(s) for each signing domain, adding the just prepared TXT resource records, not forgetting to bump up the serial number in a SOA record. Optionally add a TXT record with ADSP information (formerly SSP) if a default Author Domain Signing Practices is not appropriate. Then reload zone(s) or restart DNS server(s). 5. Test published public keys. Similar to 'showkeys', a 'testkeys' command walks through available signing keys (as declared by calls to dkim_key), generates test messages each signed with one key, and validates them by fetching a corresponding public key from a DNS server. # amavisd testkeys or more selectively, e.g.: # amavisd testkeys .org example.com (btw, if testkeys fails and you believe your DNS is correctly serving your DKIM public keys, you may need to upgrade Perl module Mail-DKIM to version 0.33 or at least 0.32_6) 6. Restart amavisd, watch the log at log level 2, searching for " dkim: ". Note that signing could be started (amavisd reload) right after completing step 2, but mail recipients would not be able to verify validity of signatures until public keys are made available by a signing domain through its DNS. Recipients are supposed to treat mail with signatures which fail verification exactly the same as mail with no signatures, so there is usually no harm done with a premature start of signing, but there is no benefit either. 7. Optional: to override default values for signature tags, one may specify by-sender signature tags through @dkim_signature_options_bysender_maps, e.g.: # @dkim_signature_options_bysender_maps maps author/sender addresses or # domains to signature tags/requirements; possible signature tags according # to RFC 4871 are: (v), a, (b), (bh), c, d, (h), i, l, q, s, (t), x, z; # of which the following are determined implicitly: v, b, bh, h, t # (tag h is controlled by %signed_header_fields); currently ignored tags # are l and z; instead of an absolute expiration time (tag x) one may use # a pseudo tag 'ttl' to specify a relative expiration time in seconds, which # is converted to an absolute expiration time prior to signing: x = t + ttl; # a built-in default is provided for each tag if no better match is found # @dkim_signature_options_bysender_maps = ( { 'postmaster@mail.example.com' => { a => 'rsa-sha1', ttl => 7*24*3600 }, 'spam-reporter@example.com' => { a => 'rsa-sha1', ttl => 7*24*3600 }, 'mail.example.com' => { a => 'rsa-sha1', ttl => 10*24*3600 }, # explicit 'd' forces a third-party signature on foreign (hosted) domains 'guest.example' => { d => 'guest.example.com' }, '.example.com' => { d => 'example.com' }, # catchall defaults '.' => { a => 'rsa-sha256', c => 'relaxed/simple', ttl => 30*24*3600 }, # 'd' defaults to a domain of an author/sender address, # 's' defaults to whatever selector is offered by a matching key } ); The result of a by-sender lookup into @dkim_signature_options_bysender_maps is a hash (a set) of DKIM signing requirements (tags), i.e. canonicalization method, hashing algorithm, domain, identity, selector and expiration time. All matching entries can participate in the result: for each tag individually the first setting (the most specific) is chosen from all matching entries. Resulting tags are then used to choose the most appropriate signing key from a set of keys as declared by calls to dkim_key. Main selection criterium is a match on tags d (domain) and s (selector), but other signature requirements must also meet the constraints of a public key (e.g. subdomain matching flag, granularity, hashing algorithm, key type). If a lookup does not find a signing key which meets requirements, no signing takes place. Also, only mail with 'originating' flag is eligible for signing. A lookup is based on either the From header field, the Sender header field, or on a mail_from address from the envelope, whichever yields a useful result first. Note that neither the Sender header field nor a mail_from address has any special meaning in the standard (RFC 4871). This results either in an author domain signature (i.e. a first-party signature, when based on a From header field), or in a third-party signature (when signing domain does not match the From, regardless of whether it is based on Sender header field or a mail_from or forced through d tag). An associative array %signed_header_fields controls which header fields are to be signed. By default it contains a standard (RFC 4871) set of header field names, augmented by some additional header field names considered appropriate at the time of a release (RFC 4021, RFC 3834). In addition a 'Sender' header field is excluded because it is frequently replaced by a mailing list, and as the RFC 2821 mandates there can only be one such header field the original one is dropped, invalidating a signature. Also the 'To' and 'Cc' are excluded from a default set because sendmail mailers are known to gratuitously reformat the list, invalidating a signature. A value in key/value pairs of %signed_header_fields is currently interpreted as a boolean, but stick to values 0 and 1 for now to allow for future enhancements. The default set of header fields to be signed can be controlled by setting %signed_header_fields elements to true (to sign) or to false (not to sign). Keys must be in lowercase, e.g.: $signed_header_fields{'received'} = 0; # turn off signing of Received $signed_header_fields{'sender'} = 1; # turn on signing of Sender $signed_header_fields{'to'} = 1; # turn on signing of To $signed_header_fields{'cc'} = 1; # turn on signing of Cc --------------------------------------------------------------------------- March 12, 2008 amavisd-new-2.5.4 release notes BUG FIXES - simplify regular expressions in parse_quoted_rfc2821() to avoid perl crashing on a long degenerated e-mail address; reported by Sébastien Aveline; - further simplify (split in two) regular expressions in parse_address_list() to avoid perl crashing on long degenerated e-mail addresses in From, To, and Cc header fields, also reported and sample provided by Tomi Lukkarinen; - incorrect parsing of header fields could let a header field to be ignored when preparing notification templates, or when adding a spam tag to a Subject header field, causing a second Subject header field to be inserted; reported by Mike Cisar; - untaint a policy bank name when it comes from an AM.PDP protocol request; symptom was a failure to insert a pen pals SQL record in a milter setup; reported by Peter Huetmannsberger; - smtp client code inappropriately concluded there is no progress being made when forwarding a message back to MTA, and exited a rw_loop when sysread returned status EAGAIN despite a 'select' setting an input-ready flag; the problem was detected on Solaris, although it could be more general; thanks to Aleksandr for a detailed problem report; - limit the number of filenames given as arguments to a file(1) utility to stay within a safe program argument space limit, run file(1) multiple times if necessary; - change the sprintf format for conversion of 64-bit SNMP-like counter values into a string (replaced %020d by %020.0f) to properly convert large values (beyond 32 bits) into strings on versions of Perl which are not compiled with support for 64-bit integers (Solaris?); reported by David Schweikert; OTHER - invoke unrar and rar without option 'av-', which is no longer valid since version of unrar 3.7.5, and was previously apparently just ignored; the rar documentation states that option 'av' is only available with registered versions, so it appears the 'av-' is redundant with rar as well. Unrar aborting with 'unknown option' error was reported by Piotr Meyer; - a new AV entry for fpscan 6.x, (F-PROT Antivirus for UNIX command-line scanner) based on information from Erik Slooff, Bruno Friedmann, and Steve); - a new AV entry for fpscand 6.x (F-PROT Antivirus for Linux/BSD/Solaris), based on initial research by Alexander Wirt, Henrik K, and F-Prot documentation, thanks also to Haukur Valgeirsson of F-Prot International; - ask_daemon_internal now keeps persistent connection to an F-PROT fpscand daemon, in addition to Sophie and Trophie daemons; - new AV entry 'bdscan' for a new version of BitDefender Antivirus Scanner for Linux and FreeBSD; thanks to Gary V; - updated README.postfix, thanks to Chris Pepper and Patrick Ben Koetter; - updated amavisd-new-docs.html; --------------------------------------------------------------------------- December 12, 2007 amavisd-new-2.5.3 release notes BUG FIXES - fix parsing an SMTP status response from MTA when releasing from a quarantine, when an MTA response did not include an enhanced status code (RFC 3463) (such as with old versions of Postfix); a parsing failure resulted in attribute "setreply=450 4.5.0 Unexpected:..." in an AM.PDP protocol response, even though a release was successful; reported by Ron Miller, John M. Kupski, investigated by Tony Caduto and Jeremy Fowler; - change parsing of addresses in From, To, and Cc header fields, avoiding complex Perl regular expressions which could crash a process on certain degenerate cases of these header fields; thanks for detailed problem reports to Carsten Lührs and Attila Nagy; - completely rewritten parsing of Received header field to work around a Perl regular expression problem which could crash a process on certain degenerate cases of mail header fields; problem reported by Thomas Gelf; - harden to some extent regular expressions in parse_message_id to cope better with degenerate cases of header fields carrying message-id; - sanitize 8-bit characters in In-Reply-To and References header fields before using them in Pen Pals SQL lookups to avoid UTF-8 errors like: penpals_check FAILED: sql exec: err=7, 22021, DBD::Pg::st execute failed: ERROR: invalid byte sequence for encoding "UTF8": 0xd864 - when turning an infection report into a spam report, avoid adding newly discovered virus names (i.e. fraud names) to a cached list if these names are already listed; previously the list would just grow on each passage through a cache, leading to unsightly long lists of spam tests in a report; based on a patch by Henrik Krohns; - fix diagnostics when an invalid command line argument is given; OTHER - reduce log clutter when certain Perl modules are loaded late, i.e. after chrooting and daemonizing, but still before a fork; now only issue one log entry by a parent process: "extra modules loaded after daemonizing: "; - slightly relax mail address syntax in subroutine split_address; - fetch additional information (tags) from SpamAssassin: TESTS, ASN, ASNCIDR, DKIMDOMAIN and DKIMIDENTITY, making them available through a macro 'supplementary_info' (if a version of SpamAssassin in use provides them); - updated DKIM section in amavisd-new-docs.html, removing the historical DomainKeys milter from examples; - declared a dummy subroutine dkim_key() and new dummy configuration variables @dkim_signature_options_bysender_maps, %signed_header_fields, $reputation_factor, @signer_reputation_maps and $sql_partition_tag, members of policy banks, in preparation for 2.6.0 - declared now for improved downgrade compatibility of 2.6.0 configuration files, if need arises. --------------------------------------------------------------------------- June 27, 2007 amavisd-new-2.5.2 release notes BUG FIXES - in a milter setup log_id was left undefined, which resulted in log lines without id, and an SQL constraint violation "Column 'am_id' cannot be null" when logging to SQL was enabled. The bug was introduced in 2.5.1; problem reported by Martin Svensson; - suppress a second quarantining attempt if the message also needs to be archived to the same location (same sql key or same local filename); reported by Wazir Shpoon; - adjust $socketname in amavisd-release to match its default counterpart in amavisd (i.e. /var/amavis/amavisd.sock); reported by Stanley Appel; NEW FEATURES - add snmp-like counters for PenPalsSavedFromKill, PenPalsSavedFromTag3 and PenPalsSavedFromTag2, which correspond to the number of messages since a program (re)start in which spam level would have exceeded a corresponding level had there not been for (negative) score points contributed by pen pals lookups. Note that for any message only one of the three counters could increment, the one corresponding to the highest level crossed. To find more information about rescued mail messages, search the log for a string 'PenPalsSavedFrom' (available at log level 2 or higher). Practical value: mail saved by pen pals from being blocked often indicate false positives by SpamAssassin; examining rules which contributed significantly to the score may indicate which rules need adjustment; - when preparing an SQL SELECT clause in lookup_sql, provide an additional placeholder %a in a clause template, which is much like the existing %k, but evaluates to an exact mail address (i.e. the same as the first entry in the %k list), which makes it suitable for SQL pattern matching; suggested by Daniel Duerr; - macro supplementary_info can supply information on two additional SpamAssassin tags: AUTOLEARNSCORE and LANGUAGES if corresponding plugins are enabled in SpamAssassin; see README.customize for the complete list; - provide two new subroutines available for calling from config files: include_config_files() and include_optional_config_files(), each take a list of filenames as arguments, and reads & evaluates them just like normal configuration files specified on a command line (option -c or a default amavisd.conf). This provides a simplified and uniform mechanism for 'including' additional configuration files, which formerly could be invoked through a perl do() function. The only difference between include_config_files and include_optional_config_files is that the former aborts if some specified file does not exist, while the later silently ignores specified but missing files. Both/each subroutine may be called multiple times, recursion is allowed (but some sanity limit to recursion is provided); based on a suggestion by Gary V. Example line in amavisd.conf: include_config_files('/etc/amavisd-custom.conf'); OTHER - provide a workaround for a crashing altermime by removing its leftover temporary file which would otherwise cause a temporary failure: TempDir::check: Unexpected file problem reported by Dennis A. Kanevsky; - add a mapping to 'doc' for a result 'Microsoft Installer' from a file(1) utility; it seems like versions 4.20 and 4.21 of file(1) (possibly earlier versions too) misclassify MS Word, Excel, and PowerPoint documents as 'Microsoft Installer'; problem investigated and a workaround suggested by Noel Jones, Mike Cappella and Michael Scheidell; - add a mapping to 'asc' for a result 'COM executable for DOS' from a file(1) utility; it seems like later versions of file(1) can misclassify a text in a GB2312 character set as a COM file; reported by Daniel J McDonald; - updated AV entry for ESET NOD32 Linux Mail Server again - command line interface (nod32cli): added a status 3 (e.g. corrupted archive) back to the list of clean statuses; the 3 was removed in 2.5.1 as the entry was substituted with the one from a NOD32 documentation; reported by Tamás Gregorics; - updated AV entry for 'F-Secure Antivirus for Linux servers' to cope with version 5.5 and later; a new entry provided by Peter Bieringer; - when a command line option -g requests changing of group ID, do so by calling POSIX::setgid, after also attempting to assign to perl variables $( and $), which may not work correctly on systems where group ID can be negative (like group 'nobody' being -2 on Mac OS X); follows a SpamAssassin problem report 3994, investigated by Sidney Markowitz; - when an AUTH command parameter (RFC 2554, now RFC 4954) is supplied on a MAIL FROM SMTP command but AUTH support has not been previously offered (like when authentication is disabled by an empty @auth_mech_avail), no longer treat the situation as a fatal error: 503 5.7.4 Error: authentication disabled but mercifully ignore the parameter and just log an informational message. This is a deviation from RFC 2554, but makes it friendlier for those insisting on running amavisd as a Postfix pre-queue smtp proxy; suggested by Alexander 'Leo' Bergolth; - adjust the list of pre-loaded perl modules required by SpamAssassin; - internal: pass a mail message to SpamAssassin as a GLOB instead of an array reference, saving one in-memory copy of a message during a SA call; - internal: make it slightly easier to switch message digest from MD5 to a Digest::SHA family by turning a hard-wired key length into a parameter (admittedly it is still ugly, requiring a change in three places for switching); also pave a transition from Digest::SHA1 to Digest::SHA; - documentation: updated files README.postfix and README.postfix.html now include a section 'Advanced Postfix and amavisd-new configuration' explaining a multiple cleanup service architecture; thanks to Patrick Ben Koetter; retired file: README.postfix.old - documentation: updated README.sql-pg to include a faster alternative to purging an SQL logging database: the alternative 'DELETE FROM maddr' on PostgreSQL runs faster by a factor of 1.5 to 2 from the one previously suggested; - suggestion: when using SpamAssassin plugin Rule2XSBody (available in more recent versions of SA), adding an entry like: Mail::SpamAssassin::CompiledRegexps::body_0 to the @additional_perl_modules list allows preloading of compiled rules. Adding the following two lines to amavisd.conf adds the directory name containing modules with compiled rules to Perl modules search path and allows Perl to find the listed module(s): my($sa_instdir) = '/var/lib/spamassassin/compiled/3.002001'; unshift(@INC, $sa_instdir, $sa_instdir.'/auto'); --------------------------------------------------------------------------- May 31, 2007 amavisd-new-2.5.1 release notes COMPATIBILITY WITH 2.5.0 - setting $bypass_decode_parts to true now also disables MIME decoding (see below); SECURITY - provides checking the number of archive members against $MAXFILES quota even when just listing an archive directory, providing some additional protection (besides a time limit) against runaway dearchivers (such as a recent Zoo archiver DoS); - please use the most recent versions of file(1) utility (currently 4.21) and recent versions of external dearchivers/decoders to avoid known security vulnerabilities in them; NEW FEATURES - introduced a variation of a message release from a quarantine, allowing a releaser to choose between forwarding a message to the back-end MTA port as usual (avoiding re-checking of a message), or to send it to MTA on its incoming port (normally 25) and let the message be rescanned, which might be useful after adjusting spam rules or antivirus database. It is implemented by: * adding a configuration variable $requeue_method (also a member of policy banks), with a default value: 'smtp:[127.0.0.1]:25' * extending the AM.PDP protocol with a 'request=requeue' attribute which can be used in place of a 'request=release', * enhancing the 'amavisd-release' utility program to choose between sending 'request=release' and 'request=requeue' based on its program name, i.e. by making a soft or hard link to amavisd-release (or its copy) named 'amavisd-requeue', the utility will send a 'request=requeue' in place of the usual 'request=release', e.g.: # ln -s amavisd-release amavisd-requeue $ amavisd-requeue spam/k/kg2P0rP9Lpu3.gz * enhancing amavisd daemon to choose between forwarding a released message either to $release_method or to $requeue_method destination based on a 'request' attribute value in an AM.PDP request; - new AV entry: ArcaVir for Linux and Unix, see below for links; - a new macro 'supplementary_info' gives access to some additional information provided by content scanners, such as a provided by SpamAssassin API routine get_tag. The macro takes two arguments, the first is a tag name (a name of some attribute which is expected to provide an associated value), the second argument is a sprintf format string and is optional, if missing a %s is assumed. Currently the only available attributes are AUTOLEARN, SC, SCRULE, SCTYPE, and RELAYCOUNTRY. These are nonempty only when an associated SpamAssassin plugin or function is enabled. BUG FIXES - fixed quarantining to an SQL database of messages with a null envelope sender address (broken in 2.5.0, causing such messages to tempfail); reported by Markus Edholm, Vahur Jõesalu and Michael Scheidell; - fixed parsing of certain broken 'From' header fields, which would result in a temporary failure and the following logged error: check_init2 FAILED: parse_address_list PANIC1 53 at /usr/local/sbin/amavisd line 3292 reported by Michael Scheidell; - avoid encoding nonprintable characters in X-Envelope-From and X-Envelope-To header fields in a quarantined message even if envelope mail addresses contain such invalid characters, so that a quarantine release is possible; (RFC 2047 allows encoding of a 'phrase' in From, To, and similar headers, as well as in comments, but not in the address specification); - avoid unnecessarily RFC 2047 -encoding of 8-bit characters in those lines of inserted X-Spam-Report (and similar) multiline header fields which only contain ASCII characters; also avoid encoding of newlines; reported by Anant Nitya; - sanitize 8-bit characters in SpamAssassin report before inserting it into an X-Spam-Report header field; - properly recognize PostgreSQL error code 'S8006' and reconnect to a disconnected server right away; thanks to Brian Wong; - call $mail_obj->finish after a SA call to allow for garbage collection and removal of SA temporary files; see: http://issues.apache.org/SpamAssassin/show_bug.cgi?id=5444 - avoid nonstandard SMTP status code 254 on discarded malware; on discarding turn status 554 into a 250 instead; violation of a SHOULD in RFC 2822 pointed out by Alexander 'Leo' Bergolth; - an informational log message was reported inappropriately: INFO: truncated ... header line(s) longer than 998 characters it didn't reflect reality, it was always reported together with the: INFO: unfolded 1 illegal all-whitespace continuation lines - when an SMTP option BODY=8BITMIME (RFC 1652) is not given on mail reception, avoid turning it on while forwarding, even if mail body contains 8-bit characters; following a garbage-in-garbage-out principle, this doesn't break anything that isn't already broken, but might prevent later conversion to 7-bit quoted-printable MIME by some downstream MTA, invalidating signatures (DKIM, S/MIME, PGP, ...) - at a risk that some overzealous firewall might block a mail transfer; - fixed a couple of documentation typos/bugs in README.customize, thanks to Mike Cappella; OTHER - modified code for checking each eval {} status: it turns out that eval is able to capture certain error conditions (e.g. certain I/O errors) but without setting the $@ variable, leaving it empty; use new idiom throughout for proper error handling and more informative reporting, showing errno in such cases; P.S.: actually what happens is the $@ gets cleared by some DESTROY method which uses eval, so the outer eval does notice an exception but is unable to capture en error message; - setting $bypass_decode_parts to true now also disables MIME decoding, not just decoders/dearchivers listed in a @decoders list, and also implicitly retains full original message for virus checking, equivalent to having a regular expression /^MAIL$/ in a @keep_decoded_original_maps list; prompted by Bill Landry; - new AV entry: ArcaVir for Linux and Unix, see: http://www.arcabit.pl/ http://www.arcabit.com/download_product.html?product=ArcaVirLinux2007 http://www.arcabit.com/products_arcavir_for_unix_2006.html the entry was kindly provided by Michal Seremak; - updated AV entry for ESET NOD32 Linux Mail Server - command line interface (nod32cli), version 2.7, thanks to Simon; - updated AV entry for Sophos sweep, adding options -mime and -oe ; - avoid repeatedly reporting the same set of modules by a log entry 'extra modules loaded:', only report it on changes to the list; repeated reports could be misinterpreted that modules were loaded with each mail task, where actually missing modules were only loaded once within each child process; - avoid reporting 'BOUNCE' in an SMTP response text when a bounce (i.e. a nondelivery status notification) was actually suppressed, such as is usually the case with infected mail or when spam score exceeds spam_dsn_cutoff_level. Previously the SMTP response text only reflected the setting of a final_*_destiny, which could mislead mail administrators into believing that excessive unconditional backscatter was being generated. The new text looks like: 250 2.5.0 Ok, id=67685-15, DISCARD(bounce.suppressed) instead of previous: 250 2.5.0 Ok, id=67685-15, BOUNCE A general note worth reiterating: to reduce backscatter pollution (sending of bounces to innocent sender addresses), please either: * set $final_virus_destiny and $final_spam_destiny to D_DISCARD or to D_PASS (_not_ to D_REJECT or D_BOUNCE), or: * carefully configure virus and spam bounce suppression by: . configuring @viruses_that_fake_sender_maps correctly (the default is fine, it suppresses all bounces to infected mail), this way one may safely set $final_virus_destiny to D_BOUNCE, it is equivalent to D_DISCARD for all infected mail containing malware matching the @viruses_that_fake_sender_maps; . and: configuring @spam_dsn_cutoff_level_maps and @spam_dsn_cutoff_level_bysender_maps, keeping levels just slightly over a kill level, have a well maintained SpamAssassin with network tests enabled and updated rules - then one may set $final_spam_destiny to D_BOUNCE, which will produce bounces for mail with spam score between kill level and cutoff level, and suppress bounces above a suppress level; some domains may still consider such practice abusive, so do not take this choice lightly; . to monitor bounces generated by amavisd, one may assign some dedicated monitoring e-mail address to $dsn_bcc, which will then receive a copy of all delivery status notifications sent out by amavisd; - dspam options changed with version 3.8.0, replacing option --feature with --tokenizer; reported by Jim Knuth; - modified syslog writing to check errno after calling Unix::Syslog::syslog, and to informatively attempt to log status when unsuccessful; an unsuccessful status is just informational, as syslog(3) routine does its own retries and leaves an unsuccessful status of a previous attempt in errno even if a subsequent logging attempt did succeed; unfortunately the system routine syslog(3) returns no value according to documentation (and according to its source code), so its completion status can not be tested; a problem of a loss of logging on a syslogd restart on OS X was reported by Paul Walker, but unfortunately can not be solved on the application side; - uncomment some debugging printouts in p0f-analyzer.pl and land them under control of a $debug variable; --------------------------------------------------------------------------- April 23, 2007 amavisd-new-2.5.0 release notes COMPATIBILITY WITH 2.4.5 The 2.5.0 is upwards compatible with 2.4.* versions, except for the following: Default notification and logging templates are enhanced to take advantage of new macros and new concepts, so it is prudent to update templates if defaults are overridden, e.g. $log_templ, $notify_*_admin_templ, ... A client-side AUTH (rfc2554: SMTP Service Extension for Authentication) is currently not available. The last version with this feature working is amavisd-new-2.5.0-pre4. The feature may be restored in a future version if sufficient interest is demonstrated. A workaround for a qmail bug (which complains when CR and LF are split across a TCP segment boundary) is no longer available, as the program has no control over IP packet splitting done by the TCP/IP stack. When pen pals feature is in use, it is worth creating an index on a msgs.message_id field. NEW FEATURES AT A GLANCE - new concept: blocking contents category; - true per-recipient defanging/sanitation of a mail body (previously a true per-recipient handling was available for mail header edits, but not for mail body modifications); - added interface code to invoke Anomy Sanitizer or the 'altermime' program allows defanging or adding disclaimers by external utilities on a per-recipient basis; - rewritten SMTP client code: get rid of the troublesome module Net::SMTP; new code now supports pipelining, client-side LMTP, IPv6, Unix sockets, more reliable error detection and handling, passes on ENVID parameter unchanged, is bare-CR-clean, tidier code (no more workarounds for rough corners in Net::SMTP), fewer context switches (handshake handovers) due to pipelining if pipelining is offered by MTA (which usually is); - makes available pedantically parsed addresses from a mail header: From, Sender, To, Cc. Addresses from mail header may be needed for deciding on inserting disclaimers, signing mail (DKIM), custom hooks (like 'vacation'-type applications), and other future applications. Get rid of inexact parsing by module Mail::Address, provide own parser; - phishing fraud as returned by ClamAV is now treated as spam, no longer as a virus; - compatible with SpamAssassin 3.2.0; - enhancements to amavisd-nanny: shows more detailed states of processes; - enhancements to amavisd-agent: shows average processing times per message; - extended AM.PDP protocol with an attribute 'policy_bank' which may be used in a client's request to require loading additional policy banks; - add support for 7-Zip archives if external utility 7z is available; - custom hooks allow custom code to be called at few strategic places; - penpals can now also match replies which reference previous outgoing mail by its Message-Id (taking into account References or In-Reply-To header field); - new key 'originating' in policy banks generalizes a MYNETS policy bank; - a documentation rewrite for setting up amavisd-new with Postfix by Patrick Ben Koetter (one of the two authors of The Book of Postfix). Previous documentation has been renamed to README.postfix.old and will be removed in the next version; the new documentation is README.postfix.html, and its automatically converted plain text version is README.postfix. A big thanks to Patrick for his efforts! BUG FIXES - if a sender is both white- and black-listed at the same time, then inserted X-Spam-* header fields were inconsistent, e.g. X-Spam-Level, X-Spam-Flag and X-Spam-Status reflected a whitelisted status (no asterisks, not a spam), while X-Spam-Score showed 64 points; now whitelisting prevails in all X-Spam-* header fields; - relax argument parsing in amavisd-release to allow releasing of quarantine id containing a body hash in a name (%b in template); reported by Ron Rademaker; - skip an SQL-logging database operation if an associated clause in %sql_clause is disabled, e.g. set to undef or ''; this allows for example to selectively disable SQL logging based on a policy bank; thanks to Riaan Kok; - let LHA decoder (do_lha) recognize also other listing formats, e.g. MS-DOS, symlinks, not just plain Unix archives; problem reported by Ryuhei Funatsu; OTHER - catch and log uncaught '__DIE__' and '__WARN__' Perl pseudo-signals which would otherwise go to stderr and not be noticed with a daemonized process; patch by Alexander 'Leo' Bergolth; - insert 'X-Spam-Flag: NO' if spam level is above tag level but below tag2 level, previously it was not inserted at all (it is still redundant, but some may appreciate an explicit statement nevertheless); - drop support for Archive::Tar; main drawback of this module is: it either loads an entire tar into memory (horrors!), or when using extract_archive() it does not relativize absolute paths (which makes it possible to store members in any directory writable by uid), and does not provide a way to capture contents of members with the same name. Use pax program instead! - abandon the use of libnet (modules Net::SMTP and Net::Cmd), replaced by own code to implement client-side SMTP and LMTP protocol support, with a full support for pipelining and IPv6. Some of my issues with Net::SMTP go back to year 2002, but the one that broke camel's back is a 3+ months status quo in not fixing a serious misfeature introduced in 1.20, which mangles 8-bit characters in mail, causing a series of support questions, and even affecting some larger service providers without them realizing there is a problem. Here are some relevant bug reports: http://rt.cpan.org/Public/Bug/Display.html?id=24835 http://rt.cpan.org/Public/Bug/Display.html?id=2608 http://rt.cpan.org/Public/Bug/Display.html?id=2607 http://rt.cpan.org/Public/Bug/Display.html?id=14875 http://rt.cpan.org/Public/Bug/Display.html?id=9394 P.S. libnet-1.21 eventually fixed the UTF8 encoding problem and added support for ENVID and AUTH options; other issues are still open (e.g. split code/text in smtp status, no pipelining support, no LMTP); - represent score in X-Spam-Status header field as a single numeric field instead of the previous explicit sum of a SA score and a score boost, which some MUAs don't know how to interpret and label mail spaminess incorrectly; - recipient notifications (in their default template) now keep the original To and Cc header fields, instead of placing a recipient envelope address into a To header field; suggested by Jorgen Lundman; - no longer removes an X-Amavis-Alert header field (as inserted by some foreign MTA), it does not hurt to keep it; - kavscanner AV entry (Kaspersky Antivirus): added a new entry to the search list of paths to a binary, kavscanner changed its default installation location again; provided by Gary V; - added a rule into a $map_full_type_to_short_type_re list, mapping a file(1) type 'RIFF...animated cursor' into '.ani'; a patch by Eric; - example rules (commented-out) to block animated cursors (ANI) and icons are added to configuration files amavisd.conf and amavisd.conf-sample; - suppress bounces not only for Precedence: (bulk|list|junk) or null return path (as before), but also for mail with a 2822.From header field matching one of: *-request | *-owner | *-relay | *-bounces | owner-* | postmaster | mailer-daemon | mailer | uucp, and for mail containing a header field List-Id (RFC 2919). Note that this new additional rule practically never triggers in practice, true bounces and common mailing list traffic is already covered by previous checks; - make available a list of detected virus names in an Amavis::In::Message object (virusnames), suggested by Tom Sommer; - split SQL documentation into three files: README.sql general SQL considerations and some examples README.sql-mysql MySQL-specific notes and schema README.sql-pg PostgreSQL-specific notes and schema (also SQLite) - README.sql-pg now adds CHECK (x >= 0) for fields that are supposed to contain unsigned integers; suggested by Hanne Moa; - repurpose/rename a contents category CC_TEMPFAIL to CC_MTA; it is turned on when MTA rejects or tempfails a mail when amavisd attempts to pass it back after checking it; (this should not normally happen, MTA should be doing most of its checks (e.g. recipient validation) before passing mail to a content filter, not after); NEW FEATURES - custom hooks allow custom code to be called at few strategic places: * during child process initialization: allows initialization of custom code, including establishing an additional SQL session or similar; * after built-in checks: allows custom code to inspect and/or modify results of checks; * before sending, allows for additional quarantining and for sending additional notifications; * at the end of message processing: allows inspecting results of checks and status of mail forwarding (e.g. for statistics or logging purposes). Mail processing sequence: child process initialization *custom hook: new() loop for each mail: receive mail mail checking and collecting results *custom hook: checks() - may inspect or modify checking results deciding mail fate (lookup on *_lovers, thresholds, ...) quarantining sending notifications (to admin and recip) *custom hook: before_send() - may send additional notifications, additional quarantining, may modify mail forwarding (unless blocked) sending delivery status notification (if needed) issue main log entry, manage statistics (timing, counters, nanny) *custom hook: mail_done() - may inspect results endloop after $max_requests or earlier If Amavis::Custom::new returns undef then no further custom calls are made. Otherwise, this method is supposed to return an object, thus enabling other custom hooks, which receive this value as their first argument. See amavisd-custom.conf for an example, and the example invocation of amavisd-custom.conf at the end of file amavisd.conf-sample; thanks to Kasscie (Yohanna Monsalvez); - formerly penpals could only match replies to previous outgoing mail where envelope sender and recipient addresses are exactly reversed. Now, in addition to this, penpals can also match replies which reference previous outgoing mail by its 'Message-ID' (taking into account the 'References' or 'In-Reply-To' header fields), even if the envelope sender address of the reply is null or does not match a recipient address of a previous outgoing mail. This covers for incoming replies to mailing list postings, incoming message disposition notifications (MDN, RFC 3798) and incoming replies from alias or role addresses. A query on a message-id is fast compared to matching on recipient id, and if it succeeds, the later one is skipped. Based on a suggestion and a patch by Alexander 'Leo' Bergolth; The %sql_clause now contains one additional SQL SELECT clause under 'sel_penpals_msgid' key, which is used in place of 'sel_penpals' when a list of references in a reply is nonempty. It is worth creating an index on a msgs.message_id field to speed up SQL lookups: CREATE INDEX msgs_idx_mess_id ON msgs (message_id); Note that a spammer can gain a (small) advantage by including a reference to a recent outgoing message in his message. With private correspondence this information is hard to come by efficiently in a timely manner, and can effectively be exploited only if a generated Message-IDs of outgoing mail is very easy to guess (e.g. generated by a very poor random number generator). With postings to public mailing lists this information is more readily available. Setting $sql_clause{'sel_penpals_msgid'} to undef or to empty disables matching on message-id, if it turns out the mechanism is being actively exploited. - added penpals snmp-like counters, which are displayed by amavisd-agent like the following: PenPalsAttempts 10222 438/h 18.3 % (InMsgsRecipsInboundOrInt) PenPalsHits 2957 127/h 48.9 % (ContentCleanMsgsInboundOrInt) PenPalsHitsMid 88 4/h 3.0 % (PenPalsHits) PenPalsHitsMidNullRPath 26 1/h 0.9 % (PenPalsHits) PenPalsHitsMidRid 917 39/h 31.0 % (PenPalsHits) PenPalsHitsRid 1926 82/h 65.1 % (PenPalsHits) Some comments on the above figures: - PenPalsAttempts shows that SQL lookups are performed only on a fraction of all incoming and internal mail (and not at all on outgoing mail) because a lookup is not performed on a high score spam (score above $penpals_threshold_high, which was 8 in this example, typically the same as kill level); - PenPalsHits shows that more than half of incoming clean messages can be associated with previous outgoing mail and can benefit from pen pals soft-whitelisting, such message is either a reply directly referencing a previous mail, or a new conversation between a sender/recipient pair which had previous conversation on a different topic; - PenPalsHitsMidRid shows that 1/3 of incoming matching replies match a previous outgoing mail by both the Message-ID as well as the exact recipient address; these are true replies to previous outgoing messages; - PenPalsHitsRid shows that 2/3 of incoming matching replies match a a previous outgoing mail by recipient address, but not by a Message-ID; these messages usually correspond to new topics among previous correspondents; - PenPalsHitsMid match only in reference to a previous Message-ID, but not by sender/recipient addresses; these are usually mailing list replies to a previous posting by a local user; - PenPalsHitsMidNullRPath are messages with null return path which are matching some previous outgoing message by Message-ID; these are usually incoming message disposition notifications (MDN, RFC 3798), or certain bounces which specify In-Reply-To or References in their header; - configuration variable %defang_by_ccat is renamed to %defang_maps_by_ccat and may now contain a list of by-recipient lookup tables (or a boolean as before for compatibility); this allows defanging/mangling to be selected on per-recipient basis; compatibility is retained by making the old variable %defang_by_ccat an alias for %defang_maps_by_ccat; - provided interface code to allow mangling/defanging/sanitation to be performed by an external utility, either by directly calling a Perl module Anomy Sanitizer (within the same process, avoiding startup cost), or by invoking a program 'altermime' (or by internal defanging code as before). Mail body defanging is only allowed for local recipients (those matching @local_domains_maps), i.e. for inbound and internal-to-internal mail. If there is more than one mangling code option available, the result of a %defang_maps_by_ccat can choose between them by returning one of the following strings, the selection can depend on mail content type and on by-recipient lookups if needed: 'anomy' chooses Anomy Sanitizer (if $enable_anomy_sanitizer is true); 'altermime' chooses a program whose path is $altermime (if found); 'attach' chooses the traditional amavisd-new defanging method which pushes an original mail message to an attachment; 'null' for testing purposes - doesn't modify mail body, but pretends it does (in logging and mail header); other non-empty and non-zero value automatically choose one of the above options depending on what is available; at least the 'attach' is always available; an empty, zero or undef value disables mail body modifications; Controls: $enable_anomy_sanitizer, @anomy_sanitizer_args, and: $altermime, @altermime_args_defang; Typical use: # with altermime: $altermime = '/usr/local/bin/altermime'; @altermime_args_defang = qw(--verbose --removeall); # with Anomy Sanitizer: $enable_anomy_sanitizer = 1; @anomy_sanitizer_args = qw( /usr/local/etc/sanitizer.cfg ); $defang_spam = 1; # old style, applies the first available mangler # to all spam-loving local recipients # unnecessarily complicated example of selective choices: $defang_maps_by_ccat{+CC_BANNED} = [ 'altermime', # use altermime for everybody (a 'constant' lookup table) ]; $defang_maps_by_ccat{+CC_SPAM} = [ { # a per-recipient hash lookup table 'user@example.com' => 1, # old style, auto-selects a mangler 'user-a@example.com' => 'anomy', 'user-m@example.com' => 'altermime', 'user-t@example.com' => 'attach', '.example.net' => 0, # no mangling }, $defang_spam, # fallback to old style setting if no match above ]; - a special case of mangling is adding a disclaimer, by invoking an external program 'altermime' (if available and enabled). This differs from mangling inbound mail in two details: * uses a separately configurable list of arguments to altermime: @altermime_args_disclaimer; and * it applies only to mail submitted from internal networks or roaming users (as recognized through a policy bank which sets: allow_disclaimers => 1), and where any of the following addresses matches local domains: author (2822.From) or sender (2822.Sender) or return path (2821.mail_from); Typically the $allow_disclaimers should be set by a policy bank which also sets the $originating flag. In addition to strings that may be returned by %defang_maps_by_ccat as described above, there are two more, only taken into account when $allow_disclaimers is true: 'disclaimer' invokes $altermime program for outgoing mail with arguments as given in @altermime_args_disclaimer; 'nulldisclaimer' for testing purposes - doesn't modify mail body, but pretends it does (in logging and mail header); Typical use: $altermime = '/usr/local/bin/altermime'; @altermime_args_disclaimer = qw(--verbose --disclaimer=/etc/altermime-disclaimer.txt); $defang_maps_by_ccat{+CC_CATCHALL} = [ 'disclaimer' ]; @mynetworks = qw( ... ); $policy_bank{'MYNETS'} = { # mail originating from our networks originating => 1, allow_disclaimers => 1, } For the moment there is one limitation: there can only be one mangler in effect at a time, it is not currently possible to both defang and to append a disclaimer on the same message: for internal-to-internal mail inserting a disclaimer takes precedence. To make it possible to provide different disclaimer texts when hosting multiple domains, there is an experimental additional configuration variable available: the @disclaimer_options_bysender_maps. It is a list of lookup tables, looked up by a sender address. The sender address is chosen from the following list, first match wins: * 'Sender:' header field, if its domain matches @local_domains_maps; * 'From:' header field, if its domain matches @local_domains_maps; * envelope sender address, if its domain matches @local_domains_maps; We already know that at least one of the above will match, otherwise adding disclaimers would be skipped at an earlier stage. The result of lookups should be one simple string, which replaces a string '_OPTION_' anywhere in @altermime_args_disclaimer elements. Typical use: @altermime_args_disclaimer = qw(--disclaimer=/etc/_OPTION_.txt); @disclaimer_options_bysender_maps = ( { 'host1.example.com' => 'altermime-disclaimer-host1', 'boss@example.net' => 'altermime-disclaimer-boss', '.example.net' => 'altermime-disclaimer-net', '.' => 'altermime-disclaimer-default' }, ); It is currently not possible to disable adding disclaimers through @disclaimer_options_bysender_maps results. This needs to be improved. The exact interpretation of the @disclaimer_options_bysender_maps lookup result may change in the future (which is why I call it 'experimental'). Note that disclaimers are pretty much useless legally. If you can help it at all, please avoid the pollution. See: http://www.goldmark.org/jeff/stupid-disclaimers/ - as mentioned above, the new SMTP/LMTP client code now supports an LMTP protocol too. This allows amavisd-new to act as an LMTP-to-LMTP content filter, possibly being inserted between MTA and an LMTP-based mail delivery agent such as Cyrus (if checking of outgoing mail is not needed). LMTP is selected when the first field of a $*_method (such as $forward_method, $notify_method, $resend_method, $release_method, $*_quarantine_method) is a 'lmtp:'. Possible uses: $forward_method = 'lmtp:/var/imap/socket/lmtp'; # over a Unix socket or: $forward_method = 'lmtp:[127.0.0.1]:24'; # over IPv4 or: $forward_method = 'lmtp:[::1]:24'; # over IPv6 If a Postfix 'lmtp' service is used to feed amavisd (instead of the more usual content filter feed through a service named 'amavisfeed' or 'smtp-amavis'), make sure not to forget to limit the number of concurrent feeds to amavisd (e.g. lmtp_destination_concurrency_limit=15) to a value same (or less) than $max_servers, or limit the maxproc field in master.cf such as: 'lmtp unix - - n - 15 lmtp' . Note that invoking amavisd as an LMTP delivery agent has a disadvantage that outgoing mail is not being checked, so infected internal hosts are able to pollute the world. Also the pen pals feature is no longer useful, as it requires the information on previous outgoing mail to be present in an SQL database. - a new command line option -i, it takes one argument which can be any string (an instance/personality name), which is then made available to amavisd.conf in a variable $instance_name (intended to be read-only); code in amavisd does not assign any semantics to this argument and does not use it for any purpose, it is purely intended for administrator's use in amavisd.conf if desired; this simple mechanism may facilitate running multiple instances of amavisd using a single configuration file, or to choose at startup time between amavisd personalities using the same config file; A possible usage is to start a test instance of amavisd while a production amavisd is still running, and letting a test instance listen on its dedicated TCP port number. Each server instance needs its own pid and lock files, its own TCP port number or socket name, and its own $db_home (nanny, cache, agent) unless bdb usage is disabled. A working directory may be shared or kept separate. An example to put by the end of amavisd.conf: if ($instance_name eq 'test') { $log_level = 5; $sa_debug = 1; $max_servers = 1; $TEMPBASE = "$MYHOME/tmp-am2"; $ENV{TMPDIR} = $TEMPBASE; $pid_file = "$MYHOME/home/amavisd2.pid"; $lock_file = "$MYHOME/home/amavisd2.lock"; $enable_db = 0; $inet_socket_port = [8888]; # listen on port 8888 } Start a test instance: # amavisd -i test debug and submit a test message to it, e.g.: $ mini_sendmail -fpostmaster@example.net \ -s127.0.0.1 -p8888 test@example.net <0.msg - policy banks now contain a new key 'originating', which generalizes a previously hard-wired policy bank MYNETS. It is a boolean variable, turned on automatically in the currently loaded policy bank when a smtp client's IP address matches @mynetworks_maps, to retain full compatibility with existing setups. When a new policy bank is loaded over a current one, the new policy bank may also modify the 'originating' key - a typical use is to turn it on by a policy bank activated by mail submission from authenticated roaming users (SASL/AUTH), so that such users are treated as locals (originating mail) even though their IP address does not match a @mynetworks_maps list of lookup tables. The current value of variable 'originating' is now the only control to some macros or decisions, which were previously controlled implicitly by a @mynetworks_maps match. These are: * macro %l (letter ell) now directly corresponds to the current value of the 'originating' variable (returning a '1' or an empty string); * some statistics counters differentiate between 'Inbound' and 'Internal' mail directly based on the value of the 'originating' variable (applies to mail with local recipients, otherwise it is 'Outbound'); * pen pals is skipped for senders claiming to be from a local domain, but 'originating' is false (i.e. unauthorized foreigners faking your domain); * only mail with 'originating' flag will be eligible for DKIM signing (starting with version 2.6.0); * there may be other uses in the future, so it is prudent to keep @mynetworks_maps and @local_domains_maps configured correctly, and (when appropriate) turn on the 'originating' flag for mail that is supposed to be treated as originating from internal or authorized roaming users; Example use: $interface_policy{'10026'} = 'ORIG'; $policy_bank{'ORIG'} = { # mail originating from our users originating => 1, # declare that mail was submitted by our smtp client allow_disclaimers => 1, # enables disclaimer insertion if available virus_admin_maps => ["virusalert\@$mydomain"], spam_admin_maps => ["virusalert\@$mydomain"], warnbadhsender => 1, # forward to an smtpd service providing DKIM signing service forward_method => 'smtp:[127.0.0.1]:10027', # force MTA conversion to 7-bit (e.g. before DKIM signing) smtpd_discard_ehlo_keywords => ['8BITMIME'], bypass_banned_checks_maps => [1], # allow sending any file names & types terminate_dsn_on_notify_success => 0, # don't remove NOTIFY=SUCCESS opt. }; - make it possible for a virus scanner to derate an infection report to a spam report, contributing to spam score and to spam report/status. A new configuration variable @virus_name_to_spam_score_maps (also member of policy banks) can turn a reported virus name into a spam score. Its default setting is: @virus_name_to_spam_score_maps = (new_RE( [ qr'^(Email|HTML)\.(Phishing|Spam|Scam[a-z0-9]?)\.'i => 0.1 ], [ qr'^(Email|Html)\.Malware\.Sanesecurity\.' => undef ], [ qr'^(Email|Html)(\.[^., ]*)*\.Sanesecurity\.' => 0.1 ], # [ qr'^(Email|Html)\.(Hdr|Img|ImgO|Bou|Stk|Loan|Lott|Cred|Job|Dipl|Doc) # (\.[^., ]*)* \.Sanesecurity\.'x => 0.1 ], [ qr'^(MSRBL-Images/|MSRBL-SPAM\.)' => 0.1 ], )); and can be replaced in amavisd.conf. To disable the feature assign an empty list to the configuration variable: @virus_name_to_spam_score_maps = (); When a virus scanner returns names of viruses, and all provided names are matched by the @virus_name_to_spam_score_maps, and no other virus scanner has anything more sinister to report, then a message is _not_ flagged as a virus, but a corresponding spam score is contributed to other spam results as returned by a normal spam scan by SA. All the usual spam rules are then followed. Phishing fraud as indicated by ClamAV is now by default treated as spam, and no longer as a virus. The log can now show entries like: amavis[26733]: (26733-03-2) Turning AV infection into a spam report: score=0.1, AV:HTML.Phishing.Auction-289=0.1 amavis[26733]: (26733-03-2) adding SA score 38.628 to existing 0.1 from an earlier spam check amavis[26733]: (26733-03-2) Blocked SPAM, ... Hits: 34.728, ... Tests: [AV:HTML.Phishing.Auction-289=0.1, ... L_AV_Phish=14, ...] The information is also available to SA rules in a form of a synthetic header field X-Amavis-AV-Status which will be seen by SA only (not inserted into passed or quarantined mail). One has a choice to adjust scores either in the @virus_name_to_spam_score_maps in amavisd.conf, or by providing rules to match on the provided header field. Doing it by SA rules has an advantage of letting other SA rules contribute their score points, possibly preventing a false positive of a ClamAV rule, or pushing score even higher for a clean bounce suppression. It also allows auto-whitelisting in SpamAssassin to account for these score points. In addition, it makes more sense when checks are cached and result reused later for some other message with the same contents in body. Here is one example of such SA rules (some long lines are wrapped, these should be unwrapped before placing them into local.cf): header L_AV_Phish X-Amavis-AV-Status =~ m{\b(Email|HTML)\.Phishing\.}im header L_AV_SS_Phish X-Amavis-AV-Status =~ m{\b(Email|Html)\.Phishing(\.[^., ]*)*\.Sanesecurity\.}m header L_AV_SS_Scam X-Amavis-AV-Status =~ m{\b(Email|Html)\.(Scam[A-Za-z0-9]?)(\.[^., ]*)*\.Sanesecurity\.}m header L_AV_SS_Spam X-Amavis-AV-Status =~ m{\b(Email|Html)\.(Spam|Bou|Stk|Loan|Lott|Cred|Job|Dipl|Doc) (\.[^., ]*)*\.Sanesecurity\.}m header L_AV_SS_Hdr X-Amavis-AV-Status =~ m{\b(Email|Html)\.Hdr(\.[^., ]*)*\.Sanesecurity\.}m header L_AV_SS_Img X-Amavis-AV-Status =~ m{\b(Email|Html)\.(Img|ImgO)(\.[^., ]*)*\.Sanesecurity\.}m header L_AV_MSRBL_Img X-Amavis-AV-Status =~ m{\bMSRBL-Images/}m header L_AV_MSRBL_Spam X-Amavis-AV-Status =~ m{\bMSRBL-SPAM\.}m header L_AV_Scam X-Amavis-AV-Status =~ m{\bScamNailer\.}i score L_AV_Phish 14 score L_AV_SS_Phish -3 score L_AV_SS_Scam 8 score L_AV_SS_Spam 8 score L_AV_SS_Hdr 6 score L_AV_SS_Img 3.5 score L_AV_MSRBL_Img 3.5 score L_AV_MSRBL_Spam 6 score L_AV_Scam 8 - added a new concept of a 'blocking contents category', which in most cases corresponds to a familiar 'main contents category' (the highest ranking category of contents pertaining to a message, e.g. virus, blocked, spam, spammy, bad header ...). The difference between the two arises when recipients are declared to be 'lovers' of some higher-ranking contents, or when a higher ranking contents category has its *_destiny set to D_PASS. For example: a message contains a banned part, but is also spam and may even have a bad header. Its contents categories are (simplified): CC_BANNED, CC_SPAM and CC_BADH, in this order. The main contents category of a message is CC_BANNED, which usually is also a reason for blocking a message, yielding a blocking ccat to also be CC_BANNED. But if some recipient is banned_files_lover (or if $final_banned_destiny is set to D_PASS), then the main ccat remains to be CC_BANNED, but the blocking ccat is CC_SPAM, i.e. the next in the list which is responsible for actually blocking the mail. If recipient would also be a spam lover, the blocking ccat might be CC_BADH (if $final_bad_header_destiny were not D_PASS); If a message is not being blocked, the 'blocking contents category' (i.e. a blocking_ccat attribute of a per-message or a per-recipient object) remains empty (undefined). For convenience some internal routines and some new macros fall back to showing the main contents category in this case. Almost all processing decisions, DSN, notification assembling, quarantining, logging etc. is now based on 'blocking contents category' when a message is being blocked, and on 'main contents category' (as before) when a message is not being blocked. There is a new macro 'ccat' which is useful in notification and logging templates, which can query the blocking contents category, as well as a main contents category. It provides access to information that was formerly available through macros ccat_maj, ccat_min, ccat_name, plus access to additional information. Macros ccat_maj, ccat_min and ccat_name are still available, but their use is deprecated, as their functionality has been incorporated into the new macro 'ccat'. Macro 'ccat' takes two optional fixed-string arguments, which are interpreted case-insensitively. In their absence it expands to a string "(maj,min)" which shows a major and a minor contents category number of a blocking ccat for a blocked message, and of a main contents category for a passed message. The first argument specifies which attribute of a ccat is to be provided, the second argument specifies whether a main or a blocking contents category is to be consulted: The first argument may be any of the following strings: name ... provide a human-readable name of a ccat (%ccat_display_names) major ... provide a number: a major contents category, values correspond to CC_* constants in the program minor ... provide a number: a minor contents category, often a 0 ... empty argument (also a default) results in a string "(maj,min)" is_blocking ... '1' if blocking_ccat is true (message is being blocked), or an empty string when a message is being passed; is_nonblocking .. the opposite: '1' if blocking_ccat false, '' otherwise is_blocked_by_nonmain .. '1' if blocking_ccat is true _and_ is different from a main contents category; The second argument may be any of the following strings: main ... provide information on main contents category when asked for name/major/minor/ blocking.. provide information on blocking contents category if it exists, otherwise it falls back to providing info on main ccat; this is also a default in the absence of this argument; For illustration, instead of a former call [:ccat_maj] use [:ccat|major] , instead of [:ccat_min] use [:ccat|minor], and instead of [:ccat_name] please use [:ccat|name] . For more examples please consult the default templates, glued to the end of file 'amavisd'. - when amavisd-nanny is given an invalid command-line argument it now shows 'Usage: ...' as well as a legend for process states; - amavisd-nanny enhanced and new process-state instrumentation added to amavisd daemon; previously only busy/idle states of child processes were shown in amavisd-nanny output, now a more detailed process state can be shown by setting a new verbosity control configuration variable $nanny_details_level to a higher than a default value of 1, e.g. to 2; The following characters in amavisd-nanny bars represent amavisd child process states as follows, in the shown order of events: A accepted a connection b begin with a protocol for accepting a request m 'MAIL FROM' smtp command started a new transaction in the same session d transferring data from MTA to amavisd = content checking just started D decoding of mail parts V virus scanning S spam scanning P pen pals database lookup and updates r preparing results Q quarantining and preparing/sending notifications F forwarding mail to MTA . content checking just finished A nanny bdb database has changed in an incompatible way, so older versions of amavisd-nanny would complain about contents of a new database. Backward compatibility is retained: new version of amavisd-nanny is able to deal with a database from older versions of amavisd-new. There is no need for conversion, a new database is created on each amavisd restart. Note that a history of process states is _not_ maintained in a nanny database, but only in a running amavisd-nanny, which is why a just-started amavisd-nanny can not show previous states of processes from time before amavisd-nanny was started - a '=' is shown instead. A display eventually catches up and all newly-entered states are shown correctly. - snmp-like database can now also store 64-bit counters data type, amavisd-agent utility modified accordingly; - amavisd-agent utility and amavisd daemon enhanced to provide and to display cumulative elapsed time by sections; currently only some of the more important sections have been instrumented, e.g.: TimeElapsedReceiving 10631 s 0.608 s/msg (InMsgs) TimeElapsedDecoding 970 s 0.056 s/msg (InMsgs) TimeElapsedVirusCheck 629 s 0.036 s/msg (InMsgs) TimeElapsedSpamCheck 75866 s 4.341 s/msg (InMsgs) TimeElapsedPenPals 2150 s 0.123 s/msg (InMsgs) TimeElapsedSending 2231 s 0.128 s/msg (InMsgs) TimeElapsedTotal 94709 s 5.419 s/msg (InMsgs) Don't be surprised if the total elapsed time exceeds amavisd uptime, 10 processes progressing slowly for 5 seconds each will accumulate 50 seconds of reported elapsed time. The average seconds-per-message figure as reported in the last column makes more sense; - amavisd-agent utility and amavisd daemon enhanced to provide and to display cumulative mail sizes in bytes, and additional message counters based on outbound/inbound/internal mail direction: message counts: InMsgs all mail received by amavisd (as in previous versions); InMsgsOutbound at least one recipient NOT in @local_domains_map; InMsgsInternal at least one recipient in @local_domains_map, and client IP address (submitter) in @mynetworks_maps; InMsgsInbound at least one recipient in @local_domains_map, and client IP address NOT in @mynetworks_maps; message sizes: InMsgsSize total mail size in bytes as received by amavisd; InMsgsSizeOutbound as above, but with at least one recipient NOT in @local_domains_map; InMsgsSizeInternal at least one recipient in @local_domains_map, and client IP address in @mynetworks_maps; InMsgsSizeInbound at least one recipient in @local_domains_map, and client IP address NOT in @mynetworks_maps; Note that a mail with multiple recipients can be both internal and outbound. P.S. in later versions a hard-wired testing against a @mynetworks_maps list is replaced by testing value of a boolean variable $originating. Example output: InMsgsSize 4332MB 109MB/h 100.0 % (InMsgsSize) InMsgsSizeInbound 3102MB 78MB/h 71.6 % (InMsgsSize) InMsgsSizeInternal 554MB 14MB/h 12.8 % (InMsgsSize) InMsgsSizeOutbound 816MB 21MB/h 18.8 % (InMsgsSize) - new configuration variables $always_bcc and %always_bcc_by_ccat, also members of policy banks, allow adding one extra envelope recipient to each message, either regardless of contents ($always_bcc), or selectively based on contents category. For example: $always_bcc = 'archiver+clean@example.com'; or selectively based on contents category: $always_bcc_by_ccat{+CC_CLEAN} = 'archiver+clean@example.com'; $always_bcc_by_ccat{+CC_VIRUS} = 'archiver+virus@example.com'; or as a member of policy banks: $policy_bank{'MYNETS'} = { always_bcc_by_ccat => { CC_BADH, 'archiver@example.com', CC_CLEAN, 'archiver@example.com', CC_CATCHALL, undef, }, }; - amavisd-nanny and amavisd-agent utilities now recognize an optional command-line option: -w , where the specified value is time in seconds between re-displays. The default interval is 2 seconds for amavisd-nanny, and 10 seconds for amavisd-agent as before. The specified interval time may be fractional; - macro 'useragent' can accept an optional argument: a string 'name' or 'body', restricting the information to be returned as follows: macro 'useragent' returns 'User-Agent: ...' or 'X-Mailer: ...' header field from a message (whichever is present, or empty); an optional argument specifies whether: an entire field is to be returned (empty or unrecognized argument), or just a field name (argument: 'name'), e.g. 'X-Mailer'; or just a field body (argument 'body'), e.g. 'Thunderbird_1.5.0.9'; - interfacing to Mail::ClamAV (a perl module to a clamav library) now performs processing in a subprocess to prevent bugs in external library from bringing down amavisd process, and to prevent virtual memory of an amavisd child process from expanding uncontrollably - at the expense of additional 20..30 ms for a fork; - extended AM.PDP protocol with an attribute 'policy_bank' which may be used in a client's request to require loading additional policy banks, e.g.: policy_bank=TLS,ORIGINATING,MYNETS Its value is a comma-separated list of policy bank names. Names of nonexistent banks are silently ignored, so are leading and trailing spaces and TABs around each name. The order of policy bank loading generally follows the order in which information about a message were obtained: - interface- or socket-based policy banks (when MTA connects to amavisd); - MYNETS (when client's IP address becomes known); - the list of policy bank names as specified in a 'policy_bank' attribute of AM.PDP protocol, comma-separated; - MYUSERS (when sender's e-mail address becomes known); - added a field 'Final-Log-ID' to a DSN report (RFC 3464), which will provide information on log_id and mail_id, e.g. '77790-10-3/uez9wtcVNTO5' in a standard way, much like the 'Our internal reference code for your message is: ...' in a DSN plain text part; - added mapping from 'RIFF...animated cursor' to ['movie','ani'] in $map_full_type_to_short_type_re, to facilitate blocking animated cursors (Microsoft Windows ANI header stack buffer overflow is being actively exploited); by Henrik Krohns; - add support for 7-Zip archives if external utility program 7z is available (under names 7zr, 7za or 7z); suggested by Bob Marcan; see: http://www.7-zip.org/ - add configurable global settings $min_servers, $min_spare_servers, and $max_spare_servers (all undefined by default, see Net::Server::PreFork documentation for their semantics), pass them to Net::Server at startup time (complementing the usual $max_servers setting) and allow to choose between Net::Server personalities Net::Server::PreForkSimple and Net::Server::PreFork - if $min_servers is defined the PreFork is chosen, otherwise the more usual PreForkSimple. The feature is mostly intended for use of amavisd as a pre-queue content filter, which is unsupported anyway. For normal post-queue use the PreForkSimple already does a good job. Based on a patch by Alexander 'Leo' Bergolth; - internal: incompatibly changed order and indirection level of arguments to routines dealing with contents categories, and some of their names; - new macro 'join', behaves like a Perl join function: the first argument is a separator string, remaining arguments are strings to be concatenated, with a separator string inserted at every concatenation point; - macro 'dquote' (as used in a default log template to protect Subject) previously escaped double quotes with \, but missed to escape \ itself, making log parsing tricky; also, as logging layer escapes \ by itself, the result was ugly and inconsistently parsable; new behaviour is to protect a double quote within a string by doubling it, so a [dquote|one"oops"two] now yields "one""oops""two", instead of "one\"oops\"two", which when logged showed as "one\\"oops\\"two"; - convenience: do not drop privileges early despite a command line option -u when an option -R is also specified with a non-empty (and non-slash) value, otherwise the requested chroot operation is not possible (root privileges are required for chrooting); - version 2.4.3 introduced some substitutions of subject tag template strings: SCORE, REQD, YESNO and YESNOCAPS; this list is now extended with few more, to facilitate cross-host troubleshooting; the full list now consists of: _SCORE_ spam score (hits), same as macro %c _REQD_ tag2_level _YESNO_ score above tag2_level? 'Yes' or 'No' _YESNOCAPS_ same, but yields: 'YES' or 'NO' _HOSTNAME_ fqdn of this host ($myhostname), same as macro %h _DATE_ rfc2822 timestamp of mail entering this amavisd, as macro %d _U_ iso8601 UTC timestamp of mail entering this amavisd, as %U _LOGID_ log id (am_id) as shown in the log, e.g. 58725-05-2, as %n _MAILID_ mail_id as used in quarantine names, e.g. jaUETfyBMJHG, as %i See also README.customize for explanation of macros. --------------------------------------------------------------------------- January 30, 2007 amavisd-new-2.4.5 release notes SECURITY - Recommended version of Convert::UUlib is 1.08 or higher to avoid processing of uninitialized data containing 'random' garbage. Note that a security hole in uulib which comes with Convert::UUlib 1.04 and older is now (as of 2006-12-05) known to be exploitable: http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2005-1349 credits to Jean-Sébastien Guay-Leroux; - p0f-analyzer.pl will no longer reply to queries coming from low-numbered UDP ports below 1024 or from nfsd port 2049, and will ignore queries with nonce longer than 1024 character or containing characters outside of \040-\177 (octal) range to limit its usefulness as a potential reflector for an attacker from internal networks. INCOMPATIBLE CHANGE WITH 2.4.4 - p0f-analyzer.pl now only binds to a loopback interface by default, instead of to all interfaces; change $bind_addr in p0f-analyzer.pl to '0.0.0.0' if p0f-analyzer.pl is running on a different host from amavisd or from other querying clients; suggested by Shaun T. Erickson and Mario Liehr; BUG FIXES - let p0f-analyzer.pl exit when a pipe on stdin is closed (e.g. when p0f is killed or crashes), instead of entering a tight loop; reported by Justin Piszcz and Henrik Krohns; - hard-blacklisting no longer skips quarantining when $spam_quarantine_cutoff_level is undefined (or is an empty string); - restart timer after Sophie times out; previously the next attempt would run with no time limit; reported by Nick Leverton and Nicklas Bondesson; - fix error reporting in open_on_specific_fd when POSIX::dup2 fails; thanks to Chris (decoder); - fix signal handling in read_snmp_variables() and register_proc(), a signal could previously get lost (not re-signaled) if it occurred within these subroutines; - fixed get_body_digest which incorrectly determined 7- or 8-bitness of mail header and body, setting body_type incorrectly (with only cosmetic ill-effects); - fixed AM.PDP code to always provide an smtp-quoted form in angle brackets in 'delrcpt' and 'addrcpt' attributes of a response, i.e. in the same form as was received in 'sender' and 'recipient' attributes. The attribute value syntax is specified in RFC 2821 as 'Reverse-Path' (i.e. smtp-quoted form, enclosed in <>); previously enclosing angle brackets were missing in a server reply; - documentation - amavisd.conf-default incorrectly stated that a default value for $prepend_header_fields_hdridx is 1; actually the default is 0 as correctly indicated in release notes; reported by Jo Rhett; OTHER - qmail interfacing notice: MTA timeout for waiting on results from amavisd should be longer than $child_timeout (8 minutes by default) with some margin, setting MTA timeout to 15 or 20 minutes is usual. With qmail however the QMQP code in qmail has hard-coded timeouts set, 10 seconds for connect and 60 seconds for read/write. If amavisd processing takes longer than 60 seconds, the MTA drops connection and retries later, yet amavisd continues processing and eventually delivers a mail (with each MTA retry), causing repeated deliveries of the same message. The following patch by Eric Huss on the www.qmail.org page: http://www.ehuss.org/qmail/qmqpc-timeout.tar.gz should be applied to qmail when interfacing it to a post-queue content filter. Problem researched by Nicklas Bondesson; - better timeout handling in interface code to daemonized virus scanners like clamd, Sophie, Trophie: allow short time (10 s) for connect and for sending a request, then allow normal (long) time to collect results; keep evidence of the initial deadline on retries; - prefer '7bit' as Content-Transfer-Encoding when attaching original message or its headers (message/rfc822 or text/rfc822-headers) to DSN or to a defanged mail, and only specify '8bit' when necessary; - remove protecting the $ and @ characters in second argument of a regexp selector macro, it is unnecessary and confusing; - macros %m, %r and header_field now return parsed and sanitized message IDs in header fields Message-ID, Resent-Message-ID. In-Reply-To, or References, void of CFWS (comments and FWS as specified by RFC 2822), through the use of new subroutine parse_message_id(); - when logging to SQL, the field msgs.message_id now contain just a message id, without CFWS and other garbage that might appear in a Message-ID header field; this facilitates queries, and pen pals matching of IDs in In-Reply-To or References header fields of a reply to an original Message-ID; - updated $map_full_type_to_short_type_re to avoid mapping file(1) result 'MS-DOS executable (built-in)' to types 'exe-ms' and 'exe'; the file(1) utility generously declares any text file starting with LZ to be a 'MS-DOS executable (built-in)'; thanks to Noel Jones, Jakob Curdes and Clifton Royston for troubleshooting; - add X-Spam-* header fields to quarantined mail if spam score is at or above tag_level. Previously message needed to be recognized as spammy or spam (tag2 or kill level) in order to receive spam header fields in quarantined copy. This also makes it more consistent with adding such header fields to passed mail; suggested by Michael Gaskins; - add X-Amavis-OS-Fingerprint header field to quarantined mail; - header field X-Spam-Score in a passed or quarantined mail now reflects score boost even when SA score is unknown (e.g. when SA was not called), and reflects white and blacklisting by pushing score to 0 or 64, to make it consistent with a bar size in X-Spam-Level header field; - resignal "timed out" after (almost) every eval {} which has no subsequent call to prolong_timer() to ensure we do not continue running with disabled timer. Exceptions are DESTROY and END handlers, and code which handles timer in some other way (e.g. by keeping evidence of a deadline); - for the purpose of looking up client IP address in @mynetworks_maps, treat unknown/unavailable IP address as 0.0.0.0; this allows treating directly submitted mail on the MTA host (not submitted through SMTP) as coming from IP address 0.0.0.0 (i.e. "This" Network - according to RFC 1700); Note that this is indistinguishable from other reasons when IP address is not made available to amavisd, e.g. when smtp_send_xforward_command option in Postfix smtp service is not enabled, which is why the default setting of @mynetworks does not include a 0.0.0.0/8 network to prevent unintentionally loading a MYNETS policy bank. One should add 0.0.0.0/8 to a @mynetworks list only when XFORWARD is known to work and if some software on the MTA host is submitting its mail to MTA directly, e.g. through a sendmail mail submission command (or its lookalike), and MYNETS policy bank loading is needed for proper processing of such mail (e.g. DKIM signing or adding disclaimers in later versions of amavisd); - report a more informative message when a file(1) utility fails to produce useful results: joins exit status with a parsing report into one message; thanks to Andres, whose file(1) utility was crashing with SEGV; - consistency: rearrange implicitly adding $X_HEADER_TAG to a hash %allowed_added_header_fields so that it is possible to turn off insertion of $X_HEADER_TAG header field by turning off associated key in %allowed_added_header_fields even when $X_HEADER_TAG is explicitly defined; - let %allowed_added_header_fields also control insertion of header fields into quarantined message; - amavisd-nanny now displays a title line indicating the semantics of columns; - Courier patch: ensure the information is stored to newly introduced recip_addr_smtp and sender_smtp object attributes, which are needed to preserve pristine address forms for DSN and ORCPT use and for logging; a patch by Martin Orr; - qmqpqq (qmail): ensure the information is stored to newly introduced recip_addr_smtp and sender_smtp object attributes; - qmail patch now activates line-by-line sending to qmail to avoid qmail bug ('bare LF' reported when CR and LF are separated by a TCP packet boundary); - tighten a regexp on matching a p0f fingerprint for Windows XP to avoid matching 'Windows XP SP1+, 2000 SP3'; suggested by Michael Scheidell; - updated AV entry for CentralCommand Vexira (vascan): removed hard-coded option '--vdb'; by Brian Wong; - internal: move code dealing with a SA call to a dedicated subroutine call_spamassassin; - internal: provide new routines to collect scalar and structured results from a subprocess (collect_results, collect_results_structured) and take advantage of them in decoding, in AV and in dspam interface routines, unifying code and providing results size sanity limit and consistent killing of runaway external programs; - experimental: taking advantage of the above, make it possible to run SA in a spawned process, requested by setting a new config variable $sa_spawned to true (it is off by default); benefits are that a mainstream child process can not be brought down by potential processing problems in SA or its external modules, and timeouts are handled cleanly by a calling process; downside is an increase of process count (worst case: doubled), with corresponding increase in memory footprint, plus about 20 .. 30 ms of additional processing time for each call to SA; - added a tuning tip on buffer sizes to README.sql for MySQL with InnoDB, by Wayne Smith; - updated URL of Sophie AV scanner; --------------------------------------------------------------------------- November 20, 2006 amavisd-new-2.4.4 release notes COMPATIBILITY WITH 2.4.3 - PostgreSQL quarantining: data type of field quarantine.mail_text should be 'bytea' (instead of 'text') to allow storing arbitrary octets without associating them with a character set. See below for a conversion of an existing database. Similarly with MySQL the data type should be 'blob'. - Note: in a sendmail milter setup with Petr Rehor's helper program amavis-milter, one should set: $prepend_header_fields_hdridx = 1; when dk or dkim signing milters are used in the same setup. See below for details. BUG FIXES: - do_ascii: fix a bug where timer was not restored after decoding of a textual mail part, so a timeout for subsequent decoding operations on the same message was limited to 10 seconds (and to 30 seconds for a call to SpamAssassin), regardless of $child_timeout setting; - don't call PerlIO::get_layers with Perl 5.8.0, the function was introduced with 5.8.1; reported by Joel Nimety; - avoid deep recursion in evaluating a regular expression in header checks which caused very slow testing for presence of a all-whitespace lines in folded header fields for degenerate cases of header; the inefficient expression was introduced with amavisd-new-2.4.0; reported and a sample provided by Kai Risku; - when spam above kill level is to be passed and spam defanging is enabled, SA summary was inserted twice (once for mail contents category being CC_SPAMMY and once for CC_SPAM), fixed. Reported by Gary V and MHahnen; - when logging directly to a file, do create a log file if it does not already exist; (bug introduced with 2.4.3) - make sure a quota limit is untainted when it is given as a command line parameter to external TNEF decoder; reported by MK; - updated Courier patch to loosen up socket protection and allow group write access to the socket; reported by Bill Taroli; - SQL logging: cleanly chop an UTF-8 octet sequence according to RFC 3629 (avoid truncating character octet sequence tail) when Subject, From or Message-Id header field is longer than 255 characters; - PostgreSQL: when storing mail text to a quarantine use pg_type=PG_BYTEA attribute on a field 'quarantine.mail_text'; previously the following error could be reported: 451 4.5.0 Storing to sql db as mail_id ... failed: writing mail text to SQL failed: Error closing, flush: sql inserting text failed, sql exec: err=7, 22P02, DBD::Pg::st execute failed: ERROR: invalid input syntax for type bytea - updated documentation in README.sql to suggest using data type 'bytea' instead of inappropriate data type 'text' for a field quarantine.mail_text To convert an existing table (when quarantining to SQL) please use: ALTER TABLE quarantine ALTER mail_text TYPE bytea USING decode(replace(mail_text,'\\','\\\\'),'escape'); If conversion of data type for 'quarantine.mail_text' is not done, the following error will be reported when storing a message to an SQL quarantine is attempted: TROUBLE in check_mail: quar+notif FAILED: temporarily unable to quarantine: 451 4.5.0 Storing to sql db as mail_id ... failed: writing mail text to SQL failed: Error closing, flush: sql inserting text failed, sql exec: err=7, DBD::Pg::st execute failed: ERROR: column "mail_text" is of type text but expression is of type bytea HINT: You will need to rewrite or cast the expression If converting quarantine table is not desirable or possible in a short term, it is possible to continue use existing SQL quarantine table without conversion by specifying the following in amavisd.conf: $sql_clause{'ins_quar'} = "INSERT INTO quarantine (mail_id, chunk_ind, mail_text)". " VALUES (?,?,encode(?,'escape'))"; $sql_clause{'sel_quar'} = "SELECT decode(mail_text,'escape') FROM quarantine". " WHERE mail_id=? ORDER BY chunk_ind"; This will allow PostgreSQL to convert data types on-the-fly, converting octets (any byte) into escaped text, and vice versa when releasing from a quarantine; Problem reported by Justin Hillyard, correct data type suggested by Nikola Milutinovic; - MySQL: updated documentation in README.sql to suggest using data type 'blob' instead of inappropriate data type 'text' for a field quarantine.mail_text. To convert an existing table please use: ALTER TABLE quarantine CHANGE mail_text mail_text blob; Seems like MySQL does not complain on incompatibility between provided data type and a data type of a field in table, but there are reports that MySQL may silently truncate data which it finds violating character set constraints, so conversion to 'blob' is highly recommended. Truncation of quarantined message at an 8-bit character reported by Lubor Kolar. OTHER CHANGES: - limit recursion in MIME::Parser to $MAXFILES to prevent MIME parser from fully traversing degenerate cases of broken MIME messages which can take excessive amount of time and memory; reported and a sample provided by Joshua Goodall, solution suggested by David F. Skoll, and requires a parser method max_parts(), available in MIME::Parser 5.417 or later; - check for already running daemon at startup time, preventing a user mistake of trying to start another instance of the daemon without stopping the currently running process; suggested by Jo Rhett; - keep sender and recipient addresses in original unparsed form (in addition to an internal form) to be able to always provide exact original address in delivery status notifications, in ORCPT, and when appending extensions in a milter setup (AM.PDP), which requires exact matching to the original form (without stripping route and without fixing poorly SMTP-quoted address forms); - new configuration variable %allowed_header_tests, also member of policy banks, allows for selectively disabling some of the header checks, e.g. checks for non-encoded 8-bit characters. The %allowed_header_tests hash contains all available header test names as its keys by default (with a value of true); removing a key, or setting its value to false, disables a test, e.g.: $allowed_header_tests{'8bit'} = 0; $allowed_header_tests{'missing'} = 0; Currently available keys (i.e. test names) are: other mime 8bit control empty long syntax missing multiple each corresponding to its own minor contents category of CC_BADH; ccat test min name description --- ------- ----------- 0 other (catchall for everything else, normally not used) 1 mime Bad MIME (sub)headers or bad MIME structure 2 8bit Invalid non-encoded 8-bit characters in header 3 control Invalid control characters in header (CR or NUL) 4 empty Folded header field made up entirely of whitespace 5 long Header line longer than RFC 2822 limit of 998 characters 6 syntax Header field syntax error 7 missing Missing required header field 8 multiple Duplicate or multiple occurrence of a header field legend: ccat min: minor contents category under a major category CC_BADH, available in templates as a macro ccat_min; test name: corresponding test name - a key in %allowed_header_tests; descr.: description of a header test or MIME subheaders/structure test; - timing report has a couple of new entries to facilitate troubleshooting: header checks section, separate entry for header and body digests, check_mail initialization, entries 'SMTP greeting' and 'SMTP response'; - when exec in a forked process fails, call POSIX::_exit with exist status 8 (ENOEXEC) instead of the more common 1 to make the failure more obvious; (a note from the future: status 6 (SIGABRT) used since 2.6.2); - initialize logging earlier so that do_log may be called earlier during program startup; also log attempts to stop and to reload, including unsuccessful ones; - avoid logging by a forked process before exec, when there is a chance the log file descriptor is in a range 0..2; - sub run_command and run_command_consumer: distinguish between undefined and empty values of argument $stderr_to, undef now prevents reopening of file descriptor 2, making it possible for the caller to keep it attached to the current stderr; this is useful when run_command is called by the master process before logging has been configured; - SQL: explicitly call DBI::bind_param to be able to specify data types of values passed in @args to Amavis::Out::SQL::Connection::execute; - bump up buffer size from 16 kB to 64 kB in some cases of copying data from/to a pipe, mostly to reduce the amount of logging; - av scanner update: 'FRISK F-Prot Antivirus' entry modified to recognize name of a 'security risk' result, thanks to Michael Renner; - in a commented-out code providing a qmail CF/LF bug workaround, replaced $smtp_handle->datasend by $smtp_data_fh->print, which is more efficient in a line-by-line writing mode needed by qmail; thanks to Ronald Vazquez; - in a (banning) check for double extensions allow for whitespace around the second filename extension (files amavisd.conf and amavisd.conf-sample); based on a sample provided by Patrick T. Tsang; - setting $max_requests to 0 disables the limit, process will not be replaced based on the number of requests it has completed (but may still be replaced for other reasons); primarily intended for testing; - bump up a default value for $max_requests from 10 to 20 to match the suggested/example value in amavisd.conf-sample; - AM.PDP/milter setup: new configuration setting $prepend_header_fields_hdridx, also a member of policy banks, with a default value of 0. It is used as an argument hdridx in an AM.PDP attribute 'insheader' which in a milter setup is passed on as an argument hdridx to a smfi_insheader call. The value of $prepend_header_fields_hdridx only affects AM.PDP protocol and only if $append_header_fields_to_bottom is false (it is false by default). If more than one milter is used, all milters should be inserting their header fields at the same index (all prepending or appending, avoiding insertion in the middle of a header), otherwise the resulting order of header fields in a modified header becomes surprising, and in combination with signing milters like DKIM or DK the signature verification will most likely fail. The default value of 0 is normal and useful in combination with other content-checking milters. Signing milters like dkim-milter and dk-milter insert their header at index 1 (just below the new Received header fields), and when amavisd-new with Petr Rehor's helper program amavis-milter is used as a milter along with dkim-milter or dk-milter, the value of $prepend_header_fields_hdridx MUST BE SET TO 1, otherwise the generated signature will fail verification at the receiving site! Discussion: when sendmail calls its milters, its Received header field is not yet created and passed on to milters, yet it is already counted as one header field for the purpose of smfi_insheader hdridx interpretation. When a milter wants to prepend its header field(s), specifying hdridx of 0 does prepend its header fields above the yet-to-be-inserted Received header field as expected, and specifying 1 inserts its header field(s) just below the yet-to-be-inserted Received header field. If some milters in a chain specify 0 and others a 1 it affects the final order of inserted header fields in unexpected ways. It would be natural to always prepend fields with an index 0, but for signing milters like dk-milter this is not acceptable, as it would be expected to include a not-yet-available Received header field in its signature. For this reason signing milters like dkim-milter and dk-milter insert their header fields (signature) at index 1, and if amavisd-milter wants to coexist in such a setup, it must also insert its header fields at index 1. The conclusion: when amavisd (with its helper program) is used in a milter setup along with other milters, it should use the same hdridx value as other milters, which in case of signing dkim-milter and dk-milter is 1. If there are no such milters, either a 1 or a 0 would do, although a value of 0 produces a more natural order of header fields, matching that of a post-queue content filtering setup. See threads: http://archives.neohapsis.com/archives/postfix/2006-10/1777.html http://archives.neohapsis.com/archives/postfix/2006-11/0467.html --------------------------------------------------------------------------- September 30, 2006 amavisd-new-2.4.3 release notes For new instructions on setting up DKIM and DomainKeys with Postfix and amavisd-new please see: http://www.ijs.si/software/amavisd/amavisd-new-docs.html#dkim COMPATIBILITY WITH 2.4.2 - in a sendmail milter setup using AM.PDP protocol (e.g. by Petr Rehor's amavisd-milter), inserted header fields are now prepended to mail header by a new AM.PDP protocol attribute 'insheader', so an upgrade of the milter helper is needed to support the change (e.g. amavisd-milter 1.1.3), otherwise header field insertions will be ignored; - due to enhanced header checks (checking for missing required fields and checking for multiple occurrences of header fields which are allowed to occur only once), new cases of invalid mail may pop up in bad headers category, e.g. mail posted by MUA like Eudora; - a string %i in quarantine filename templates (such as $spam_quarantine_method) now uses a 'T' as a date/time separator, e.g. 20060814T173846 (conforming to iso8601), instead of a former '-'; - when loading a policy bank, its entries of type hash are now merged with existing hash, a key at a time; previously a newly loaded hash replaced a previous one entirely; BUG FIXES AND WORKAROUNDS - fixed a bug (introduced with amavisd-new-2.4.0): when receiving mail from MTA through an LMTP protocol (not SMTP) and with D_BOUNCE as a final*destiny setting, a suppressed non-delivery notification (e.g. spam above cutoff_level) did not turn LMTP status into a success, so an undesired bounce was generated by MTA in a post-queue filtering setup, contributing to excessive bounce backscatter; reported by Michael Scheidell, thanks to Gary V for analysis; - bug fix to amavisd-release: a regexp needs to be relaxed to allow quarantine names like Y/spam-Y5y7A3J5r2Ax.gz, reported by Rob Chanter; - fix a bug in LDAP lookups which could lead to an infinite loop while expanding %m in the filter; reported by Petr Vokac; - add "LOCAL_STATE_DIR => '/var/lib'" to the SA object initialization for versions of SA 3.1.4 or older, so that SpamAssassin would see additional rules provided by sa-update and placed to its default location; the SA 3.1.5 provides its own default so this becomes unnecessary; - bug fix: don't reject mail when mail size restriction is in force, the limit is exceeded, and $final_destiny_by_ccat{+CC_OVERSIZED} is not D_REJECT; - treat blacklisting as high spam score when considering suppressing quarantining (@spam_quarantine_cutoff_level_maps) or suppressing sending a DSN (@spam_dsn_cutoff_level_maps); - calling do_quarantine() multiple times on the same message would accumulate header edits from each invocation, fixed; (such situation can only happen with a modified program); - when defanging mail or releasing mail from a quarantine, with a goal of not breaking DKIM Sender Signing Practices (SSP) and DomainKeys policy, do not copy existing Sender header field to a new header, and insert our own Sender field (configurable by %hdrfrom_notify_recip_by_ccat); - explicitly set PerlIO layer to ":bytes" on a temporary file handle for email.txt (just in case); based on a problem report by Alexander Schäfer; - in a string produced by a macro %c remove a decimal dot if score happens to be an integer; - reduce $sa_mail_body_size_limit from 512 kB to 400 kB in amavisd.conf and amavisd.conf-sample for the time being, while the SA folks work on http://issues.apache.org/SpamAssassin/show_bug.cgi?id=5041 (MS Outlook Express seems to be chopping long mail in approx 500 kB chunks); - another workaround for Perl taint bug: IO::Handle::_open_mode_string taints the $1 when mode string to IO::File::open is '+<', use O_RDWR instead; thanks to Ryan Frantz; - abort if a specified syslog facility name is unknown, instead of switching to LOG_DAEMON as before; - change the code which selects defanging so that defanging is triggered if any applicable contents category of a message chooses defanging; counterintuitive behaviour reported by Tapani Tarvainen; - fix example in amavisd.conf-sample to use +CC_SPAM instead of CC_SPAM as a key to a hash, e.g. $final_destiny_by_ccat{+CC_SPAM}, otherwise Perl would implicitly turn CC_SPAM into a string when used in such a context. Note that any Perl expression syntax would do, as long as the argument does not look like a plain variable which receives implicit quoting; possibilities include $xx{&CC_SPAM}, $xx{+CC_SPAM}, $xx{CC_SPAM()}, $xx{(CC_SPAM)} and similar; a more obvious &CC_SPAM is avoided because it prevents subroutine call inlining optimization in Perl; Note that the '+' trick does not prevent implicit quoting with the '=>' operator, so this is wrong: %h = (+CC_SPAM => 1); use '&' or a '()', e.g. %h = ( CC_SPAM() => 1 ); or %h = ( &CC_SPAM => 1 ); or avoid the '=>' operator and use a comma: %h = ( CC_SPAM, 1 ); - qmail: update amavisd-new-qmqpqq.patch to be compatible with Net::Server version 0.91 or later; thanks to mr from DBA Lab S.p.A.; - AM.PDP protocol: change the order of attributes returned in an reply: delete and edit header fields before adding new header fields; problem of deleting just-inserted header fields in a sendmail milter setup reported by Petr Rehor; - AM.PDP protocol change - with version 2 of the protocol the following changes to the protocol were made: * "version_server=2" is provided in a server response as the first attribute, older versions did not provide such attribute (assumed version on the server side was 1); * delheader and chgheader now stand in a response before insheader and addheader, assuming that milter MTA will execute these in the same order; * new attribute: "insheader=hdridx hdr_head hdr_body" (where hdridx as used by amavisd will always be 0 for now), making it possible to prepend header fields in a sendmail milter setup (instead of appending them, breaking compatibility with DomainKeys); problem noted by Adam Gibson and Petr Rehor; * new attribute: "quarantine=reason" to place message on hold or to a quarantine maintained by MTA, and supply a reason text (e.g. client may call smfi_quarantine milter routine); For future use - it is currently (2.4.3 or earlier) never used. OTHER CHANGES AND SMALL FEATURES: - turn UTF8 warnings into fatal errors by: use warnings FATAL=>'utf8'; - reduced default $spam_check_negative_ttl to 10 minutes (from 30 minutes) - when defanging, enforce 998 characters line length limitation imposed by RFC 2822 by truncating long lines and appending a "..."; - incompatible change: a string %i in quarantine filename templates (such as $spam_quarantine_method) now uses a 'T' as a date/time separator, e.g. 20060814T173846 (conforming to iso8601), instead of a former '-'; - enhance header checks with checking for missing required fields (Date, From) and with checking for multiple occurrences of header fields which are allowed to occur only once: Date, From, Sender, Reply-To, To, Cc, Bcc, Message-ID, Subject, In-Reply-To, References. Added minor content types to major category CC_BADH: 7=missing, 8=multiple header field; - use enhanced status code 5.7.0 instead of 5.7.1 for blocking spam and viruses, and use 5.6.0 for blocking mail with invalid header; - macro SCORE now returns a single number instead of an explicit sum of SA score and boosts (contributed by soft-w/b-listing or pen-pals). When a single number is preferred in the log (Hits: ...), use this macro instead of %c in a log template: , Hits: [:SCORE]# based on a problem report by Ed Lucero; - updated Panda pavcl AV entry to match the new version called 'Panda CommandLineSecure 9 for Linux', thanks to Andrzej Kukula; - updated AV scanner entries for ESET NOD32 for Linux Mail servers, ESET NOD32 for Linux File servers, F-Secure Antivirus for Linux servers and clamscan, thanks to Anders Norrbring: - updated AV entry for Kaspersky AV version 5.5, thanks to Harrie Overdijk, Anders Norrbring and Gary V; - 'reload' and 'stop' command line options now report a process id of the previous daemon that was killed; - allow command-line option -d to specify a list of SA debug areas (also called 'facilities'); some useful examples: amavisd -d plugin,dkim,dk,spf debug-sa amavisd -d auto-whitelist,bayes,learn debug-sa amavisd -d dcc,razor2,pyzor,util debug-sa amavisd -d dns,uri,uridnsbl debug-sa amavisd -d received-header,metadata debug-sa amavisd -d rules,check debug-sa - substitute string _SCORE_ in spam_subject tag templates with actual score, _REQD_ with tag2 level, and _YESNO_ with 'Yes' (_YESNOCAPS_ with 'YES') if score is above tag2 level, and 'No' (or 'NO') otherwise; this is a quick-fix measure for often demanded feature; currently the mechanism is a simple string substitution and not a true macro expansion, so other macros are not currently available - more refined solution is expected for some future release; - reshuffled the order of rules (with minor adjustments) in example files amavisd.conf and amavisd.conf-sample to make it easier to permit certain files within archives; - introduce new variable @spam_dsn_cutoff_level_bysender_maps (also member of policy banks), complementing an existing by-recipient list of lookup tables @spam_dsn_cutoff_level_maps. The new variable serves to make it possible to trim down spam bounces to domains sending their own bounces with non-null return path (envelope sender address) and without DSN NOTIFY=NEVER option, but also to frequently abused domains, or to those sending marginal spam. When spam level exceeds either the @spam_dsn_cutoff_level_bysender_maps or the @spam_dsn_cutoff_level_maps level, (non)delivery status notification is suppressed even with $final_spam_destiny set to D_BOUNCE; - introduce new variables $resend_method and $release_method (also members of policy banks), both are undefined by default. If defined and nonempty, $resend_method overrides forward_method on forwarding a defanged mail, and $release_method overrides notify_method on releasing a message from quarantine. The $resend_method might be useful when a modified mail requires local DKIM or DomainKeys re-signing; - added global configuration variables $sql_lookups_no_at_means_domain and $ldap_lookups_no_at_means_domain, both false by default. They control whether a database mail address field with no '@' character represents a local username, or a domain name. By default (value false) it indicates a username in SQL and LDAP lookups (but represents a domain in hash and acl lookups), so domain names in SQL and LDAP should be specified as '@domain'. Setting these to true will cause 'xxx' to be interpreted as a domain name, just like in hash or acl lookups, which may facilitate interoperability with databases from other applications; - added global configuration variable $sql_quarantine_chunksize_max, which determines a maximum size (in bytes) for data written to a field 'quarantine.mail_text' when quarantining to SQL. Must not exceed size allowed for a data type on a given SQL server (e.g. maximum size for data type 'blob' in MySQL is 65535 bytes). It also determines a buffer size in amavisd. Too large a value may exceed process virtual memory limits or just waste memory, too small a value splits large mail into too many chunks, which may be less efficient to process; defaults to 16384; - added configuration variables @archive_quarantine_to_maps and $archive_quarantine_method, allowing for archival quarantine of all mail (configurable by recipient and by policy banks) regardless of its contents category. This archive is independent from other quarantining, i.e. if spam quarantining and archival quarantining are both enabled, two copies will be stored to quarantine. When quarantining for archive one has two choices: archive_quarantine would store all mail addressed to recipient, whereas enabling clean quarantine as in: $quarantine_method_by_ccat{+CC_CLEAN} = 'local:clean-%m'; $quarantine_to_maps_by_ccat{+CC_CLEAN} = 'clean-quarantine'; would quarantine only clean mail, no spam, no viruses, no banned, no badh. Note that logging to SQL has only one field to store quarantine location, so in case of multiple quarantine locations only the first is remembered. The usual logging however reports all quarantine locations with the main log entry. - added a global configuration variable @additional_perl_modules, which is a list of additional Perl module names or absolute file names that should be compiled/executed (by calling 'require') at a program startup time by a master parent process, before chroot-ing and before changing UID takes place. Its purpose is to pre-load additional non-standard SpamAssassin plugins and similar modules that a standard SpamAssassin initialization would miss, causing them to be loaded later by each child process, which is inefficient and may not work in a chrooted process. Example: @additional_perl_modules = qw( /usr/local/etc/mail/spamassassin/FuzzyOcr.pm /usr/local/etc/mail/spamassassin/ImageInfo.pm /usr/local/etc/mail/spamassassin/WebRedirect.pm String::Approx Net::HTTP Net::HTTP::Methods URI URI::http URI::_generic URI::_query URI::_server HTTP::Date HTTP::Headers HTTP::Message HTML::HeadParser HTTP::Request HTTP::Response HTTP::Status LWP LWP::Protocol LWP::Protocol::http LWP::UserAgent LWP::MemberMixin LWP::Debug ); Make sure these files are owned by root and not writable by unprivileged users such as amavis! - added a global config variable $enforce_smtpd_message_size_limit_64kb_min, true by default; when true an rfc2822 requirement that a limit on mail size must not be below 64 kB is enforced, so that any specified limit below 64 kB is treated as 64 kB; setting this variable to false disables this check, so mail size restrictions below 64 kB can be used and are effective; - added a by-contents-category setting %subject_tag_maps_by_ccat, unifying former separate settings @spam_subject_tag_maps, @spam_subject_tag2_maps, @spam_subject_tag3_maps and $undecipherable_subject_tag, and making it possible to specify subject tags (strings to be inserted into Subject:) for other categories, such as viruses, banned, and bad headers. Note that now only one such tag is inserted - previously if passed mail was both spam and undecodable two tags were inserted; - when spam level is at or above tag_level, turn on contents category CC_CLEAN with a minor category 1, making it easier to configure actions and settings (like subject_tag) through %*_by_ccat variables; - treat empty string in tag_level the same as undef, i.e. lets X-Spam-* header fields always be inserted for local recipients; - new configuration variable: $allow_fixing_improper_header (also a member of policy banks) is a more-general big brother of an older configuration variable $allow_fixing_improper_header_folding. It controls fixing of a mail header in passed and released mail; it currently controls truncating of header lines longer than 998 characters, and is a pre-condition for $allow_fixing_improper_header_folding, controlling removal of all-whitespace continuation lines. The $allow_fixing_improper_header defaults to true for backward compatibility. Fixing header may protect poorly written mail readers, but may break DomainKeys/DKIM validation of messages with illegal header if verification is done after content filtering, so if this is of concern, one has a choice of turning it off; - added macro sprintf, which invokes Perl's sprints with the usual arguments semantics, e.g. [:sprintf|%%s %%.1f %%%%|text|[:SCORE]] based on suggestion (but swapped arguments) from Joel Nimety; - added macros min and max, which returns minimal or maximal value from their arguments, ignoring empty string arguments, e.g. [:min|100|[:SCORE]] - extend a set of saved header fields in a hash $msginfo->orig_header_fields, which is accessible through a macro 'header_field'; currently the set of kept header fields consists of: From, To, Cc, Sender, Subject, Received, Message-Id, Resent-Message-Id, Precedence, User-Agent, X-Mailer, DKIM-Signature, DomainKey-Signature, Authentication-Results; - added macro useragent, which returns a 'User-Agent: ...' or 'X-Mailer: ...' header field (whichever is present); note that this is an entire field, including a header field head, unlike macros header_field and x-mailer; - added macros dquote and uquote to facilitate sanitation of header fields in logging: dquote encloses its argument in double quotes and replaces existing double quotes by \" (suitable to sanitize Subject header field); uquote replaces one or more consecutive space or tab characters by '_', but does not protect existing underlines, which makes it a lossy transformation (suitable for From or To header fields); provisional - exact interpretation may change; - updated notification templates to make use of new macros; - insert X-Amavis-OS-Fingerprint header field (if available) into a passed message to local recipients (not just to a message copy submitted to SpamAssassin for checking); suggested by Jeff Noxon; - insert X-Amavis-PenPals header field (if information is available) into a passed message, showing time interval (age) since the last message sent in the opposite direction, i.e. from the current recipient to the sender of the current message; - insert X-Amavis-PolicyBank header field into a message that is passed to SpamAssassin for a check (but not to a passed message as seen by recipient). A header field body is a slash-separated list of all policy banks loaded, e.g.: X-Amavis-PolicyBank: AM.PDP-SOCK/MYNETS/MYUSERS (but more usually just: MYNETS). If no policy banks are loaded, the header field will not be inserted. This information may be used by SA rules to add score points based on policy bank, or to countermeasure or conditionalize other rules, for example: header L_MYNETS X-Amavis-PolicyBank =~ m{(^|/)MYNETS(/|$)}m # apply score directly: score L_MYNETS -0.8 # use rule to countermeasure other rules: meta L_MYNETS_UNDISC_RECIPS UNDISC_RECIPS && L_MYNETS score L_MYNETS_UNDISC_RECIPS -0.841 score UNDISC_RECIPS 0.841 # use rule to conditionalize other rules: meta L_OTHERS_UNDISC_RECIPS UNDISC_RECIPS && !L_MYNETS score L_OTHERS_UNDISC_RECIPS 0.841 score UNDISC_RECIPS 0.001 - a new hash variable %allowed_added_header_fields (also member of policy banks) is consulted for each header field insertion, and if the result is false the header field is not inserted into passed mail. Can be used to suppress inserting header fields such as X-Virus-Scanned, X-Spam-Report, X-Spam-Level, X-Amavis-PenPals, X-Amavis-OS-Fingerprint, X-Amavis-Modified, Received, etc. Only applies to passed mail, not to mail that is being written into a quarantine or to a copy submitted to SA for checking. Keys (header field names) must be in lowercase. Example use - disables insertion of certain header fields: $allowed_added_header_fields{lc('X-Amavis-OS-Fingerprint')} = 0; $allowed_added_header_fields{lc('X-Amavis-PenPals')} = 0; $allowed_added_header_fields{lc('X-Spam-Report')} = 0; $allowed_added_header_fields{lc('X-Spam-Status')} = 0; $allowed_added_header_fields{lc('X-Virus-Scanned')} = 0; Note that turning off 'X-Spam-Report' through %allowed_added_header_fields is equivalent to having $sa_spam_report_header at false, turning off 'Received' is equivalent to having $insert_received_line at false, and the last line in the above example is equivalent to setting $X_HEADER_TAG to undef or $X_HEADER_LINE to undef. For compatibility the $X_HEADER_TAG is treated somewhat specially: if explicitly set to a nonstandard value, it is implicitly added to the %allowed_added_header_fields in the base policy bank. No automatism is provided for $X_HEADER_LINE in other policy banks. Example use in a policy bank: $policy_bank{'ALT'} = { allowed_added_header_fields => { lc('X-Amavis-PenPals') => 0, # turn it off lc('X-Amavis-OS-Fingerprint') => 0, # turn it off }, }; - when loading a policy bank, its entries of type hash are now merged with existing hash, a key at a time; previously a newly loaded hash replaced a previous one entirely; this is relevant for policy bank entries such as %allowed_added_header_fields and %*_by_ccat, making it more natural and specifying just keys that need to be changed in a new policy bank instead of entire hashes; - kill external tnef decoder if running for too long; - abort Convert::UUlib::LoadFile or other Convert::UUlib processing in do_ascii() if running for too long; problem case provided by Martin Grimm; - add Net::Server hooks post_configure_hook() and post_bind_hook(), making it easier to affect protection of Unix sockets created by Net::Server (like allowing write access for group) by uncommenting a call to umask in post_configure_hook(); suggested by Mike Gaskins; - internal: wrap most top-level initializations in BEGIN blocks, so that compiled initialization code will be discarded after it did its job, saving about 100 kB of process memory footprint; - update and add several comments - either to clarify code, or to fix typos; - document that a method 'smtp:' can be used for quarantining to a dedicated mailbox, much like the older way of using 'local:anything' with a fully qualified email address in quarantine_to, see: http://www.ijs.si/software/amavisd/amavisd-new-docs.html#quarantine traditionally: $notify_method = 'smtp:[127.0.0.1]:10025'; $quarantine_method_by_ccat{+CC_SPAM} = 'local:%m'; $quarantine_to_maps_by_ccat{+CC_SPAM} = ['quar@example.com']; now more obvious and preferred: $quarantine_method_by_ccat{+CC_SPAM} = 'smtp:[127.0.0.1]:10025'; $quarantine_to_maps_by_ccat{+CC_SPAM} = ['quar@example.com']; - make @viruses_that_fake_sender_maps a member of policy banks to make it possible to bounce certain types of viruses if originating from inside; - added configuration variable @smtpd_discard_ehlo_keywords (also member of policy banks), which is a case-insensitive list of EHLO keywords (AUTH, DSN, 8BITMIME, PIPELINING, SIZE, etc.) that the SMTP/LMTP server will not send in an EHLO response to a remote SMTP client. It is equivalent to a Postfix configuration variable of the same name. Practical use example - DKIM/DomainKeys signing of locally-originating mail after it has passed a content filter: # Configure MTA to send to port 10026 mail originating from our users # (from mydomains or authenticated roaming users, it will be returned # to port 10027 after checking), and to send to port 10024 all the rest # (incoming mail), which will be returned to port 10025 after checking: $forward_method = 'smtp:[127.0.0.1]:10025'; # MTA with non-signing service $notify_method = 'smtp:[127.0.0.1]:10027'; # MTA with DKIM signing service $inet_socket_port = [10024,10026]; # listen on two ports $interface_policy{'10026'} = 'ORIGINATING'; # switch policy bank on 10026 $policy_bank{'ORIGINATING'} = { # mail originating from our users # force MTA to convert mail to 7-bit before DKIM signing # to avoid later conversions which could destroy signature: smtpd_discard_ehlo_keywords => ['8BITMIME'], # forward to an smtpd service providing DKIM/DomainKeys signing service: forward_method => 'smtp:[127.0.0.1]:10027', # other special treatment of locally originating mail, e.g.: spam_admin_maps => ["virusalert\@$mydomain"], # warn of spam from us banned_filename_maps => ['ALT-RULES'], # more relaxed rules... }; master.cf: # mail return from a content filter (non-signing) 10025 inet n - n - - smtpd -o content_filter= ... # mail from our users returning from a filter (DKIM or DK signing service) 10027 inet n - n - - smtpd -o content_filter= ... -o milter_macro_daemon_name=ORIGINATING -o smtpd_milters=inet:127.0.0.1:4444 Note that the same effect (making Postfix convert outgoing mail to 7-bits before DKIM signing) could be achieved by a Postfix setting smtp_discard_ehlo_keywords=8bitmime on an smtp service feeding mail to be signed to amavisd, but this would require setting up two such services, one with the option and one without. - README.postfix: suggest options "-o local_header_rewrite_clients=" and "-o smtpd_milters=" on smtpd at port 10025; thanks to Noel Jones; - README.sendmail-dual: add FEATURE(`nocanonify') on MTA-TX, and FEATURE(`nocanonify',`canonify_hosts') on MTA-RX; thanks to Ricardo Stella; --------------------------------------------------------------------------- June 27, 2006 amavisd-new-2.4.2 release notes SUMMARY OF CHANGES: - new feature: "pen pals soft-whitelisting" lowers spam score of received replies to a message previously sent by a local user to this address; - new feature: added command line options to override certain configuration settings from a config file, see below; - documentation bug fixes, especially on the use of SQL data type TIMESTAMP; - zoo decoder interface routine can now use utility unzoo(1) or zoo(1); COMPATIBILITY WITH 2.4.1: There are no incompatible changes since 2.4.1, but please notice below the fixes to SQL and to LDAP documentation, which may affect you. BUG FIXES AND CHANGES since 2.4.1: - LDAP.schema: add missing LDAP attribute amavisSpamQuarantineCutoffLevel to the list of allowed attributes in objectclass amavisAccount; pointed out by Paolo Cravero; - README.sql PostgreSQL notes: fixed incorrect advice in README.sql which suggested to declare a field msgs.time_iso as TIMESTAMP WITHOUT TIME ZONE instead of the correct TIMESTAMP WITH TIME ZONE. Previous instructions were also contradictory to suggested data type on ALTER TABLE msgs ALTER time_iso. Using inappropriate WITHOUT TIME ZONE when comparing time_iso to now() (which is aware of a time zone) offsets results by a current time zone offset, which unexpectedly includes too many or too few records in maintenance purging operation. A workaround if WITHOUT TIME ZONE continues to be used is to: SET TIME ZONE 'UTC'; before purging. If you were misled by previous documentation choosing WITHOUT TIME ZONE for time_iso, and decided now to convert it to WITH TIME ZONE, the following clause can convert time_iso to proper universal time by manually providing appropriate time offset: ALTER TABLE msgs ALTER COLUMN time_iso TYPE TIMESTAMP WITH TIME ZONE USING time_iso [-+] INTERVAL '[offset]'; Thanks to Brian Wong for a problem description and advice. - README.sql MySQL notes: fixed incorrect advice in README.sql which suggested to declare a field msgs.time_iso as TIMESTAMP instead of the correct TIMESTAMP NOT NULL DEFAULT 0. The "DEFAULT 0" is mandatory to prevent MySQL from overwriting mail reception timestamp with current local time when other fields are updated at the end of processing of a message. Also not to be forgotten: $timestamp_fmt_mysql *MUST* be set to 1 in amavisd.conf with MySQL when msgs.time_iso data type is TIMESTAMP... ! - README.sql MySQL notes: if using field msgs.time_iso to select records for purging (instead of msgs.time_num), and its data type is TIMESTAMP... (as opposed to CHAR...), one should use function utc_timestamp() in place of now() in the DELETE clause to make it work correctly regardless of time zone. Alternatively, now() can continue to be used, provided that SQL client time zone is set to UTC in the purging SQL script: SET time_zone='+00:00'; thanks to Gary V for investigation; - README_FILES/README.sql: added short for-the-impatient sections: * BRIEF MySQL EXAMPLE of a log/report/quarantine database housekeeping * BRIEF MySQL EQUIVALENT EXAMPLE based on time_iso if its type is TIMESTAMPS * BRIEF PostgreSQL EXAMPLE of a log/report/quarantine database housekeeping - a message with only a header, without empty separator line and with no body, lost the last line of a header on forwarding or writing to quarantine; observed by Elias Oltmanns, reported through Debian bug tracking; - header validity checks inappropriately reported 'header field syntax error' as a 'header field too long' (BadHdrLong) instead of BadHdrSyntax; - ensure that notification would not be sent if notification template is empty, solving the following problem: when recipient notifications for bad headers is enabled, and a message is spam with bad headers, recipient would receive an empty notification message (because message contents category is spam and recipient notification template for spam is empty); reported by Alex; - changed SMTP status code 550 to 554 when rejecting mail contents, the 550 is not envisioned in RFC 2821 as a valid reply code to a "." after data transfer; thanks to Victor Duchovni; - fixed case mismatch when storing e-mail address to SQL table maddr, which wasted one unnecessary failed attempt on INSERT; - ignore $timestamp_fmt_mysql if SQL database driver (DBD) is not 'mysql'; - perl taint workaround in lookup_sql() where SQL select clause could become tainted; problem reported by Christer Borang; - fixed amavisd.conf-default which stated incorrect default values of keys 'ins_rcp' and 'ins_quar' in %sql_clause; reported by Glenn Sieb; - limit reported boost score to three decimal places; long fractions observed by Gary V; - Postfix since version 20060610 uses xtext-encoded (rfc3461) strings in XCLIENT and XFORWARD attribute values, previous versions used plain text with neutered special characters - amavisd-new now xtext-decodes value if it looks xtext encoded, and encodes it on sending; the change could affect exotic host names (e.g. with a plus in host name) from broken mailers or DNS; thanks to Ralf Hildebrandt for pointing out the recent change in Postfix; - improve regular expressions in the $map_full_type_to_short_type_re list to cope better with different versions of the file(1) utility regarding recognition of various MS executables; based on a problem report by Misha; - use stricter suggested regular expression in amavisd.conf for matching CLSID (Class ID extension); previous expression was loose and too easily matched file names with braces in the name; suggested by Martin Schuster through Debian bug tracking; - zoo decoder interface routine (do_zoo) can now use utility unzoo(1) or the traditional zoo(1); the unzoo(1) recognizes some additional parameters which makes it more resilient (but still not watertight) against some attempts to hide archive contents or to extract members to unexpected locations, but unfortunately does not recognize all zoo compression schemes ("error, LZD not yet implemented"), and the relative modes "-j ./" or "-j X" do not protect against all malicious cases - so it is a mixed blessing. The way amavisd calls zoo(1) (piping members to stdout, which can be slow) avoids some of the security problems with zoo (writing to arbitrary directories), which were probably the main reason for ClamAV project deciding to switch to unzoo(1); - zoo sucks, unzoo (v4.4) sucks more: considered, but decided against changing zoo entry in @decoders to ['unzoo','zoo'] in amavisd.conf, as was suggested by Gábor Kövesdán. It would not necessarily be an improvement (see previous item, misses extracting members from my test cases), so feel free to choose between the two poor choices, I still prefer zoo(1), partly also because it covers cases which clamd decoding misses; - kill external zoo or unzoo decoder if running for too long; - internal: saving recipient addresses to SQL table maddr is now done earlier to make information available to pen pals code; - explicitly test if SQL 'prepare' silently fails to return a statement handle, just in case; - adjusted list of pre-loaded SA modules to cater for SA 3.1.3; NEW FEATURES: - new feature: added command line options which override some configuration settings from a config file (an option to override pid_file suggested by Paul Murphy and Gábor Kövesdán): -d log_level ... overrides $log_level -m max_servers ... overrides $max_servers -L lock_file ... overrides $lock_file (Net::Server serialization) -P pid_file ... overrides $pid_file -H home_dir ... overrides $MYHOME directory -Q quarantine_dir ... overrides $QUARANTINEDIR directory, empty disables -T tempbase_dir ... overrides $TEMPBASE directory -S helpers_home_dir ... overrides $helpers_home directory (SA workplace) -D db_home_dir ... overrides $db_home, empty arg turns off $enable_db -R chroot_dir ... overrides $daemon_chroot_dir, empty avoids chroot -p listen_port_or_socket ... overrides $inet_socket_port as well as $unix_socketname, argument may be a decimal TCP port number, or an absolute path name of a Unix socket; may be specified multiple times: daemon can listen on multiple inet sockets and/or multiple Unix sockets; example: -p 10024 -p 9998 -p /var/amavis/amavisd.sock -V ... shows version and exits -h ... shows version and command line options, then exits For completeness, here are remaining options, unchanged from previous versions: -u user ... overrides $daemon_user -g group ... overrides $daemon_group -c config_file ... config file name, may be specified multiple times - new feature: "pen pals soft-whitelisting" lowers spam score of received replies (or followup correspondence) to a message previously sent by a local user to this address; Pre-requisites: * both the outgoing and the incoming mail must pass through amavisd (although outgoing mail may have checks disabled or made more permissive if desired); * SQL logging must be enabled (@storage_sql_dsn) and records should be kept for at least several days (some statistics (2006-11 update): 90% of replied mail (or followups) is sent within 2 weeks since previous correspondence, 40% within 24 hours, 20% within 3 hours, 10% within 30 minutes, 5% within 12 minutes); * @mynetworks and @local_domains_maps must reflect reality, allowing amavisd to distinguish between outgoing, incoming and internal-to-internal mail; * the information about client IP address must be available to amavisd, i.e. Postfix XFORWARD protocol extension must be enabled, or AM.PDP+milter; * configuration variable $penpals_bonus_score must be set to a positive value (such as 1.0, increase to perhaps 5 or 8 after seeing that it works), zero disables the feature and is a default; * $sql_clause{'sel_penpals'} must contain a SELECT clause (which by default it does, unless overridden by an old assignment to %sql_clause in amavisd.conf); * sender/recipient address pair must exactly match recipient/sender pair of previous correspondence (except for allowed case-changes in domain part), which means that care must be taken when canonical and/or virtual mapping is performed by MTA (such as mapping between internal and external address forms) - if external address forms of local addresses are to be seen by a content filter then canonical mapping (int->ext) must be done *before* filtering and virtual mapping (ext->int) *after*; alternatively, if internal address forms are to be seen by a content filter, then canonical mapping should be done after filtering, and virtual mapping before; see README.postfix, section "TO DO 'VIRTUAL ALIAS' MAPPING AND OTHER POSTFIX CLEANUP PROCESSING BEFORE OR AFTER CONTENT FILTERING?" (P.S. later renamed to 'Advanced Postfix and amavisd-new configuration'); How it works: * SQL logging stores records about all mail messages processed by amavisd, their sender, recipients, delivery status, mail contents type (no changes there, this feature was introduced with amavisd-new-2.3.0); for the purpose of pen pals scheme only records with local-domain senders matter; * when a message is received, an SQL lookup against an SQL logging database is performed, looking for previous messages sent in reverse direction, i.e. from a local user (which is now a recipient of the current mail) to the address that is now the sender of the message being processed; A SELECT clause in $sql_clause{'sel_penpals'} is used, which by default only considers records of previous messages that were actually delivered (not rejected, discarded or bounced), and were not infected. SQL lookup returns a timestamp of the most recent such message (if any), the difference (in seconds) between the current time and the timestamp is an 'age' as used in the following formula; * an exponential decay formula calculates score points to be deducted from the SA score: weight = 1 / 2^(age/penpals_halflife) score_boost = -penpals_bonus_score * weight i.e. penpals_bonus_score is multiplied by 1, 1/2, 1/4, 1/8, 1/16, ... at age 0, 1*halflife, 2*halflife, 3*halflife, 4*halflife ... weight is a continuous function of age (actually, in steps of one second); * main configuration variables, members of policy banks: $penpals_bonus_score ... a maximal (positive) score value by which spam score is lowered when sender is known to have previously received mail from our local user from this mail system. Zero or undef disables pen pals lookups, and is a default. $penpals_halflife ... exponential decay time constant in seconds, defaults to 7 days; pen pal bonus is halved for each halflife period since the last mail sent by a local user to the current message's sender; * auxiliary configuration variables, global settings: $penpals_threshold_low ... SA score below which pen pals lookups are not performed to save time, defaults to 1.0; undef lets the threshold be ignored (useful for testing and statistics gathering); $penpals_threshold_high ... when (SA_score - $penpals_bonus_score > $penpals_threshold_high) pen pals lookup will not be performed to save time, as it could not influence blocking of spam even at maximal penpals bonus (age=0); usual choice for value would be kill level or tag2 level, or other reasonably high value; undef lets the threshold be ignored and is a default (useful for testing and statistics gathering); Caveats / notes / exceptions with "pen pals soft-whitelisting": * pen pals soft-whitelisting aids incoming mail, and internal-to-internal mail, but has no effect on outgoing mail; * if SQL logging was not used so far and you are considering enabling it for a busy site, you would appreciate PostgreSQL 8.1 compared to MySQL, as purging old records seems to be *much* faster than in MySQL 4.1, which could lock down mail processing for an hour or more during a weekly (or daily) purge, as opposed to minutes or seconds; * infected messages are exempted from pen pals checks; * mail with (unadjusted) SA score below $penpals_threshold_low (1 by default) is exempted from pen pals check to save time and lighten the load on SQL; similarly for high score spam which would not have a chance of being 'saved' even by a maximal pen pals bonus score; * non-delivery notifications have null sender address, so can not match previous correspondence and can not receive a pen pal bonus; * unauthenticated sender address matching local domains but coming from outside is not trusted and is exempted from pen pals checks; * messages from a local user to self are exempted from pen pals check; * outgoing messages (i.e. to non-local recipients) are exempted from pen pals checks to save some time and simplify reasoning (which reverse mail transaction to trust?); assuming that local users rarely send spammy mail, outgoing mail would rarely need help from pen pals checks; * messages received from mailing list typically use ML bounce or admin address (possibly VERPed) as the sending address, so they would not be considered replies to postings to a mailing list from a local user (to be addressed in future version 2.5.0); * underlying assumption is that a local-domains sender address in mail coming from inside can be trusted not to be faked; if this is not the case, an internal user cooperating with a spammer can widen spam tolerance for another internal user (but it probably does not pay off, too much trouble for too little effect); * if a spammer knows or can guess that a local user is frequently sending mail to some address (e.g. a mailing list unprotected by DKIM or SPF), he can gain few bonus score points by using such sending address in his spam; * there may be multiple MTA+amavisd servers, but all must use the same logging SQL database; * forwarding is compatible with the pen pals scheme; * a forwarding scheme like SRS (with SPF), where envelope sender address is replaced by a forwarding mailbox address is counterproductive; for example: a local user may also have an external mailbox at some remote provider with poor spam protection; forwarding from the remote to a local mailbox is set up and a forwarding MTA misguidedly substitutes original sender address with a mailbox address; spam reaching remote mailbox is forwarded to a local site with a sender address rewritten, making it look like it is coming directly from a user's remote mailbox, and inappropriately benefiting from pen pals bonus of user's previous correspondence with his remote mailbox; Testing: * set $penpals_bonus_score initially to a low value such as 1 to avoid surprises; * set $penpals_threshold_low and $penpals_threshold_high to undef to perform pen pals lookups regardless of the score; * at log level 2 (or higher) search the log for a string "penpals: " (only shows on incoming mail sent by a non-local sender); the log also shows mail_id of the referenced message (previous communication), and Subject header fields of previous and current message; Based on a feature request by Aaron P. Martinez, thanks to Gary V for suggestions and prompting and to Michael Scheidell and Richard Bishop for feedback. --------------------------------------------------------------------------- May 8, 2006 amavisd-new-2.4.1 release notes INCOMPATIBLE CHANGE WITH 2.4.0: - notification templates incompatibility with 2.4.0 (but not with versions 2.3.3 or older): major contents category numbers are renumbered due to a newly inserted category CC_SPAMMY; it affects the use of macro ccat_maj in templates (one field added), and only affect users which provide non-default templates based on 2.4.0 templates; older templates (2.3.3 or earlier) are unaffected as they do not use macro ccat_maj; OTHER CHANGES: - revert a change introduced with 2.4.0, which was adding address extensions at CC_SPAM, i.e. when score exceeds kill level. Previously (2.3.3) address extensions were inserted at tag2 level. Implemented by a new mechanism: a new major contents category CC_SPAMMY is inserted just below the CC_SPAM, where CC_SPAMMY is controlled by tag2_level and CC_SPAM continues to be selected at kill_level. Also spam defanging (if enabled) is now activated at CC_SPAMMY and no longer at CC_SPAM (which was on a TODO list for some time); undesired change in 2.4.0 reported and changes tested by Mario Liehr; - fixed old nuisance bug (probably present since 2.3.0) when an external decoder program for self-extracting archives (rar/unrar, lha, arj/unarj) is defined but the program does not exist, which resulted in logged non-fatal errors like: run_command: failed to exec SCALAR(0x8598550) lq ... run_command: failed to exec REF(0x85985c8) v -c- -p- -av- -idcdp -- ... run_command: failed to exec ARRAY(0x89e5f0c) l ... No such file or directory reported by Martin Baertl, Maurizio Marini, boka, and Donald Teed, investigated by Gary V; - bug fix in a Courier setup: add a missing reset of per-recipient data to prevent previous message check affecting the next one performed by the same process; fix by Martin Orr, reported by Bowie Bailey; - the amavisd-new-courier.patch now requires Net::Server version 0.90 or later (preferably 0.93 or later); to use older version of Net::Server please apply the older amavisd-new-courier-old.patch and follow README.courier-old - both will go away with next version of amavisd-new; - updated amavisd-new-qmqpqq.patch patch (qmail interface) to work with the current code, by Martin Solciansky, testing by Nicklas Bondesson; - fix error handling when a problem occurs during temporary directory cleanup; - when defanging mail make a 'Subject' header field be editable by header edits, so that Subject tags like ***UNCHECKED*** can still apply; - modify unquote_rfc2821_local so that it appends an '@' as a domain name only if localpart contains '@', so that read_array() can still be used to read a list of networks in CIDR notation; a change in 2.4.0 to properly handle addresses like "aaa@bbb" (local part in quotes, no domain) made read_array unsuitable for reading list of networks; pointed out by Petr Vokac; - add another round of local($1) declarations as a workaround for already familiar Perl taint bugs, popping up again on some Perl installations; reported by Jaap Struyk; reported symptoms were: Insecure dependency in chown while running with -T switch at /usr/lib/perl5/site_perl/5.8.7/Net/Server.pm line 488 Insecure dependency in eval while running with -T switch at /usr/lib/perl5/site_perl/5.8.7/Mail/SpamAssassin/PluginHandler.pm line 91 - added config variables: @spam_subject_tag3_maps, @spam_tag3_level_maps (and $sa_tag3_level_deflt), which makes it possible to split contents category CC_SPAMMY into two sublevels (minor categories) and give each its own Subject tag text; the "CC_SPAMMY,0" contents category still corresponds to tag2 level, and "CC_SPAMMY,1" contents category corresponds to tag3 level (if defined). Only static maps are available (also members of policy banks), but no corresponding SQL and LDAP attributes are provided. Example: @spam_tag2_level_maps = (5.5); @spam_tag3_level_maps = (12); @spam_subject_tag2_maps = ('***LIKELY*SPAM*** '); @spam_subject_tag3_maps = ('***BLATANT*SPAM*** '); based on suggestion from Benedict White; - add LDAP attributes: amavisSpamSubjectTag, amavisSpamSubjectTag2, amavisSpamDsnCutoffLevel, amavisSpamQuarantineCutoffLevel to match equivalent SQL lookup fields; missing amavisSpamQuarantineCutoffLevel noticed by Paolo Cravero; - presence of LDAP attributes is now tested with 'defined', no longer as Perl booleans; - mail_via_bsmtp: storing mail in BSMTP format now saves DSN information, as permitted by RFC 2442; - apply the concept of separate timers $child_timeout and $smtpd_timeout as used in an SMTP session to AM.PDP and AM.CL protocols; - apply the concept of separate timers $child_timeout and $smtpd_timeout as used in an SMTP session to Courier patch; by Martin Orr; - new macros: remote_mta, smtp_response, remote_mta_smtp_response and score_boost available to log templates and notification templates; - enhanced regexp selector macro [~string|regexp|then|else], which can now capture parenthesized regexp subexpressions and make them available as %1, %2, ... %9 to 'then' and 'else' replacements; a copy of the first argument (a string) is available to replacements as %0; - extend the semantics of the regexp selector macro, which can now take more than one pair of regexp+then arguments, catering for a nested 'if then elseif then elseif then else' structure: [~string|regexp1|then1|regexp2|then2|...|regexpN|thenN|else] - enhanced iterator macro, which can now take a long macro name as its first argument, and imply a %x as iterator name; - make use of the new macro remote_mta_smtp_response and add it to a default $log_templ, so that a Postfix queue-id of a forwarded message shows up like 'queued_as: DCF2A17B9E4' in the main log entry, facilitating search for a related log entry in an MTA log. In case of a mail split, all the MTA responses would now be shown, e.g.: queued_as: F3DBD17B847/F3DBD17B847/F3DBD17B847 (customizable by the use of macros in $log_templ); - sophos_savi_internal (SAVI module): don't include errno ($!) in the error message, it may be misleading; reported by Matthias Ivers; - internal - programming style: use more predictable $b=1 instead of $b++ where variable $b is supposed to be a boolean and not a counter; --------------------------------------------------------------------------- April 3, 2006 amavisd-new-2.4.0 release notes The most important changes since 2.3.3 at a glance: Delivery status notifications (DSN) are now supported, both as an SMTP protocol extension and in notifications. Header fields like X-Amavis and X-Spam are now prepended to mail header for DomainKeys compatibility. Configuration variables can be chosen based on mail contents category, which is now represented explicitly. A built-in macro expander is enhanced, providing new macros and call types. Added support for passive operating system fingerprinting with the use of p0f, supplying collected information as a header field to SpamAssassin. Provide compatibility with Net::Server 0.91 and later. INCOMPATIBLE CHANGES SINCE 2.3.3: - incompatible change when logging or quarantining to SQL: added field 'quar_loc' to table 'msgs' to facilitate quarantine release, and added FOREIGN KEY constraint for data consistency and simplified purging; see below for a simple database modification; - inserted header fields like X-Amavis-* and X-Spam-* are now _prepended_ to mail header instead of being appended, and occupy position just above the inserted Received header field; this pairing with Received makes it easier to identify which MTA/content filter inserted them, makes it consistent with position of Resent-* header fields as required by RFC 2822, and avoids the possibility of breaking DomainKeys, DKIM, and similar mail signing schemes. SpamAssassin implemented the same change with 3.1.0. To achieve former behaviour, specify: $append_header_fields_to_bottom=1; - trailing whitespace is no longer trimmed by default from SQL fields, from LDAP attribute values and from associative array righthand-sides (hash values) as read by read_hash(); see below if trimming is really still needed; - SMTP server side: no longer allow e-mail address without enclosing angle brackets in MAIL FROM and RCPT TO smtp commands; such syntax is illegal according to RFC 2821 and RFC 821, no compliant MTA is using it, so the change should not effect anyone (except perhaps sloppy testers); - changed defaults for banned & bad header administrator address to: $banned_admin = undef; $bad_header_admin = undef; @banned_admin_maps = (\$banned_admin, \%virus_admin,\$virus_admin); @bad_header_admin_maps = (\$bad_header_admin); In other words, if $banned_admin is left at a default value (undefined), banned admin falls back to %virus_admin or $virus_admin. If $bad_header_admin is left at a default value (undefined), bad header admin has no default, admin notifications for bad headers are not sent; MAJOR NEW ENHANCEMENTS: - support for DSN (RFC 3461) in the SMTP protocol (parameters NOTIFY and ORCPT in ESMTP RCPT commands, parameters RET and ENVID in ESMTP MAIL command), with corresponding updates to Delivery Status Notifications (RFC 3462, RFC 3464); (about a missing support for option ORCPT in Net::SMTP please see http://rt.cpan.org/Public/Bug/Display.html?id=18456 ); - represent mail contents category more explicitly internally, and provide new configuration variables: %final_destiny_by_ccat %lovers_maps_by_ccat %defang_by_ccat %quarantine_method_by_ccat %quarantine_to_maps_by_ccat %notify_admin_templ_by_ccat %notify_recips_templ_by_ccat %notify_sender_templ_by_ccat %warnsender_by_ccat %hdrfrom_notify_admin_by_ccat %mailfrom_notify_admin_by_ccat %hdrfrom_notify_recip_by_ccat %mailfrom_notify_recip_by_ccat %hdrfrom_notify_sender_by_ccat %admin_maps_by_ccat %dsn_bcc_by_ccat %warnrecip_maps_by_ccat %addr_extension_maps_by_ccat gradually phasing out separate configuration variables for each category; the change is fully backward compatible, existing variables are referenced through default values of the new variables, and no longer used directly; The chain of lookups adhere to the following evaluation sequence for settings with an associated *_by_ccat mechanism: * policy bank chooses a *_by_ccat associative array (by TCP port or by client's IP address (MYNETS)); * the most relevant contents type of the message chooses an entry in a _by_ccat associative array; the entry can be a final settings value, or a ref to an array of by-recipient lookup tables (*_maps); mostly for compatibility reasons an entry can also be a ref to CODE, which allows for delayed evaluation through legacy *_maps settings (which may again be members of policy banks); * the chosen list of lookup tables is queried based on recipient address, producing a final setting; Note that currently only settings which are applicable _after_ the mail contents type has already been determined, have their associated _by_ccat associative array. Settings like @bypass_spam_checks_maps which need to be evaluated _before_ mail contents is assessed, do not have their associated _by_ccat variable; - added ability to explicitly kill externally running decoder process or a command-line virus scanner process if running for too long; - enhanced built-in macro expander now allows long macro names (previously limited to one character), neutral and active macro calls, dynamically defining macros, new regexp matching built-in macro, more robust and explicit bookkeeping of quoting levels, as well as speedups achieved by pre-tokenization; details in see README.customize; - improved wrapping of inserted header fields, fields in DSN, and in generated text sections of the new notification templates; - improved text of notification templates, taking advantage of new macros; - compatible with Net::Server 0.90, 0.91, 0.92 and 0.93 by providing workarounds; thanks to Paul Seamons, the author of Net::Server, for his cooperation, the 0.93 solves problems introduced by a change in 0.91 (but it remains incompatible with version of amavisd-new 2.3.3 and older); - experimental support for passive operating system fingerprinting with the use of externally running utility p0f, supplying collected information as a header field to SpamAssassin, making possible to add rules to score SMTP client hosts based on educated guess about their operating system type and IP distance; see below for details; - make variable $myhostname a dynamic variable, member of policy banks, likewise for syslog parameters facility, priority and ident; details below; - added config options to enable quarantining (archiving) of clean mail; - lots of cleanups and generalizations in the code; SECURITY: - fix insufficient sender address sanitation when storing quarantined or forwarded files as BSMTP files _and_ having a %s in the corresponding *_method template; potential security vulnerability (with limited scope) in versions of amavisd-new 2.3.1, 2.3.2 and 2.3.3 discovered by Thomas Jarosch; - recognize result "ms-windows metafile" (or "ms-windows metafont") from a file(1) utility and provide short type 'wmf' for it; added two example rules to amavisd.conf (and amavisd.conf-sample) to block files containing Windows Metafiles, based on US-CERT Alert TA05-362A; OTHER CHANGES: - incompatible change when logging or quarantining to SQL is enabled (as mentioned above, here is a more detailed description of the change): * added column 'quar_loc' to table 'msgs' to store quarantine file name (the same string as in macro %q, normally seen in the main log entry); based on input from Andrew A. Neuschwander, Brian Wong and Craig Herring; * add constraint FOREIGN KEY ... ON DELETE CASCADE to keep database consistent (free of orphaned records) and simplify maintenance deletions and possibly speed them up; suggested by Brian Wong; The following clause must be executed for upgrading pre-2.4.0 amavisd-new SQL schema to the 2.4.0 schema: ALTER TABLE msgs ADD quar_loc varchar(255) DEFAULT ''; The following clause should preferably be executed to take advantage of the ON DELETE CASCADE: ALTER TABLE msgrcpt ADD FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE; ALTER TABLE quarantine ADD FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE; See updated suggested set of DELETE clauses at the end of README.sql. The following clause can optionally be used to create an index on field msgs.time_num to speed up deletions in MySQL; CREATE INDEX msgs_idx_time_num ON msgs (time_num); or if purging is based on field msgs.time_iso instead of msgs.time_num: CREATE INDEX msgs_idx_time_iso ON msgs (time_iso); (compatibility note with pre-releases of 2.4.0: there were added fields msgrcpt.time_num and quarantine.time_num in pre-release versions of 2.4.0, which are now dropped in favor of FOREIGN KEY constraint; these fields are no longer set by the program and should not be relied-on when purging records, they may be removed from tables); - solve compatibility issues brought up by changes in file descriptors usage as introduced with Net::Server version 0.91; thanks to Ralph Seichter, Matt Jackson, Jim Knuth and Paul Seamons (the author of Net::Server) for help; - bug fix with LDAP lookups: if an LDAP connection to the server drops (i.e. after being idle for some time) amavisd is unable to reconnect; a symptom in the log is: 'do_search: failed again'; a fix by Petr Vokac, and later independently fixed by Matteo Brancaleoni and Mike Hall; problem also reported by Paolo Cravero; - bug fix with LDAP amavisBannedRuleNames lookups, failing to looking up a set of banned rules names and referencing them in the users lookup table, like in other lookup tables. The LDAP lookup was returning an array reference for the 'amavisBannedRuleNames' attribute since it was a list (multivalued) and the reference wasn't being dereferenced down the line. The fix is to make the attribute single valued, the value can be a comma-separated list of names. This brings it in-line with SQL lookups which also uses a single field of comma-separated names. The included LDAP.schema and documentation files are fixed accordingly. Also moved the LDAP stuff out of README.lookups into its own README.ldap and updated it accordingly for the banned rules stuff. Fixed by Michael Hall; problem reported by Jérôme Schell, Aury Fink Filho and Brian Wong; thanks also to Jack Stewart and Willi Gruber; - bug fix: properly disconnect SMTP session with 421 response if it times out; watchdog timer needs to be nudged during DATA transfer as well; pointed out by Victor Duchovni; - introduced new configuration variable $smtpd_timeout (default 8*60 seconds) which controls the amount of time we are willing to wait for slow/idle client during incoming SMTP session before disconnecting a session. Previously the $child_timeout was covering complete elapsed time, both our processing and waiting for client, now the $child_timeout only still limits our processing, and $smtpd_timeout only limits waiting time. With Postfix after-queue setup the $smtpd_timeout should be higher than Postfix setting max_idle (default 100s). Some other setups (like a pre-queue setup) may demand substantially higher $smtpd_timeout values; inflexibility pointed out by Martin Schmitt; - added ability to kill externally running decoder process or a command-line virus scanner process if running for too long; currently implemented for all command-line virus scanners and for more common and/or more troublesome external decoders: do_unrar, do_unarj, do_uncompress, do_pax_cpio, do_lha, and partly for do_arc, do_zoo; allowed time is calculated as 2/3 of the remaining time (initially at $child_timeout), but at least 10 seconds; - use the same timeout calculation as above for calls to SA, taking $sa_timeout instead if that value is bigger than the calculated time, thus making $sa_timeout pretty much redundant; - let do_pax_cpio recognize (and ignore) a single character in place of a date in more exotic cases of a pax listing; reported by Ralf Hildebrandt; - standards compliance: recognize (and discard) source route in mail address as required by rfc2821; - no longer bother to convert addresses like <""@yahoo.com> to <@yahoo.com>, both forms are invalid anyway, and recent versions of Postfix treat them the same. It is probably a good idea to set strict_rfc821_envelopes=yes in main.cf to reject such non-replyable sender addresses straight away, otherwise we end up processing such mail with inability to bounce it when needed, effectively losing it; - make address with '@' in the localpart but without a domain (such as <"aaa@bbb"> ) distinguishable from by appending an empty domain ('@' only) to the internal (unquoted) address form; also, we used to strip off empty domain on rfc2821-quoting, but this leads Postfix to interpret an address with an '@' in the local part like <"hhh@example.net"> as (subject to the 'resolve_dequoted_address' Postfix setting), which is not what the sender requested (perhaps unintentionally) so we no longer do that. Both measures together, along with the new address parsing code, solve the inconsistency problem reported by Les Ault; - fix string_to_mime_entity() to properly split header from body even in some corner cases (empty header or empty body); as a bonus a tiny speedup in template message splitting is gained; - mail header of a 'defanged' message should not contain broken original headers (with illegal characters or whitespace lines); now sanitize such header fields; reported by Ivers Matthias; - do not fix illegal all-whitespace continuation header lines when writing to quarantine (or when submitting notifications) in order to preserve the original bad header; only fix the header when such message is forwarded or released from a quarantine; masking problem brought up by Michael Scheidell; - when quarantining in a Unix-style mbox format, replace null return path in a delimiting 'From ' line with a string 'MAILER-DAEMON', like Postfix and sendmail local delivery agents do, otherwise some mbox-reading clients do not recognize the line as a message delimiter; - when quarantining to a mbox file, ">"-escape all /^From / lines, not just the ones following a blank line; this is more universal and does not break on more sloppy mail readers (thunderbird, kmail, mutt and pine); MUAs like elm and mail(1) (the later usually comes with the OS) are more robust, treating as a message delimiter only /^From / lines following a blank line, these did not mind the more compact approach used by amavisd-new so far; - new config variable $syslog_ident makes it possible to configure syslog ident string, its value defaults to 'amavis'; suggested by Andrzej Kukula; - instead of the old config variable $SYSLOG_LEVEL (default value 'mail.debug') there are now two config variables $syslog_facility and $syslog_priority, defaulting for compatibility to the before-the-dot and after-the-dot substrings of the variable $SYSLOG_LEVEL. The variable $SYSLOG_LEVEL still exists, can still be used, but is considered obsolete; - make variables $syslog_ident, $syslog_facility and $syslog_priority dynamic variables, members of policy banks. This makes it possible for each policy bank to use its own specific syslog settings, for example to log to a different file (by using a different syslog facility like 'LOCAL3' for mail originating from inside (policy 'MYNETS')), or to change syslog ident to 'outgoing-amavis' for certain policy bank, or to rise syslog priority for releases from a quarantine. Note that switching syslog_ident or syslog_facility is done only when necessary, and involves closing and reopening syslog connection, which involves some (quite small) cost for each change. Dynamically changing syslog_priority however is for free; - make variable $myhostname a dynamic variable, member of policy banks. This makes it possible for each policy bank to exhibit a different identity in notifications, in inserted header fields and in certain log entries. A quick overview of where myhostname may appear: From, Resent-From, Resent-Sender, Message-ID, Resent-Message-ID, X-Amavis-Modified, Reporting-MTA, macro %h, SQL logging in field msgs.host, log identity, and can be incorporated in variables $smtpd_greeting_banner, $smtpd_quit_banner and $hdrfrom_notify_* by specifying a substring ${myhostname}, which will be replaced by a current value of $myhostname just before use; - new configuration variables for finer control on propagation of DSN options: $propagate_dsn_if_possible and $terminate_dsn_on_notify_success (both part of policy banks). One or the other may be used to hide internal mail routing from outsiders if desired. Although Postfix can be configured to selectively announce or not announce DSN smtp extension based on client address (e.g. announcing it for internal clients but not for the rest of the world) letting DSN options reach a content filter may be desirable because a content filter can provide a more informative delivery status notification, and perhaps more importantly, it can suppress sending a DSN when it suspects the sender address is faked (viruses, high score spam). This means the front-end Postfix smtpd service should not be terminating DSN chain, but unfortunately the second instance of smtpd service at port 10025 can no longer distinguish between internal and external clients, because it only sees an IP address of a content filter. One solution is to turn on the $propagate_dsn_if_possible within policy bank MYNETS, and turn it off globally, e.g.: $propagate_dsn_if_possible = 0; $policy_bank{'MYNETS'} = { propagate_dsn_if_possible => 1, ... }; Turning off $propagate_dsn_if_possible is exactly equivalent to the case where MTA on the return port (10025) does not announce support for DSN extension to the SMTP protocol. The only difference is that the amavisd-new setting can be controlled more selectively by a policy bank. Turning on $terminate_dsn_on_notify_success is similar, but more refined. It tells amavisd not to pass NOTIFY=SUCCESS option when submitting checked mail back to MTA, which lets amavisd behave as if talking to a non-DSN compliant server, so that DSN success notification will be generated by amavisd itself (unless suppressed for other reasons). This is similar to $propagate_dsn_if_possible=0, the difference is that other DSN options (if any) *will* be passed to MTA, so options like NOTIFY=NEVER or RET, ENVID or ORCPT will not be lost. Such behaviour is not strictly by the book (RFC 3461), but is still in its spirit :) Here is the most useful setting: $terminate_dsn_on_notify_success = 1; $policy_bank{'MYNETS'} = { terminate_dsn_on_notify_success => 0 }; or if you prefer this syntax, changing only one key in an existing p.bank: $policy_bank{'MYNETS'}{terminate_dsn_on_notify_success} = 0; - new configuration variable $dsn_bcc (also part of policy banks) and a corresponding %dsn_bcc_by_ccat (by-contents-category settings) allows to specify an additional e-mail address to receive a copy (blind-CC) of a delivery status notification (DSN) that is being sent to the envelope sender address; it allows administrator to monitor what DSN messages (and how many) are being sent out by amavisd-new; Possible uses: $dsn_bcc = 'admin@example.com'; # applies to all content categories or more selectively (overrides $dsn_bcc): %dsn_bcc_by_ccat = ( CC_VIRUS, 'admin+infected@example.com', CC_BANNED, 'admin@example.com', CC_BADH, 'admin@example.com', CC_SPAM, undef, ); or perhaps: %dsn_bcc_by_ccat = ( CC_SPAM, undef, CC_CATCHALL, 'admin@example.com', ); Note the use of comma as a delimiter (instead of '=>'), as _values_ of CC_* constants must be used as hash keys, not constant _names_ as strings; (the Perl '=>' operator implicitly quotes its left operand); - passive operating-system fingerprinting (p0f) support lets SA gain information about SMTP client's operating system and estimated IP distance, and can reduce the number of bounces: * preconditions are: $os_fingerprint_method must be configured, the p0f-analyzer.pl process must be running, and amavisd must be receiving client IP address information from MTA, which in a Postfix case means the XFORWARD protocol extension to SMTP must be enabled in the Postfix service feeding mail to amavisd, e.g. "-o smtp_send_xforward_command=yes", or in a sendmail/milter setup the more sophisticated AM.PDP protocol must be used; * find and install the p0f utility: http://lcamtuf.coredump.cx/p0f.shtml or in FreeBSD ports collection as 'net-mgmt/p0f'; * start a p0f process on the same host where MTA (MX) is running, making it listen only to incoming TCP sessions (to reduce its workload) to the IP address and TCP port (25) where MTA is accepting incoming mail from outside (it doesn't hurt to let it see other traffic too, it just isn't needed); after testing p0f alone and seeing that it works, you may start it up, feeding its output to program p0f-analyzer.pl that comes with amavisd-new package, e.g.: p0f -l 'tcp dst port 25' 2>&1 | p0f-analyzer.pl 2345 & on multi-homed boxes one may need to specify interface and IP address where MTA is listening, the filter syntax is the same as in tcpdump, e.g.: p0f -l -i em1 'dst host 192.0.2.66 and tcp dst port 25' 2>&1 \ | p0f-analyzer.pl 2345 & * the program p0f-analyzer.pl reads p0f reports on stdin, keeps a cache for a limited time (10 minutes, configurable) of data about incoming TCP sessions organized by remote IP address, and listens on UDP port 2345 (specified as its command line argument) for queries; only queries from allowed IP addresses are accepted and responded to, other queries are silently ignored - configure @inet_acl accordingly, defaults to 127.0.0.1; * adding the following line to amavisd.conf, matching the chosen port number to the one specified on the command line to the p0f-analyzer.pl: $os_fingerprint_method = 'p0f:127.0.0.1:2345'; makes amavisd send queries to p0f-analyzer.pl (on the supplied IP address and UDP port number) to collect information about remote SMTP client's OS; collected response is then supplied as a header field when SpamAssassin is invoked; query/response is very quick and imposes no burden on amavisd process nor does its extend its processing time. The $os_fingerprint_method setting is also a member of policy banks to make it more flexible to disable fingerprinting for mail from site's own SMTP clients, e.g: $policy_bank{'MYNETS'}{os_fingerprint_method} = undef; * one may now add scoring rules to SA local.cf file, e.g.: describe __L_P0F_EXISTS A header field X-Amavis-OS-Fingerprint does exist header __L_P0F_EXISTS exists:X-Amavis-OS-Fingerprint describe L_P0F_WXP Remote system is truly a Windows XP, not Windows 2000 header L_P0F_WXP X-Amavis-OS-Fingerprint =~ /\AWindows XP(?![^(]*\b2000 SP)/m score L_P0F_WXP 2.3 describe L_P0F_W Remote system is some Windows variant, except Win. XP header L_P0F_W X-Amavis-OS-Fingerprint =~ /\AWindows(?! XP)/m score L_P0F_W 1.3 describe L_P0F_UNKN P0f was unable to determine remote OS type header L_P0F_UNKN X-Amavis-OS-Fingerprint =~ /\AUNKNOWN/m score L_P0F_UNKN 0.8 describe L_P0F_Unix Remote system is running Unix, not Linux header L_P0F_Unix X-Amavis-OS-Fingerprint =~ /\A((Free|Open|Net)BSD|Solaris|HP-UX|Tru64|AIX)/m score L_P0F_Unix -1.0 describe L_P0F_Linux Remote system is running Linux header L_P0F_Linux X-Amavis-OS-Fingerprint =~ /\ALinux/m score L_P0F_Linux -0.1 It is also possible to add score based on estimated IP distance, for example to slightly favorize nearer hosts (this is probably good for Europe or academic/university networks, and possibly less useful elsewhere): header L_P0F_D1234 X-Amavis-OS-Fingerprint =~ /\bdistance [1-4](?![0-9])/m header L_P0F_D5 X-Amavis-OS-Fingerprint =~ /\bdistance 5(?![0-9])/m header L_P0F_D6 X-Amavis-OS-Fingerprint =~ /\bdistance 6(?![0-9])/m header L_P0F_D7 X-Amavis-OS-Fingerprint =~ /\bdistance 7(?![0-9])/m header L_P0F_D8 X-Amavis-OS-Fingerprint =~ /\bdistance 8(?![0-9])/m header L_P0F_D9 X-Amavis-OS-Fingerprint =~ /\bdistance 9(?![0-9])/m header L_P0F_D10 X-Amavis-OS-Fingerprint =~ /\bdistance 10(?![0-9])/m header L_P0F_D11 X-Amavis-OS-Fingerprint =~ /\bdistance 11(?![0-9])/m score L_P0F_D1234 -0.5 score L_P0F_D5 -0.5 score L_P0F_D6 -0.5 score L_P0F_D7 -0.5 score L_P0F_D8 -0.5 score L_P0F_D9 -0.4 score L_P0F_D10 -0.3 score L_P0F_D11 -0.3 It can be useful to tame false positives from some other rules, e.g. # tame a Botnet plugin, reducing its false positives score BOTNET 0.1 meta BOTNET_W !DKIM_VALID && (L_P0F_WXP || L_P0F_W) && BOTNET score BOTNET_W 2.8 meta BOTNET_WU !DKIM_VALID && L_P0F_UNKN && BOTNET score BOTNET_WU 2.0 meta BOTNET_OTHER !BOTNET_W && !BOTNET_WU && BOTNET score BOTNET_OTHER 0.5 * make sure the @mynetworks is configured correctly, otherwise you will be inappropriately penalizing mail from internal hosts running Windows! Other methods to turn off fingerprinting for our own SMTP client hosts is to put $os_fingerprint_method in policy banks, and/or to specify more selective packet filter on the p0f command line; * based on statistics, less than 0.7 % of mail coming from external Windows XP -based hosts is ham, yet 20 % of all spam is coming from external Windows XP hosts; amavisd-new suppresses bounces to external Windows XP hosts, reducing bounce pollution. The amavisd-agent utility now provides some additional statistics based on p0f information. Some statistics collected from our logs in February 2006: p0f OS guess ham : spam ----------------------------- Windows-XP 0.7 % : 99.3 % Windows-2000 5.8 % : 94.2 % UNKNOWN 16.5 % : 83.5 % Linux 58.8 % : 41.2 % Unix 80.3 % : 19.7 % (Unix+Linux 66.5 % : 33.5 %) (ham: mail with score below 3, spam: score above 6) - new configuration variable $allow_fixing_improper_header_folding (also a member of policy banks) controls fixing improperly folded header fields made up entirely of whitespace (prohibited by RFC 2822) by removing all-whitespace continuation lines; defaults to true for backward compatibility; fixing such header fields is desirable and can protect poorly written mail readers, but may break DomainKeys/DKIM validation of messages with illegal header, so if this is of concern, one has a choice of turning it off (if local MUAs can't be fixed); - make config variable $insert_received_line a member of policy banks; - removed mail header (macro %H) from the default template of the plain text part of the virus or banned sender notifications; these headers are available in the third MIME part of the DSN, so having them twice was redundant; - new macros: date_unix_utc, date_iso8601_utc, date_iso8601_local, date_rfc2822_local, x-mailer, header_field, ccat_name, ccat_maj, ccat_min, wrap, lc, uc, substr, index, len, incr, decr and a couple of SpamAssassin lookalike macros - see README.customize for details; - new macro ccat_min expands to a minor contents category; this makes it possible for a notification template to distinguish (for example) between cases of bad message header: 1=bad MIME, 2=8-bit char, 3=NUL/CR, 4=empty, 5=long, 6=syntax error; See also README.customize for the list of currently available macros; see %ccat_display_names for minor ccat numbers currently in use, feel free to add new ones; - edited text of default notification templates to make them tidier and more informative; also make a good use of new macros; older templates are still compatible with this version of amavisd-new; - new global variable $timestamp_fmt_mysql, defaults to false; specific to MySQL, when logging to SQL is enabled and field msgs.time_iso is declared as TIMESTAMP instead of char(16); setting $timestamp_fmt_mysql=1 changes the timestamp format written to field msgs.time_iso to avoid ISO 8601 standard delimiter 'T' and trailing timezone 'Z', which MySQL does not like. Don't turn on $timestamp_fmt_mysql when using PostgreSQL! Should not turn on $timestamp_fmt_mysql with MySQL unless msgs.time_iso is of type TIMESTAMP. - new config variable $trim_trailing_space_in_lookup_result_fields controls trimming of trailing whitespace from SQL fields, from LDAP attribute values and associative array righthand-sides (hash values) as read by read_hash(); disabled by default; turn it on for compatibility with pre-2.4.0 versions. Note that trailing spaces may still be trimmed by SQL itself (contrary to SQL-99 specification). Trailing spaces in MySQL version up to and including 4.1 are removed from values when stored in a VARCHAR column; this also means that spaces are absent from retrieved values. Starting with MySQL V5.1, trailing spaces are retained when values are stored and retrieved. Investigated by Gary V; - treat exit status 2 as a warning when returned by bzip2, gzip and other decompressors handled by do_uncompress; problem with decoding of corrupted bzip2 file reported by Kim Leandersson; - when determining file short type, match "Microsoft Cabinet file" result from a file(1) command case-insensitively; reported and patch provided by ap at zip com au via Debian bug tracking system; also recognize "Microsoft Cabinet archive data" as .cab; - do_unzip: set attribute 'U' (undecodable) if zip archive fails to be decoded; based on a patch by Oliver Geisen; -- well, on a second thought, perhaps not, this line is now commented out; it flagged too many bounces containing chopped-off ZIP attachments as ***UNCHECKED***; - added a minimalistic decoder interface routine to call a command line unpacker from stuffit.com to decode Macintosh StuffIt archives. Not tested extensively, program source is not available for inspection, use at YOUR OWN RISK (and the risk is non-negligible!). If using non-default assignment to @decoders, the following entry can be added to the list of decoders: ['sit',\&do_unstuff,'unstuff'], thanks to Oliver Geisen for the suggestion; - keep X-Spam-Level bar empty if sender is whitelisted; - untaint recip_score_boost when writing a log report entry to SQL; 'Insecure dependency in parameter 7' could have been reported when SQL-based score_sender lookup table is used; reported by Jim Knuth; - treat undefined spam level as 0 for the purpose of comparing it to tag/tag2/kill levels, e.g. when spam scanning is skipped due to large message size; this now allows score_sender_maps to push non-checked messages over a tag2/kill limit by its score boost if desired; - header fields X-Spam-Status, X-Spam-Score, X-Spam-Level and X-Spam-Flag in a quarantined message now take into account also the score_sender boost and white- and blacklisting (using 'any' and 'max' to summarize in case of multiple recipients); - in passed and quarantined mail a header field X-Spam-Status now shows score as an explicit sum of SA score and a by-recipient score_sender boost (when the boost is nonzero); the X-Spam-Score header field still shows a sum of both as a single number so as not to confuse MUA filters which may operate on that header field; - insert X-Quarantine-ID header field into passed mail if a passed message was also quarantined (e.g. *_lover or final_*_destiny=D_PASS), suggested by Pavel Urban; - when folding is needed for long new or edited header fields, use TAB instead of a space on fold points; also: take into account the apparent display size of expanded TABs instead of the actual character count, to make header look prettier; use of TABs suggested by Debian community; - improved wrapping of inserted header fields and in generated text sections of notification templates; - automatic wrapping of long header lines from notification templates; - standards compliance: wrap Diagnostic-Code field in message/delivery-status section of delivery status notifications according to rfc3461 section 9.2; - added config options to enable quarantining (archiving) of clean mail; defaults settings are: $clean_quarantine_method = undef; $clean_quarantine_to = 'clean-quarantine'; quarantining clean messages is disabled by default; to enable: $clean_quarantine_method = 'local:clean-%m'; - reports "Blocked TEMPFAIL" instead of "Blocked CLEAN" in case of a temporary 4xx failure; - in generated MIME parts (notifications, defanging) replace suggested file names like 'message.txt', 'dsn_status.txt' and 'header.txt' with names without an extension, i.e. 'message', 'dsn_status' and 'header'; Reportedly Outlook Express 6.0 (but not Outlook) determines how to present a MIME part based on its name extension, instead of based on MIME type. Depriving it of name extensions makes it obey a MIME type. This solution has been successfully tested with Outlook (5 & 6), Outlook Express (2k, 2k3), Thunderbird, JavaMail and Squirrelmail. Investigation and suggestion by Ivers Matthias; - clamscan AV entry: change test for status 1 by a test for /:.*\sFOUND$/ to prevent system errors like failed load (e.g.: 'ld-elf.so.1: Shared object "libgmp.so.6" not found', which also produce exit status 1) from causing all mail to be treated as infected; suggested by Tomasz Kojm in response to my problem report; - add three AV entries for avast products (FreeBSD and Linux): 'avast! Antivirus daemon', 'Client/Server Version' and the command-line avastcmd; kindly provided by Frantisek Mensik, ALWIL Software; later refined based on testing and feedback from Bill Landry; - updated regexp in McAfee uvscan entry to cope with spaces in virus name; reported by, and output samples provided by Andreas Schulze; - updated bdc AV entry: newer versions of BitDefender don't use option --all any longer; thanks to Max Matslofva, Andreas, Gary V and Bill Landry; - updated NOD32 AV entry based on NOD32 documentation and advise from Willi Gruber about adding status code 3 to the list of success values; - better handle failed decoder attempts when checking an executable file for self-extracting archive (SFX), avoid decoder 'dry runs' where possible; - require minimal version 0.32 of Net::LDAP, a subroutine Net::LDAP::Util::escape_filter_value() is needed; reported by Harry Hoffman; - allow to specify option 'deref' in calls to Net::LDAP->search to control dereferencing of aliases to locate the base object for the search; the default remains 'find' as before. Use a key 'deref' in the default_ldap hash to specify a different value. The values are those documented in the Net::LDAP manpage in the search function under the deref section; (btw, dereferencing in LDAP is supported in both Postfix and Courier); a patch provided by John Allman; - added a timing report entry fwd-data-cmd for time it takes MTA to respond with a status to a DATA command; - read_hash: trim whitespace off the right-hand side data only if followed by a comment (#), otherwise keep the rhs as it is; - include a currently chosen dataset name (dsn) in the log entry when quarantining to SQL, to be able to retrieve a quarantined message from the correct SQL server; suggested by Cami Sardinha; - dwell in $tempdir (chdir to) instead of in $TEMPBASE most of the time; - edit_header() may now be called multiple times for the same header field to provide iterative header edits - provided mostly for completeness; - Courier code (the patch) now supports D_BOUNCE and D_DISCARD message destinies and adding and deleting recipients (due to differing per-recipient configuration or addr_extension_*_maps); it also supports DSN (RFC 3461); by Martin Orr; - prepend (!!) or (!) to log messages at levels below 0 to facilitate log parsing and make critical messages stand out; based on a patch by Henrique de Moraes Holschuh (Debian port maintainer); - silence logging of AM.PDP commands delivery_care_of, tempdir_removed_by and tempdir when using amavisd-milter; a patch by Petr Rehor; - silence Perl warnings about uninitialized values; - optimization: about 15% speedup in macro expansion due to pre-tokenization of template strings at startup (affects preparing main log entry and each notification message); - modified old amavis.c client program to return status 0 (success) even when LDA command-line arguments are not specified (i.e. when amavisd daemon is in charge of delivery), and amavisd daemon returns status 99 (=discard); simplified code in amavisd/check_amcl_policy() to take advantage of it; - internal/coding: a much needed and appreciated hefty patch from Martin Orr introduces class Amavisd::TempDir and collects there existing code from various places dealing with maintaining a temporary directory and files within; it also makes possible for the Courier interface code to use this module and avoid duplicating code; - internal/coding: the Amavis::In::Courier package has been tidied up and split into multiple methods, of a hopefully manageable length. Several new features in amavisd-new core code, which required changes to Amavis::In::Courier to take advantage of, have also been dealt with (in particular MYNETS and MYUSERS policy banks and SNMP counters); by Martin Orr; - require minimal version 1.43 of DBI, working last_insert_id is needed; (actually last_insert_id is no longer needed, but the requirement stays :) - internal/coding: when logging to SQL avoid a need for last_insert_id() by doing SELECT after INSERT when adding a new e-mail address to table maddr; this also avoids a rollback/retry when more than one process tries to insert the same new address into a database; - internal/coding: merge subroutines do_spam and do_virus into a single do_notify_and_quarantine, and use the same code to prepare spam-describing headers as in subroutine add_forwarding_header_edits_per_recip; - internal/coding: new $msginfo data object: contents_category, holding a reference to a sorted (descending order) array of entries, each one corresponding to one contents category under which a message can be classified, e.g. it can be both a CC_VIRUS and a CC_BANNED. Supporting subroutines are: add_contents_category, main_contents_category, is_in_contents_category, setting_by_contents_category; thanks for suggestions and feedback to Gérald Macinenti; The contents_category list is a sorted list of strings, each of the form "major" or "major,minor", where major and minor are numbers, representing major and minor category type. Sort order is descending by numeric values, major first, and subordered by a minor value. When an entry "major,minor" is added, an entry "major" is added automatically (minor implied to be 0). A string "major" means the same as "major,0". See CC_* constants for major category types. Minor category types semantics is specific to each major category, higher number represent more important finding than a lower number; - internal/coding: merge sections in sub check_mail dealing with different content types one-at-a-time into a single section 'decide_mail_destiny', taking advantage of the new information on contents category, improving consistency and simplifying code; - internal/coding: new $msginfo data objects: spam_level, spam_status, spam_report, autolearn_status, avoiding ugly global variables; - internal/coding: separate SpamAssassin-specific code from general anti-spam code - new module Amavis::SpamControl::SpamAssassin; based on suggestion and patch by Felix Schwarz; - internal/coding: use File::Spec::catfile to splice full file name from its components (in mail_to_local_mailbox, more needed); a patch by Felix Schwarz. Actually, on a second thought, comment that out and revert to previous code: there are so many other similar cases which were not generalized, that it makes no sense to generalize (through File::Spec) one percent of them (and drag-in yet another Perl module), and leave the rest hard-coded; - collect the most commonly needed header fields into an associative array $msginfo->orig_header_fields, removing the need to call ensure_mime_entity() in certain cases, saving on mime decoding when it is not really needed; - internal/coding: do_log now takes optional arguments, and if they are present, the message text is treated as a format string to snprintf. Take advantage of this in most calls to do_log. The message (format) argument should not be tainted (not enforced), but arguments may be. In the absence of additional arguments, do_log behaves as before; - updated README.postfix to explicitly override (just in case) two newer Postfix options: smtpd_data_restrictions and smtpd_end_of_data_restrictions, thanks to Noel Jones for the suggestion; - documentation: updated README.sendmail-dual - added custom rules to reject unknown users outright; provided by Matej Vela, thanks to Simone Marx; added a reference to the 'milter-ahead' project info; thanks to Adam Gibson; - documentation: fixed README.exim_v4 (don't let messages with null return path get through unchecked), by Igor D'Astolfo; - documentation: updated README.customize, describing new features of a built-in macro expander, and describing new macros; - documentation: updated README.sql, describing new SQL log purging recommendations, improved PostgreSQL instructions, and the (optional) use of data type TIMESTAMP in field msgs.time_iso; --------------------------------------------------------------------------- August 22, 2005 amavisd-new-2.3.3 release notes Version 2.3.3 is a maintenance release over 2.3.2. Besides fixing known problems and providing some optimizations, no new features were added. If using SpamAssassin older than 3.1, an upgrade of either SA to 3.1, or an upgrade of amavisd-new to 2.3.3 is recommended. - privacy: add a safety fuse / workaround around calls to SA to detect SA's failure (in SA versions before 3.1) to catch a failed exec() in a forked process, which could produce runaway process clones. See SA bug report #4370. An incident of a mail copy being delivered to unrelated recipient reported by Joel Nimety; - privacy: turn warning into a fatal error when a quarantine ID of a message requested for a quarantine release does not match the requested mail_id; - security: require minimal version 1.35 of Compress::Zlib to avoid vulnerability in the zlib compression library; - the dsn_cutoff_level should have been ignored if undefined according to documentation, but was not, causing DSN to be suppressed regardless of spam level; discovered by Gary V; - ensure the banned check is not performed if all recipients agree it is not needed, even in presence of $banned_namepath_re; undesired behaviour (not strictly incorrect) reported by Joel Nimety; - missing import of lookup_ip_acl in module Amavis::In::AMCL caused failure in sendmail milter setup when using the new AM.PDP protocol; reported by Mic And; - document and explicitly define handling of syntactically invalid IP address in lookup_ip_acl: it matches a zero-length-mask net, a constant lookup table, or a hash entry with an undef key, but no other entries in IP lookup tables; syntactically invalid IP addresses are now logged; - fix parsing of IPv6 address in $notify_method and $forward_method in case of dynamic destination override (the use of '*' in method fields); - check during startup that $myhostname is a fully qualified domain name (or 'localhost', if you must), and abort if it isn't, otherwise a non-FQDN can end up in places where RFC 2822 does not allow it; if uname(3) does not provide a FQDN, then an assignment to $myhostname must be done explicitly in amavisd.conf; - when quarantining to a single file in mbox format the 'From ...' line needs an English date, regardless of current locale; fixed by globally setting locale LC_TIME to "C"; - pass on the parameter BODY=8BITMIME on MAIL FROM when submitting to MTA when original message reception indicated it is needed (RFC 1652). Note that mail forwarding may now fail if the feeding MTA requests BODY=8BITMIME SMTP service extension (or just passes data with msb set), but the MTA on the output side does not allow the use of the BODY parameter in SMTP. In case of Postfix this may only happen when receiving service on port 10025 is misconfigured and does not announce ESMTP capability and support for the SMTP service extension 8BITMIME; - RFC 2554 requires auth_param to be xtext-encoded addr-spec (no angle brackets) or "<>", not the xtext-encoded addr-spec enclosed in angle brackets (when specifying submitter during authentication); fixed; - apply some sanity limit on collected bad-header samples to ensure that a grossly broken mail does not unnecessarily fill up memory; - when sending recipient warnings for viruses, banned files, or bad headers, recipient address must not be rfc2822-quoted twice; fixed; - fix interpretation of $defang_all to really imply all; previously it only affected clean messages; - in quarantined mail the reported spam score in X-Spam-Status header field now includes maximum of all by-recipient score boosts (less surprising when soft-whitelisting through @score_sender_maps is in use); suggested by Mike Cappella and Gary V; - when a policy delegation protocol attribute "request" is not "AM.PDP" (perhaps it is a Postfix policy delegation request) don't attempt to find and open a mail file; - do_ascii and do_unarj: set environment variable TMPDIR on a command line temporary directory option to "$tempdir/parts" instead of $TEMPBASE to minimize possible pollution of top level directory; - don't abort even if amavisd.conf returns undef as a final value, as long as there are no errors reading or interpreting it; - if during 'amavisd stop' or 'amavisd reload' the old running daemon does not go away for one minute after sending it a SIGTERM, use a bigger hammer and send it a SIGKILL; suggested by Sven Riedel; - extend LDAP lookups to allow multiple search attributes (multiple occurrences of %m in a query); a patch by Michael Hall (and a similar one by Matthias Bandemer); - LDAP lookup on an empty envelope address (e.g. a null return path) adds another lookup key "<>", as it is difficult if not impossible to have LDAP attributes with empty string as a value; by Michael Hall; - LDAP.schema: drop "MUST ( mail )" from objectclass 'amavisAccount'; suggested by Michael Hall; - updated comments and documentation, most notably the README.chroot; - contributed file Macintosh.tar.gz updated by Dale Walsh; COMPATIBILITY - replaced 'hits=' with 'score=' in inserted X-Spam-Status header field (and in some internal log entries) for compatibility with a changed default in SpamAssassin 3.1; - insert X-Spam-Score header field for compatibility with SA (previously insertion of this header field was commented-out because the information is redundant, as the score already appears in X-Spam-Status); OPTIMIZATION - speed up sending a mail header of full defanged (rewritten) mail over SMTP back to MTA by a factor of 4 by buffering header fields into large chunks to avoid bottleneck in Net::Cmd::datasend, which has lots of overhead for line-by-line writes. Previously slow writes mostly affected mail messages with extreme header lengths (such as results of a broken mail loop), or when delivering defanged messages, particularly at sites with large MTA mail size limits, sometimes to a point of exceeding timeout limits; reported by Dominik Weber and Ralf Hildebrandt; - move subroutine lookup_ip_acl() and associated ip_to_vec() into its own dedicated new package Amavis::Lookup::IP; provide a constructor to pre-parse IP lookup tables to speed up IP lookups in lookup_ip_acl; prepare pre-parsed commonly used IP lookup tables (@mynetworks_maps, @publicnetworks_maps, @inet_acl); - optimized reading loop in SMTP DATA state, receiving data is now about 35% faster when mail size limit is not enforced (which is a default); no speedup (nor slowdowns) when mail size limit _is_ enforced; - cache results of evaluated macros during a single call to expand(), as macro calls often come in pairs, like: [?%e||\[%e\] ] or [? %#T ||, Tests: [%T|,]]; together with the above optimization in pre-parsed IP lookups it shaves off 25% of time in preparing main log entry; - set locale LC_TIME to "C" globally, avoid changing and restoring locale for every log write and when generating RFC2822 timestamps; - added an optimization note in README.sql about indexes and about SELECT count(*) in MySQL with InnoDB; investigation by Paolo Cravero; --------------------------------------------------------------------------- June 29, 2005 amavisd-new-2.3.2 release notes INCOMPATIBILITY with 2.3.1 and earlier versions: If running amavisd daemon in chroot please note: Each child process now opens its own syslog connection or a file descriptor to a log file, and no longer inherits a connection from its parent. When running in chroot jail and logging to syslog, the syslog client routines need syslogd socket to be present in the chroot subtree to be able to establish a connection with syslogd, otherwise logging output may be lost. Additional syslogd sockets (to be made available in the jail) may be requested from the syslogd daemon, see its documentation. This requirement is equivalent to the requirement of chrooted Postfix services (see Postfix documentation file BASIC_CONFIGURATION_README). BUG FIXES since 2.3.1: - do not enforce $MAXFILES limit during top-level MIME decoding to avoid tempfailing mail; MIME parts are still counted, so a limit exceeded may still be reported during subsequent decoding, but this is handled more gracefully and does not cause preserved temporary directories to be left behind; reported by Marcin Lemanski; suggested by Stephane Lentz and Robert LeBlanc (noted in the 2.0 release notes); - use recv() instead of read() to get results from daemonized virus scanners in an attempt to avoid a bogus Perl I/O status on some Linux installations (reported by Sander Steffann); we now get a meaningful status codes like ECONNRESET instead of a bogus EBADF (Bad file descriptor); - ignore status ECONNRESET when reading results of a daemonized virus scanner from a socket, specific to some Linux versions; thanks to Sander Steffann for the initial report and extensive help in debugging the Perl problem; - run_av and other similar code sections: replace line-by-line reads by block-by-block reads wherever possible to avoid inappropriate status report EBADF (Bad file descriptor) caused by Perl I/O bug when last line is not terminated by a newline. The problem was affecting reading response from some command line virus checkers; reported by Sander Steffann; Perl bug tracking: #39060: readline of a not NL-terminated last line results in Bad file descriptor; - ignore status EAGAIN when reading results on a pipe from a forked process; the status EAGAIN seems to be an artifact of Perl I/O on some installations; reported by several people to cause problems on FreeBSD with Perl 5.8.7 (but Perl 5.8.6 is fine); thanks to Bart Matterne for testing and feedback; - allow one level of indirection when collecting %needed_protocols; global setting $protocol='COURIER' did not work, a workaround was needed with previous version, e.g.: $policy_bank{'QMQPqq'}={protocol=>'QMQPqq'}; reported by Nicklas Bondesson and Martin Orr; - fix a bug (introduced with 2.3.0) in Courier and QMQPqq setups, where global information about processed message wasn't always reset and could leak into processing of a subsequent message; reported by Nicklas Bondesson; - SQL: fix arguments in calls to last_insert_id(), failing under PostgreSQL (MySQL didn't mind); pointed out by Henrik Krohns; - if module SAVI is loaded, insist it is version 0.30 or later; incompatibility with earlier versions reported by Andrzej Kukula; - make use of the new Net::Server 0.88 hook run_n_children_hook() to reload SAVI database; removes a need to apply SAVI patch to Net::Server; the Net::Server hook was suggested by Paul B. Henson and others, and incorporated into Net::Server 0.88 by Paul Seamons; - reopen log file or syslog connection in each child process to make it use its own file descriptor; also minimizes transients when syslogd is restarted and its socket re-created, as reported by Les Ault. When running in chroot please make sure a syslogd socket is also available in the chroot jail, see README.chroot for syslogd options (and BASIC_CONFIGURATION_README in Postfix documentation for the Postfix equivalent); - close log file or syslog in forked process before exec, just to play nicely; - do_lha: fix extracting archive member filename in case of broken archive or empty name (avoid interpreting creation date as a file name); do not increment OpsDecByLha counter for empty archives, which are most likely not lha archives at all; - obey $final_bad_header_destiny D_DISCARD or D_REJECT even for messages with bad headers from mailing lists or with a null envelope sender (DSN); previously such messages were passed; undesired behaviour reported by Cami Sardinha. Such messages are still let through with $final_bad_header_destiny set to D_BOUNCE, as otherwise they will be lost because a bounce is suppressed for null sender messages and for mail from mailing list. This behaviour is retained for backward compatibility, but may need to be reconsidered. - fix regexp for extracting am_id from amavis-milter helper program requests; - if fork/exec fails, try to commit suicide in forked process with POSIX::_exit(1) first, before trying kill('KILL',$$) as a last resort; - updated $log_templ example in amavisd.conf-sample to match the default; pointed out by Gary V; - further reduce a couple of more frequent Perl warnings about the use of uninitialized values in expressions; - pre-load additional Perl modules required by SA 3.1 plugins; - require minimal versions of modules: Time::HiRes 1.49, Archive::Zip 1.14; - replaced nonexistent variable @sa_spam_modifies_subj_maps by @spam_modifies_subj_maps in commented-out example in amavisd.conf-sample; noticed by Joachim Schoenberg; LDAP CHANGES by Michael Hall: All the LDAP changes are transparent to the user. - rewritten some of the code similar to the restructuring of the SQL code in version amavisd-new-2.3.0. A new package Amavisd::LDAP::Connection was added which is an LDAP connection object, and the old connection-related code in Amavis::Lookup::LDAP has been moved to the new package. Amavisd-new will now try to reconnect (once) while processing a message, similar to SQL; - added the ability to specify a '%d' (domain) token in the LDAP base DN; based on idea from Alexander Wittig; - updated default LDAP port based on whether SSL/TLS is being used or not; based on idea from Timo Veith; - updated the search code to query for multiple records and return the results sorted in 'make_query_keys' order versus doing a query for each key. As a result performance is enhanced, and the tweaks 'ldap_get_all', and 'use_query_keys' (recently added) are no longer applicable or needed and have been removed; - improved LDAP error reporting and misc changes to multivalued attributes; - documentation changes (amavisd.conf-default, README.lookups); MINOR IMPROVEMENTS: - macro %c (commonly used in a log template) reports spam score no longer as a single number, but as an explicit sum of a SA score and a by-sender boost score (from @score_sender_maps) when boost score is nonzero; suggested by Ed Walker; - enhancement to amavisd-release: if its only command line argument is '-', then read arguments from stdin, one release request per line, ignoring empty lines; input lines have the same format as command line arguments, i.e.: mail_file mail_file secret_id mail_file secret_id alt_recip1 alt_recip2 ... - better handle cases where a persistent temporary file email.txt as prepared by the SMTP server module gets replaced as a result of some user program modification (e.g. when invoking altermime); problems reported by Dinesh Shah and Leonardo Rodrigues; --------------------------------------------------------------------------- May 9, 2005 amavisd-new-2.3.1 release notes INCOMPATIBILITY with 2.3.0 and older versions: - command line option 'foreground' no longer automatically redirects logging to STDERR; to request logging to STDERR turn off the $DO_SYSLOG and $LOGFILE; the improved flexibility suggested by Matthias Andree and Ralf Hildebrandt; BUG FIXES since 2.3.0: - don't test errno when line-by-line reading loop is exited before eof, it was inappropriate and Perl on some versions of Linux does not like it, possibly reporting "Error reading mail header: Bad file descriptor"; Besides fixing the loops that needed the fix, modified also all remaining reading loops for consistency. Reported by Ralf Hildebrandt; - don't call $per_msg_status->get_autolearn_status with SA older than 3.0; reported by Ian Abbott; - pre-load some additional SpamAssassin modules, needed by SA 3.1 (CVS); - reading from dspam process used wrong variable, producing empty lines for SA checking; reported by Chris Lewicki; - SAVI-Perl: libsavi option for turning on mime parsing is 'Mime', not 'MIME'; libsavi is case-sensitive since version 3.93.0 and was rejecting incorrect option name; thanks to Paul B. Henson; - fixed disabling of SQL wblist ($sql_select_white_black_list=undef); bug reported by Tom Sommer; - do_tnef: extract data from attribute 'Attachment' in addition to data from a more common attribute 'AttachData'; example data provided by Goetz Rieger; - avoid some more frequent Perl warnings on the use of uninitialized variables; - add prototypes for decoding subroutines; prototype mismatch warnings reported by Michael Muenz; - fixed prototype for add_forwarding_header_edits_common(), thanks to Ian Abbott; - replace a simple-minded logic for loading input protocol modules by a slightly more sophisticated code which takes into consideration field 'protocol' in all defined policy banks; reported by Brian Wong; - when replacing existing address extensions don't treat the whole localpart as an extension if the address happens to start with a $recipient_delimiter; pointed out by Kaj J. Niemi; MINOR IMPROVEMENTS: - unfold obsolete-syntax folded header fields composed entirely of white space (RFC 2822 section 4.2); suggested by Ian Abbott and others. Note that such unfolding breaks DomainKeys/DKIM "simple" canonicalization algorithm (but is transparent to "nofws") if the affected header is included in the signature; - do_tnef_ext: add support for decoding TNEF containers by an external program 'tnef' if available; selectable by an entry in the @decoders list; - mail_via_bsmtp enhancement: substitute %s in quarantine filename template by a defanged sender name; based on a patch by Thomas Jarosch; - lookup_ip_acl enhancement: when a supplied lookup table is an associative array (a hash) and the looked-up address is an IPv4 address, allow simple classful subnet specification by repeatedly truncating the trailing byte from the looked up IPv4 address until a match is found or until further truncation is not possible. Note that this does not apply to IPv6 addresses. For more flexible CIDR subnet specifications please use lookup arrays. - provide a routine read_array, which is modelled after read_hash, but stores lines read from a file into an array lookup table, instead of a hash. --------------------------------------------------------------------------- April 24, 2005 amavisd-new-2.3.0 release notes QUICK OVERVIEW: Provides more flexible configuration of decoders. Allows recipients to have individual banning rules. Assigns a long-term unique id to each message, reducing clashes and facilitating retrieval of information. The daemon can store information to an SQL database for logging, reporting and quarantine retrieval, optionally storing entire message to an SQL database. File-based quarantine can disperse files to 62 subdirectories. Provides a quarantine release mechanism. Reconnects to SQL if connection is broken. Can skip quarantining high-score spam. Compatibility with IPv6-enabled Postfix is improved. SECURITY: - require minimal version 1.05 of Convert::UUlib to avoid a security problem in the underlying uulib: http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2005-1349 which is now known to be exploitable (2006-12-05), credits to Jean-Sébastien Guay-Leroux; INCOMPATIBILITY with 2.2.1 and older versions: - structure of @banned_filename_maps config variable has changed in incompatible way to allow per-recipient banned rules: it is now a two level map indexed by recipient address (similar to the structure of @score_sender_maps). See further down for details. The change will not affect existing installations which either: * leave @banned_filename_maps at its default value and use the traditional $banned_filename_re configuration variable to specify banned rules, which most installations do and remains the most commonly used method; * or, assign to @banned_filename_maps an empty list to disable it; * or, use the alternative mechanism $banned_namepath_re and disable $banned_filename_re or @banned_filename_maps. - macro %i (used in log and other templates) now always shows mail_id (see below) which is often but not necessarily also the name of a quarantined file (like before, available as a macro %q); previously the %i reflected the actual file name, which was longer/different than now; - default values of $virus_quarantine_method, $spam_quarantine_method, $banned_files_quarantine_method and $bad_header_quarantine_method now specify shorter file names based on the newly-introduced mail_id: virus-%m, spam-%m.gz, banned-%m and badh-%m respectively; - a config variable $file now defaults to a string 'file' instead of being undefined; this makes it unnecessary to be explicitly assigned to in the configuration file; - SQL fields virus_lover, bypass_virus_checks and bypass_spam_checks in table 'policy' are now optional, and if missing their value is treated as undef (same as if the field is present but is NULL) - instead of being interpreted as false. This makes it consistent with the interpretation of other missing fields. The change is unlikely to affect existing installations, because these fields were considered non-optional in previous versions. BUG FIXES or missing functionality: - avoid modifying directory which is being read by readdir, it can cause premature termination of the directory traversal; reported by Dale Walsh; - minimize deep recursion in regexp evaluation while parsing some degenerate cases of Received header subfields via/with/id/for, which could result in a process crash; - turned loops in banned checks inside out to make 'allow' rules useful and let them behave as one may expect: when checking mail parts against lookup tables in @banned_filename_maps the evaluation order of checking part's attributes against rules has changed: on each rule in a list, all attributes of a part are now checked together; previously all rules were evaluated (inner loop) for each attribute (outer loop), which made 'allow' rules hardly ever useful; - ignore dsn_cutoff_level if undefined, instead of treating it as 0; - fix generating the positive delivery notifications (requested by $warn*sender settings): the DSN was missing entirely in case of Courier or sendmail-milter setups and the chosen template was not the most appropriate; Courier problem reported by Sander Holthaus; - fix the text in notification templates 'neutral' and 'virus/banned sender' to claim non-delivery when the message was truly not delivered, and to claim delivery for delivered messages; - when per-recipient subject tag strings are different for each user, the mail forwarding clustering algorithm should have split the forwarded mail into separate deliveries but did not do so, resulting in all recipients of a multi-recipient mail to get the same string inserted into Subject; reported by Michael Goth; - at last: when mail is received through LMTP protocol, gracefully handle a temporary failure 4xx reply from MTA to a RCPT TO command and pass it back to an LMTP client for tempfailed recipients only, instead of returning 450 for _all_ recipients (needed the sending routine to be aware of the receiving side capabilities, which was previously not available); - stricter and more consistent error checking and better error reporting on Perl read, sysread and getline operations; - use O_CREAT|O_EXCL when creating files that are not supposed to pre-exist, to be able to detect a potential race condition and other programming errors; - fixed reporting of virus names for av scanner Perl-SAVI; - with Sophos Perl-SAVI module, avoid setting 'MaxRecursionDepth' if $MAXLEVELS is undefined or zero, matching its semantics to other uses in the program; debugging and a fix by Paul B. Henson; - fixed sloppy regexps when parsing SMTP commands; - fixed a typo in README.lookups leading to confusion between fields spam_subject_tag and spam_subject_tag2, making one to believe there is only one such field; pointed out by Max Clark and others; - handle special case: Postfix hates ""@domain but does not mind @domain; NEW FEATURES: - new config variable @decoders (with its policy banks counterpart) makes it possible to enable/disable decoding of individual file content types from the configuration file, and to adjust the external decoders paths and search order, all in one place. This list now makes the following config variables obsolete: $arc, $gzip, $bzip2, $lzop, $uncompress, $unfreeze, $lha, $unarj, $unrar, $zoo, $pax, $cpio, $ar, $rpm2cpio, $cabextract, $ripole; although they are still observed for compatibility reasons if the @decoders list is left at its default value; - banned filename/filetype rules can now be specified on a per-recipient basis Structure of @banned_filename_maps config variable has changed in incompatible way to allow per-recipient banned rules: it is now a two level map, similar to the structure of @score_sender_maps. Lookup keys used at the first level are recipient addresses, results from this lookup can be either a ref to a list of second-level lookup tables, or a string which is interpreted as a comma-separated (or whitespace-separated) list of _names_ of second-level lookup tables. These names are mapped to actual second-level lookup tables through an associative array %banned_rules. The indirection by names through %banned_rules is primarily intended for SQL and LDAP lookups, which can not return complex data structures and Perl code (and should not do so for security reasons), but names may just as well be used by static lookups. The resulting list of second-level lookup tables (which in most cases is a list containing a single element $banned_filename_re, which makes it compatible with existing setups) is looked up by using keys such as part file name and part file type, exactly like in previous versions. The previous default assignment: @banned_filename_maps = ( \$banned_filename_re ); is now incorrect, it can be rewritten (if explicit assignment is desired) as: @banned_filename_maps = ( {'.' => [$banned_filename_re]} ); or (by giving a name to a lookup table): @banned_filename_maps = ( {'.' => 'MYRULES'} ); %banned_rules = ('MYRULES' => $banned_filename_re); which is equivalent to the default setting of both variables: @banned_filename_maps = ( {'.' => 'DEFAULT'} ); # proper two-level struct. @banned_filename_maps = ( 'DEFAULT' ); # same as previous, but shorter %banned_rules = ('DEFAULT' => $banned_filename_re); The SQL table 'policy' may now contain an optional field 'banned_rulenames', which is a comma-separated (or whitespace-separated) list of _names_ of second-level lookup tables, with semantics as described above for static lookups. The configuration variable @banned_filename_maps is a member of policy banks as before. The associative array %banned_rules is global and is not a member of policy banks. The alternative 'banned file' mechanism $banned_namepath_re hasn't changed and is still not merged into @banned_filename_maps, which means it can not be used when per-recipient banned rules are needed. Perhaps in the next version... - introduce a concept of 'mail_id', which is similar to the am_id as reported in the log and elsewhere (e.g. 92598-11-5), but has much stronger long-term uniqueness property and can be used for the purpose of uniquely identifying a quarantined mail, or for other uses. The mail_id is a 12-character string consisting of characters [A-Za-z0-9+-] (like base64, except for a '/' being substituted by a '+') [a note from the future: since amavisd-new-2.7.0 the character set is [A-Za-z0-9-_] and follows rfc4648 base64url, derived from base64 by substituting '+' -> '-' and '/' -> '_'], guaranteed to start and to end with an alphanumeric character (i.e. not with '+' or '-' or '_'). It is derived by a cryptographically strong method (MD5), cumulatively collecting entropy during the life of child processes, folding-in entropy from processed mail and other inexpensively accessible sources, collected when an opportunity arises (e.g. file system file-IDs, SA results etc), without placing a burden on system sources of randomness (see RFC 4086). Note that MD5 has been demonstrated to have some weaknesses, but we are not talking about cryptographic attacks here, but rather about spreading message identity codes which have no inherent intention of causing collisions. The mail_id carries 71.9 bits of information (subject to quality of sources of entropy). For a high-end system that keeps a year's worth of mail messages in evidence (e.g. in quarantine) and receives 10e6 messages each day (20..200 TB of yearly mail contents), the probability of a mail_id collision happening during one year (while gradually displacing an entire collection with a new set of IDs) is n^2/m = 0.3 % (10e6 * 365)^2 / (62 * (64^(12-2)) * 62) = (10e6 * 365)^2 / 2^71.9 = 0.003 Eventual clash is still possible and needs to be detected by testing each generated code against the set of message IDs currently kept in a database. On a smallish system receiving 10.000 mail messages daily an 8-character mail_id would suffice, but savings are not worth the trouble of providing a configuration flexibility. Paired with a mail_id there is its companion secret_id generated for each message, such that mail_id can be derived from secret_id and pairing checked, but not the other way around. The secret_id can serve as a 'ticket', granting user a right to release a quarantined message addressed to him. - SQL: can store information about every processed mail to SQL; the information is similar to level 0 log entries, but more detailed; an SQL database can be used as a basis for searching for a particular mail, for preparing reports and to facilitate quarantine management (searching and releasing). Enabled by configuring the @storage_sql_dsn list which contains information about an SQL server and dataset name, just like the @lookup_sql_dsn does for the SQL lookups. If @storage_sql_dsn is the same as the @lookup_sql_dsn, a single connection to SQL database will be used, otherwise separate and independent connections are established, possibly to different SQL servers. Loosely based on ideas from Maia Mailguard by Robert LeBlanc and a patch by Brian Wong. Thanks to Brian Wong for testing and valuable feedback. See README.lookups for an SQL schema. See new file amavisd-sql-maintain (incorporated into README.sql in later versions) for ideas on database housekeeping (expiring old entries). - SQL: can quarantine to an SQL database; selected by setting config variables $*_quarantine_method to 'sql:' The @storage_sql_dsn list of dataset names is used to choose SQL server and dataset name, and must be nonempty when $*_quarantine_method is 'sql:'; When $*_quarantine_method is set to 'sql:' the SQL logging must be enabled as well; - SQL: clean the mess of needing SQL lookup objects to be aware of each other, by separating SQL connection information (Amavis::Out::SQL::Connection object) from objects holding SQL statement handles. Statement handles are invalidated on reconnect, and are dynamically 'prepared' as needed. - SQL: thanks to a reorganization of SQL objects an automatic reconnect to an SQL server is done without temporary failing a processed message; - SQL: new configuration variable (an associative array) %sql_clause, also part of policy maps, allows SQL clauses to be switched with policy banks. The components of %sql_clause under keys 'sel_policy' and 'sel_wblist' perform the duty of legacy configuration variables $sql_select_policy and $sql_select_white_black_list. Compatibility with older configuration files is maintained when %sql_clause is left at its default value; - can add one layer of 62 subdirectories to the quarantine directory for more efficient file system use by uniformly distributing quarantined mail; enabled by setting a new config variable $quarantine_subdir_levels to 1; - choosing mail_id as a quarantine file name now greatly reduces the likelihood of two quarantined messages processes by the same child process shortly one after another from attempting to get quarantined under the same filename, leading to a temporary failure ("File...already exists, refuse to overwrite") and leaving behind a preserved temporary directory; reported by Martin Svensson; - release from quarantine functionality is now a built-in feature; a message release can be requested via enhanced AM.PDP protocol. There is a new utility program 'amavisd-release', which currently mostly serves to demonstrate how to request releasing of a quarantined file. Currently the supported quarantine types are: plain file, gzipped plain file with a name ending in .gz, and an SQL-based quarantine. Currently not supported is a release from a BSMTP-encoded plain file or from a mbox (Unix-style) mailbox quarantine file. Example use: $ amavisd-release spam/V/V5htXBh0y0Hr.gz H2huh4wfrSyC or providing a replacement list of recipients: $ amavisd-release spam/V/V5htXBh0y0Hr.gz H2huh4wfrSyC user@example.com The first argument is a (relative) quarantine file name, as reported in the log. It must include a 12-character mail_id which is automatically recognized. The second argument is a secret_id, which can be fetched from an SQL database if @storage_sql_dsn is enabled (see README.sql), for example by the command: $ mysql amavis -e 'SELECT secret_id FROM msgs WHERE mail_id="V5htXBh0y0Hr"' or (preferably) by some other more advanced utility program. Current simple-minded heuristics in the amavisd-release program is to assume a message is stored in an SQL database when the file name (first argument) consists only of a 12-character mail_id. Please adjust the program if this assumption is not true, e.g. when $virus_quarantine_method='local:%m' instead of a default $virus_quarantine_method='local:virus-%m'; If secret_id is not available, administrator may choose to skip checking of secret_id in the amavisd daemon by setting a configuration variable $auth_required_release to false (it defaults to true), and supplying an empty secret_id or not supplying it at all in the AM.PDP release request. The variable $auth_required_release is also part of policy banks. If the release client program specifies a nonempty secret_id in the request, the secret_id will be validated and a request will fail if not valid, regardless of the setting of $auth_required_release. Release requests contend for the same amavisd child processes as regular mail processing. This may cause delays in responses to release requests, especially when Postfix feeds mail to amavisd-new via LMTP which is more persistent in keeping connections open than the Postfix SMTP client service. To ensure one child process is always available for processing extra requests such as release requests, one may choose to set $max_servers larger (by one) than MTA's maxproc setting. To enable recognition and processing of AM.PDP protocol requests in amavisd daemon, a dedicated Unix socket or a TCP port needs to be opened, for example by the following assignments in amavisd.conf: $unix_socketname = "$MYHOME/amavisd.sock"; $interface_policy{'SOCK'} = 'AM.PDP'; $policy_bank{'AM.PDP'} = { protocol => 'AM.PDP' }; or similarly for connections through a dedicated TCP port 9998, and restricts it to accepting IP connections from local IP address only: $inet_socket_port = [10024,9998]; $interface_policy{'9998'} = 'AM.PDP'; $policy_bank{'AM.PDP'} = { protocol => 'AM.PDP', # Amavis policy delegation protocol inet_acl => [qw( 127.0.0.1 )], # log_level => 3, }; - new configuration variable @spam_quarantine_cutoff_level_maps turns off quarantining if spam score is at or above this level; suggested by Gary Verchick, MrC and others; - more informative logging and SMTP status generation in smtp client code; - IPv6: allow optional brackets around IPv6 address in lookup tables and configuration variables, e.g. [FE80::]/10 is treated the same as FE80::/10 Allow (and ignore) link-local scoped address in extended numeric IPv6 syntax (interface specification) when parsing link-level IPv6 addresses, e.g. fe80::1%lo0; - IPv6: adjust the default @mynetworks to include link-local and site-local address ranges [FE80::]/10 and [FEC0::]/10, and add (optional) brackets around [::1]; (although the use of site-local address ranges seems to be deprecated nowadays); - IPv6: tested sending mail via slightly modified Net::SMTP through an IPv6 socket to an IPv6-enabled Postfix; updated README.postfix; To experiment, replace the: @ISA = qw(Net::Cmd IO::Socket::INET); with: use IO::Socket::INET6; @ISA = qw(Net::Cmd IO::Socket::INET6); in Net/SMTP.pm, to make use of the: $notify_method='smtp:[::1]:10025'; Don't forget to add [::1] to mynetworks at MTA smtpd service on port 10025; - bring earlier the initialization of %local_delivery_aliases and %builtins so that the config file can override the defaults; - add SA autolearn status to the logged spam status (log line: "SPAM, ..."), as well as full TESTSCORES info to the list of SA tests (including score points for each SA test); contributed by John Sivak; - new small wrapper module Amavis::IO::Zlib around Compress::Zlib allows for reading back compressed quarantine files for a mail release, and allows for writing compressed quarantine files without having to fork a gzip process. This makes the utility program gzip(1) optional, which may be appreciated when running in a chroot environment; - modified do_gunzip to use the new Amavis::IO::Zlib module; - added LDAP lookups for the following attributes: amavisVirusAdmin, amavisNewVirusAdmin, amavisSpamAdmin, amavisBannedAdmin, and amavisBadHeaderAdmin; by Michael Hall; added attribute amavisBannedRuleNames by Brian Wong and Michael Hall; - a policy bank name 'MYUSERS' now has a special semantics: this policy bank gets loaded whenever the sender matches @local_domains_maps. This only makes sense if local sender addresses can be trusted -- for example by requiring authentication before letting users send with their local address; the feature requested and a patch provided by Steffen Hansen; (a note from future: starting with 2.6.0 an additional requirement for loading a policy bank 'MYUSERS' is that 'originating' flag is on, which typically means that mail is coming from internal networks or authenticated roaming users); - add cumulative percents to the TIMING report to make it easier to locate large contributors to elapsed time; - updated interface code to Sophos Perl-SAVI module, taking advantage of its new ability to reload virus data: amavisd-new will initialize the SAVI object in the parent, which will be inherited by the children. The children will detect whenever the virus data is stale and automatically exit (reducing the number of messages processed with out-of-date protection), and the parent will reload the virus data before spawning new children; update provided by Paul B. Henson. To have this feature fully functional a small patch to Net::Server is needed: http://www.csupomona.edu/~henson/www/projects/SAVI-Perl/dist/ Net-Server-0.87-parent_fork_hook-1.diff.gz (patch is no longer necessary since Net::Server 0.88 and amavisd-new 2.3.2) - provide a routine dump_hash for debugging purposes: given a reference to an associative array (a Perl hash) it writes its contents to a log. Note: if called within amavisd.conf the log is still directed to STDERR. For example: @local_domains_maps = ( read_hash("$MYHOME/local_domains") ); dump_hash($local_domains_maps[0]); OTHER CHANGES: - new documentation file README.sql; - tightened up a sample regular expression in amavisd.conf for catching Class ID (CLSID) extensions in file names; - restrict the 'double extension' banning rule to require at least one letter in the next-to-last dot-delimited field; this allows filenames such as prog.33.22.01.exe not to be blocked by this rule; - change hash lookups code to stop a hash search when a matching key exists, even if the matching result (corresponding hash value) is undef; this reverts the change made in a release 2.1.0 to a previous behaviour which is consistent with regexp lookups (but leaves SQL lookups to be different, continuing the scan to more general entries on a NULL field value; SQL lookup is specific because all attributes live in the same record); inconsistency reported by Gary Verchick; - as a workaround for some versions of unrar (sparc64?) which are unable to create a subdirectory parts/rar by themselves, do_unrar now prepares the subdirectory explicitly; suggested by Andrzej Kukula; - provide a new CentralCommand Vexira 'vascan' av entry (distinct from the 'Antivirus', which remains H+BEDV AntiVir -based) to work with the new Vexira scanner, thanks to Brian Wong and Norman C Rice; - Vexira vascan: added status codes 3 (password protected) and 9 (unsupported format), recognize "sequence found"; thanks to Brian Wong; - F-Prot Antivirus: enhance the pattern to capture virus names; - renamed AV entry "H+BEDV AntiVir" to "Avira AntiVir", reflecting the company name change; thanks to Patrick Ben Koetter; - change the default $sql_select_policy and $sql_select_white_black_list to use the LEFT JOIN ... ON instead of WHERE for the 'join' relation; should be functionally equivalent, but makes the join operation explicit; - changes to LDAP Schema to make it import and play nicely with Novell NDS, by Michael Tracey, SONOPRESS USA, LLC April 07 2005 (changes are included in LDAP.schema but commented out by Mark) - remove special handling in unmangle_sender() for ancient viruses Magistr and Badtrans, leave decisions on suppressing DSN entirely to @viruses_that_fake_sender_maps; - rise a limit on the number of logged matching SA tests from 10 to 50 (still some arbitrary sanity limit); based on observation of Bojan Zdrnja; - add a minimal version requirement 2.22 for Digest::MD5, we need the new 'clone' method; reported by Thomas Jarosch; - a command line option 'debug-sa' now sets variable $sa_debug to a string '1,all' instead of 1, in anticipation of the next version (3.1) of SA which changed interpretation of its debug option; the '1,all' seems to be compatible both with the old and the new interpretation, despite producing a warning in pre-3.1 versions of SA; - documentation note: Macintosh.tar.gz installation instructions for Macintosh are not recent, they apply to Mac OS X 10.2.0-10.3.9 --------------------------------------------------------------------------- December 22, 2004 amavisd-new-2.2.1 release notes SECURITY: - add support for the pax(1) archive decoder, which can handle tar/cpio/pax archives (including legacy format variants). Due to limitations in cpio (and in Archive::Tar), for security reasons it is preferred to decode such archives with pax and no longer with cpio; please add a line: $pax = 'pax'; to amavisd.conf and verify that the program pax is installed on the system (and in the jail if running in chroot); problem reported by Ron Ogle; - perform additional tests at startup time on proper protection of the configuration file; - add file name extensions wmf, emf and grp to the example list of banned extension, according to recent Microsoft security bulletins; suggested by Stephane Lentz; - introduces 'clean but inconclusive' av scanner result to avoid a specialized or quick partial av scanner like jpeg checker to claim mail is clean when all other general purpose av scanners fail (see below); INCOMPATIBILITY: - removed some legacy $*_ldap variables, as they are no longer needed; These variables were still declared but ignored in 2.2.0 for compatibility with older amavisd.conf files. Such variables need to be removed from the amavisd.conf if they are still present there from older versions, otherwise Perl will complain with 'Global symbol ... requires explicit package name"; OTHER FIXES: - files_to_scan and decompose_mail are now able to remove unexpected directories which may have been left behind by some failed decoding and were causing temporary failures and mail delivery retries; error recovery problem after failed unarj reported by Ralf Hildebrandt, and a related problem with tar, reported by Ron Ogle; - error recovery code in files_to_scan and rmdir_recursively now tries to change protection on directories and files, and retry if the first attempt to access them fails because of denied permission; - pre-load some additional Perl modules needed by SA when running in chroot; - add module Net::LDAP::Search to a list of pre-fetched modules; omission pointed out by Paul Jacobson; - when quarantining is disabled by keeping $QUARANTINEDIR undefined, the log entry and administrator notification message inappropriately suggested that mail was quarantined, which in fact (appropriately) it was not. Setting $QUARANTINEDIR='' did work as expected. Reported by Sascha Lucas; - avoid the use of Encode::is_utf8 due to a Perl bug (still present in 5.8.8, Encode::is_utf8 on tainted utf8 character string produces false); Perl bug tracking: #32687: Encode::is_utf8 on tainted UTF8 string returns false; - modify safe_encode() to guarantee the result is a string of octets, not a string of UTF-8 characters; it saves some unnecessary work in further processing and keeps MIME::Entity from UTF swamp when running in chroot; problem pointed out by Branko F. Gracnar; - avoid braindead Perl default where an empty regexp implies the last successfully matched regexp, which (if not being very careful) brings in some completely unrelated last-executed regular expression; - change kill 'TERM' into kill 'KILL' when a forked process within run_command and run_command_consumer gets into deep trouble, to avoid exit handlers being invoked in the subprocess (which could lead to two processes trying to clean the same set of temporary files); - in an old sendmail setup using the amavis(.c) helper program without LDA arguments, avoid inappropriate warning: "WARN: no recips left (forgot to set $forward_method=undef using milter?) and return status 0 instead of 99 when message is to be blocked, as the helper program amavis(.c) does not recognize status 99 in this situation and inappropriately passed it on to sendmail; reported by The Mindflayer; - the @bypass_header_checks_maps is now able to also bypass the bad header checks as provided by MIME::Parser; inconsitency reported by CRivera; - avoid some Perl warning messages; thanks to Bill Landry; CHANGES AND MINOR NEW FEATURES: - add configuration variable @newvirus_admin_maps (and $newvirus_admin, along with corresponding SQL field 'newvirus_admin') which works like the existing @virus_admin_maps (and $virus_admin), except that it sends virus administrator notification to specified e-mail address only for newly encountered viruses which have not yet been encountered since the amavisd startup. It makes use of by-virusname counters in the SNMP counters database. If more than one child process starts working on infected message containing a not-yet-accounted-for virus, there might be more than one 'first time' notification, this is not a malfunction. Both the @newvirus_admin_maps and the @virus_admin_maps may be enabled, each (possibly both) would receive their notifications as appropriate. A useful setting is to globally enable only the new virus notifications, and additionally enable _all_ administrator notifications for internally originating mail only (by the use of policy banks); - provide separate configuration variables @banned_admin_maps and @bad_header_admin_maps, along with corresponding SQL fields 'banned_admin' and 'bad_header_admin'; their function was previously covered by @virus_admin_maps, which now only still controls administrator notifications in case of viruses; - introduces 'clean but inconclusive' av scanner result to avoid a specialized or quick partial av scanner like jpeg checker to claim mail is clean when all other general purpose av scanners fail: in av scanner entries (lists @av_scanners and @av_scanners_backup) give an extended meaning to undefined fourth argument (the 'match for clean' list or regexp). The interpretation of the fourth argument is now: 4. an array ref of av scanner exit status values, or a regexp (to be matched against scanner output), indicating NO VIRUSES found; a special case is a value undef, which does not claim file to be clean (i.e. it never matches, similar to []), but suppresses a failure warning; to be used when the result is inconclusive (useful for specialized and quick partial scanners such as jpeg checker); Also modified example jpeg checker entry in amavisd.conf accordingly. - NOD32 av scanner: changed @av_scanners entry to match the new version of the scanner; thanks to Nejc Skoberne; - added @av_scanners entry for File::Scan; - when preparing an SQL SELECT clause for white/blacklisting lookup, take into account a relative position of ? and %k in the $sql_select_white_black_list template to improve flexibility of specifying the clause; suggested by Matt Petteys; - reduce the log level of some more common and harmless log messages; - macro %p and the log entry now reports full policy bank path, not just the last loaded policy bank name; - added LDAP attributes amavisWarnVirusRecip, amavisWarnBannedRecip, and amavisWarnBadHeaderRecip; by Joel Nimety and Michael Hall; - renamed LDAP attribute name amavisSpamModifiesSubject to amavisSpamModifiesSubj in order to match the documented LDAP schema; noticed by Kees Bos, patch by Michael Hall; - add support for ripOLE decoder, which attempts to extract embedded documents from MS OLE documents (MS Office) (http://www.pldaniels.com/ripole/, by Paul L Daniels)); ripOLE is still experimental/alpha code; To make amavisd-new find the installed program 'ripole', add the: $ripole = 'ripole'; to the amavisd.conf. Suggested by David Wilson and Noel Jones; - allow multiple occurrences of command line option: -c config_file and execute the provided configuration files one after the other; based on a subset of functionality provided as a patch by Davor Ocelic; - a slight improvement in classifying mpeg and some other multimedia files (in the default $map_full_type_to_short_type_re); - several minor code cleanups; - add a recommendation by Daniel J McDonald to a documentation file INSTALL: If different UID is preferred for an AV scanner, a solution for ClamAV is to add user clamav to the amavis group, and then add AllowSupplementaryGroups to clamd.conf; - enclosed a simple demonstrational Perl program amavis.pl, which is functionally much like the amavis.c helper program, but talks the new AM.PDP protocol with the amavisd daemon. See README.protocol for the description of AM.PDP protocol. To be placed in amavisd.conf: $protocol='AM.PDP'; $unix_socketname='/var/amavis/amavisd.sock'; Usage: amavis.pl sender recip1 recip2 ... < message.txt - documentation updates; --------------------------------------------------------------------------- November 2, 2004 amavisd-new-2.2.0 release notes SECURITY: - modified MIME entity traversal to include MIME container parts (e.g. multipart/*) as pseudo parts. Such pseudo-parts do not carry any body contents but do show up (with their MIME content type only) in the tree structure as seen by banned names checks. This makes it possible to specify more complex banned rules based on the placement of leaf nodes within or outside of mime multipart containers. This also re-enables the possibility to check such MIME wrappers for banned MIME Content-Types (most notably for the message/partial and message/external-body), which was lost with the change of internal representation of mail parts in version 2.0 (amavisd-new-20040701); - preserve original zip archive for virus scanners to see, if the archive contains any zero-length members (Archive::Zip module chooses not to extract members with declared zero size, even if the size does not correspond to the actual size); based on a patch by Dirk Datzert; - add tests to mime_traverse to verify that the files produced by MIME::Parser really do exist and are readable; (and sent a patch adding I/O checks for MIME::Tools to its maintainer; please use MIME::Tools 5.414 or later from CPAN to avoid possibility of full /tmp partition causing infected or spam mail to pass through); - changed recommendation in INSTALL to choose the latest version of MIME-Tools from CPAN - the 5.415 at the time of this writing; - do_unrar: recognize encrypted entire archives (not just their individual members), and flag mail as undecipherable; - recognize file(1) report/^MS Windows\b.*\bDLL\b/ as short type 'dll'; add 'dll' to example patterns in amavisd.conf and amavisd.conf-sample; add name extension '.cpl' to the list of basic banned names; INCOMPATIBILITY: - incompatible change: the default value for $recipient_delimiter is now undef and no longer '+'; adding address extensions must now be explicitly enabled; - minor change in the default X-Virus-Scanned: header field, see below; OTHER CHANGES SINCE 2.1.2: - rewritten LDAP modules, by Michael Hall; - improvements in handling of double errors (errors that occur while handling consequences of some previous error); be permissive on failures in DESTROY methods; prevent some cases for child process not being able to sign off from the nanny database; - enforce $max_requests more strictly, dropping SMTP session after task count is exceeded by one; this is in response to the new smtp session caching behaviour in Postfix, which is now much more persistent in keeping the session open on a busy mailer; although dropping session at the server side is discouraged by RFC 2821, this change was recommended on the postfix-users mailing list; - added a site-wide mail size limit $smtpd_message_size_limit, and a per-recipient mail size limit lookups @message_size_limit_maps along with SQL and LDAP fields, making it possible to reject mail based on its size. The list of lookup tables maps recipients to mail size in bytes, e.g.: @message_size_limit_maps = ({'user1@example.com' => 20_000_000, 'user2@example.com' => 15*1024*1024, 'user3@example.com' => 0, # uses global limit '.example.com' => 10*1024*1024 }); $smtpd_message_size_limit = 20*1024*1024; # global limit if nonzero A value of 0 or undef disables the check and is a default. A per-recipient limit is bound on the high side by the $smtpd_message_size_limit, and on the low side by 64kB, which is a minimal allowed size limit imposed by RFC 2821. This limiting really belongs to MTA and is only partially supported here (no admin notifications, no quarantine, no final_*_destiny configurability). It is mostly provided here to be able to place some sanity limit on runaway or malicious clients, or if someone insists on using amavisd-new in a pre-queue filtering setup; suggested by Tuomo Soini; - internal: add new object $msginfo->mail_text_fn to hold the file name of the original mail, decouple the location of file email.txt from the temporary directory, which was implied until now. This is presently needed for the Courier interface. Add optional attribute 'mail_file' to the AM.PDP protocol, see README.protocol; - in banned parts descriptor strings which are used in banned name checking, provide a 'T=empty' short type for empty mail parts, including for empty MIME parts (instead of omitting T=... altogether). This can be used in banned rules to test for empty parts, generally or restricted to empty MIME parts; suggested by Ricardo Stella and Stephane Lentz; - a banned lookup result (which is interpreted as boolean for most purposes: zero or empty for false, anything else for true) may give a result 'DISCARD' (which is true as well), which will disable DSN if it turns out the mail was blocked by such banned rule (akin to viruses_that_fake_sender and spam_dsn_cutoff_level). Here is an example rule in $banned_namepath_re: [ qr'(?# BLOCK EMPTY MIME PART APPLICATION/OCTET-STREAM ) ^ (.*\t)? M=application/octet-stream \t(.*\t)* T=empty (\t.*)? $'xmi => 'DISCARD' ], Suggested by Ricardo Stella and Stephane Lentz; - fix 'Insecure dependency in unlink' in sub files_to_scan which could happen when some decoder left non-regular files in the directory; reported by Andrzej Kukula; - bug fix: only insert LDAP and SQL lookups objects into lists of maps at the first task of a child process, later tasks should not insert duplicates; - new subroutine do_ar and new configuration variable $ar to handle standard Unix archives, including Debian binary packages; - recognize a Unix archive and give it a short type .a; - recognize a Unix relocatable binary and give it a short type .o; - do not penalize SMTP status "554 5.1.1 Error (DATA): no valid recipients", the situation arises regularly when pipelining is in effect and all recipients happen to be rejected; - protect spaces and newlines when logging broken Message-ID and Resent-Message-ID header fields (macros %m and %r) to facilitate log parsing; protect newlines in logged Subject header field (macro %j); parsing difficulty pointed out by Chris Lee; The present solution uses =XX (hex) encoding and is a quick and dirty fix. A cleaner solution to avoid double sanitation of special characters is needed, but would involve a deeper reorganization; - updated example list of "banned extensions - long" in amavisd.conf and amavisd.conf-sample; - change the default lock file name from "$MYHOME/amavisd-$$.lock" to "$MYHOME/amavisd.lock", to avoid inventing a new name at every restart and leaving old files behind; pointed out by Dale Walsh and Martin Orr; - updated av entry for nvcc (Norman Virus Control v5 / Linux) to include statuses 10,11, and 2,14 to the status lists according to documentation; password protected or corrupted archive (status 11) was not recognized as non-infected status; thanks to Michael Ramke of Norman Data Defense Systems GmbH; - updated DrWebD entry to recognize and ignore flag DERR_SKIPPED; - support Mail::ClamAV 0.12 and 0.13 or later, which is incompatible with 0.11 due to change of constant names in the underlying ClamAV library; - added 'check-jpeg' example entry to the @av_scanners list and provide the associated module JpegTester.pm; it offers a fully-fledged check for jpeg comment field buffer overflow attempts; should serve mainly as an example for adding similar quick responses to new threats; - added 'check-jpeg-simple' example entry to the @av_scanners list (only in amavisd.conf-sample); it offers a quick check (and not very exact one: checks only the first 32kB) for jpeg comment field buffer overflow attempts; should serve mainly as an example for adding similar quick responses to new threats; - relax too restrictive sanity check on temporary directory name when accepting requests from a helper program or via AM.PDP protocol (e.g. with sendmail milter setup); reported by Babu Kanagala; - relax allowed set of characters when receiving XFORWARD attribute values, it turns out that characters like '=' and '+' are allowed; - when using "bsmtp:" delivery method suppress X-Envelope-From and X-Envelope-To header fields, as the addresses are already available in the envelope; - when using the "bsmtp:" quarantine method the *_quarantine_to was completely ignored, which made it impossible to turn off quarantining selectively for certain users by specifying an empty or undef value. Since 2.2.0 an empty *_quarantine_to turns off quarantine for a recipient regardless of the quarantine method. A nonempty string in *_quarantine_to (the exact value is still ignored) must now be used even with "bsmtp:" to enable quarantining. Inconsistency discovered by Sean Doherty; - suppress leading $QUARANTINEDIR string from the value of macro %q, thus hiding the absolute file path from notifications; - add configuration variable $local_client_bind_address (and equivalent policy bank key), to allow for explicitly binding local socket address to a specific interface in SMTP client; suggested by Wouter de Jong; - keep whatever (if any) file results from gunzip and family (do_uncompress) even if the decompressor's exit status is nonzero; reason: gzip returns status 2 when decompressing file with trailing garbage; reported by Tobias Reckhard; - collect declared original file name from gzipped (and friends) files if reported by file(1), making them available to banned name checking; - avoid unnecessarily checking white/black lists if spam checks will not be used (e.g. infected mail); - use qquote_rfc2821_local to properly quote e-mail addresses in the most visible log entries; - if there is more than a single (or less then one) occurrence of %k in the SQL SELECT clause template, multiplicate the set of query keys accordingly, making possible more complicated custom SELECT clauses; - don't forget to load amavis policy delegation protocol support code if AM.PDP is explicitly requested in $protocol, even if not listening on a Unix socket; - add 'queue_id' attribute to the AM.PDP protocol; equivalent to a change in the Courier support code by Martin Orr; - include the declared (faked) sender address in the virus recipient notification template, in addition to the originator IP address; - add macro %Q and method Amavis::In::Message::queue_id, holding a MTA queue ID if available (in Courier and milter/AM.PDP setup); by Michael Musikhin (through Martin Orr); - add macro %y to show elapsed processing time; suggested by Ed Walker; - sanitize newlines and spaces (and some other characters) when moving syntactically invalid Message-ID and Resent-Message-ID to macros %m and %r for logging purposes; resulting wrapped main log entry reported by CRivera; - bring up syslog priority to LOG_NOTICE when debug or debug_oneshot is in effect; - make a product name, version ID and version date available as separate variables to avoid the need to parse $myversion for the purpose of customizing e.g. the setting of $X_HEADER_LINE; based on suggestion by Dale Walsh; the re-introduction of a date ($myversion_date) also suggested by Stephane Lentz. Added variables: $myproduct_name, $myversion_id, $myversion_id_numeric, and $myversion_date, which serve to assemble the $myversion. Modified the default templates of $smtpd_greeting_banner and $smtpd_quit_banner to take advantage of the new variables. Changed $X_HEADER_LINE default from "by amavisd-new at $mydomain" to "$myproduct_name at $mydomain" and added an example of a $X_HEADER_LINE setting with version number included to the amavisd.conf-sample; - added SQL fields 'virus_admin' and 'spam_admin' to lookup lists @virus_admin_maps and @spam_admin_maps; patch by Robert Collier; - add a log message 'SPAM-KILL, ...' (at log level 3) for not-passed mail, to complement the 'SPAM-TAG' log message for passed mail; - add Mail::SpamAssassin::Plugin::SPF to a list of modules that SA fails to load at init time; - prevent sending the same SMTP response more than once, if the first attempt failed due to disconnected SMTP session; - fix a double-@ formatting buglet in the log message "adding address extension _spam to user@@domain", reported by Vincenzo; - add kill('TERM',$$) to the arsenal of attempts to get rid of unwanted forked process; - includes rpm spec file, including the init script, contributed by Marius Andreiana, based on previous work by Dag Wieers; - document the localization template directory contents (in file amavisd.conf-sample) when read_l10n_templates is used; thanks to Joël Brogniart; - includes file Macintosh.tar.gz, which contains auto-startup scripts and installation instructions for Mac OS X, contributed by Dale Walsh of the Dale Enterprise L.L.C. --------------------------------------------------------------------------- September 6, 2004 amavisd-new-2.1.2 release notes - fixed (hard)black- and white-listing on static lookup tables which failed to match any sender; reported by Derck Floor; - use $hdrfrom_notify_recip address in the From: field for recipient notifications, instead of $hdrfrom_notify_admin; inconsistency pointed out by Ekkehard Burkon; - the 'neutral' sender notification template was joining the Subject and the Message-ID header fields into one longer Subject when it was reporting a nondelivery other than the 'invalid characters in header'. Likewise the first body line of this same DSN was eaten up: "This nondelivery report was generated by the amavisd-new program" (the problem was introduced in amavisd-new-20030616 and never reported); - in amavisd-agent, amavisd-nanny, amavisd: extend the signal and error handling in code sections holding bdb locks from just ignoring the SIGINT, to controlled catching and re-signaling several signals and error conditions; problem reported by Tom Mulder; - suppress duplicate names from the list of virus names in macro %V; by Gregor Weiss; - fine-adjusted log level of some log messages; - discard leading and trailing whitespace from the macro %t (Received trace); - extend the search for IP in the Received trace from 4 to a maximum of 6 entries; - ignore private IPv6 addresses (RFC 3513: link-local, site-local, multicast) when searching through Received trace for the origin of mail; - place mail header field X-Envelope-From in front of the X-Envelope-To in quarantined mail; also changed case of X-Quarantine-id into X-Quarantine-Id for consistency with other header fields; - provide new macro %e which evaluates to our best guess of the originator IP address collected from the Received trace, complementing similar macros %t, %a and %g; suggested by Gregor Weiss; - add the result of macro %e to the default 0-level log entry; - provide new macros %u and %U to evaluate to a timestamp of the message reception similar to an existing macro %d (RFC 2822 local date-time); the (%u) as Unix time (seconds since 1970-01-01T00:00Z as a decimal integer, suggested by Gregor Weiss), and (%U) as ISO 8601 (EN 28601) UTC date-time; - avoid some empty lines in default DSN templates and fix some inconsistencies in their formatting; - internal: collect existing common code for time formatting as new subroutines iso8601_timestamp and iso8601_utc_timestamp; collect existing common code to find IP address in the Received trace as a new subroutine best_try_originator_ip; - bump up the version number in $myversion - the 2.1.1 still presented itself as 2.1.0; - add a note about a data structure difference between @score_sender_maps and $per_recip_blacklist_sender_lookup_tables (amavisd.conf-sample, amavisd) --------------------------------------------------------------------------- August 24, 2004 amavisd-new-2.1.1 (amavisd-new-20040824) release notes - unconditionally initialize @banned_filename_maps to (\$banned_filename_re), otherwise $banned_filename_re is ignored by default (unless amavisd.conf explicitly assigns to @banned_filename_maps); a patch by Thomas Jarosch; - fixed inappropriate log entry in SQL whitelisting: wbl: (SQL) recip whitelisted sender <...>, unexpected wb field value: "1"; reported by Carlos Horowicz; - added missing import of &ca to the amavisd-new-courier.patch; by Martin Orr; - produce a warning when there is an unknown field in the policy bank to be loaded; - with delivery method 'bsmtp:' prepend a directory $QUARANTINEDIR to the file path if not explicitly specified, to behave like the 'local:' delivery method, making it possible to hide full path from the X-Quarantine-Id and notifications; a patch by Thomas Jarosch; - pre-load SA 3.0.0 module Mail::SpamAssassin::Plugin::Hashcash to make it available in the chroot jail; - pre-load modules Mail::SpamAssassin::SpamCopURI and URI::* for SA older than 3.0.0; - enhancement to amavisd-nanny: when terminating a process and SIGTERM produces no result for some time, try SIGKILL; contributed by Philip Engdahl; --------------------------------------------------------------------------- August 15, 2004 amavisd-new-2.1.0 (amavisd-new-20040815) release notes The 2.1.0 release is mostly a maintenance release over 2.0, with only a handful of smaller features added. Based on a manual code audit the number of smaller internal code changes is rather extensive, some changes dealing with long-standing known deficiencies, minor bugs, documentation problems and typos. Only a few fixes are for new bugs introduced in 2.0. The files amavisd.conf, amavisd.conf-default and amavisd.conf-sample have been extensively reworked, with the hope to suit better the new installations, while possibly causing some head-scratching for existing installations when looking at a diff. The file amavisd.conf is the one that should serve as a sound base for the initial config file, while keeping an eye on the list of all variables and their defaults in amavisd.conf-default. The amavisd.conf-sample is being phased out of active maintenance, and should serve mostly as a set of examples and the source of documentation until better documentation is available. Two nice features are available: - the use of BerkeleyDB and libdb is now optional; controlled by variables $enable_db and $enable_global_cache; - a new program 'amavisd-nanny', with the accompanying instrumentation in amavisd, displays the general health of all amavisd child processes, reports crashed ones and attempts to kill long overdue processes; It is still experimental and minimalistic, problem reporting is currently only to stdout. Other changes: - incompatible change since 2.0: the use of BerkeleyDB is now off by default; The use of BerkeleyDB and libdb is made optional, controlled by variables $enable_db and $enable_global_cache, both false by default. $enable_db: enables the use of BerkeleyDB/libdb (for SNMP counters database and nanny, and optionally for cache); $enable_global_cache: enables the use of libdb-based cache when $enable_db is also true; If either the $enable_global_cache or $enable_db are false, cache of mail body MD5 digests is kept in child-local memory as in pre-2.0 versions, and is therefore local and short-lived, with lower expected cache hit rate; - incompatible change: DSPAM 3.0 or better is needed (if $dspam is enabled), no longer works with 2.x; - incompatible change: changed name of the (hardly ever needed) configuration variable auth_supported_out to auth_required_out, to better reflect its semantics (should be true if MTA server to which amavisd is sending notifications and forwarding mail requires authentication (AUTH smtp command)); - a new small program included: amavisd-nanny is a program to show the status and keep an eye on the health of amavisd child processes (experimental); - fixed a bug in lookup_acl where a "." did not act as a catchall; thanks to JP; - fixed a problem in SQL lookup which could return undef even when not all the matched records had NULL in the field; - fixed compatibility with old 'amavis' helper program ('delivery_care_of' defaulted to 'client', instead of depending on the presence of ldaargs), reported by Charlie Schluting and Christer; - fixed long standing problem in do_ascii, which could return without calling Convert::UUlib::CleanUp, occasionally spilling state into subsequent mail checks within the same process; - fixed macros %D, %O and %N when log_recip_templ is being expanded; a patch by Ed Walker; - fixed recognition of separators in a nested call during macro expansion; - pre-load missing modules Net::LDAP, Net::LDAP::Schema, Net::LDAP::Search, and Net::LDAP::RootDSE; suggested by Paul Jacobson; - fix locale-related bug in rfc2822 date generation, where we were restoring the saved LC_TIME value to LC_CTYPE (!); a patch by Henrique de Moraes Holschuh / Debian support team; - protect from signals while bdb cursor holds a lock; - new subroutine inherit_header_edits() and slight code restructuring makes possible for spam_scan() and other code before the final delivery to start submitting common header edits into $msginfo->header_edits, avoiding the need for passing them through global variables; - now a loglevel-based automatic syslog priority assignment can no longer lower a message syslog priority below the syslog priority specified in the $SYSLOG_LEVEL, it can only increase it; the violation of the least-surprise principle pointed out by Andy Dills; - a small optimization in logging: a new subroutine ll allows to save time in preparing complex log entries when we know their log level exceeds the current log level and won't be logged; - in default macro templates $log_templ and $log_recip_templ: * placed 'spam' condition before 'bad header' for consistency with program behaviour; * added reporting of tag/tag2/kill levels in $log_recip_templ (experimental: macro names may change in future versions); * rewritten templates using negation (i.e. [?x|1] ) to avoid unsightly selector nesting; - MIME decoding is now allowed to exceed the decoding quota, avoiding the problem when a small quota settings might not allow even a plain mail through; - override LC_TIME to "C" on every log message, to work around issues with Unix::Syslog, which would log stuff with the date stamp localized, which syslog would dislike and add its own, and the resulting mess is not recognized by amavis log processors; a patch by Henrique de Moraes Holschuh / Debian support; - changed dspam command line options to work with dspam 3.0 (no longer with 2.x), a patch by Ron Ohmer, Nabil Sefrioui, and Reech; - dspam header fields are now inserted into passed mail if all recipients are local; - supply better defaults for $hdrfrom_notify_sender, $hdrfrom_notify_recip, $hdrfrom_notify_admin and $hdrfrom_notify_spamadmin, similar to defaults from amavisd-new-20030616-p10; - when parsing output from the 'AVG grisoft' virus scanner, don't include CR in virus name; reported by Vernon A. Fort; - new file 'amavisd-new-qmqpqq.patch' provided by Martin Solciansky, (similar to fixes by Christopher Odenbach) making it work again with the current version of amavisd-new; - use lstat instead of stat, and test for soft links wherever appropriate; - remember inode and device number when creating temporary directory and temporary file, and test for change before removing them; - enhanced security: certain tainted values are allowed to enter deeper into program, untainting them only where and when really necessary; - avoid a taint problem in Mail::ClamAV; - added AV entry for CAI eTrust Antivirus; by Stephane Lentz (requires a suid shell wrapper around inocmd32); - added status 9 to the set of infected statuses for the drweb command line scanner (DrWeb Antivirus); - use our subroutine q_encode to Q-encode header fields from the notification templates, instead of MIME::Words::encode_mimeword (a similar fix in 2.0 applied only to encoding of modified headers in passing mail); - add attribute 'x-spam-type=original' to the Content-Type header field (the SpamAssassin's code to recognize an original email) when defanging spam, facilitating reporting spam via SA to other spam fighting tools; a patch by Brian May; - add a note to amavisd.conf that $sa_auto_whitelist has no effect on SA since 3.0.0 - SA has now a configuration file option 'use_auto_whitelist'; - turn off timer in post_process_request_hook() to avoid periodically recreating child processes on an idle machine; - added @mynetworks_maps and enhanced lookup_ip_acl() to take a list of lookup tables: currently members can be an array ref (as before), or a hash ref (new) or a plain constant (new); - generalized @debug_sender_acl into @debug_sender_maps along the lines of other lookup tables, and make it part of policy banks; - added @warnvirusrecip_maps, @warnbannedrecip_maps, @warnbadhrecip_maps; - added @spam_subject_tag_maps and @spam_subject_tag2_maps, to allow per-recipient spam tags string; suggested by Ed Walker; Note, there is an inconsistency in names of legacy variables and the new @*_maps, in an attempt to rectify an unfortunate choice of name for seldomly used variable $sa_spam_subject_tag1: @spam_subject_tag_maps = (\$sa_spam_subject_tag1); # exotic @spam_subject_tag2_maps = (\$sa_spam_subject_tag); # in common use Note also that corresponding SQL fields are 'spam_subject_tag' and 'spam_subject_tag2'; usually only the 'spam_subject_tag2' would be used; - added configuration variable $auth_reauthenticate_forwarded, which directs amavisd to apply its own credentials ($amavis_auth_user and $amavis_auth_pass) to unauthenticated forwarded (passed) mail, besides using them for submitting original messages (notifications and quarantine). This is similar to how mailing list managers are allowed to work (rfc2554). Note that the Perl module Net::SMTP in its current version is unable to specify the 'submitter' in its 'MAIL FROM' command, this should be rectified; --------------------------------------------------------------------------- July 1, 2004 amavisd-new-20040701 / amavisd-new-2.0 release notes MAJOR NEW FEATURES (since amavisd-new-20030616-p10): - security improvements: no shell required in chroot jail, checks performed to see if dropping privileges was successful, can drop privileges before reading config file; - the 'amavisd reload' command reimplemented, it now works even if running chroot-ed; - new feature: policy banks hold sets of configuration variables that may be switched with another predefined set based on incoming port number or original SMTP client IP address, avoiding the need to run more than one instance of amavisd daemon; - new feature: @score_sender_maps is a soft variant of black- and whitelisting; - extended semantics of SQL field wblist.wb for soft black/white-listing; - redesigned mail structure representation allows better control over 'banned' names and types; - MIME defanging can wrap the entire original mail in a MIME container; - more flexible control on lookups: configurable list of lookup tables observes the specified order of tables and permits arbitrary number of tables of any available type; - level-0 logging either by-message or by-recipient; - syslog priorities are now dynamically derived from the log level; - constantly updates a small database of SNMP-style counters, providing real-time measurements for status monitoring and statistics reporting, reducing the need for analyzing a log file; includes a sample/demo program 'amavisd-agent'; - new 'policy delegation protocol' between helper program and the daemon can pass more information to the daemon and allows the daemon to pass more instructions back to MTA (useful for sendmail milter setup); - persistent cache of recent virus and spam checks, common to all child processes, improves the cache hit rate; - support for IPv6 address formats; - provisional/experimental support for DSPAM spam checker; - support for ClamAV virus scanner via Perl module Mail::ClamAV; - cleaned amavisd.conf : amavisd.conf configuration file with the essentials amavisd.conf-default lists all configuration variables with their defaults amavisd.conf-sample traditional-style commented amavisd.conf with examples ---- INCOMPATIBLE CHANGES since amavisd-new-20030616 (any patch level) - requires Perl module BerkeleyDB with libdb version 3.1 or later (tested with db 4.1); This requirement will be made optional at a later date (in amavisd-new-2.1.0); - a directory at $db_home (default /var/amavis/db) must be manually created to store cache and snmp DB files. It should be writable by user running amavisd. The db files within are removed and re-created at each restart to avoid having to deal with db recovery (but need not be, as far as the program logic is concerned); - sending signal HUP in order to restart amavisd no longer works (previously it only worked in non-chrooted environment and relied on guessing amavisd absolute path); please use 'amavisd reload', or 'amavisd stop' and restart; If the HUP method is really still needed, please replace the line commandline => [], # disable by: commandline => ['/usr/local/sbin/amavisd','-c',$config_file], in file amavisd, adjusting the path if necessary. - due to changed names of temporary files, the old 'DrWebD' av entry needs to be replaced with the current one; - changed $final_virus_destiny default from D_BOUNCE to D_DISCARD - changed $final_spam_destiny default from D_REJECT to D_BOUNCE - changed defaults for variables $virus_quarantine_to, $banned_quarantine_to, $bad_header_quarantine_to and $spam_quarantine_to from undef (no quarantine) to values 'virus-quarantine', 'banned-quarantine', 'bad-header-quarantine' and 'spam-quarantine' respectively. Set them to undef or '' (empty string) to disable quarantine; - add address extension at tag2 level, not at kill level as before; suggested by Jacob Elder and others; - because of the reorganization of lookup tables, a new way of quickly disabling virus or spam checks in amavisd.conf is used. Instead of: # @bypass_virus_checks_acl= qw( . ); # uncomment to DISABLE anti-virus code # @bypass_spam_checks_acl = qw( . ); # uncomment to DISABLE anti-spam code the new recipe is: # @bypass_virus_checks_maps = (1); # uncomment to DISABLE anti-virus code # @bypass_spam_checks_maps = (1); # uncomment to DISABLE anti-spam code - @virus_admin_maps and @spam_admin_maps now take as lookup keys recipient addresses, not sender address as before. The new semantics was often requested, the old semantics was not useful because modern viruses and spam fake sender address, so the choice was made to incompatibly change semantics and use the same config variable names, rather than introduce new names and leave behind useless variables; An implication is that with multiple-recipient mail it is now possible to have more than one admin notification generated when recipients have different admin addresses associated. Still, each distinct admin address receives only one admin notification; - removed old compatibility variable $mailto. Use $virus_admin and $spam_admin variables instead, as suggested in amavisd-new-20021116 release notes, or the more recent inventions @virus_admin_maps and @spam_admin_maps, or their equivalents in policy banks; - removed support for old configuration variable $mailfrom. Use variables $mailfrom_notify_admin, $mailfrom_notify_sender, $mailfrom_notify_recip, and $mailfrom_notify_spamadmin as introduced in amavisd-new-20020630 for the purpose; - removed ancient variable @local_domains, use @local_domains_acl instead, or the more general @local_domains_maps - removed old compatibility code which allowed for a couple of traditional variables to treat value "no" as false. The use of "no" for false has been deprecated since amavisd-new-20021116. - revoke an old compatibility measure where a missing (undef) tag2 level would fall back to the kill level value; - lookup_hash incompatible change (but hardly anyone will notice): a key presence in the Perl hash but with undefined hash value used to be interpreted as true, but now it is treated as undef, causing search to continue with remaining lookup tables (if any). This is more general and more in spirit with other lookup mechanisms; - changed a default to initialize SAVI-Perl every time a child process is started, no longer at master process startup time only. This is to avoid the need to restart amavisd every time the Sophos IDE database is changed. One can revert to the previous behaviour by uncommenting a call to Amavis::AV::sophos_savi_init in subroutine fetch_modules_extra; SECURITY - no longer invoke shell to call gzip for compressing quarantined files or to call the sendmail command for submitting messages; the most important consequence is that a shell is no longer needed in a chroot environment and should preferably be removed; - not to forget what was introduced in -p10: inserted a security check for a missing Net::Server patch, and abort if vulnerable; - new command line options '-u user' and '-g group' are available. These are pretty much equivalent to doing a su(1) to the specified user first (in which case the use of these options is redundant). By doing 'su' or by specifying a command-line option '-u username' one can prevent a potential security risk on misconfigured sites where amavisd.conf is writable by UID running amavisd (e.g. not owned by root). If a (non-root) username or UID is specified, privileges are now dropped _before_ opening and evaluating a configuration file. The consequence is that the configuration variables $daemon_user and $daemon_group (in amavisd.conf) can not have an after-effect (a warning is issued if different). If -u is not specified, the behaviour is as before, i.e. the config file is read and evaluated under the current UID (as root unless 'su' was done), and the values of $daemon_user and $daemon_group from the config file are passed to Net::Server, which changes UID during its startup after chroot-ing (if requested). If chroot is desired, the -u must not be used: the root privilege is required to do chroot, and the config file must be read _before_ doing chroot. A case of Catch-22. Be doubly careful of who can modify the configuration file. Another consequence of specifying -u is that any external files (e.g. templates, lookup hashes) as possibly read from amavisd.conf, are now accessed as unprivileged user and no longer as root. The same goes for opening the log file when not logging via syslog. MAJOR NEW FEATURES - policy banks hold sets of configuration variables controlling most of per-message settings, including: static lookup tables, IP interface access rules, forwarding address, log level, templates, administrator addresses, spam trigger levels, quarantine rules, lists of anti-virus scanner entries (or just a subset), banned names rules, defang settings, etc. The whole set of these settings may be replaced with another predefined set based on incoming port number, making it possible for one amavisd daemon to cope with more diverse needs of served user communities which could so far only be implemented by running more than one instance of the amavisd daemon, each with its own configuration file; This mechanism brings new potentials for the future: in principle policy banks could be swapped not only based on port number or SMTP client IP address, but on any characteristics pertaining to a mail message as a whole (not specific to each of its recipients), or to characteristics of a connection from a mailer (e.g. the interface address or protocol); Until a better mechanism is available, a policy bank named 'MYNETS' has special semantics: this policy bank gets loaded whenever MTA supplies a SMTP client IP address (Postfix XFORWARD extension or a new AM.PDP protocol) and that address matches the @mynetworks list. A hash %$interface_policy is a current mechanism of assigning a policy bank to an incoming TCP port number (port must be in the list @$inet_socket_port). Whenever the connection from MTA is received, first a built-in policy bank with an empty name - the $policy_bank{''} gets loaded, which bringings in all the global/legacy settings. Then it is overlaid by whatever configuration settings are in the bank named in the $interface_policy{$port} if any, and finally the bank 'MYNETS' is overlaid if it exists and the SMTP client IP address is known (by XFORWARD SMTP extension command from MTA) and it matches @mynetworks. See amavisd.conf-sample for examples. When a new policy bank is overlaid over an existing set of configuration variables, variables not present in the new policy bank retain their value. The built-in policy bank (with empty name) is predefined, and includes references to most other variables (the dynamic config variables), which are accessed only indirectly through the currently installed policy bank. Overlaying a policy bank with another policy bank may bring in references to entirely different variables, possibly unnamed. Configuration variables are referenced from a built-in policy bank (which is implemented as a perl hash, i.e. an associative array) by keys of the same name, e.g. { log_level => \$log_level, inet_acl => \@inet_acl, ...}. For scalars one level of indirection is allowed, e.g. a policy bank { log_level => \$log_level }; $log_level=2; is equivalent to { log_level => $log_level } or { log_level => 2 }, but in the first example with an indirect reference, the $log_level may be assigned to even _after_ the policy bank has already been formed. A word of caution: the syntax of entries within a policy bank hash is slightly different from assignments to configuration variables. This is because entries within policy bank are not assignments, but key=>value pairs as in any Perl hash. And these pairs are delimited by commas, unlike statements, which are delimited by semicolons. Value is separated from its key by '=>' (or by a comma), whereas the assignment operator is '='. Keys of a policy bank are without leading $ or @ or %, unlike variable names. Values of a hash can only be scalars (e.g. strings or references). Compare: - value of a policy bank is a reference to a Perl hash, e.g.: { log_level => 3, forward_method => 'smtp:[127.0.0.1]:10025', spam_admin_maps => ["spamalert\@$mydomain"], } - normal assignments look like: $log_level = 3; $forward_method = 'smtp:[127.0.0.1]:10025'; @spam_admin_maps = ("spamalert\@$mydomain"); And a final note: Perl can detect and report typing mistakes in variable names, but mistyped key is just some unused hash entry lurking in a hash, never used and never reported as mistyped/useless. - @score_sender_maps is a soft variant of black- and white-listing, contributing positive or negative score points based on sender e-mail address. Btw, the 'score' in the name '@score_sender_maps' is meant as a verb, recipient is scoring a sender (= to grade, to determine the merit of); Whitelisting is becoming deprecated because it is often and easily abused (but blacklisting can still be useful); # ENVELOPE SENDER SOFT-WHITELISTING / SOFT-BLACKLISTING # Instead of hard black- or whitelisting, a softer approach is to add # score points (penalties) to the SA score for mail from certain senders. # Positive points lean towards blacklisting, negative towards whitelisting. # This is much like adding SA rules or using its white/blacklisting, except # that here only envelope sender addresses are considered (not addresses # in a mail header), and that score points can be assigned per-recipient # (or globally), and the assigned penalties are customarily much lower # than the default SA white/blacklisting score. # # The table structure is similar to $per_recip_blacklist_sender_lookup_tables # i.e. the first level key is recipient, pointing to by-sender lookup tables. # The essential difference is that scores from _all_ matching by-recipient # lookups (not just the first that matches) are summed to give the final # score boost. That means that both the site and domain administrators, # as well as the recipient can have a say on the final score. - the MIME defanging (defang: to make harmless or less powerful) wraps the entire original mail in a MIME container of type 'Content-type: multipart/mixed', where the first part is a text/plain with a short explanation, and the second part is a complete original mail, enclosed in a 'Content-type: message/rfc822' MIME part. Defanging is only done when enabled (selectively by malware type) and the malware is allowed to pass (*_lovers or *_destiny=D_PASS). The feature is global, i.e. not available on a per-recipient basis. Conventional mail header fields are retained, and header fields Resent-{From,Date,Message-ID} are added. A header field X-Amavis-Modified is inserted to indicate that the mail body has been modified. Note that defanging changes mail body and makes subsequent DomainKeys/DKIM verifications to fail. The DomainKey-Signature header field is not retained in defanged mail (but is retained in the attached original mail). It is an experimental feature (disabled by default, except for mail bombs), and not very efficient for large mail. MIME defanging is unconditionally done for mail bombs, i.e. when the X-Amavis-Hold header field is added. The text in the first MIME part describes the reason, e.g.: WARNING, possible mail bomb, NOT CHECKED FOR VIRUSES: Exceeded storage quota 29089500 bytes by do_unzip; ... When MIME defanging is enabled for passed spam, the first MIME part contains the full SpamAssassin report. The MIME defanging feature is not available in the sendmail milter setup. - new parameter: @mynetworks It is an IP access list which determines if the original SMTP client IP address belongs to our internal networks. It is much like the Postfix parameter 'mynetworks' in semantics and similar in syntax, and its value should normally match its Postfix counterpart. It affects the value of a macro %l (=sender-is-local), and the loading of policy 'MYNETS' (if present). Note that '-o smtp_send_xforward_command=yes' (or its lmtp counterpart) must be enabled in the Postfix service that feeds amavisd, otherwise client IP address is not available to amavisd-new and new features based on @mynetworks do not work (the %l macro works as before and bases its decision on sender e-mail address matching local_domains); LOGGING - syslog priorities are now dynamically derived from the log level of each log message (the first argument of sub do_log). The priority as specified in the $SYSLOG_LEVEL configuration variable is ignored in 2.0 (no longer in 2.1), but the 'facility' is not ignored. This makes it possible to influence the log verbosity by syslog.conf settings. Here is an example of a useful syslogd.conf setting (some levels may be left out; assumes the $log_level is high enough, e.g. 2, to produce any low-priority log messages at all): mail.err /var/log/messages mail.notice /var/log/amavisd.log mail.info /var/log/amavisd-info.log mail.debug /var/log/amavisd-debug.log (On linux do not forget to prefix filenames in syslog.conf with a '-' to disable sync, which has much impact on syslog performance!) At the moment the mapping of message log levels to syslog priorities is hardwired: level <= -3: LOG_CRIT level <= -2: LOG_ERR level <= -1: LOG_WARNING level <= 0: LOG_NOTICE level <= 1: LOG_INFO (in version 2.0) level <= 2: LOG_INFO (since 2.1) else: LOG_DEBUG Not to be confused with the $log_level configuration variable setting, which still works as before, suppressing generation of all log messages with log levels above $log_level. For efficiency reasons one should not specify unnecessarily high $log_level and then discard low syslog priority messages in the syslogd. - added $log_recip_templ variable, which is similar to $log_templ, but gets called for every recipient (the $log_templ is evaluated only once per message). Normally one or the other log template should be disabled by assigning undef or an empty string to the corresponding variable; - the default $log_templ no longer shows quarantine ID if quarantining is disabled; - added a macro %. (a dot), which might be useful in the $log_recip_templ; Its value is empty when $log_templ is expanded, and is a recipient counter (starting by 1) when $log_recip_templ template is expanded. Based on this macro one can provide a single template for both the $log_templ and the $log_recip_templ if needed, or perhaps let the log entry for the first recipient be more verbose that the rest; - added a macro %T which expands to a list of triggered SA tests, but only when $log_templ and $log_recip_templ are expanded. In notifications the %T is still a list of To: addresses. An overlaid semantics, but we are running out of letters and a macro expander rewrite would be needed; - new macros %k, %1, %2, %O, please see README.customize LOOKUPS - new configuration variables make it more flexible to specify arbitrary list of lookup tables. Legacy configuration variables are still available and are referenced from the default values of @*_maps lists. If these lists are redefined, legacy variables are not used. The new variables (lists) are: @local_domains_maps @bypass_virus_checks_maps @bypass_spam_checks_maps @bypass_banned_checks_maps @bypass_header_checks_maps @virus_lovers_maps @spam_lovers_maps @banned_files_lovers_maps @bad_header_lovers_maps @virus_admin_maps @spam_admin_maps @virus_quarantine_to_maps @banned_quarantine_to_maps @bad_header_quarantine_to_maps @spam_quarantine_to_maps @spam_quarantine_bysender_to_maps @banned_filename_maps @spam_tag_level_maps @spam_tag2_level_maps @spam_kill_level_maps @spam_dsn_cutoff_level_maps @spam_modifies_subj_maps @whitelist_sender_maps @blacklist_sender_maps @score_sender_maps @addr_extension_virus_maps @addr_extension_spam_maps @addr_extension_banned_maps @addr_extension_bad_header_maps @keep_decoded_original_maps @map_full_type_to_short_type_maps @viruses_that_fake_sender_maps - for more informative logging of lookup operations, a new object type Amavis::Lookup::Label can be inserted to lists of lookup tables for the purpose of labeling the main purpose of the list; - all lookup* subroutines can now return matching key when called in a list context; - lookups can now return a list of all matching entries (not just the first match); used for the new soft- white/blacklists (@score_sender_maps); - sub lookup() now allows for one level of list elements dereferencing, which makes possible the construction of the argument list and later still be able to modify its members (e.g. creation of regexp lookup table objects in the configuration file); It facilitates transition from old hard-wired lists of lookup tables to new @*_maps list variables which permits specifying an arbitrary number of lookup tables and to specify their search order; - simplify and unify calls to lookup() by collecting arguments (references to lookup tables) in lists, e.g. @local_domains_maps, @virus_lovers_maps, @virus_admin_maps. These array variables default to lists of legacy variables, which are now never directly used by the program. Either the individual legacy variables may be assigned to, or the entire list replaced, in which case the legacy variables no longer have any effect. - lookup_acl: respect $localpart_is_case_sensitive setting; - lookup_hash and lookup_sql: rewritten lookup_hash and factored out the common code from lookup_hash and lookup_sql to make_query_keys(); - lookup_hash bug fix: avoid splitting address literal as if it were a domain name; (a bug with key '.' not being tried for address literals fixed thanks to Uwe S. Fuerst); SQL, LDAP LOOKUPS - extended semantics of SQL field wblist.wb, which can hold a score value boost, which is interpreted as soft black/white-listing (the same semantics as the value in @score_sender_maps); - recognize SQL server error 2013/"Lost connection to" and treat it the same as 2006/"MySQL server has gone away"; by Max Kalika; - full domain stripping: @.sub.example.com @.example.com @.com @. - lookup_hash and lookup_sql: limit the list of subdomain search keys to 10 levels as a sanity measure; e.g. for address user@14.13.12.11.10.9.8.7.6.5.4.3.2.com the subdomains keys search list starts at .9.8.7.6.5.4.3.2.com; (domain names are limited by standard to 127 levels); - prepare SELECT statements on demand; - recognize all-zero and all-null boolean fields as false; - recognize new (optional) fields in the table 'policy': spam_dsn_cutoff_level virus_quarantine_to, banned_quarantine_to, bad_header_quarantine_to addr_extension_virus, addr_extension_spam, addr_extension_banned, addr_extension_bad_header (the addition of virus_quarantine_to was suggested independently by Harald Kapper and by Dipl.Ing. Martin Boeck); - consider the SQL user id a string (no longer required to be numeric); thanks to Max Kalika / Gentoo support; - LDAP white/black list support by Jacques Supcik (similar to contribution from Scott Dier and Eric Dorland, which I forgot about, sorry); - added amavisSpamQuarantineTo to the LDAP schema; new version of LDAP schema; by Jacques Supcik: In the previous schema, the tag levels have been defined as integer. This is too restrictive, and have now been changed to strings (there is no float type in LDAP); - added LDAP attributes for completeness: amavisBadHeaderLover, amavisBypassBannedChecks, amavisBypassHeaderChecks, amavisVirusQuarantineTo, amavisBannedQuarantineTo, amavisBadHeaderQuarantineTo; by Jérôme Schell; DECODING / DEARCHIVING - provide optional ability to retain complete email message in its un-decoded form (alongside its decoded parts) for a virus scanner to see (enabled if $keep_decoded_original_re matched string 'MAIL'); suggested by Tomasz Papszun; (partly backported to amavisd-new-20030616-p8); - rewrite code that generates new file names (Amavis::Unpackers::Part->new), and rewrite code dealing with banned names. Keep information about each part organized as a tree, matching the descendence of each part, new package/object Amavis::Unpackers::Part to collect such information; - make Amavis::Unpackers an optional module: the interface code to external decoder/unpackers/dearchivers does not get compiled and does not consume virtual memory if $bypass_decode_parts is true; (previously it just didn't get called, but was sitting in memory nevertheless); - decode RPM archives by converting them to cpio, if rpm2cpio and cpio are available; - do_tnef: extract $tnef->message if it exists, not just $tnef->attachments; - support extracting MS cabinet files (CAB) by calling cabextract, if enabled and found. Beware: Lars Hecking warns that cabextract 0.6 is quite buggy and the author has been notified. VIRUS SCANNING - support for ClamAV virus scanner via Perl module Mail::ClamAV, based on code by Roberto Pereira da Rosa; - don't call virus scanners if there are no files in the directory to be scanned (e.g. mail with an empty body); some virus scanners don't like to be given an empty directory (e.g. Symantec newer savsecls); reported by Marco Bicca; - rewritten/unified/generalized subroutines ask_daemon and sophos_savi based on the new subroutine ask_av; - scan parts directory for file names exactly once regardless of the number of virus scanners and their arguments (containing '{}/*' or not); - supply full original mail to virus scanners in case of MIME parse errors (in addition to any possibly decoded parts); - when collecting file names to be virus scanned, prepare a hash which maps base file names to Amavis::Unpackers::Part objects, and make it available to virus scanner interface routines, which may benefit from the additional information about the file to be scanned. In particular, the new interface to Mail::ClamAV now turns on the option CL_MAIL, and the interface to SAVI turns on the option MIME, when entire mail is passed to AV scanner for checking. This enables ClamAV and SAVI to attempt MIME decoding the file by itself. TODO: The same option (--mime) would need to be specified when calling 'clamscan' and supplying a non-decoded mail for checking; pointed out by Riccardo Ghiglianovich and Michael Boelen; OTHER EXTERNALLY VISIBLE CHANGES - The 'amavisd reload' command is now implemented differently: old: signals SIGHUP to a running amavisd process and exits immediately; the running amavisd process (under control of Net::Server) when it receives a SIGHUP starts its own copy with same arguments and exits; new: signals SIGTERM to a running amavisd process, waits for it to finish, then continues (same as 'amavisd start') to become a new daemon; The new method works even when chrooted, and is more reliable when the existing process is slow to terminate, as it actively waits for the previous daemon to finish before proceeding to promote itself to become a new daemon. - a simple demo program 'amavisd-agent' is provided, allowing for continuous inspection of SNMP counters; a path to the /var/amavis/db is hardwired in the program, modify it if necessary. - server-side support for optional Postfix SMTP/LMTP command XFORWARD: information about the original SMTP client IP, its DNS name, HELO name and protocol used is now made available to the amavisd program for logging and other purposes. The same information can also be obtained from the 'Amavis policy delegation protocol (AM.PDP)' if the helper program supports it (useful for sendmail milter setup); - client-side support for optional Postfix SMTP command XFORWARD: if MTA announces in its SMTP EHLO response that it supports XFORWARD, amavisd will provide additional information about the original SMTP client if the information is available (either from XFORWARD on the receiving side, or by the 'Amavis policy delegation protocol'; - server side support for the new amavis helper protocol AM.PDP which allows for header modifications, removal of recipient addresses (e.g. non- spam lovers) or rewrite of recipient addresses (e.g. adding address extensions), and specification of full SMTP response; - modified search logic for matching mail parts against $banned_filename_re; The old search order for names did not result in what one might expect when pattern list elements with a value of false were used in $banned_filename_re. Namely, all three components were searched independently (Content-Type, declared name, and file(1) type) and a logical or was used. Because searches for each mail part were independent, it was not possible to specify for example that anything within a zip would be allowed. If any of these searches returned true, mail was blocked. To make this useful, a complete rewrite of mail unpacking was needed and all information be made available in one place after the unpacking is over, so that checking for banned names can be done all at once. The search order is now much the same as used in rsync and its server, see man rsync, section 'EXCLUDE PATTERNS'. The new comments in amavisd.conf-sample explain the new situation. - replaced $relayhost_is_client by a more flexible specification: To make it possible for several hosts to share one content checking daemon, the IP address and/or the port number in $forward_method and $notify_method may be specified as an asterisk. An asterisk in the colon-separated second field (host) will be replaced by the SMTP client peer address (i.e. the MTA host). An asterisk in the third field (tcp port) will be replaced by the incoming SMTP/LMTP session port number plus one. This obsoletes the previously used less flexible configuration parameter $relayhost_is_client. An example: $forward_method = 'smtp:*:*'; $notify_method = 'smtp:[127.0.0.1]:10025'; The same functionality can also by achieved by using a bigger hammer, the policy banks. These may completely replace the global settings for $forward_method and $notify_method, based on incoming port number; - turn address extension variables (the so called "plus addressing") into recipient-based lookup tables, including the SQL lookups. For example: @addr_extension_virus_maps = ('virus'); # defaults to empty @addr_extension_spam_maps = ('spam'); # defaults to empty @addr_extension_banned_maps = ('banned'); # defaults to empty @addr_extension_bad_header_maps = ('badh'); # defaults to empty or perhaps: @addr_extension_virus_maps = ( {'sub.example.com'=>'infected', '.example.com'=>'malware'}, 'virus' ); suggested by Gentoo modification, Jacques Supcik, and others; - log and report hits and tag/tag2/kill levels rounded to three decimal places (trailing zeroes trimmed), no longer rounded to one decimal place; - added @spam_dsn_cutoff_level_maps, making it possible to specify different DSN cutoff levels for different recipient domains or users. In multi-recipient messages where recipients can specify different values, the maximum value is used for deciding whether DSN should be suppressed; suggested by Ales Casar; - configuration variable $gets_addr_in_quoted_form is no longer used; knowledge about address form (quoted or not) is now implicit in the receiving protocol; - if tag level turns out to be undef, it will not be shown in X-Spam-* header fields, and will be interpreted as having a value lower than any spam score when deciding whether to insert X-Spam-* header fields or not; - added macros %a and %g: * %a original SMTP session client IP address (empty if unknown) * %g original SMTP session client DNS name (empty if unknown) (like macros %I and %M that were once proposed by Dibo in his 2002-07 patch) This information may be available from Postfix when XFORWARD protocol extension to SMTP is enabled, and can be made available by helper program (e.g. from sendmail milter) when the new AM.PDP protocol is used; - added macro %p, expanding to a current policy bank name (or empty if a built-in policy bank is still in place); - added macro %r, expanding to the contents of the first Resent-Message-ID header field, or empty if no such field exists. Include reporting the Resent-Message-ID in the log and in the sender notification; suggested by Oliver Gorwits; - new configuration variable $addr_extension_bad_header for completeness; - added $bad_header_quarantine_to, @bad_header_quarantine_to_maps, $bad_header_quarantine_method, and $warnbadhrecip for completeness; suggested by Robin Lynn Frank; - MIME::Parser errors now contribute to bad-header checks, so that the header checking is now conceptually extended to MIME sub headers (Postfix similarly considers MIME subheaders part of mail headers); MIME::Parser 6.1xx or later is recommended. - allow $*_quarantine_method to be undef as a quick way of disabling some quarantine (it also can be disabled as previously, by using method 'local:' and following its rules); - persistent cache of recent virus and spam checks, global to all child processes, can improve the cache hit rate. Uses BerkeleyDB database (hash and queue) and its interlocking mechanisms (Berkeley DB Concurrent Data Store) for the purpose. The V3.1 or better is required, V4 is recommended. - include version information in the 'Usage' text; - rewritten lookup_ip_acl() and added ip_to_vec() to allow for IPv6 address syntax as specified in rfc3513 to be used in IP lookups; - @inet_acl now defaults to ('127.0.0.1', '::1'), i.e. it adds the IPv6 loopback address to the list; - new configuration variable $sa_spam_level_char (defaults to '*') allows specifying another character for X-Spam-Level bar. Empty or undefined value disables inserting the X-Spam-Level header field; - added configuration variable $sa_spam_report_header to enable/disable inserting the X-Spam-Report header; patch by Craig Sanders; - added $banned_quarantine_to configuration setting to make possible the quarantining of banned mail to a different place from viruses; - don't insert virus-, banned- and bad headers- related headers for passed mail to recipients with corresponding bypass_*_checks, making them believe the mail was not spam-checked (as they are not expecting such headers anyway); This was already done in version amavisd-new-20030616-p6 but only for spam-related headers; - for choosing address extensions use the same criteria as for adding header fields, e.g. pretend to not know the result of a certain test (virus, spam, ...) when recipient chooses to bypass such test, even if the result of the test is known; - added variable $sa_spam_subject_tag1 (undef by default). If $sa_spam_subject_tag will not be inserted (at tag2 level), and $sa_spam_subject_tag1 is nonempty, this string (e.g. '***possible SPAM*** ') will be inserted into the Subject header field for spam levels above tag level; suggested by Immo Goltz; - added separate configuration variables $banned_files_quarantine_method and $bad_header_quarantine_method. Quarantining of banned files and bad headers were previously controlled by $virus_quarantine_method; - rewritten read_hash, it is now possible to specify key value (right-hand side) for each key. If value is not specified, a '1' is assumed as before; - SMTP server support for rfc2554 authentications (PLAIN and LOGIN only) allows client authentication to be relayed to the MTA when message is forwarded. Might be useful if amavisd-new is used in a Postfix SMTP proxy setup, but is not needed for other setups. Disabled by default, see variables $auth_required_inp and @auth_mech_avail. - SMTP client support for rfc2554 authentications (any authentication method as supported by Net::SMTP and Authen::SASL Perl modules). Authentication of forwarded mail (PLAIN and LOGIN only) is carried over from the incoming mail, authentication to be used when submitting notifications is controlled by configuration variables $amavis_auth_user and $amavis_auth_pass. Disabled by default, see variable $auth_supported_out (later renamed to $auth_required_out). - when passing envelope sender address to SpamAssassin, supply it as a rfc2822-standard header field Return-Path, and no longer as X-Envelope-From (the change came with a pre-release amavisd-new-20040301). - provisional/experimental support for DSPAM spam checker (pre 3.0): if configuration variable $dspam is nonempty and represents a path to a 'dspam' program, a message is passed to dspam and its inserted headers of the form X-DSPAM-* are axtracted and then made available for SpamAssassin rules to check and score if desired. All messages are currently presented to dspam as the same user, affecting how its database is being built. False negatives and false positives (based on SA assessment) are fed back into DSPAM as a simple form of auto-learning. Works reasonably well, but do not expect miracles. See subroutine spam_scan. Here is how DSPAM can be installed to be able to be used by amavisd-new: dspam 2.x: ./configure --enable-alternative-bayesian \ --with-userdir=/var/amavis/dspam --enable-signature-headers \ --without-local-delivery-agent --without-quarantine-agent dspam 3.0.0: ./configure \ --with-dspam-home=/var/amavis/dspam --enable-signature-headers \ --without-delivery-agent --without-quarantine-agent make install chmod u-s,a+rx /usr/local/bin/dspam chown vscan:vscan /var/amavis/dspam User 'vscan' may need to be added to DSPAM trusted.users file. The following can be inserted into the SA config file (local.cf) to make it recognize and incorporate DSPAM's assessment: header DSPAM_SPAM X-DSPAM-Result =~ /^Spam$/m describe DSPAM_SPAM DSPAM claims it is spam score DSPAM_SPAM 0.5 header DSPAM_HAM X-DSPAM-Result =~ /^Innocent$/m describe DSPAM_HAM DSPAM claims it is ham score DSPAM_HAM -0.1 Eventually DSPAM support should be removed from amavisd-new, as soon as SA will be able to call it on its own. INTERNAL CHANGES - reformatted the whole program, reducing indentation from 4 to 2 and replacing TABs with spaces (with some dubious help from perltidy, plus lots of manual adjustments); - completely rewritten code to handle both the old and the new amavis helper protocol, as well as Postfix 'TCP client/server table lookup protocol' as specified in the Postfix documentation: tcp_table(5); (process_policy_request, prepare_policy_query, check_amcl_policy) - type_short may now be a list of short types, not necessarily just a single value. Typical use is to classify a MS executable as both an 'exe' and as 'exe-ms', which makes more specific banned rules possible without unnecessary complication in regexps; - parts now carry attributes, which can be inspected for banned checks; current attributes are U for undecodable, and C for (en)crypted; - opened another can of Perl worms (taint bugs): turn on Perl pragma "use re 'taint'" in all modules, and selectively turn it off where needed. It replaces cumbersome manual preservation of taintedness when regexp saved ranges are used without intention to untaint. Because of Perl bugs, strategically placed local($1,$2,...) are needed, otherwise previous taint flag in $1, $2, ... can be brought on to new variables, which can all of a sudden become tainted out of nowhere; - catch and report throws (die) in pre_loop_hook() to properly report problems during initialization; - introduced subroutine exit_status_str and unify reporting of subprocess status; - enhanced sub best_try_originator to ignore IP addresses from private, local and dedicated IANA networks (rfc3330) and look for the first public address in the 'Received' path; - examine first four (chronologically) Received header fields (instead of first two) when looking for an originator IP address, and ignore those with private IP addresses; - moved code dealing with body digest cache to a new package Amavis::Cache to facilitate transition to shared or persistent cache; - new explicit cache expiration time controls (time to live in seconds): $virus_check_negative_ttl $virus_check_positive_ttl $spam_check_negative_ttl $spam_check_positive_ttl $cache_entry_ttl - discard cache db ($db_home) and recreate it during restart; - more informative changes of child process $0, which may show in the ps(1) output; - store tempdir of a current message to the Amavis::In::Message object; - gather some statistics about idle time; - reorder and adjust mapping from file(1) results to file type classes; - optimization: instead of invoking file(1) utility program for each part to be analyzed, now call it once for each round of currently available parts, giving it the list of all available parts as arguments; - shorten the names of parts from part-..... to p..., to be able to stash more files names into a command line, e.g. when calling file(1) or external virus checker which can not deal with directories; - use regexp lookup table mechanism (table $map_full_type_to_short_type) to match long types (output of file(1) utility) to short types (.exe, .jpg, .doc, ...). The default table can be replaced by a customized table in amavisd.conf; - replace $(?!\n) with \z in regular expressions throughout; replace most of the remaining $ with \z in regular expressions. The regular expression primitive \z is available since perl 5.005. - TODO: disregard $MAXFILES during initial MIME unpacking; reported by Stephane Lentz and Robert LeBlanc (done in 2.3.2); Some un-edited notes on the new banned rules mechanism: (wrapped log lines, and replaced \\ by \ for clarity: | Feb 24 19:07:29 hauptpostamt amavis[29847]: (29847-04-5) p.path BANNED: | | "P=p002,M=application/octet-stream,T=zip,N=document.zip | | P=p003,T=exe,T=exe-ms,N=document.htm .scr", part p003 is of type (file(1)) MS executable, with suggested name "document.htm .scr" (lots of spaces in the name) its parent resides on temp file p002 (i.e. p003 was extracted from it), which is of type (T) zip archive, with suggested (MIME) name (N) "document.zip", and has a MIME type (M) "application/octet-stream". such a component p003 lying within such p002 is considered banned by the following regexp rule (one rule within the $banned_namepath_re list): | matching_key="(?mix-s:^ (.*\t)? N= [^\t\n]* \. [^./\t\n]* \. | (exe|vbs|pif|scr|bat|cmd|com|dll) (\t.*)? $)" which says that any component at any level must not have a name (N) matching a pattern: any number of characters, a dot, any number of non-dot and non-slash characters a dot, and ending with: exe or vbs ... (basically: double extension ending with listed extensions) The complications such as using [^\t\n]* instead of .* are there to keep regexp contained within fields and ancestors/descendents. There is one detail to remember when comparing logged p.path log entries and the actual matching rules: - for the sake of readability the logged entry has \n (newlines) converted into ' | '. The \n is a separator between components in the tree from the root (the mail itself, hidden) to the leaf component which can not be further expanded (i.e. not an archive) - for the sake of readability the logged entry has \t (a tab) converted into comma, separating information fields such as P=... M=.. T=.. M=.. So the above logged string: P=p002,M=appl...,T=zip,N=document.zip | P=p003,T=exe,T=exe-ms,N=document.htm.scr is actually a single string: P=p002\tM=appl...\tT=zip\tN=document.zip\nP=p003\tT=exe\tT=exe-ms\tN=document.htm.scr and a Perl regexp is applied directly to it. The raw string is rather unsightly, but the \n and \t were chosen to minimize clash with valid characters within file names. If a \n or \t is present in a name of the components, such character is converted into a space to avoid clashing with separators. | Feb 24 19:11:58 hauptpostamt amavis[31505]: (31505-01-5) p.path BANNED: | "P=p002,M=application/octet-stream,T=zip,N=paypal.zip | | P=p003,T=exe,T=exe-ms,N=paypal.scr", a MS executable named "paypal.scr" within a zip archive "paypal.zip" | matching_key="(?mix-s:^ (.*\t)? N= [^\t\n]* \. | (exe|vbs|pif|scr|bat|com) (\t.*)? $)" block component at any level with a name (N) terminating by dot followed by any of the listed extensions. | Feb 24 19:18:25 hauptpostamt amavis[32159]: (32159-01-2) p.path BANNED: | "P=p002,M=application/octet-stream,T=zip,N=text.zip | | P=p003,T=exe,T=exe-ms,N=text.txt .exe", a MS executable named "text.txt .exe" (with lots of spaces in the name) within a zip archive named "text.zip" | matching_key="(?mix-s:^ (.*\t)? N= [^\t\n]* \. [^.\t\n]* \.\n | (exe|vbs|pif|scr|bat|cmd|com|dll) (\t.*)? $)" blocked by the double-extension rule. | Feb 24 19:30:15 hauptpostamt amavis[1690]: (01690-02-8) p.path BANNED: | "P=p002,M=application/octet-stream,T=zip,N=jokes.zip | | P=p003,T=exe,T=exe-ms,N=jokes.doc .exe", | matching_key="(?mix-s:^ (.*\t)? N= [^\t\n]* \. [^.\t\n]* \.\n | (exe|vbs|pif|scr|bat|cmd|com|dll) (\t.*)? $)" same thing --------------------------- --------------------------------------------------------------------------- June 29, 2004 Patch: amavisd-new-20030616-p10 - insert a security check to test for a missing Net::Server patch, and abort if vulnerable; - provide and use our own subroutine q_encode to do the Q-encoding when editing an existing invalid header field with non-encoded 8-bit characters, e.g. when inserting ***SPAM*** or ***UNCHECKED*** into Subject. The MIME::Words::encode_mimeword() does not encode spaces and does not limit encoded words to 75 characters, which violates the RFC 2047 and breaks mail readers; reported by Sebastian Hagedorn and Gregor Hoffleit; - fixed a bug in inserting the tag_level header field, which was missing if sender was blacklisted and tag_level was greater or equal to 0; thanks to Joerg Thaler; - amavis-milter.c * log envelope sender address at the same log level (DBG_INFO) as recipient addresses; * remove the log message "(mlfi_eom) header already present", it was inappropriate, the call to smfi_chgheader succeeds even if no such header was already present; * relax permissions on created directory and temporary file to allow group read access (needed if virus scanner runs under a different user id but within the same group); by Adam C. Migus; * add queue id (when available) to most log messages; - do not preserve evidence just because a message gets an X-Amavis-Hold header field; - when logging directly to a file and started as root, change UID of a log file to $daemon_user to avoid restart problems; based on a patch by Carsten Hoeger and Gregor Weiss; - added configuration variable $first_infected_stops_scan to stop anti-virus scanning when the first scanner detects a virus; the default is false, all scanners in a group are called (as usual); - in the AV entry for clamscan added the option '--tempdir=$TEMPBASE'; - in the AV entry for 'Norman Virus Control v5 / Linux' changed the command name 'nvccmd' into 'nvcc'; correction by Michael Ramke of the Norman Data Defense Systems GmbH; - insert debug reports into sub ip_addr_to_name to help recognize DNS problems; - revoke deleting an existing 'X-Scanned-By' header field, which was introduced in -p8; removing it gets in a way when more than one content filter is being chained; - load a missing SpamAssassin v3.0 module, needed when running in chroot; - contributed (for now in the form of an optional patch): support for the incoming qmqpqq protocol over a TCP socket - to be used with qmail. Apply the provided patch 'amavisd-new-qmqpqq.patch', it updates files amavisd.conf and amavisd in the current directory; by Martin Solciansky; - documentation updates; --------------------------------------------------------------------------- April 2, 2004 Patch: amavisd-new-20030616-p9 - avoid choking on undefined $banned_filename_re; thanks to Ales Casar, Sebastian Hagedorn and E. Falk; - if Subject mail header field got $undecipherable_subject_tag inserted, it would also receive the spam tag $sa_spam_subject_tag unconditionally; fixed, thanks to Francis Stevens; - updates to the @av_scanners list in amavisd.conf: * DrWebD now works with a new DrWeb daemon 4.31, thanks to information provided by Krzysztof Cegielski, DrWeb Polska; * updated BitDefender bdc, thanks to Alfredo Milani Comparetti; * clamscan: use documented option --no-summary instead of the undocumented/old(?) option --disable-summary; by Georgy Salnikov; * updated Kaspersky aveclient and F-Secure fsav, thanks to Tomi Hakala; * recognize that KasperskyLab antivirus in demo mode turns on the bit 0x10 in status; avp and avpdc now use the same two sets of status codes; avp: statuses 3 and 6 moved to infected (to match avpdc); statuses 2 and 5 considered infected; all suggested by Georgy Salnikov; - when original undecoded mail is to be kept for virus scanners (requested by patterns /^MAIL$/ or /^MAIL-UNDECIPHERABLE$/ in $banned_filename_re), the preserved file is now named parts/part-00000 instead of parts/email.txt to preserve the size of the name, upon which some virus scanners depend (e.g. DrWebD); A note to Courier users: due to the way a file name in Courier is created and passed to amavisd-new, it is currently not possible to use the triggers /^MAIL$/ and /^MAIL-UNDECIPHERABLE$/ in $keep_decoded_original_re, or a failure to create a hard link occurs; Thanks to Bowie Bailey for helping to troubleshoot the problem; (this limitation is lifted in amavisd-new-2.2.0) - remove option -c when calling gzip, bzip2, compress, lzop and unfreeze (as has long been done in the development version); the option -c is not needed when no file argument is present, and some implementations of gzip and compress may choke on it; - make loading of Perl module Carp::Heavy optional, versions of Carp that came with Perl 5.005 did not have it; reported by Jefferson Pizzolatti; - load module Mail::SpamAssassin::BayesStore::DBM, required by SA 3.0 running in chroot jail; - look for program 'gcpio' ( $cpio = ['gcpio','cpio'] ), as on OpenBSD the plain cpio does not support required options (the --no-absolute-filenames is essential), but GNU cpio does; thanks to Manfred Gloiber; - relax parsing of file(1) output to allow tab as well as space to follow a file name; the file(1) on Solaris uses tab instead of space; suggested by Glen Harris; - make LHA understand self-extracting archives (SFX); patch by Georgy Salnikov: Although lha unpacks any non-SFX archive independently if its extension, it unpacks the SFX lha archives only if they have the extension .exe. Now the file is symlinked to $part.exe for checking by lha: this will now work for SFX, and will still work for non-SFX. Also, do_lha will return 0 if $exec and the part cannot be de-archived, the same as in do_unrar. The executable file formats are checked for being zip/rar/lha SFX archives. However, if the SFX archive is not a zip archive, do_unrar always returns success, so that if the archive is also not a rar, it will be never checked by lha. Now do_unrar returns 0 if $exec and the corresponding part can not be un-RARed; - do_executable and do_unarj: added checking for SFX arj; by Georgy Salnikov: (commented out the call to do_unarj in do_executable until more experience is gained on how well unarj survives certain mail contents; Mark); - do_unarj: let arj/unarj work on file named part*.arj; - when calling IO::File::open() use '+>' instead of 'w+' to avoid Perl taint bug ($mode turns tainted) (bug still present in 5.8.2) triggered by expression in IO::Handle::_open_mode_string(); - attack the Perl 5.8.0/5.8.1/5.8.3 taint bug (once variables $1,$2,etc get tainted they start spreading taintedness to other variables): * insert local($1,$2,$3,...) in blocks of code which call external modules which trigger the bug (Mail::SpamAssasin, MIME::Parser, ...) * insert local($1,$2,$3,...) in blocks of code which depend on these variables to be clean, and which demonstrated through bug reports and experience with various version of Perl that these variables were not always taint-clean; The last taint incident triggered by SA 3.0.0 (svn) reported by Luc de Louw; - recognize status EX_NOUSER when forwarding via pipe to sendmail (old setup); previously it was treated as a temporary failure; patch by The Mindflayer; - turn error message 'error reading from client socket' back into 'client broke the connection without a QUIT' for consistency with P7; - when %virus_admin lookup table is used, prefer $msginfo->sender_source (unmangled sender domain) over $msginfo->sender; suggested by Pawel Golaszewski; - nicety: in virus recipient notifications now supply the To: header field with the true recipient address instead of "undisclosed-recipients:;" in case of single-recipient mail; suggested by several people; - amavisd would insert differently capitalized header fields (either X-AMaViS-Alert or X-Amavis-Alert), depending on the reason being reported; now use the X-Amavis-Alert throughout; reported by Carsten Hoeger; - fetch Perl auto-loaded modules auto::POSIX::setgid and auto::POSIX::setuid if they exist; they are needed by older versions of Perl when running in chroot. A manual change was needed until now (documented in README.chroot), which should no longer be needed; a FreeBSD problem report 64636. - helper program amavis-milter.c (used in the sendmail milter setup): * amavis-milter.c only included the client IP and client host name on the first mail transaction of a multiple-transaction SMTP session, but not in subsequent transactions; pointed out by Stephane Lentz. The solution was once already provided back in July 2002 by Radoslav Dibarbora - Dibo, and forgotten; credits where credits are due: http://marc.theaimsgroup.com/?l=amavis-tech&m=102569946910555 * make a clear distinction between message data and connection data, which required code reshuffling and revealed previous unclean solutions; * add error checking and reporting to mkdir/rmdir/open/unlink/write system calls; previously an error could pass by unnoticed or just caused a tempfail without an explanation; * change final milter status ACCEPT into CONTINUE to allow further milters in the milter chain to examine the mail; * code cleanup; * regenerated helper-progs/configure with Autoconf 2.57 to make it capable of detecting mkdtemp and still be able to find sendmail libmilter files; thanks to Sebastian Hagedorn for a problem report, for testing, and for revealing a bug in the pre-released version; * use different syslog priority for different internal message log levels; by Sebastian Hagedorn; * adjust log levels of messages, set default verbosity 1 (DBG_WARN); - documentation updates, including the updated comments in amavisd.conf regarding whitelisted senders, to reflect the change indicated in the amavisd-new-20030616-p4 release notes; --------------------------------------------------------------------------- March 9, 2004 Patch: amavisd-new-20030616-p8 - be compatible with SpamAssassin version 2.70 and 3.0 as well as 2.6x; SA changed its API, replacing Mail::SpamAssassin::NoMailAudit with $spamassassin_obj->parse and belonging objects; - as a stop-gap solution to the W32/Bagle-{F,...} detection problem (password-protected zip archives), three new measures are available: * ability to present the full non-decoded original message to virus scanners was partly back-ported from the development version. Enabled by adding qr'^MAIL$' or qr'^MAIL-UNDECIPHERABLE$' to the list in $keep_decoded_original_re, as illustrated in amavisd.conf; similar to a patch by Ted Cabeen. The following keys are used for a lookup into $keep_decoded_original_re: 'MAIL' always provide a full original message to virus scanners (besides its successfully decoded components); 'MAIL-UNDECIPHERABLE' same as for 'MAIL', but only if it contains undecipherable components such as password protected archive members, unsupported compression methods or encrypted parts (e.g. with PGP). Don't put too much trust into this, as some more exotic file formats may not be understood and not flagged as undecipherable; * a key 'UNDECIPHERABLE' is matched against $banned_filename_re when mail contains any undecipherable components, and if lookup returns true, mail will be banned. For example: $banned_filename_re = new_RE( qr'^UNDECIPHERABLE$', # contains any undecipherable components ... ); * a string can be prepended to Subject (for local recipients only) if mail could not be decoded or checked entirely, e.g. due to password-protected archives or non-decodable mail bombs: $undecipherable_subject_tag = '***UNCHECKED*** '; # undef disables it NOTE: this solution is a quick-fix response to popular demand. Although the same or similar functionality will probably remain in future versions, the syntax and exact semantics may be refined. - bring do_unarj in line with the rest of de-archivers: provide the same degree of mail bomb protection that was available in do_unrar; retain original archive for inspection by a virus scanner if it contains any members that can not be extracted (e.g. password protected members or unsupported compression schemes). (The .arj size checking deficiency was mentioned in the AMaViS Security Announcement (ASA) 2004-01-19); - don't call a virus scanner if there are no files (no mail parts) to scan; it caused problems with certain scanners like aveclient, which expect a list of file names as arguments and complain if the list is empty; reported by Daniel Luttermann; - a much needed feature: can specify $sa_dsn_cutoff_level in amavisd.conf to suppress sending a DSN (delivery status notification) when spam level is above this value, effectively turning D_BOUNCE into D_DISCARD for this message; undef disables the feature and is a default. A good first approximation value is 10, or with some risk go down to 8. The parameter has no effect if DSNs are already disabled (e.g. when $final_spam_destiny is D_DISCARD or D_REJECT); - do_unrar: double check the archive size (against summary line as well); - derive the value of macro %l from sender_source instead of from a declared sender address, so as to not rise false alarms when sender address is known to be faked; patch by Joerg Friedrich; - reduce the number of retries to connect to a daemonized virus scanner from 3 to 2, so that a fallback to backup scanners occurs sooner; - updated $banned_filename_re example in amavisd.conf, modify it to will; - updates to the @av_scanners list in amavisd.conf: * KasperskyLab AVP - aveclient: *IMPORTANT* updated entry for the exit status only reflects the result of the last file scanned, we must use regular expression to detect viruses; fix provided by Andreas Triller; * Trend Micro FileScanner - vscan: added option -za, otherwise some broken archive may sneak-in a virus; suggested by Stephane Lentz; * Sophos sweep: added options -cab -tnef --no-reset-atime; * Dr.Web command line scanner: updated options, recognize exit status when using evaluation license; * ClamAV clamd: change socket location to a more usual value, adjust to will; - amavisd.conf: added new virus names to the $viruses_that_fake_sender_re list, and uncommented the [qr/.*/=>1] line, so that by default any unknown virus will be treated as a sender-faking virus; adjust to will; - do not send bad header notifications in response to messages from mailing lists; - added header check for folded header field lines made up entirely of whitespace (a 'header space gap' violation to rfc2822); a check is only made when other header checks (bad character in header) are enabled; - distinguish an empty string from undef in $mailfrom_to_quarantine, making it possible to specify a null return path when quarantining to a mailbox; - helper-progs/amavis-milter.c (sendmail) enhancement based on work by Stephane Lentz: * the name of a temporary directory is derived from the sendmail queue ID, making it easier to match sendmail and amavisd-new log entries; the queue ID is also a part of the quarantine file name; * a phantom 'Received:' header field is prepended on the temporary file to preserve the information on the original SMTP client IP address, host name and queue ID. This trace header field does not propagate to recipients (is not inserted into the original message), but is available in the quarantined messages and is visible to SpamAssassin. Adding a milter macro {b} on ENVFROM is advised to preserve the MTA timestamp in the log, although not mandatory (falls back to current time): define(`confMILTER_MACROS_ENVFROM', confMILTER_MACROS_ENVFROM``, {b}'')dnl If the new amavis-milter.c gives you trouble, switch the soft link to the previous version in the same directory. Keeping macro {b} does not hurt. - catch possible errors in pre_loop_hook and report them properly; - better check for I/O errors on SMTP input socket; - add the following modules to the list of pre-fetched modules: Carp::Heavy, IO::Handle, IO::Socket::UNIX, IO::Socket::INET. The absence of Carp::Heavy could mute error report or a backtrace when running chroot-ed; the rest are for completeness only; - added a macro %z which expands to the original mail size (in bytes); it could be useful in the $log_templ; thanks to Nick Leverton; - make SpamAssassin timeout value configurable: variable $sa_timeout; by Henrique M. Holschuh (Debian); - 'Received:' header field cosmetics: use 'unknown' in case the HELO argument was empty, and put-in a 'unix socket' if message was received from a helper program; by Henrique M. Holschuh (Debian); - remove more pre-existing X-Spam* header fields from other scanners: X-Spam-Tests, X-Scanned-By; by Henrique M. Holschuh (Debian); - helper-progs/amavis.c updates (old sendmail setup): report errors, log to syslog, change default dir to something obvious for bug reporting; by Henrique M. Holschuh (Debian); - updated documentation; --------------------------------------------------------------------------- January 5, 2004 Patch: amavisd-new-20030616-p7 - do_unzip and do_unrar: retain an archive if any of its components is password protected or encrypted (plus unpack what can be unpacked, as before). This gives virus checkers a chance to examine the original unpacked archive as a whole (e.g. scanning for variants of W32/Mimail), matching it in non-decoded form against virus patterns even if containing password-protected components. As a consequence, some virus scanners may now log their complaint when encountering such protected archives which previously didn't reach them. Such log entries should be considered informational only; - add module Net::DNS::RR::AAAA to the list of Perl modules to be fetched before chroot takes place; thanks to Per olof Ljungmark; - preload Perl modules DBD::*, based on @lookup_sql_dsn; required when using SQL lookups from a chroot jail; - updated example dsn in @lookup_sql_dsn (file amavisd.conf) to use the new DBD::mysql syntax, and to show how to force accessing SQL server via inet socket which makes it easier to use from a chroot jail; - disregard cached spam results for mail with small body. The most pronounced undesirable effect was in mail with an empty body where spam score was derived from header only; - change the default value for local_domains_sql lookup for the catchall key '@.' under conditions: when using SQL lookups and user record with key '@.' is present in the database and a field 'local' is not present. Previously it surprisingly defaulted to true, now it falls back to static lookup table defaults, the same as if the record '@.' were not present in the table; - fixed ugly text formatting in recipient notifications template; reported by Florian Effenberger; - fixed parsing of 'Received:' header field for some unusual cases; - updated amavisd-new-courier.patch to apply cleanly against -P6 (now P7); by Martin Orr; - added av_scanners entry for 'AVG Anti-Virus', kindly provided by Grisoft s.r.o. from Czech Republic, http://www.grisoft.com/ - make Dr.Web Daemon av entry work with evaluation or regular license; thanks to Andrew I Baznikin; - added av_scanners entry for new AVP client (aveclient) that is shipped whith avp 5.x.x.x. Moved kavscanner to the @av_scanners_backup list (it is presumably slow and less reliable than aveclient), updated other AVP/Kaspersky entries; thanks to Nabil Sefrioui; - added status codes 10 and 15 to the list of ok statuses for kavscanner (10=Password-protected archives, 15=Corrupted files); - add an example dummy virus scanner 'all-clean' to the @av_scanners_backup list which always succeeds, always returning false (= status clean). Uncomment it if desired to avoid mail requeue when all other scanners fail, and to just pass the mail unchecked; - disregard rfc2821 recommendation that 552 smtp response code should be treated as 452. It is unnecessary in amavisd-new setups, and it is wrong because 552 has other meanings assigned to it besides "too many recipients"; - allow IP address in $forward_method and $notify_method to be bracketed, which is needed for IPv6 addresses containing colons. Both formats are now allowed: 'smtp:127.0.0.1:10025' and 'smtp:[127.0.0.1]:10025', the later is now preferred; - change IP address bracketing in log entries 'SEND via SMTP' and 'FWD via SMTP' from [127.0.0.1:10025] into [127.0.0.1]:10025 to match the syntax in the connect log entry and to facilitate parsing of IPv6 addresses from the log; - update received_line() to generate valid Received header field even for IPv6 client addresses; - modified helper program amavis.c to allow it to be run non-root and to set temp file mode for group accessibility. Change its default log level to DBG_WARN; - fix an incorrect SELECT example in README.lookups; thanks to Nabil Sefrioui; - documentation updates; --------------------------------------------------------------------------- November 10, 2003 Patch: amavisd-new-20030616-p6 - change SQL lookup code to better handle SQL database server restarts. After an SQL server restart amavisd-new would previously TEMPFAIL (4xx) all messages until amavisd child process would run down, then resume normal operation with the new child birth. Now the SQL server reconnect is done when the next mail arrives, so only one mail with each amavisd child process TEMPFAILs during SQL server restart, and normal operation resumes faster; - in flatten_and_tidy_dir and in do_ascii: sanitize protection of files and directories which may otherwise be made inaccessible to virus scanners; based on patch by Henrique de Moraes Holschuh (Debian amavisd-new support); problem reported by Aspa and by Tomasz Papszun; - fix a potential security problem: don't let rmdir_recursively and rmdir_flat follow symbolic links; this might be exploited by attempting to delete some foreign file using privileges of the amavisd process (which should not be root); - use cpio option -d when unpacking cpio archives; - don't insert spam-related headers for passed mail to recipients with bypass_spam_checks, making them believe the mail was not spam-checked (as they are not expecting such headers anyway); - text added to the banned mail sender notification template (at the end of file amavisd), explaining to the sender what happened and how to avoid the restriction; edit to will; - amavis-milter.c program in the helper-progs subdirectory is now based on the most recent version 1.1.2.3.2.40 from the AMaViS CVS (maintained by Lars Hecking), but hacked a bit to make its options mostly compatible with the previous version. Start it with option -h to get current usage text. See helper-progs/README if using sendmail milter setup; Please revert to the old one if the .40 gives you trouble; both version are included in the helper-progs subdirectory. - fix parsing of unrar info lines; - consistency with other virus/banned logic: don't send recipient notification (reporting banned name) if mail contains both a virus and a banned name, but $warnvirusrecip is false; reported by Nathan G. Grennan and Urška (Brinar) Mlakar; - check for possible I/O errors when reading from SMTP socket, and distinguish error condition from normal TCP session teardown; - do not redirect stderr to /dev/null when calling file(1). This way the diagnostics from file(1) will now at least be visible in the debug session; - determine_file_types: make 'bzip compressed' pattern match older bzip format (v1) as well; thanks to Davaeron; - sub run_command: explicitly close STDERR before reopening it; this way the reporting of possible problems in each operation would be separate, and it seems to avoid a rare problem (open STDERR '/dev/null' failing) reported by Sam Hart; - don't attempt to do lookups in regexp table $viruses_that_fake_sender_re if it is undefined; reported by Chris Paul; - remove existing X-Spam-Score along with other X-Spam* header fields if spam scanning is enabled; - do not skip inserting 'X-Spam-Status: No, hits=- ... WHITELISTED' for whitelisted senders which caused SA to be bypassed - artificially assume score is -10 for the purpose of comparing it to tag_level; reported by Mike Vanecek; - explicitly qualify wblist.rid field in $sql_select_white_black_list just in case someone has a field 'rid' in the other table; - fix showing the value of LC_TYPE environment variable in the log, show LC_CTYPE as well; - SAVI-perl: remove MIME option from the default set of options. Even with recent versions of Sophos SAVI there are cases where the library goes into a spin while trying to decode broken MIME message (same applies to Sophie - one may want to change its configuration); - updated vfind entry in @av_scanners to work with the new version, changing '--vexit {}' to '--vexit {}/*'; thanks to Lowell Filak; - updated kavdaemon entry in amavisd.conf; thanks to Michael Hall and Daniel Melanchthon; - added option '-packed' to 'FRISK F-Prot Daemon' entry; thanks to Manfred Gloiber; - NAI uvscan entry: commented pre/post actions in the entry to show how to remove environment variable LD_PRELOAD after finishing, suggested by David Tilley; added option --mime, suggested by Max and Kevin W. Gagel; added note on how to treat password-protected files as viruses, by Seth Parker; - added 'Dr.Web Daemon' entry which talks directly to Dr.Web daemon over its Unix socket, speeding up a single-file check more than 200 times; provided by Andrew I Baznikin; - another entry for Symantec AntiVirus Scan Engine provided by Guido R. Rolon A.; I'm not sure which is which, check your documentation; - added 'dumaru', 'parite', 'gibe' and some other virus names to $viruses_that_fake_sender_re; - add examples to $viruses_that_fake_sender_re to show how to make a default result true, and only list exceptions; - placed a comment in amavisd.conf pointing out the proper syntax for $hdrfrom_notify_* variables; thanks to Wouter de Jong; - bump up the size of $sa_mail_body_size_limit to 150 kB in amavisd.conf - documentation updates; fixed typos and spelling mistakes in the documentation files and in amavisd.conf comments; - new documentation file README.protocol, specifying the new (and the old) protocol between helper programs and amavisd daemon, to be made available with the next major release. The description of the current (traditional) protocol was contributed by Stephane Lentz. --------------------------------------------------------------------------- August 25, 2003 Patch: amavisd-new-20030616-p5 - fix 'Modification of non-creatable array value attempted' bug when no 'Received' header field was present in an infected mail; reported by Paul Miner; - caching of SQL lookups on white/blacklist was based on sender address only, instead of sender _and_ recipient. This could lead to white-/blacklisting of one recipient to affect other recipients of the same message; reported and debugged by Paul Gamble; - added LDFLAGS to helper-progs/Makefile.in. It is needed at least on NetBSD. Patch by Julian C. Dunn (the NetBSD package maintainer for amavisd-new); - more obvious logging of HOLD reason in sendmail/milter setup (reported by Pascal Martinez); - $MAXLEVELS zero or undef should disable the limit according to docs, but was not honoured; reported by Rob Hutton; - if notifications delivery encounters a temporary failure (4xx), propagate this status to the final result instead of only logging a warning; - amavisd.conf: * VirusBuster entry changed to match newer version of the product; information from Marcus Schopen; * another entry in amavisd.conf for KasperskyLab kavscanner (v4.5?), contributed by Simone Marx; --------------------------------------------------------------------------- August 12, 2003 Patch: amavisd-new-20030616-p4 - revert to using alarm() instead of Time::HiRes::alarm(). It is nonstandard to mix the two, and is causing problems on some operating systems (e.g. Solaris); thanks to Geoff Gibbs; - rise log level for log entries on intentional mail drops in case a DSN (a bounce) should be sent, but will not be; such as on rejected bounces, viruses with forged names, and spam from mailing lists. The new log entries now say: '... Not sending DSN ...' and provide more information on the reason for dropping DSN; - if sender is whitelisted, don't insert 'X-Spam-Flag: YES' header field, don't append spam address extension, and don't quarantine. This makes it less surprising, although previous behaviour was according to documentation. NOTE: the documentation still describes former behaviour, this needs to be fixed; - report deaths of command line scanners and some external programs distinctly from normal exits with nonzero exit status; - fix replacing * or {}/* in the pattern with actual file names, causing MkS_Vir (mks32), VFind, Dr.Web Daemon, and KasperskyLab aveclient to receive its file arguments glued together; based on patch by Rafael J. Wysocki; - update 'Panda Antivirus for Linux' entry to work with new (and older) versions of pavcl; updated entry kindly provided by Panda Software; - stop the timer after SMTP transaction is over to better behave in persistent SMTP/LMTP sessions; start the timer at the beginning of a SMTP transaction, in addition to restarting it when DATA mode is entered; - sub mail_via_smtp_single: properly report SMTP response code when all recipients are rejected by MTA, instead of logging a 'mail_via_smtp:' without a value. The problem was commonly seen with the W32/Mimail-A virus which fakes an often invalid local sender address and gets rejected by Postfix outright; reported by Turgut Kalfaoglu; - use SMTP response code 554 (instead of 550) for rejecting syntactically invalid header (according to rfc4409 (ex rfc2476)); - add am_id to SMTP response code generated by one_response_for_all() to make it easier for MTA log to be correlated with amavisd-new log; some cosmetic improvements in the generated SMTP response text; - added 'Return-Path:' in notifications to make it more obvious to see envelope address from reports; - the 'Message-ID:' in neutral DSN notifications template was inadvertently pushed into the DSN body; - report undefined spam score in X-Spam-Status header field as 'hits=-' instead of 'hits=0.0' which can be misleading; - indicate blacklisting in X-Spam-Status header field of quarantined messages; - add X-Envelope-From header field to quarantined messages; - added virus names: tanatos, lentin, bridex (alternative names for bugbear, yaha and braid) to the $viruses_that_fake_sender_re; thanks to Harrie Overdijk; - set environment variables LINES and COLUMNS to sensible defaults to avoid some external program get puzzled about the terminal settings (e.g. older versions of pavcl from Panda Software); - another attempt at fixing the Subject header field duplication. The patch amavisd-new-20030314-p2 fixed the case of entirely missing Subject header field, but did not fix the case of Subject header field present but with an empty text. Reported by Steven Cobb and Francois Rolland; - rewritten 'Received' header fields parsing to better cope with valid, as well as with more common cases of broken syntax; used when trying to report originator IP address for believed-to-be-faked senders; - more permissive parsing of SMTP addresses and options on MAIL FROM and RCPT TO commands; - 'neutral' (=space) field in SQL black/whitelists now terminates the lookup search, avoiding fallback to static black/whitelists. It enables recipient to explicitly express its neutral stance towards the sender, overruling the site default; - $sa_mail_body_size_limit now takes into account some portion of the mail header size so that huge mail headers that can cause slow SA calls are avoided (such degenerate cases were reported by Ralf Hildebrandt); - taint fix in read_l10n_templates (as used by the Debian distribution), patch by Henrique de Moraes Holschuh; - don't send recipient notifications to recipients that have bypass_virus_checks/bypass_banned_checks; suggested by Joe Breu; - replace /bin/false with a more usual /usr/bin/false as a last resort exit; - fetch modules 'Net::Ping' and 'bytes', which seem to be needed in certain chrooted setups; - change log level from 0 to 1 for the log entry 'BAD HEADER from'; reported by Thomas Lamy via Debian bug reports; same for 'unrar: all %d members are encrypted'; - fix typo in variable name: $spamassasin_obj -> $spamassassin_obj - explicitly set pipes and sockets to binmode, as is a default since Perl 5.8.1 - documentation updates: * new file LDAP.schema, by Jacques Supcik, PhD * updated README.chroot to tell that /dev/urandom is needed in chroot jail (otherwise creation of MIME notifications fails); thanks to Lynn Duerksen and Jimmy Porter; * updated README.sendmail-dual, thanks to Robert LeBlanc and Stephane Lentz; * added URLs of external programs to INSTALL; * small updates to other doc files; --------------------------------------------------------------------------- June 28, 2003 Patch: amavisd-new-20030616-p3 - avoid 'savemail: cannot save rejected email anywhere' sendmail panic when feeding mail via LMTP and using D_BOUNCE settings in dual-sendmail setup. --------------------------------------------------------------------------- June 27, 2003 Patch: amavisd-new-20030616-p2 - when running chrooted, fix pre-loading of modules needed by SpamAssassin and Razor agents, which SpamAssassin forgets to pre-load by itself; reported by Neil Camara; - updated KasperskyLab AVPDaemonClient entry to protect against carriage returns in collected virus names; thanks to Harrie Overdijk; - updated 'Symantec CommandLineScanner' entry to look also for the new scanner name 'savsecls'; thanks to Guido Rolon; - documented extended uses of read_hash() in the release notes; - updated README.chroot; --------------------------------------------------------------------------- June 24, 2003 Patch: amavisd-new-20030616-p1 - bug fix: allow stderr to be joined with stdout (>&1), instead of creating a file with a name "&1" when calling command-line scanners or doing LHA decoding (it got broken in 20030616); noticed by Henrik Larsson; - bug fix: the X-Spam-Level header field would carry 64 asterisks instead of none when spam is configured to pass (mail tagging only) and when SA was not called (e.g. sender is whitelisted, ...) and tag_level or tag2_level is set to 0 or less; reported by Ralf Hildebrandt; - bug fix: untaint e-mail addresses when forwarding via pipe; reported by Sam Tilders; - modify code to match documentation: $relayhost_is_client should influence $notify_method too, not just $forward_method (adding an extra argument to mail_dispatch() was necessary to make this possible); reported by Zhu Yicun; - SQL failure modes consistency: when initial connect to SQL failed, previous versions of amavisd-new would fall back to static defaults, disabling SQL lookups. Now initial connect failure is fatal, just like if connect failure occurred during operation - mail flow just stops during SQL servers outage, and is resumed when servers become reachable again; - commented-out adding of 'X-Spam-Report' header field to retain behaviour from amavisd-new-20030314 - I find this header too long and intrusive for regular use. Uncomment it if adding X-Spam-Report is desired; - support lzop (.lzo) and freeze (.F) compressors if available; - use external program cpio (if available) to efficiently and safely handle the following archive formats: cpio binary, HPUX binary cpio, cpio crc, old ASCII cpio, new ASCII cpio, and HPUX old ASCII cpio, as well as POSIX.1 tar (also GNU tar) and old tar if allowed (see below) without reading whole archive members into memory; - if your cpio(1L) can read tar format (as is common on FreeBSD and Linux), it is recommended to uncomment the following line (in file 'amavisd'): # /^\.tar$/ && defined $cpio && return do_cpio($part,$tempdir); which will cause tar archives to be safely and efficiently decoded by cpio(1L) instead of Archive::Tar, which loads the whole archive into memory; - use separate file(1) 'classifications' ".uue" and ".hqx" (instead of ".asc") for uuencoded and binhex formats; - merge 'H+BEDV AntiVir' and 'CentralCommand Vexira Antivirus' entries into a single entry - they are basically the same product (the Vexira entry didn't work for new version of Vexira); thanks to Vivek Khera; - do_unrar: some switches to rar/unrar are not recognized by older versions of rar/unrar; issue a more informative log when this happens, and retry without newer switches for compatibility; thanks to Geoff Gibbs; - crop backtrace on SA timeouts at 980 characters; - preload Perl modules Net::DNS::* if spam scanning is enabled; they could not be loaded if running in a chroot jail, so SpamAssassin did not use Razor; - documentation: collected documentation pieces pertaining to LDAP into README.lookups; fix SQL example table data in README.lookups; ( the patched MIME-tools by David F. Skoll is recommended over 5.411, as it better handles broken/bad MIME syntax: http://www.mimedefang.org/ -> Download section ) --------------------------------------------------------------------------- June 16, 2003 amavisd-new-20030616 release notes This is mostly a maintenance release. Only a handful of new features were added that were small or easy enough to provide. - revised the incoming SMTP session abort procedure to properly shut down the remote SMTP client in case of unexpected trouble (e.g. disk full) during SMTP data reception; - subroutine mail_via_smtp_single() overhaul to produce cleaner diagnostics and to properly abort outgoing SMTP session in case timeout occurs in SMTP 'DATA' mode (avoids recipients repeatedly receiving partial message in successive delivery attempts when the receiving MTA responds very slowly); problem reported by Scott Vintinner; - fixed the generation of LMTP response code when D_BOUNCE was chosen, but DSN was not actually sent because sender was thought to be faked or mail came in from a mailing list. Since DSN was not really sent, the final response remained at 5xx (causing bounce by MTA) instead of being converted to 2xx as D_BOUNCE would suggests; reported by Peter Bates; - security: avoid inadvertently untainting values (e.g. mail addresses) at several places throughout the program; - work around a Perl 5.8.0 taint bug where global variable $1 could become tainted, making further untainting attempts unsuccessful; - work around another Perl 5.8.0 taint bug where Encode::encode fills up all available memory when given a tainted string with a non-encodable character. New subroutine safe_encode() provides a wrapper around the Encode::encode; - treat 'Maximum number of files exceeded' the same as 'Exceeded storage quota', i.e. inserting the X-Amavis-Hold header field; noted by Christopher Odenbach; - call exec with explicitly specified program path, preventing exec from even considering calling a shell; - catch and better report the Perl's failure to fork on open; - avoid inappropriately reporting 'Illegal seek' in run_av (and elsewhere) even though nothing went wrong; reported by William Yodlowsky; - new subroutine flatten_and_tidy_dir() recursively descends into a directory containing potentially unsafe files with unpredictable names, soft links, etc., rename each regular nonempty file giving it a generated name, and discard all the rest. - do_unrar and do_unarj now let the archiver itself recursively unpack the archive and then use flatten_and_tidy_dir() to tidy up the result. With previous method it was not possible to extract archive members with names containing non-ASCII characters from rar archives due to a bug in unrar and rar programs. Problem reported by Pan Bambaryla. Protection from mail bombs in do_unrar is retained. - do_unrar: use program rar or unrar, whichever is available. Both rar and unrar recognize the same options to extract. (amavisd.conf: $unrar=['rar','unrar'] ). Under some circumstances unrar falls in a loop while rar extracts the archive correctly. - do_unrar: don't bail out on exit status 1 (warning) from unrar after collecting file names from the archive. Be more careful when extracting file names, such as archive members with names starting with '-' or spaces; Disable showing archive comment and file comments which can easily break member-name parsing. Avoid extracting files in subdirectories twice: once when directory itself is listed, and the second time when each file in a directory is listed. Licence note: unrar is free, rar is not ( see http://www.rarlab.com/ for fresh unrar sources ). - do_unarj: prefer more versatile arj over unarj (amavisd.conf: $unarj=['arj','unarj'] ). Both programs support subcommand 'e' that we use; suggested by Guillem Jover (via Debian support); ARJ since 2.78/3.10 is Open Source (GPL license). When arj is available (recognizing several new archive types, archive versions and options), put these options to good use, recovering several archive members that are unaccessible by the demo program unarj, such as old member versions, memebers with equal names, etc. Disable showing comments. - add X-Virus-Scanned header only if a mail was actually scanned for viruses (i.e. av scan not bypassed for this set of recipients); pointed out by Phil Regnauld; - updated entry for 'KasperskyLab AvpTeamDream' virus scanner; by Daniel Melanchthon; - truncate instead of round the spam level when producing the bar in the X-Spam-Level header field for compatibility with SA. Clip the level bar at 64 characters instead of 60 (more would not be allowed by RFC 2822, although SA used to crop at 100); - subroutine hdr() incorrectly fixed an illegal header field body which did not have a space after a line fold; noticed by Virna Gupta; (the buglet did not show unless one modified the program); - replace several remaining regular expressions /...$/ with /...$(?!\n)/ as a matter of principle (as was done in amavisd-new-20030314-p2 for the more urgent cases). The use of simple $ was almost always subtly semantically wrong, but fortunately in most cases without consequences; - specify open mode explicitly in several 'open's, and protect special characters in file names from being 'too cleverly' interpreted by Perl open (still not all cases done); - SpamAssassin changes umask to 0077 - restore our mask after the SA call; - be a bit more careful regarding platform-independency, distinguishing between use of \012 and \015 against \r and \n (note the \n on Mac is CR); - change log level of several less important av scanner log messages from 2 to 3; - small documentation updates in amavisd.conf; - updated README.lookups: the new substring capture and reference mechanism in regexp lookups explained. The SQL example modified to work under PostgreSQL, thanks to Phil Regnauld; - updated documentation: README.sendmail-dual, README.postfix, README.chroot, new file: README.exim_v4_app2 - by Louis Erickson; NEW FEATURES - provide a command line option 'debug-sa', which is equivalent (but more convenient) then setting $sa_debug to true and starting as '# amavisd foreground'; - list version numbers of the more important Perl modules at startup; - lookup_re(): RE pattern now allows for capturing of parenthesized substrings, which can then be referenced from the result string using the $1, $2, ... notation, as with the Perl m// operator. The number after the $ may be a multi-digit number. To avoid possible ambiguity the ${n} or $(n) form may be used. Substring numbering starts with 1. Nonexistent references evaluate to empty strings. If any substitution is done, the result inherits the taintedness of the key. Keep in mind that the $ character needs to be backslash-quoted in qq() strings (but not in q() strings). Example: $virus_quarantine_to = new_RE( [ qr'^(.*)@example\.com$'i => 'virus-${1}@example.com' ], [ qr'^(.*)(@[^@]*)?$'i => 'virus-${1}${2}' ] ); - read_hash() now returns the hashref (first argument) also as a return value, making it easier to create new hashes (instead of only adding values to existing ones). Also the first argument (hashref) is now optional, and if missing a new hash is created. The following three cases are now equivalent: a) $sa_tag2_level_deflt = {}; read_hash($sa_tag2_level_deflt, '/var/amavis/tag2_levels.dat'); b) $sa_tag2_level_deflt = read_hash({}, '/var/amavis/tag2_levels.dat'); c) $sa_tag2_level_deflt = read_hash('/var/amavis/tag2_levels.dat'); - extend the semantics of the configuration variable $inet_socket_port, which can now either be a scalar as before (i.e. a single port number), or a ref to a list of port numbers, e.g: $inet_socket_port=[10024,10026,10028]; # accept SMTP on all these TCP port When $relayhost_is_client is true, the semantics of $forward_method has changed slightly: instead of taking the port number from $forward_method, it is now calculated as being one higher than the port number on which the incoming SMTP connection came in (one from the $inet_socket_port list). This allows for multiple MTA pairs on the same host to share the same amavisd daemon without having to use multiple loopback interfaces. Useful for example if incoming and outgoing mail is handled by separate mailers on the same host. - add field 'spam_quarantine_to' to SQL table 'policy' - a patch provided by Vivek Khera. For compatibility a missing field defaults to undef, causing lookup search to proceed with the next lookup map; - add fields: spam_lover, banned_files_lover, bad_header_lover, bypass_banned_checks and bypass_header_checks to SQL table 'policy' for completeness; For compatibility a missing field defaults to undef, causing lookup search to proceed with the next lookup map; - add a 'SPAM-TAG, ...' log entry at log level 2, which is produced when inserting spam-related header fields for each cluster of recipients with same settings. It complements the 'SPAM, ...' log entry, which is triggered at kill level. Feel free to adjust their log levels if you think one or the other is redundant; - insert a 'X-Spam-Report' header field at tag2 level when spam is passed. Feel free to comment out the ...append_header('X-Spam-Report'... line if this is undesired. - fold-in the 'bad_headers.patch' to the main code (available in the distribution since amavisd-new-20030314-p1), but renamed *bad_headers_lovers* to *bad_header_lovers* and added $final_bad_header_destiny and *bypass_header_checks*. To refresh memory: Enables checking headers of each mail for invalid (non-encoded) 8-bit characters, and produces a bounce (non-delivery status notification, or an SMTP REJECT if desired) with the full explanation of the problem, with offending header fields trimmed, sanitized and included in the text. New setting $final_bad_header_destiny, defaults to D_PASS for backward compatibility, but a value D_BOUNCE is suggested. Similarly to other *lovers* settings, a hash and an ACL lookup %bad_header_lovers and \@bad_header_lovers_acl are available, and a setting $warnbannedsender. Certain recipients may be exempt from the checking (*bad_header_lovers*). Similarly mail from mailing lists (Precedence: list or bulk), and mail with null reverse-path mail (e.g. bounces) is passed, even if violating the RFC 2822 header syntax. A log entry is produced nevertheless. Postfix users: this is similar to the Postfix strict_7bit_headers=yes functionality, but produces a much more informative problem report (non-delivery notification) to the sender. One difference is that amavisd-new header check takes into account only the RFC 2822 header, not MIME headers in the mail body. It is reasonably efficient to use amavisd-new as a header checker only, without any anti-virus or anti-spam checks, if desired. - added configuration settings $warnbannedsender and $warnbannedrecip, separating this function from $warnvirussender and $warnvirusrecip, which previously applied to both virus and banned files; - new macro %c which evaluates to spam level/hits (mnemonic: sCore) as provided by SpamAssassin; useful in $log_templ; - new configuration settings *bypass_banned_checks* and *bypass_header_checks*; - new parameter $spam_quarantine_bysender_to (in contrast to the usual $spam_quarantine_to) makes possible collecting quarantined spam by sender names or domains, instead of (or in addition to) the more usual by-recipient map; - new configuration setting $remove_existing_spam_headers (defaults to true); - bring-in the localization (l10n) support contribution from Debian Linux, by Henrique de Moraes Holschuh; see README.l10n, and comments at the call to read_l10n_templates() in file amavisd.conf; - supply X-Envelope-From and X-Envelope-To header fields containing envelope addresses to the messaged passed to SpamAssassin for checking; - there is now an official OID: 1.3.6.1.4.1.15312.2.2 under which LDAP schema for amavisd-new LDAP lookups is to be defined (iso.org.dod.internet.private.enterprise. 'ijs.amavisd-new.ldap' ); --------------------------------------------------------------------------- May 6, 2003 Patch: amavisd-new-20030314-p2 - blacklisted sender only caused spam headers to be inserted, but did not cause rejecting the message and other 'evasive actions' due to over-optimization; reported by Lawrence Farr, Robin Elfrink and Eric Vollmer; - fix improper evaluation of bypass_spam_checks* in multi-recipient mail where some recipients match 'bypass_spam_checks*' and others do not. Spam-check was bypassed if _any_ (instead of _all_) recipient matched 'bypass_spam_checks*'; thanks to Joseph W. Breu; (it got broken in amavisd-new-20030314, the amavisd-new-20021227 was ok); - adding address extensions was not done for *_lovers, only for *_destiny=D_PASS; reported by Steve Khoo. The "deal_with_spam" section needed to be overhauled for this reason. For consistency with the documentation and with the change in adding address extensions, quarantining is now done if mail is considered spam, regardless of it being delivered (e.g. to spam lovers), or not; - if only spam checking is performed, but no virus scanning nor banned file names checking is enabled, and the spam is detected but configured to pass, and the Subject is configured to be edited, the Subject header field was duplicated because the existing Subject header field was not detected. Solution: make MIME entity available even if virus and banned files scanning is not performed. Header information is needed by add_forwarding_header_edits_per_recip() when deciding whether to add or replace the Subject header field in case spam is passed. (workaround for previous versions: don't leave $banned_filename_re undefined. If not needed, set it to an empty list: new_RE() ); - when quarantining to a mbox file, excape 'From ' with '>' only if it follows an empty line, as sendmail mail.local man page prescribes; - added parameter $notify_xmailer_header to control the body of the header field X-Mailer as placed in the notification messages (e.g. bounces) generated by amavisd-new; by default this header is no longer inserted, as it was felt it revealed too much information. Choose from: $notify_xmailer_header = undef; # no X-Mailer inserted (default) $notify_xmailer_header = ''; # X-Mailer: MIME-tools x.xxx (Entity x.xxx) $notify_xmailer_header = 'your-text-here'; # X-Mailer: your-text-here - replace regular expressions /...$/ with /...$(?!\n)/ or with string equality operator in some places (many more to come in the next major release); the use of simple $ was almost always subtly semantically wrong, but in most cases without consequences; - enable timeouts after SMTP connection has been established, to be able to handle cases where MTA is very slow in accepting checked mail, which could result in multiple deliveries if MTA on the input side times out; - debugging aids: call stack backtrace is logged if Mail::SpamAssassin::NoMailAudit::check exceeds allowed time; - LDAP enhancement: allow "hostname" to be a string or a reference to an array, in which case each entry will be tried in order until a connection is made; by Jacques Supcik; - X-Spam-Status now says WHITELISTED if sender was whitelisted; contributed by Joseph W. Breu; - if mail rejection was due to sender blacklisting, say so in the SMTP response; thanks to Robin Elfrink; - updated file bad_headers.patch to apply cleanly against this version. The bad_headers.patch will be folded-in with the next major release, please try it out, comments welcome. For patch description see release notes for amavisd-new-20030314-p1; - specify option 'ExactAddresses=>1' in a call to Net::SMTP::new() in anticipation for the next release of Net::SMTP; - amavisd.conf: updated documentation / comments; - amavisd.conf: removed option '-ni' in the DrWeb virus scanner entry; thanks to Mike Boev and the FreeBSD community ( http://www.freebsd.org/cgi/query-pr.cgi?pr=50893 ) - amavisd.conf: added entry for 'MkS_Vir daemon', thanks to Dariusz Grzegorski; - amavisd.conf: updated entry for 'F-Secure Antivirus' to cope with fsav 4.50 as well as with 4.1x; solution by Juhan Tamsalu; - documentation: updated README.chroot; new README.sendmail describing dual-MTA sendmail setup, small updates to other doc files; March 21, 2003 Patch: amavisd-new-20030314-p1 - fix passing of remote MTA reject code back to the input side (DSN was not sent if MTA rejected the message); - fix broken logic which allowed a virus to pass if a recipient was a 'banned file lover' and the same message also contained a 'banned file'; (or the other way around: pass a banned file if containing a virus and sent to virus lovers); thanks to Ortwin Gentz, I'm glad somebody is looking over my shoulder; - remove a log entry: one_response_for_all: SHOULDN'T HAPPEN The test was inappropriate, but harmless; - enhanced sub run_av() to fix a problem with MkS_Vir virus scanner, where a '*' in the template command was not expanded to actual file names; reported by Pan Bambaryla; - trim virus scanner output (in sub run_av) to a manageable size of 900 characters so as not to clutter log entries and notifications too much; provoked by the Panda pavcl; - move the check for 'Precedence:' mail header field from sub delivery_status_notification() to its caller for flexibility; - replaced the text in notification templates: The message WAS delivered to: into: The message WILL BE delivered to: to reflect the truth. The message may still be rejected by the MTA when forwarding takes place. - small documentations edits/updates; - cosmetic: trim trailing whitespace in the program; - cosmetic: make shorthand subroutine nf() locally scoped; by Jacques Supcik; - make available an experimental patch (file: bad_headers.patch, not applied by default), which, when applied (with patch(1) utility) to amavisd-new, makes it check headers of each mail for invalid (non-encoded) 8-bit characters, and produces a bounce (non-delivery status notification, or an SMTP REJECT if desired) with the full explanation of the problem, with offending header fields trimmed, sanitized and included in the text. Certain recipients may be exempt from the checking (*bad_header_lovers*). Similarly mail from mailing lists (Precedence: list or bulk), and mail with null reverse-path mail (e.g. bounces) is passed, even if violating the RFC 2822 header syntax. A log entry is produced nevertheless. Postfix users: this is similar to the Postfix strict_7bit_headers=yes functionality, but produces a much more informative problem report (non-delivery notification) to the sender. One difference is that amavisd-new header check takes into account only the RFC 2822 header, not MIME headers in the mail body. It is reasonably efficient to use amavisd-new as a header checker only, without any anti-virus or anti-spam checks, if desired. The patch is experimental in the sense that it may change in future versions, but is fully functional when applied; - Jacques Supcik cleaned up and generalized his LDAP lookups support code (which is off by default). In his words: The major change is in the definition of ldap lookups. In the config file, you have to enable ldap first: | $enable_ldap = 1; Then you can define defaults for your ldap queries: | $default_ldap = { | hostname => 'localhost', tls => 0, | base => 'ou=hosting,dc=example,dc=com', scope => 'sub', | query_filter => '(&(objectClass=amavisAccount)(mail=%m))' | }; And then the lookups themselves: | $virus_lovers_ldap = {res_attr => 'amavisVirusLover'}; | $banned_files_lovers_ldap = {res_attr => 'amavisBannedFilesLover'}; | ... With this method, you can define every parameter individually. You could have a different ldap server for each lookup! Like that, the configuration is closer to the one in postfix or courier. The hashes are converted into lookups objects in the Amavis::Lookup::LDAP class method. --------------------------------------------------------------------------- March 14, 2003 amavisd-new-20030314 release notes MAIN NEW FEATURES AT A GLANCE - per-user white- and blacklisting, including an SQL lookup mechanism; refined semantics for white/blacklists; - easier to run in chroot jail, see README.chroot; - can specify two sets of virus scanners: the main list and a backup list, which gets consulted only if all scanners from the main list fail; - split REJECT into separate destiny settings: REJECT and BOUNCE, giving you a choice of who is responsible for sending a non-delivery notifications to the sender; see comments in amavisd.conf; - edited configure file amavisd.conf, providing more informative comments, cleaner examples and simplified common settings; - made it work better with the Unicode-aware Perl 5.8.x (not against it) and distinguish between character data / byte data / octet data (binary), provide UTF-8 (Unicode) support for notification messages, while avoiding some possible nasty surprises in the UTF-8 locale (which is enabled by default in Red Hat 8.0); - SQL SELECT clause settable from the config file; - can store quarantined or forwarded messages as BSMTP files (RFC 2442); Changes since amavisd-new-20021227 patch level 2 BUG FIXES - fixed caching problem when bypass_*_checks are used selectively: when only one of the virus/spam tests were performed but the other bypassed due to a per-recipient setting, the result of a non-performed test was inserted into cache as clean. This could cause tests to be bypassed for subsequent message with the same body but different recipients with different bypass* settings, but only within a lifetime of a child process (10 messages by default). Noticed by Jürgen Louis Fluk, thanks! - changed caching of SQL lookups to match the documented behaviour, i.e. SQL lookup result is cached (for the benefit of field lookups) for a single message only, and no longer for the child lifetime. It is possible to revert to the previous behaviour by commenting-out one line (look for $sql->clear_cache). Pointed out by Paul Miner; - see also further down about the field 'banned_files_lover' in the SQL table 'policy', and the UTF-8 (Unicode) workarounds. SECURITY - running amavisd-new in chroot jail is now easier to set up due to added configuration variable $MYHOME and its simplification effects on other settings in the config file. Relevant settings are better commented; - avoid the need to have a shell in chroot jail (use of the new subroutine run_command() instead of Perl functions qx() and system() ); - new file README.chroot with guidelines on how to run amavisd-new in a chroot jail, to prevent possible vulnerabilities from affecting the rest of your system; - added a security notice (web page and other places) that the Unix utility file(1) 3.39 and older contains an exploitable security vulnerability, which can cause system access with access rights of the user running amavisd-new daemon (not root, unless you violated the recommendations). Please upgrade file(1) to 3.41 or newer: ftp://ftp.astron.com/pub/file/file-3.41.tar.gz - new section (Security considerations) on the web page; CLEANUPS AND CHANGES IN amavisd.conf - improve explanation texts (comments) in amavisd.conf, edit the file to improve legibility and structure, change some examples. The daemon is thought to be compatible with amavisd.conf from previous version, but it may be worthwhile to switch to the new file to avoid asking FAQ; - introduce a configuration variable $MYHOME, which is never used directly by amavisd daemon, but is a handy way of setting default values for other configurable variables in the config file. Also makes default settings more chroot-friendly. - introduce a configuration variable $mydomain, which is never used directly by amavisd daemon, but is a handy way of setting default values for other configurable variables when there is only a single domain (with subdomains) behind the mailer. This avoids the need to edit half-a-dozen configuration variables in simple setups; - moved debugging-related settings to a separate section (Section VIII - Debugging); - make SQL SELECT clause settable from the config file; - renamed variable @local_domains to @local_domains_acl for consistency; The old name became a synonym for @local_domains_acl, so compatibility with existing config files is retained; - comment out the example: read_hash(\%whitelist_sender, ...); it slipped into the amavisd-new-20021227 released code from my tests; - comment out the setting $relayhost_is_client=1; It is only useful for multi-access daemon setup, and potentially confusing for others; VIRUS SCANNERS - introduced a list of backup virus scanners: @av_scanners_backup, complementing the normal list of virus scanners. If no virus scanners from the @av_scanners list produce 'clean' nor 'infected' status (e.g. they all fail to run or the list is empty), then all scanners in the @av_scanners_backup list are tried. When there are both daemonized and command-line scanners available, it is customary to place slower command-line scanners in the @av_scanners_backup list; - removed the Mac and SafeMacDfHandling options from SAVI-Perl, as they are no longer supported by Sophos library since 2003-03; set new option NamespaceSupport to SOPHOS_DOS_AND_MAC_FILES; thanks to Paul B. Henson; - changed a default to initialize SAVI-Perl only at startup time, and no longer every time the child process is started. This is because starting with SAVI 3.0 (March 2003) the initialization is slower. NOTE: it is now necessary to restart amavisd master process after SAVI database is updated, to make it notice new virus descriptions. - in 'MkS_Vir for Linux' entry replaced options -e -c with -s, to make mks work, suggested by Pan Bambaryla; - added status 8 ('corrupted') to the list of clean statuses for 'KasperskyLab AVPDaemonClient', as is already the case for some other virus scanners. See amavisd-new web page (Security considerations) if the idea bothers you; - RAV command line scanner no longer supported off-the-shelf, as interfacing to it seems to be violating their license terms. Btw, if RAV command line scanner does not work as user amavis, check permissions and ownerships of /usr/local/rav8/* or reinstall RAV as the user that will run amavisd; thanks to Oyku Gencay; - RAV does not like 'new ask_daemon same as Amavis::AV::ask_daemon sophos_savi same as Amavis::AV::sophos_savi amavisd.conf file is changed to use the short names, but full subroutine names are still valid. - Provide protection mechanism for SMTP/LMTP server code to limit the number of recipients in a mail transaction to some sane value ($smtpd_recipient_limit, default 1000). Even if the number is exceeded, mail will still be normally delivered in more than one transaction (provided your MTA is rfc2821 compliant). - SMTP client code now correctly handles the situation where remote SMTP server returns '452 Too many recipients'. When this happens the mail forwarding or submission is split into several SMTP transactions, and as long as the remote SMTP server is willing to accept at least one recipient per transaction, mail delivery is assured. - Auto white-listing (AWL) by SpamAssassin may be turned on by setting $sa_auto_whitelist to true (contributed by Hamish Marson). This only works as one global database, the per-recipient AWL is presently not available. - New configuration variables $hdrfrom_notify_sender, $hdrfrom_notify_admin, and $hdrfrom_notify_spamadmin, to remove the overloading (double semantics) from variables $mailfrom_notify_*. Now $mailfrom_notify_* control ONLY envelope addresses, while $hdrfrom_notify_* only controls From header in notifications. The default values for $hdrfrom_notify_* are chosen to preserve compatibility. - The spam headers in quarantined message now reflect the lowest tag and kill levels when more than one recipient is given, instead of reporting the levels for the first specified recipient. - Tested with SpamAssassin 2.50 and with its new Bayesian classification (used as a global database by amavisd-new). Given two sets of spam and non-spam (=ham) messages, one can build a database by using new SpamAssassin utilities sa-learn-spam and sa-learn-nonspam as user amavis, and the database built this way will be used by amavisd-new/SpamAssassin. - Initialize SpamAssassin as $daemon_user (no longer as root), as a safety measure, and to ensure the files it creates are still accessible after the UID change. Incompatible changes to 20021116: - Different parameters in @av_scanners list for daemonized antivirus scanners (as described above). - lookup_hash: an undefined hash value was previously treated as a special case, resulting in value true. This is no longer the case, which brings it in line with other lookup methods. It is not likely anyone will be affected, as this practice was only documented but not encouraged by any example. --------------------------------------------------------------------------- Here is a brief description of patches that needed to be applied to amavisd-new-20021116, and were available from the amavisd-new web page: patch5 (2002-12-09): - fix a simple test to decide if the anti-virus and anti-spam code is needed; - use the /m regexp modifier by default when parsing av-scanner output for virus names - seems like (?m) in qr is ignored (have we found a Perl bug?); - fix DSN to report original smtp response in case reporting MTA is remote (thanks to Chris Hastie for his keen eye in understanding DSN); - fix reporting quarantine mailbox name(s) in case of per-recip quarantine or quarantine forwarding to MTA; - macro %q (quarantine id) is now a list of quarantine mailbox names or addresses (may have more than one entry in case of per-user quarantines); - macro %v (av-scanner output) is now a list of scanner output lines, to facilitate formatting of multi-line av-scanner output; - fix spam admin notifications, which used the virus template instead of the spam template; - fix problem of undefined entity causing spam notification failure in case other content checking (virus/banned) is not enabled; - make SQL lookups (DBI access) optional - no DBI code is loaded if @lookup_sql_dsn of empty; - support SAVI-Perl module 0.10 or later (direct access to the Sophos library engine) ( http://www.csupomona.edu/~henson/www/projects/SAVI-Perl/ ); - add macro %j that evaluates to message 'Subject:' header field; - fix minor SMTP protocol inconsistency when multiple MAIL FROM commands are issued by the client; - more informative timing report when more than one av-scanner is used; - make sendmail initial submission happy when null reverse-path is specified (instead of the correct empty argument, which works but makes sendmail issue a warning, we now use <> in arguments to specify null paths), (thanks to Sebastian Hagedorn and Dale Perkel); - replace log entry "spam from=<%s>, to=%s" with "SPAM, <%s> -> %s ...". - dropped (redundant) historical %bypass_checks @bypass_checks_acl (use bypass_virus_checks and %bypass_spam_checks instead); patch4 (2002-11-21): In amavisd: - SQL lookup fix; - finished per-recipient header edits and split-forwarding, where one multi-recipient message may be forwarded as several messages if different headers need to be inserted for different recipients (not with milter); - the check for banned file contents now looks for both the short and the original full type classification as provide by 'file' utility (thanks to Wolfgang Lumpp for the idea); - #!/usr/local/bin/perl => #!/usr/bin/perl - cosmetics In amavisd.conf: - suggested group name change $daemon_group 'sweep'=>'amavis'; so that non-Sophos users are not puzzled where 'sweep' group comes from; - added path and fixed args for KasperskyLab AVPDaemonClient (thanks to Mike Hall); - added path for Trend Micro FileScanner vscan (thanks to Eduardas Paulavicius); - added 'use strict;' and '1;' at the end; prevent read_config from failing if the config file does not return true (thanks to Vivek Khera); - cosmetics, fixed typos in comments; patch2 & patch3 (2002-11-18, 2002-11-19): - fixes a problem with 'recip_done' object method (causing TEMPFAIL) when $warnvirusrecip is enabled (thanks to Chris Hastie, Dale Perkel, and Didi Rieder); - pass null return path to sendmail in such a way to make it happy (thanks to Sebastian Hagedorn and Dale Perkel); - added a safety precaution to prevent somebody shooting himself in the foot by specifying an empty $forward_method while using the SMTP in/out setup (e.g. Postfix), which could send mail down the bit bucket; patch1 (2002-11-17): - fixes a problem in milter setup where per-recipient response codes needed to be handled differently (the problem was causing a TEMPFAIL) (thanks to Stephane Lentz and Didi Rieder for the initial milter testing and problem reports); --------------------------------------------------------------------------- November 16, 2002 amavisd-new-20021116 release notes - provide a mechanism to load only required code sections (anti-virus scanning, anti-spam scanning, SMTP/LMTP server module, traditional amavis client & milter server module), resulting in reduced memory usage and less installation dependencies; - introduce the per-recipient status handling and make possible some per-recipient functionality that was not possible before, such as handling the per-recipient spam thresholds. This required a major rewrite and cleanup of some sections, hopefully providing cleaner mechanism for possible future advances in this area. Consider for example a previously incorrectly handled situation where there is more than one recipient, and some (but not all) of them get a reject from the (outgoing) relay MTA - it is not possible for a single SMTP response or status code to describe the situation; - generate proper (non-)delivery status notifications (DSN), compliant with rfc1892 (now rfc3462) and rfc1894 (now rfc3464). This form supersedes the simple virus/spam sender notifications, but also covers the area of genuine delivery failures, such as selective recipient rejects by the outgoing relay MTA, which could previously lead to multiple deliveries; NOTE: to avoid sender getting two non-delivery messages (one from MTA, and another from amavisd), do not set $warnvirussender and $final_virus_destiny=-1 (REJECT) at the same time (and equivalently for spam settings); - enhanced mail system status codes (rfc1893 (now rfc3463), and rfc2034) are now included with all SMTP responses and DSN notifications; - added checking for banned MIME types and names. If any mail part matches, the whole mail is rejected, much like the way viruses are handled. A list in object $banned_filename_re can be defined to provide a list of Perl regular expressions to be matched against each part's: * Content-Type value (both declared and effective mime-type), including the possible security risk content types message/partial and message/external-body, as specified by rfc2046; * declared (recommended) file names as specified by MIME subfields Content-Disposition.filename and Content-Type.name, both in their raw (encoded) form and in rfc2047-decoded form if applicable; * file content type as guessed by 'file' utility and classified into short type names such as .asc, .txt, .html, .doc, .jpg, .pdf, .zip, .exe, ... - see subroutine determine_file_types(). This step is done only if $bypass_decode_parts is not set. NOTE: by default the $final_banned_destiny is set to 1 (pass), so detected banned file names only cause a header line to be added, quarantining, and added address extension - but the mail is delivered nevertheless. This default is set so that we can get initial experience without being too obtrusive. Change as you prefer. - besides SMTP/ESMTP protocol, the server now also accepts LMTP protocol (rfc2033). This now explains why I had to implement PIPELINING, ENHANCEDSTATUSCODES and 8BITMIME SMTP extensions, as these are required by rfc2033. One advantage of using LMTP to feed content filter is that LMTP uses per-recipient status response, as opposed to 'one-size-fits-all' SMTP status response, which require that the SMTP client (e.g. content filter) is capable of generating DSN. Another advantage is specific to the Postfix setup, as Postfix LMTP client supports multiple transactions per session, saving on connection teardown/reconnect for every message being checked. To enable LMTP feed from Postfix, add 'max_use=10' to main.cf, and replace last argument 'smtp' with 'lmtp' in the master.cf line: 'smtp-amavis unix - - n - 2 lmtp'. No changes are needed in amavisd-new, both protocols can coexist, the distinction is based on HELO/EHLO vs. LHLO command. Btw, older versions of Postfix lmtp client inappropriately lowercased the envelope addresses. This is fixed in Postfix Snapshot 1.1.11-20021015, and in the regular Postfix release 1.2 (when available). Lowercasing can be a problem for addresses where local part is case-sensitive, although such setups are rare. - improved per-recipient SMTP response code handling when sending mail via SMTP. Previously one rejected recipient (by MTA) in a multi-recipient message caused the whole message to me rejected. This is now correctly handled with the new ability to send DSN; - added command line option '-c config-file' so one can override the default location of the configuration file (/etc/amavisd.conf); - explicitly set PATH and HOME environment variables (settings: $path and $helpers_home) - added another form of lookups: Perl regular expression matching. See README.lookups for details. Corresponding new variables in amavisd.conf are: $virus_lovers_re, $spam_lovers_re, $bypass_virus_checks_re, $bypass_spam_checks_re, $local_domains_re, as well as $banned_filename_re, $viruses_that_fake_sender_re, $keep_decoded_original_re, $whitelist_sender_re, $blacklist_sender_re. - besides whitelist_sender* lookup tables, there is now also a blacklist_sender* set of tables, which causes mail to be declared spam and to skip remaining spam checks. See examples in amavisd.conf. - provide a configurable lookup table $keep_decoded_original_re of file types, for cases where unpacker is not very trustworthy. The lookup key is what 'file' utility returned. If the part contents match the lookup table, we keep both the original and the unpacked file for virus scanner to check; - provide a configurable list of regular expressions $viruses_that_fake_sender_re, which recognizes viruses that usually fake envelope sender address. Don't send sender notification if a match is found. For syntax of the new regexp lookup tables see README.lookups; - specifying per-recipient quarantine address or location is now possible by setting $virus_quarantine_to and $spam_quarantine_to be a ref to a hash lookup table. Thanks to Vivek Khera for the idea; - no 'configure' and 'make' for the daemon; all configuration is done via amavisd.conf configuration file at the daemon startup time; Also the ./mta, ./av and ./notify configuration subdirectories are now gone; - provided a sample init shell script amavisd_init.sh (edit to will, and move it to /etc/init.d/amavisd if you want). Based on the script from Wil Cooley; - one-shot debugging mechanism: if envelope sender matches @debug_sender_acl lookup table, turn debugging fully up just for this one message and cause temp file and directories not to be cleaned for this message. This facilitates debugging a particular problem even in the presence of regular traffic; - cleaner debug log entries for multi-transaction SMTP/LMTP protocol sessions; - cleaner log entries - avoid misleading and incomplete text when quarantine is disabled; thanks to Michael Leone; - avoid using (nonstandard) field width in the %e format specifier when calling strftime, which lead to bad date syntax on some systems; - local time zone offset is now automatically computed, no more '-0000' in rfc2822 dates; - put Message-ID field value in angle brackets as required by rfc2822 when generating notifications; - add a In-Reply-To field to notifications when original Message-ID is known; - don't bounce a virus (or spam) back to a mailing lists even if $final_virus_destiny (or $final_spam_destiny) is set to REJECT; a patch by Brian May from the Debian support crew, thanks; - because some external module may play games with STDIN and STDOUT (like SpamAssassin seems to do when local_tests_only=>0) run the input protocol directly on the Net::Server's socket, not on STDIN and STDOUT, which are aliases to the socket; - a patch to amavis-milter.c by Didi Rieder to support REJECTing mail, by instructing sendmail to return non-delivery notification to sender; - a modified amavis.c helper program to make possible to invoke local delivery agent from it, for those still using such a setup; - a new macro %l is available for use in notification messages (via 'expand'); it evaluates to true (1) if the sender matches @local_domain, and returns empty otherwise; by default it is now used in creating a 'Subject:' line, inserting word 'LOCAL' before the sender name when appropriate; - new macros %D and %N expand to lists of recipients that got the mail delivered (%D), or not-delivered (%N). Union of both sets gives %R, i.e. a list of all recipients as specified in the envelope; - dropped macro %a - improved parsing of e-mail addresses according to rfc2821 (full address literals syntax, etc.); - proper line folding for generated rfc2822 header fields; - proper SMTP response wrapping (for very long responses such as the ones that include a trouble report) according to rfc2821; - allow for obsolete rfc822 syntax of permitting whitespace before colon in header field name; - added optional spam-sender nondelivery notifications, based on patch from Lazslo E. Miranda (lazslo@dcc.ufmg.br) and Fernando F. Morais (frota@cecom.ufmg.br); - dynamically change process name (Perl variable $0) to reflect the process state; suggested by Chip Paswater. Not all operating systems make this process state visible by ps(1); - determine location of external programs (or their absence) at startup time; in amavisd.conf one may specify absolute path or just rely on PATH. This mechanism is also used to determine absolute path of the daemon itself, making reload (after HUP) more predictable; - explicitly specify lock file for serialization to be used by the Net::Server::PreForkSimple module, instead of relying on default provided by POSIX::tmpnam. The default approach has a possibility for a minor security problem, because the lockfile is created with open()..., so it will follow symlinks. Observed, and a patch provided, by Jarno Huuskonen; - new defense against mail bombs: for the cumulative total of all decoded mail parts we set max storage size. The formula is: quota = max($MIN_EXPANSION_QUOTA, $mail_size*$MIN_EXPANSION_FACTOR, min($MAX_EXPANSION_QUOTA, $mail_size*$MAX_EXPANSION_FACTOR)) In plain words (later condition overrules previous ones): allow MAX_EXPANSION_FACTOR times initial mail size, but not more than MAX_EXPANSION_QUOTA, but not less than MIN_EXPANSION_FACTOR times initial mail size, but never less than MIN_EXPANSION_QUOTA - if the permitted quota is exceeded (or the defense triggered by other similar safeguards), the virus scanning is skipped to protect the virus scanner from tripping over the mail bomb, a header field is inserted: X-Amavis-Hold: ... reason ... and a log entry 'Placing on HOLD: reason' at level 0 is produced. Also the temporary directory is preserved. The 'X-Amavis-Hold: ...' header field can be used by your MTA to put the message 'on hold' (freeze). If MTA is not set up to catch messages with this tag (the default), they are passed normally to recipients. This is likely to be the best action under the circumstances. - tested with razor-agents-2.20 and SpamAssassin-2.43 (Razor2 is now called by SpamAssassin, and no longer directly by amavisd-new). New configuration variable $helpers_home, which defaults to $TEMPBASE. Thanks for hints by Chris Hastie and John Stewart; - to avoid CPU loop in SpamAssassin-2.43 and earlier, my patch needs to applied to SA - see amavisd-new web page; - to avoid taint problem in Razor 2.20 (if SpamAssassin-2.43 is configured to call it), my patch needs to be applied to it - see amavisd-new web page; - chroot available (but not well tested): $daemon_chroot_dir = '/var/amavis' - provide a fail-over mechanism for SQL database connect - given a list of SQL servers/databases, pick the first that is available. Thanks to Ken McKittrick for making available the patch, and to Ben Ransford for writing it. - remove existing virus-related and spam-related headers (some of them optionally) if we'll be providing our own; suggested by Borut Mrak; - avoid 'insecure dependency' in lookup_sql when calling DBI::execute (thanks to ric* at mpc.com.br) - fixed macro %H to provide original header lines, not the ones stored internally by MIME::Entity, which may have been modified; (thanks to Chris Hastie for noticing the problem); - no longer inserts X-Razor-id header field; INCOMPATIBLE CHANGES: - when specifying boolean values to variables in the amavisd.conf file, please specify 1 (or old style "yes") for true, and 0 or "" or undef for false. The old style "no" yields true for Perl, and is only still supported (converted to 0) for some traditional variables for compatibility with amavisd(-snapshot); - removed variables $sendmail_wrapper*, and changed the syntax for specifying $forward_method/$notify_method, with the intention to do all mail sending settings at one place. See amavisd.conf for examples; - variable $mailto is now deprecated (but still works as a fallback default for compatibility with previous version). Use $virus_admin and $spam_admin lookup tables instead, they also offer a mechanism to specify per-domain administrator address; - variable $warnadmin is no longer used. Use $virus_admin and $spam_admin lookup tables instead. Not specifying administrator address (e.g. leaving $virus_admin, $spam_admin and $mailto undefined) turns off admin warnings. - dropped variable $LOGDIR, the variable $LOGFILE now specifies the full path; - renamed: $warnsender -> $warnvirussender, $warnrecip -> $warnvirusrecip - dropped variables $enable_relay, $sendmail_cf_orig, $QMAILDIR; they were never used in amavisd-new; - dropped $MAX_ARCHIVE_NESTING, which is replaced by new storage limitations: $MIN_EXPANSION_QUOTA, $MIN_EXPANSION_FACTOR $MAX_EXPANSION_QUOTA, $MAX_EXPANSION_FACTOR - SQL database is now specified differently (@lookup_sql_dsn) MTA-SPECIFIC: sendmail/milter - $notify_method now specifies deferred delivery mode ('-odd') by default, when submitting notifications to sendmail. This is to avoid calling milter immediately during submission, which in turn calls amavisd-new, possibly leading to a deadlock situation when the number of amavisd-new child processes is small. Seems like this change is needed since Sendmail 8.12 or so. The following recommendation is from mimedefang-filter man page: | You MUST run a client-submission queue processor if you use | Sendmail 8.12. We recommend executing this command as part of the | Sendmail startup sequence: | sendmail -Ac -qp1m DECODERS-UNPACKERS: - rewritten decode_parts() to allow for retaining source text if the unpacker sw is considered unreliable. This more cleanly resolves the problem reported on the amavis-user mailing list on 2002-06-06: "Amavisd passing through VBS/VBSWG.gen@MM" A nice side-product is that a directory search is avoided for each nested unpacking step; - refined do_unzip to control and limit the size of decompressed members (among others it defends against the 42.zip-type bombs); - rewritten external decompressors interface to the gzip/bzip2/compress family. Instead of using a 'system' call, they are now called through fh_copy, making possible to control and limit the size of the decompressed contents on the fly, avoiding denial-of-service attacks. Affected: b(un)zip2, g(un)zip, (un)compress; - fh_copy now uses IO::Handle object to assure the forked process gets reclaimed even in case of aborted contents extraction; stdin gets redirected to /dev/null or to a specified input file for the exec'd process; - Convert::UUlib is called again (the amavisd-new-20020630 removed its usage due to problems with improper decoding). This time the originals are kept, so that virus checker sees both the original and the attempted-decoded part; - completely rewritten do_ascii and its usage of Convert::UUlib to fix: * coding error (bitwise op treated as logical op and improperly negated): ... if (!$uu->state || !FILE_OK || -z $newpart); * it never reset the state, so if a successfully decoded ASCII file contained another ASCII file, each decoding level would decode all previously decoded parts again, plus add new ones at this level; recursion would not stop until the hard limit, resulting in TEMPFAIL; * it never checked nor reported errors that should have been detected (I/O errors, out of memory, trouble accessing or creating files); * more informative log entry; - save MIME preambles and epilogues (if nonempty) as extra (pseudo)parts to be scanned. This also mitigates the problem of syntactically-incorrect MIME mail as produced by some user agents, which (rightfully) gets treated as one long preamble by MIME-Tools, and previously went by unchecked; - supports unpacking arc archives using 'nomarch' (by Russell Marks, http://rus.members.beeb.net/nomarch.html), (thanks to David D. Kilzer for the initial code). Using 'nomatch' fixes a nasty habit of arc which gratuitously appends a form-feed at the end of file when using the 'p' (pipe) option, which might mislead a virus scanner. Besides, 'nomarch' is GPL licensed; - fixed a 'broken pipe' problem when calling unrar, thanks to Ricardo Campos Passanezi and Rainer Link; VIRUS SCANNERS: - rewritten interfacing for most command-line virus scanners. All settings for them is now done in amavisd.conf. New ones may be added without having to modify the daemon source. More complex scanners (e.g. daemonized scanners) still need to have a corresponding interface routine in the daemon; - rewritten Sophie and Trophie interfaces to be more resilient to Sophie/Trophie daemon restarts during virus database reloads, avoiding an unnecessary retry (TEMPFAIL) - thanks to Cor Bosman for the suggestion and code, and to Dale Perkel for testing the Trophie interface; - make sophos sweep tolerant to encrypted attachments: if all files are password protected, then the scanner failure is ignored and the message is allowed to pass. Based on patch by Radu Greab; - updated nai uvscan interface to recognize the result 'Found trojan or variant Exploit-CodeBase !!!', a patch by Anton Berezin; - Clam Antivirus supported; - fixed a problem in 'avp' scanner interface ($TEMPBASE not imported), thanks to Joshua E Warchol; - avpdc (KasperskyLab AVPDaemonClient): recognize additional exit codes, based on patch from Christian Hammers; - Panda new regexps, no TERM vt100 setting (thanks to Benjamin Zwittnig); - Trend vscan exit code seems to be the number of infected parts. Updated the test to reflect that, based on observation from Stephane Lentz; - MksVir scanner interface returns error code 2 if viruses are removed (if --clean option is passed) - a patch by Robert Litwiniec; ---------------------------------------------------------------------------- June 30, 2002 amavisd-new-20020630 release notes Since it seems like several people are adapting amavisd-new in details to their requirements, and certain improvement requests have much in common, I'm making available the 20020630 release, shortly before leaving on vacation. As this means my support will be absent for the coming few weeks, and this release is perhaps by few days premature, please consider it primarily a development and new features release. It is available at the usual location at: http://www.ijs.si/software/amavisd/ (or ask Google about 'amavisd-new') Having said that, it is still a fully functional and tested version, and it is running in production at a couple of sites now. Also it is my version of choice in view of dependability, having it running at our site while I'm away. If you get into trouble, you can still go back to amavisd-new-20020517 with which it is fully upwards compatible. The main changes and features since amavisd-new-20020517 are: - the code is thoroughly rearranged, interfaces cleaned, separated into namespaces (packages), several sections generalized (e.g. lookups, appending/editing header lines). The AV scanner and unpacking sections are still mostly the same and compatible with amavisd, so whatever improvements and new AV scanner support becomes available for amavisd, applies almost without a change to amavisd-new; - SMTP on the input side (used with Postfix and Exim) now talks ESMTP (rfc2821) and not just rfc821, including some SMTP extensions: command pipelining (rfc2920), message size declarations (rfc1870), and 8bit-MIME transport (rfc1652). The main reason for this was the change in recent Postfix versions which can now do MIME transformations to support 7bit transports (implied by SMTP). To ensure the transparent 8bit path and avoid message transformation by MTA, amavisd-new needs to declare it does present an 8bit-clean path. A side benefit is a little speedup in passing chunks of mailing list addresses due to pipelining support; - split certain previous amavisd.conf settings (variables) into several variables or lookup tables: * sender address for notifications: $mailfrom -> $mailfrom_notify_admin, $mailfrom_notify_sender, $mailfrom_notify_recip, $mailfrom_notify_spamadmin (these may also be empty to specify null reverse path <>, which is most useful for sender notification); * administrator address for notifications: $mailto -> $virus_admin, $spam_admin (per-sender lookups) * %bypass_checks -> %bypass_virus_checks, %bypass_spam_checks - supports SQL database lookups via Perl module DBI (interface to popular database types). Some examples are provided, if you need other SQL lookups just modify the code by analogy. SQL lookups are most useful for per-user settings of virus_lovers, bypass_virus_checks, bypass_spam_checks, spam thresholds etc. when the user base is large and subgroups can not be identified through their (sub)domains. Another use is for dynamically changing settings without having to restart amavisd-new; - can optionally insert 'Received:' header if acting as a mail relay (not with milter); does loop detection as required by rfc2821 section 6.2; - notification messages now contain 'Date:' and 'Message-id:' headers; - quarantined viruses contain X-AMaViS-Alert header line with names of detected viruses; - quarantined spam contains X-Spam-Status and X-Spam-Level header lines; - optionally send spam admin notifications, which include the full SpamAssassin spam report and message header; - when started as root, changes UID and GID to $daemon_user, $daemon_group; - to facilitate startup scripts and debugging, supports few simple command line parameters: amavisd ... standard run: changes uid/gid and daemonizes amavisd start ... same thing amavisd debug ... starts with full debug level, stays attached amavisd reload ... finds amavisd master process and sends it a HUP amavisd stop ... finds amavisd master process and sends it a TERM - some more (minor) configurable options: $daemonize, $pid_file, $replace_existing_extension, $localpart_is_case_sensitive - no longer calls Convert::UUlib for uuencoded, xxencoded, and binhex attachments. The first two are handled by MIME::Parser, the remaining are likely to be handled by anti-virus scanners, especially when some virus would use such encoding. The use of Convert::UUlib is dropped because it was causing recent problems with garbling virus so that it could no longer be recognized by AV scanners, and because the underlying library does not seem very dependable. See the thread 'Amavisd passing through VBS/VBSWG.gen@MM' from the beginning of June 2002 in amavis-user mailing list archives; - quarantine files now include internal amavis id in the file name, instead of the process number; previous naming scheme could stumble across a name contention on a busy system; - only a recommendation: logging via syslog is now preferred to direct logging to a file. It serializes the logging, and avoids locking/unlocking and reopening a log file by amavisd* for every log entry. The syslog daemon does it more efficiently and reliably. NOTE for Linux users: make sure you prefix the file name in syslogd.conf with a '-' tag to disable fsync after every write; this is most necessary for heavy logging such as from MTA and/or amavisd-new. The downside is that you may lose the last few log entries in case of machine crash. See man page of your syslogd for details. - changed examples according to rfc2606 and recent complaints on the postfix-users list. CAVEATS: - header rewriting is only available in SMTP-in/SMTP-out setup, i.e. with Postfix and Exim, but not with sendmail milter setup; - the SQL lookups are a very recent addition and not so well tested as the rest of the program. Also since SQL lookups are supposed to introduce per-user settings (e.g. spam thresholds), the code still does not accommodate it, and spam thresholds of the first recipient in a message affects the whole message; - as Razor2 is still pretty much unstable, I left the spam_scan() routine much as it was in the May version. Contributions welcome; - as I ran out of my time for this release, I didn't prepare a separate version without SpamAssassin and Razor 1.20 support, so you will need to strip it out if you do not need it. Similarly for the required DBI Perl module. I'm very sorry; - the time stamp used in the 'Date:' header in notifications, and in an optional 'Received:' header, does not contain true time zone offset, but -0000 instead (standard meaning for unknown). I did not want to include a fat Perl package for handling time zones. For now just edit the subroutine rfc2822_timestamp() if you want to change that; - amavisd.conf settings $daemon_user and $daemon_group are not automatically set by ./configure options. Please set them manually. I would like to thank many people on the amavis-user list, on the postfix-users list, and in private conversations, who contributed valuable ideas and improvements, and offered much appreciated encouragements. --------------------------------------------------------------------------- May 17, 2002 Available at: http://www.ijs.si/software/amavisd/ amavisd-new-20020517 is primarily a response to popular demand for Mail::SpamAssassin support. If amavisd-new-20020424 meets your needs, there is no urgent need to upgrade. There are no incompatible changes between these two versions, except some new (optional) amavisd.conf variables, so you may keep old amavisd.conf file if you wish. A new file README.exim is now provided, so that Exim 4.x is now a supported and tested configuration, besides the usual Postfix and sendmail/milter MTAs. Also included is a brand new qmail amavis client by Lars Hecking (untested, please try it), plus his updated version of amavis.c (no longer needed in recommended Postfix and Exim configurations, and Sendmail milter and qmail configurations use a different client). Several files are unchanged from the base amavisd CVS release 2002-05-13 and do not reflect the amavisd-new state: FAQ, HINTS, INSTALL, BUGS, NEWS, TODO, ChangeLog, tests/, doc/. Please start with this file README.amavisd-new-RELNOTES and follow it. More recent instructions and last-minute changes are available from the web page. Changes since amavisd-new-20020424: - supports Mail::SpamAssassin and Vipul's Razor (1.20 required) for spam checking (but not for modifying mail body - only add headers and/or address extension, or reject/discard/quarantine spam). NOTE: spam checks are off in the default amavisd setup. Copy file ./amavis/amavisd.in.all to ./amavis/amavisd.in before running ./configure and make, to use the spam-check-enabled version! Amavisd calls Mail::SpamAssassin directly, avoiding the need to set up spamc/spamd or to chain filters. This is more efficient, one daemon less to worry about, although maybe less flexible for some taste. At the moment the SA per-user database is not used. Feel free to experiment with it and let me know what you came up with. The usual SA config files are observed, but remote tests are disabled by default (Razor is handled directly, RBL lookups can/should be performed early by MTA which knows what IP address mail came from, and that information is (mostly) lost afterwards). If you feel otherwise, change the hard-wired settings in the call: $spamassasin_obj = Mail::SpamAssassin->new( { dont_copy_prefs => 1, local_tests_only => 1 } ) (also the Razor score contribution is hard-wired, modify: '$razor_spam_found ? 3 : 0' to will). Amavisd-new handles Razor checks directly to be able to exercise more control over it than would otherwise be available through SA: timing, signatures are needed for insertion into header, skips one-liner body checks which are common Razor false-positives. To avoid SA calling Razor again, either keep the default setting 'local_tests_only=>1', or set 'score RAZOR_CHECK 0' in the SA configuration. Added are 'whitelist_sender' hash and ACL lookups (see README.lookups), which approves spam from specified SMTP originator addresses - SpamAssassin can only check and whitelist rfc822 headers, not the envelope addresses, and I see no way of passing envelope addresses to it - which is a pity, as important information is lost. SpamAssassin checks are computationally quite expensive compared to other amavisd activities, and the time needed for SA check goes up significantly with the message size. SA check are skipped (but not the Razor check) if mail size exceeds 64k. My analysis shows that presently less than 1% of spam exceeds 64k characters, and this is probably well below the false-negative SA rate, so it is not worth wasting time to check large mail. remaining changes: - new file README.exim (thanks to Jochen Erwied, Patrice Fournier and Igor D'Astolfo); - updated README.postfix to describe how to avoid running header_checks, body_checks and dns_lookups in Postfix twice; - put back file README.customize, which was omitted from amavisd-new-20020224 by mistake (but available with previous version and on the web); - added introductory paragraph to README, the rest still needs to be updated; - added missing last-step check for '.' in hash lookups to match examples and make it more useful; - added timeouts to certain tasks to make it better suitable for unattended operation (less, but still subject to certain DoS attacks, similar to the official amavisd). The main reason for adding timeouts is that we don't want spam checking to slow down amavisd operation too much: if it takes too long, just skip it (assume not spam) and move on; - change unmangle_sender() to believe sender address for Klez viruses; this is sometimes wrong, but seems like people prefer to sacrifice few false accusations in favour of some warranted sender notifications. Feel free to hack this routine at will (and publish good ideas), it is intended to be modified; - use Perl module Errno instead of errno.ph to avoid using broken file errno.ph on some popular platforms; - make possible to have per-sender-domain administrator e-mail address for admin notifications (hash lookup %mailto, sub warn_admin() ) - relax temporary file/dir protection to allow them to be readable by the group. This makes possible for anti-virus (daemon) checker to run under a different user (but in the same group). Doing so should be safer as it makes impossible for virus-checking daemon to clobber files. It is only supported in the all-SMTP configuration. To use it with traditional amavis clients (e.g. milter), you will have to modify their sources to change umask and mods for file/dir they create; - avoid changing sender address <> to <""> under certain circumstances (not strictly wrong, but still a bad idea); - replaced header 'X-Razor-Warning:' with SA-compatible 'X-Spam-Status:' ; the 'X-Razor-id:' is still provided to facilitate user spam reporting; - added '--' between options and argument when calling $sendmail_wrapper to be triply sure we avoid problems with some mailers (note that $sendmail_wrapper is only still needed in the sendmail setup, Postfix and Exim do not need it in the recommended configuration); - tidy the %local_delivery_aliases mechanism for local delivery / quarantine; - allow MIME::Parser to decode uuencoded parts, if it feels it can (and should) do it; - write warnings from MIME::Parser to the log (at log level 1 or higher), instead of discarding them. The package is available at the usual location at: http://www.ijs.si/software/amavisd/ where also the most up-to-date version of FAQ and certain other fresh documentation files live. --------------------------------------------------------------------------- April 24, 2002 amavisd-new-20020424 is primarily a maintenance release to summarize one week's worth of experience with amavisd-new-20020418 and to implement some good ideas from the amavis-user mailing list. It also brings one or two new features. It is available at: http://www.ijs.si/software/amavisd/ or more specifically, at: http://www.ijs.si/software/amavisd/amavisd-new-20020424.tar.gz Changes since amavisd-new-20020418: - removed Perl 'my' declaration from configurable variables which prevented them from being changed in amavisd.conf (thanks to Sebastian Hagedorn and Wouter de Jong for reporting) - introduced child timeouts to prevent bad amavis client from monopolizing a child forever (thanks to Sebastian Hagedorn for reporting the problem) - supported and documented (in README.postfix) configuration where multiple remote or local SMTP-in/SMTP-out MTAs (e.g. Postfix) can use the same amavisd server, by making it deliver checked mail back to the same IP address it came from (see variable $relayhost_is_client in amavisd.conf) (thanks to Wouter de Jong for the splendid idea which fitted naturally into the overall scheme) - in certain log messages include the SMTP-in and SMTP-out MTA IP address - new access list checking for IP addresses - used to limit SMTP access to authorized MTAs only: @inet_acl; access control is now enabled by default - slightly more sophisticated hash-based access list lookups, modeled after Postfix map lookups. The sequence now goes through the following steps: - hash lookup for user+foo@do.ma.in - hash lookup for user@do.ma.in (only if $recipient_delimiter is nonempty) - hash lookup for do.ma.in - hash lookup for .do.ma.in - hash lookup for .ma.in - hash lookup for .in - hash lookup user+foo@ - hash lookup user@ (only if $recipient_delimiter is nonempty) - updated README.lookups - cleaner quarantine code; new variable $mailfrom_quarantine allows to choose either the original envelope sender, or admin-specified fixed sender address; include a special per-user quarantine example (look for 'trouble-user-quarantine' in the amavisd) - fixed problem with localized system error messages - use numeric errno instead of strings in Sophie and Trophie clients (thanks to Igor D'Astolfo for reporting the problem with Italianized version of Linux and for a good suggestion) - successfully tested with Exim (thanks to Igor D'Astolfo for testing and for reporting the <> sender (but not recipient!) problem) - wrong variable used in the spam section ($final_virus_destiny instead of $final_spam_destiny) (thanks to Wayne Smith for reporting and for testing Vipul's Razor 'plugin' patch) - changed pattern match in the ./av/oav to support new version of OpenAntiVirus ScannerDaemon (thanks to Rainer Link) - changed 'configure' to make it recognize Sophie 1.33rc1 (thanks to Igor D'Astolfo and Lars Hecking) - include acinclude.m4/acx_pthread.m4 macro (thanks to Rainer Link) - updated README.postfix: includes instructions on how to avoid body_checks and header_checks for reinserted mail (contributed by Wayne Smith, works nicely, requires Postfix version 1.1.7-20020331 or later) - clarified comments in amavisd.conf - optionally keep existing X-Virus-Scanned: header lines, or remove them before adding our own header line - see $remove_existing_x_scanned_headers (requested by Darryl Harvey) - avoid historical (misleading) parameter name $localhost_ip; use $relayhost and $relayhost_port instead, but take old variables into account for backward compatibility with existing amavisd.conf files - Here is an overall picture (sequence of events) of how pieces fit together: bypass_checks? ==> PASS no viruses? ==> PASS log virus if $log_templ is nonempty quarantine if $virus_quarantine_to is nonempty notify admin if $warnadmin notify sender if $warnsender notify recips if $warnrecip final_destiny==pass? ==> PASS virus_lovers? ==> PASS DISCARD or REJECT (depending on final_*_destiny) --------------------------------------------------------------------------- April 18, 2002 This it to announce the second release of amavisd-new-20020418, available at: http://www.ijs.si/software/amavisd-new-20020418.tar.gz It is a version of amavisd (a daemonized AMaViS, which is an interface between MTA and virus scanners), based on amavisd CVS from today (20020418) (same configuration, amavis clients in C, instructions, AV client code), while also being a successor of the initial release of amavisd-new-20020329, which is a performance-enhanced pre-forked Net::Server -based amavisd with SMTP-in/SMTP-out capability, written in Perl. (I tried to make this release based on amavisd-snapshot-20020300, but failed, as that version still has $errval semantics bug (the bitwise-operations problem was discussed on the AMaViS-user list some time ago). This was fixed in the CVS version and most AV clients were changed then. As the CVS version contains AV client code for new AV scanners, this makes it incompatible with config stuff from amavisd-snapshot-20020300. If you have problems with ./configure or make, these problems would be common to both versions. Also the unpackers and decompressors code is mostly the same for all recent amavisd versions, so any problems in this arena (like DoS mishandling) are most likely to be common for both the official amavisd and the amavisd-new version. Compared to the first release of amavisd-new-20020329, the second release brings further significant performance improvements especially in the SMTP-in/SMTP-out configuration, e.g. with Postfix, but also brings some interesting new features and new configuration possibilities. No important bug fixes were needed, so upgrade at your leisure if you are running my initial version. The Postfix users would perhaps want to rush a bit though, to put new performance improvements into use. The summary of changes since the initial version: - significant SMTP-in speedups (25% with fast AV scanner), file reuse - pass reject reason to MTA on the input side - more informative MTA log entries in the SMTP-in/SMTP-out setup - amavis internal id (am_id) in log entries and passed to MTA in SMTP response - ISP features: specify subgroups of users who want to receive viruses - address extensions: e.g. user@domain -> user+virus@domain if virus detected - can specify final_virus_destiny: reject, discard, pass - quarantine new options: save to individual file, save to mailbox, pass to MTA - new headers in quarantined viruses preserve envelope addresses and quarantine id (similar to the suggestion from Furio Ercolessi - see code) - detailed timing breakdown report for each passed message - anti-spam hooks and examples, example patch to integrate Vipul's Razor client - body cache now always enabled - heavy speedup for mailing list bursts - Sophie 1.33-ready - rewritten Trophie client, based on new Sophie client code - rewritten README.postfix, describing new setup possibilities - new file README.lookups (to be used with virus_lovers and bypass_checks) - new file README.customize (same as in the initial version) - new file README.performance (unfinished) - new amavisd.conf options, documented there - code heavily commented, cleaned, generalized again - does not accuse innocent users of sending viruses if we are suspicious of sender address (see FAQ below) (same as in the initial version) CAVEATS: - no test mode; - only Postfix and sendmail milter are fully supported and tested; to integrate with other mailers one would need some understanding of their operation to set up properly; contributions are welcome; (P.S. note: this is probably a non-issue, related to the file system problem: All amavisd versions seem to share one still unresolved problem, probably with amavis-milter.c client, its use of libmilter, or perhaps even in the libmilter code itself - see recent thread on the AMaViS-user list (subject: Leftover email.txt files with amavisd 'standard' also). The problem is most pronounced in burst of heavy traffic. ) FAQ: - Net::Server 0.82 triggers a Perl 5.005 bug (the problem is obvious: you get syntax errors). Either upgrade to Net::Server 0.83, or upgrade your Perl - 5.6.x should be ok. - if you intend to play with customized notifications, it is wise to remove the '-t' option from $sendmail_wrapper_args in amavisd.conf. That way you are free to screw up notification mail headers any way you want, and the message would still be delivered to the correct recipients. Removing '-t' is now the recommended setting, but both variants should work. If you use SMTP-out method for notifications (e.g. Postfix recommended and default setting), this does not affect you. - if you see virus notifications claiming the virus originator is or and sender notifications are not sent, this is not a bug, but a feature - see comments at the subroutine unmangle_sender(). The original idea comes from Furio Ercolessi: as some viruses tend to use forged or corrupted sender or 'From:' addresses, we try to determine the true virus sender, and if we can not do that, we avoid accusing innocent users of sending viruses. - if you kill or HUP amavisd, temporary directories may be left undeleted; this is normal and mails are not lost; - if amavisd does not restart after receiving HUP, a possible reason may be that amavisd can not be found in the path as set in the $ENV{PATH} variable (near the beginning of amavisd program). Another reason may be a syntax error if you changed the amavisd.conf file. Try to start it manually: $ su vscan -c amavisd If that does not make you wiser, set $DEBUG = "yes" and retry. This is also the recommended first-time start method. - after changing $inet_socket_bind in amavisd.conf, you must stop amavisd and start it anew. The HUP method causes amavisd to stumble over its feet. Below is my announcement notice for the initial version of amavisd-new, just slightly edited to remove some mistakes. --------------------------------------------------------------------------- Date: Sat, 30 Mar 2002 04:13:25 +0100 From: Mark Martinec Subject: [AMaViS-user] ANNOUNCE: new amavisd - leaner and meaner To: amavis-user@lists.sourceforge.net Message-id: <01KFYI1DJ2O200AMKT@CATHY.IJS.SI> I would like to announce an updated version of amavisd, based on recent CVS code of amavisd (which is not far from the February amavisd snapshot). It is a result of my three weeks work on the code, caused by our needs for: - better reliability; - higher throughput (less overhead); - versatility (e.g. separating amavis and MTA hosts, load sharing), Available at: http://www.ijs.si/software/amavisd-new-20020329.tar.gz This is now finally a version which I can recommend to friends :) TODO: better DoS handling in unpackers. Main features - in brief: - pre-forked reusable children - saving on process creations; - persistent connections to certain AV scanners, e.g. Sophie, saving on forks; - both SMTP and pipe (sendmail wrapper) interfaces independently available and configurable on all three sides: input, output, notifications; - sendmail Milter interface supported and tested; Postfix supported and thoroughly tested (Exim untested; classical sendmail untested, no qmail); - customizable notification messages; - compatibility with existing configurations; - cleanups, generalizations, speedups, fixes, better code documentation; - HUP signal causes restart with new configuration; - ISP feature: certain recipients may be allowed to receive viruses (with alert header line added (not with milter), notifications are still generated); - anti-spam hooks, caching-ready, Vipul's Razor interface in a form of a patch included - should make integration with SpamAssassin easier. Install: Unpack the tar over the checked-out CVS version of amavisd branch (or February snapshot). It overlays some files, the rest is unchanged. For milter interface make sure you use amavis-milter.c from CVS, as the snapshot version contains a bug which can cause message loss if amavisd dies. Details: - child and socket handling is now delegated to Perl module Net::Server, which gives us pre-forked children which are able to do more than one mail-check during their lifetime, saving on process creations and giving better response time; - Net::Server controls number of children, does signal handling, takes care of dead children, handles listening on multiple sockets (both Unix and TCP/UDP), delegating tasks and synchronizing 'accept's; - as a consequence, certain AV-scanner interfaces (most notably Sophie, and soon to follow Trophie) can keep persistent connections to the AV checking daemon, saving on AV scanner process creations (forks on accept), and socket setups/teardowns; - on the input side: both SMTP (TCP) and traditional amavis client protocol (Unix socket) are now accepted - even both at the same time (by default) to ease transition to SMTP interface; - new input-side SMTP interface (SMTP server) is easier to set up (no need to worry about file ownerships, UIDs and GIDs), and more versatile (e.g. SMTP responses carry more information that sysexits.h-based status codes, SMTP can talk to remote host); It is based on Perl native I/O and it is quite fast - no OO overhead, _not_ SMTP::Server-based (which I do not consider production quality); - a nice by-product of SMTP input interface is the extra information available in MTA logs, e.g. Postfix log: postfix/smtp[7656]: DA7B147FA6: to=, relay=localhost[127.0.0.1], delay=5, status=sent (250 Ok, discarded - VIRUS: EICAR-AV-Test) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - notifications messages can be delivered either via SMTP or as traditionally via pipe to sendmail wrapper / MTA pickup. This means that in the Postfix setup for example, one needs to worry only about one output interface (SMTP or pipe). This also saves unnecessary work of checking the just-generated notification messages for viruses; - the output pipe interface is now more careful with status checking and forking; - output SMTP interface error handling more compact; - sender notifications can be sent to more than one contact address (not used at the moment, but the mechanism is available); - sender address unmangling patch incorporated (but I'm willing to take it out if considered inappropriate for the base distribution); - see file ./amavis/mta/postfix_init for comments describing how to select SMTP- or pipe-based output interface (for re-injection and notifications); - customizable notification messages and log entry text (see README.customize); - clean notion of when mail addresses are in their quoted and when in unquoted form (RFC2821). All internal handling uses unquoted form, addresses get quoted as required by the output interface, and quoting gets stripped away as required by each input interface; - besides traditional choices where a mail can be forwarded/accepted, discarded, or temporary failure/retry indicated, there is now a fourth choice: REJECT. Depending on the MTA this requires a message rejection to be done by the input side MTA itself. This comes handy in cases when amavisd accepts a non-infected mail, but outgoing MTA does not want to take it back for final delivery, e.g. in case of some policy violation. Traditionally amavisd would indicate temporary failure on its input side, causing the message to be retried and re-scanned over and over again, without having a chance of ever being accepted; - all existing virus scanners are still there (one little change in each: replaced a call to do_virus with return 1); Sophie client rewritten to take advantage of persistent connections; - decoding sections are mostly unchanged. This area needs more work in the future; - includes exit status codes from sysexits.ph instead or having them hard-wired; - anti-spam code easier to integrate into amavisd due to some code rearrangements; a patch to integrate Vipul's Razor is included as an example, but it may be better to tie amavisd with SpamAssassin. Anti-spam code will NOT be integrated into amavisd, but the least we can do is making it easier for people to add their own code; - cache-ready (example in the included Razor patch). This works by calculating a message digest (hash, signature) of the message body and keeping it in storage for a short while, e.g. for the lifetime of a child process (10 consecutive requests by default). If another message with the same body content arrives in the near future we can skip a virus check. This comes handy where mailing list traffic is frequent, especially if we have to deal with poorly done mailing list managers or heavy spam traffic; To support this concept the functions of MIME decoding and unpacking of archives are now separated. - MIME parsing is now supplied with our own Filer subclass. This was necessary to avoid MIME parser complaints when it tried to reconstruct file name extensions from file names in strange character sets. Its work was completely unnecessary and harmful, since we want to supply our own file names and do not care for file name extensions. - as a consequence of our own Filer subclass, we now avoid the first (or the only) directory traversal (reading) in the first decoding pass; Not much, but every little bit counts. - when using sendmail wrapper it is no longer necessary to supply the sendmail -t option. If -t is not specified, sendmail wrapper receives addresses via command arguments (exec, no shell), which is more reliable than having to parse mail headers - which are now more error-prone due to user-customizable notifications. The -t option is still supported though, but not recommended. - do_unzip no longer complains with multi-line backtrace when it dies; just a single message is issued, like with other decoders; - HUP signal causes restart and re-reading of config file; - many new comments, code unifications, supplied some missing error checks, code generalizations; MTA support: - thoroughly tested with Postfix in all combinations of input, output, and notifications interfaces; in production use; - big thanks to Sebastian Hagedorn who helped to test the milter interface on his Solaris! - Exim and traditional sendmail interface untested, but should work without much work. - Qmail interface is still missing, like in current CVS amavisd. I believe the amavis client for qmail can be written as a few-dozen line Perl program. Anybody want to try? A quick cookbook on how to set up Postfix / amavisd interface using SMTP on both amavisd input and output side (including notifications). Amavisd by default now accepts both Unix socket and SMTP on the loopback interface, so the transition is easier: first install new amavisd, then at some other time change Postfix configuration (if desired) to: master.cf: # MTA -> amavisd smtp-amavisd unix - - n - 2 smtp # amavisd -> MTA localhost:10025 inet n - n - - smtpd -o content_filter= main.cf: # choose transport to amavisd content_filter = smtp-amavisd:localhost:10024 [see new file README.postfix for details] The amavisd can now easily be located on a different host than MTA, also Postfix load-balancing transport methods can be used (e.g. multiple MX records). Although amavisd now talks SMTP and incorporates some rudimentary defenses against malicious SMTP clients, do not expose its SMTP server directly to the world - always front-end it with MTA. By default it binds to the local interface only, but other access restrictions are also available. Happy amavising! Experiences and comments are most welcome. amavisd-new-2.7.1/amavisd-new-courier.patch000640 000621 000620 00000047703 11747106037 020406 0ustar00markcmi000000 000000 --- amavisd.ori 2012-04-29 02:31:15.655242548 +0200 +++ amavisd 2012-04-29 02:31:46.672242261 +0200 @@ -101,5 +101,5 @@ # Amavis::In::AMPDP # Amavis::In::SMTP -#( Amavis::In::Courier ) +# Amavis::In::Courier # Amavis::Out::SMTP::Protocol # Amavis::Out::SMTP::Session @@ -224,5 +224,5 @@ fetch_modules('REQUIRED BASIC MODULES', 1, qw( Exporter POSIX Fcntl Socket Errno Carp Time::HiRes - IO::Handle IO::File IO::Socket IO::Socket::UNIX IO::Socket::INET + IO::Handle IO::File IO::Select IO::Socket IO::Socket::UNIX IO::Socket::INET IO::Stringy Digest::MD5 Unix::Syslog File::Basename Compress::Zlib MIME::Base64 MIME::QuotedPrint MIME::Words @@ -10829,5 +10829,5 @@ # sub post_configure_hook { -# umask(0007); # affect protection of Unix sockets created by Net::Server + umask(0007); # affect protection of Unix sockets created by Net::Server } @@ -10856,4 +10856,34 @@ ### Net::Server hook +### This hook takes place immediately after the "->run()" method is called. +### This hook allows for setting up the object before any built in configuration +### takes place. This allows for custom configurability. +sub configure_hook { + my($self) = @_; + if ($courierfilter_shutdown) { + # Duplicate the courierfilter pipe to another fd since STDIN is closed if we + # daemonize + $self->{courierfilter_pipe} = IO::File->new('<&STDIN') + or die "Can't duplicate courierfilter shutdown pipe: $!"; + } +} + +### Net::Server hook +### This hook occurs just after the bind process and just before any +### chrooting, change of user, or change of group occurs. At this point +### the process will still be running as the user who started the server. +sub post_bind_hook { + my ($self) = @_; + if (c('protocol') eq 'COURIER') { + # Allow courier to write to the socket + chmod(0660, $unix_socketname); + } + if ($self->{courierfilter_pipe}) { + # Watch for courierfilter telling us to shut down + $self->{server}->{select}->add($self->{courierfilter_pipe}); + } +} + +### Net::Server hook ### This hook occurs in the parent (master) process after chroot, ### after change of user, and change of group has occured. @@ -10908,4 +10938,15 @@ } $spamcontrol_obj->init_pre_fork if $spamcontrol_obj; + if ($courierfilter_shutdown) { + # Tell courierfilter we have finished initialisation by closing fd 3 + # But make sure it's a pipe (and not the courierfilter shutdown pipe) + # first: if we have been started using filterctl (i.e. not when + # courierfilter itself starts) then there is no initial pipe on fd 3 so + # it could be assigned to another file + open(my $fh3, '<&3'); + if (-p $fh3 && $self->{courierfilter_pipe}->fileno() != 3) { + POSIX::close(3); + } + } my(@modules_extra) = grep(!exists $modules_basic{$_}, keys %INC); if (@modules_extra) { @@ -11347,5 +11388,7 @@ $ampdp_in_obj->process_policy_request($sock, $conn, \&check_mail, 0); } elsif ($suggested_protocol eq 'COURIER') { - die "unavailable support for protocol: $suggested_protocol"; + # courierfilter client + $courier_in_obj = Amavis::In::Courier->new if !$courier_in_obj; + $courier_in_obj->process_courier_request($sock, $conn, \&check_mail); } elsif ($suggested_protocol eq 'QMQPqq') { die "unavailable support for protocol: $suggested_protocol"; @@ -11426,4 +11469,24 @@ } +### Net::Server hook +### Called in child process when any filehandle Net::Server is selecting on +### becomes readable +### Used to detect EOF on the courierfilter pipe +sub can_read_hook { + my($self, $fh) = @_; + if ($self->{courierfilter_pipe} + && fileno($fh) == $self->{courierfilter_pipe}->fileno()) + { + do_log(0, "Instructed by courierfilter to shutdown"); + kill('TERM', $self->{server}->{ppid}); + # Wait for the parent to kill us + sleep(1); + # Still here? Close down ourselves + $self->child_finish_hook; + exit; + } + return undef; +} + ### Child is about to be terminated ### user customizable Net::Server hook @@ -15669,4 +15732,9 @@ undef $Amavis::Conf::log_verbose_templ; +# courierfilter shutdown needs can_read_hook, added in Net::Server 0.90 +if ($courierfilter_shutdown && Net::Server->VERSION < 0.90) { + die "courierfilter shutdown needs Net::Server 0.90 or better"; +} + if (defined $desired_user && $daemon_user ne '') { local($1); @@ -16235,4 +16303,6 @@ host => $bind_to[0], # default bind, redundant, merged to @listen_sockets listen => $listen_queue_size, # undef for a default + # need to set multi_port for can_read_hook + multi_port => $courierfilter_shutdown ? 1 : undef, max_servers => $max_servers, # number of pre-forked children !defined($min_servers) ? () @@ -19286,5 +19356,424 @@ no warnings 'uninitialized'; -BEGIN { die "Code not available for module Amavis::In::Courier" } +BEGIN { + require Exporter; + use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); + $VERSION = '2.102'; + @ISA = qw(Exporter); + import Amavis::Conf qw(:platform :confvars ca c); + import Amavis::Util qw(do_log am_id untaint debug_oneshot snmp_counters_init + switch_to_my_time switch_to_client_time + read_text orcpt_encode xtext_decode); + import Amavis::Lookup qw(lookup); + import Amavis::Lookup::IP qw(lookup_ip_acl); + import Amavis::rfc2821_2822_Tools qw(quote_rfc2821_local qquote_rfc2821_local + unquote_rfc2821_local); + import Amavis::Timing qw(section_time); + import Amavis::TempDir; + import Amavis::In::Message; +} + +use IO::File; + +# Amavis::In::Courier->new() +# Creates a new Amavis::In::Courier object +sub new() { + my($class) = @_; + my $tempdir = Amavis::TempDir->new; + bless { tempdir => $tempdir }, $class; +} + +# courier_in_obj->process_courier_request(socket, conn, check_mail) +# Processes a request from Courier to check a single message +# socket: the socket to communicate with courierfilter +# conn: Amavis::In::Connection object +# check_mail: reference to the MTA-independent function called to check the +# message +sub process_courier_request($$$) { + my($self, $socket, $conn, $check_mail) = @_; + + # Save the policy bank so that it can be restored at the end + my %baseline_policy_bank = %current_policy_bank; + + eval { + $self->init_request($conn); + $self->read_courierfilter_socket($socket); + $self->open_mail_text(); + $self->change_policy_bank(); + $self->call_check_mail($conn, $check_mail); + $self->process_result(); + 1; + } or do { # An exception occurred + my($eval_stat) = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; + my $msg = "Error in processing: $eval_stat"; + do_log(-2, "TROUBLE in process_courier_request: 451 4.5.0 %s", $msg); + # Close the mail text file + $self->{msginfo}->mail_text->close() if ($self->{msginfo}->mail_text); + $self->{msginfo}->mail_text(undef); + # Send a temporary failure to Courier + $self->{smtp_resp} = "451 4.5.0 $msg"; + }; + + # Send the SMTP reponse back to Courier (done outside the eval to ensure that + # it always happens exactly once, whether or not there is an exception) + do_log(3, "Mail checking ended: %s", $self->{smtp_resp}); + send($socket, "$self->{smtp_resp}\n", 0); + + # Record time + section_time('send response'); + do_log(2, Amavis::Timing::report()); + + # Restore the policy bank + %current_policy_bank = %baseline_policy_bank; + + # Clean up object + $self->{per_recip_data} = undef; + $self->{control_files} = undef; +} + +# courier_in_obj->init_request( ) +# Begins processing for a single request: initialises global variables and +# creates msginfo object +# conn: Amavis::In::Connection object +sub init_request($) { + my($self, $conn) = @_; + + # Set up globals + am_id("$$-$Amavis::child_invocation_count"); + Amavis::Timing::init(); + snmp_counters_init(); + + # Create msginfo object + $self->{msginfo} = Amavis::In::Message->new; + $self->{msginfo}->rx_time(time); + $self->{msginfo}->log_id(am_id()); + $self->{msginfo}->conn_obj($conn); + + $conn->appl_proto('courierfilter'); +} + +# courier_in_obj->read_courierfilter_socket(socket) +# Reads the courierfilter socket, which specifies the path to the mail text and +# the control files, storing the path to the mail text in the msginfo object +# Also reads the control files and stores their data in msginfo +# socket: The courierfilter socket +sub read_courierfilter_socket($) { + my($self, $socket) = @_; + + # Just make sure + local $/ = "\n"; + + # Read the path to the mail text + switch_to_client_time("start receiving message text path"); + my $text_path = $socket->getline; + switch_to_my_time("received message text path"); + $text_path || die "Can't read message text path: $!"; + chomp($text_path); + $text_path = untaint($text_path) if ($text_path =~ m{^[A-Za-z0-9/._=+-]+\z}); + $self->{msginfo}->mail_text_fn($text_path); + + # Read control files + $self->{control_files} = []; + my $path; + switch_to_client_time("start receving control file paths"); + for ($! = 0; defined($path = $socket->getline); $! = 0) { + chomp($path); + # courierfilter indicates end of control files by sending a blank line + $path || last; + + switch_to_my_time("received control file path"); + $path = untaint($path) if ($path =~ m{^[A-Za-z0-9/._=+-]+\z}); + push(@{ $self->{control_files} }, $path); + $self->read_control_file($path); + switch_to_client_time("receiving control file paths"); + } + switch_to_my_time("finished receiving control file paths"); + + # Check we did actually get a control file + @{ $self->{control_files} } || die "No control files specified"; + # Record the recipients in msginfo + $self->{msginfo}->per_recip_data($self->{per_recip_data}); + + # Record time + section_time('read control'); +} + +# courier_in_obj->read_control_file(path) +# Reads a single Courier control file, adding its recipients to +# $self->{per_recip_data} and storing other information in msginfo. +# $self->{per_recip_data} is an array of Amavis::In::Message::PerRecip objects. +# (Note that this method will overwrite any previous settings for sender, etc, +# but if there are multiple control files they should contain the same +# information.) +# path: the path to the control file +sub read_control_file($) { + my($self, $path) = @_; + + do_log(3, "Reading Courier control file %s", $path); + + # Read the file + my $control_data = read_text($path); + my ($rcpt_idx, $recip) = (0, undef); + foreach (split(/\n/, $control_data)) { + # Parse a line of the control file + # Sender + do_log(4, "Courier control file line: %s", $_); + if (/^s ( .*? (?: \[ (?: \\. | [^\]\\] )* \] + | [^@"<>\[\]\\\s] )* + ) \z/xs) { + my $sender_quoted = $1; + my $sender_unquoted = unquote_rfc2821_local($sender_quoted); + $self->{msginfo}->sender_smtp('<'.$sender_quoted.'>'); + $self->{msginfo}->sender($sender_unquoted); + } + + # Recipient + if (/^r ( .*? (?: \[ (?: \\. | [^\]\\] )* \] + | [^@"<>\[\]\\\s] )* + ) \z/xs) { + $recip = Amavis::In::Message::PerRecip->new; + my $addr_quoted = $1; + my $addr_unquoted = unquote_rfc2821_local($addr_quoted); + $recip->recip_addr_smtp('<'.$addr_quoted.'>'); + $recip->recip_addr($addr_unquoted); + $recip->courier_control_file($path); + $recip->courier_recip_index($rcpt_idx); + $recip->recip_destiny(D_PASS); # Default destiny + push(@{ $self->{per_recip_data} }, $recip); + $rcpt_idx++; + } + + # Original Recipient (RFC 3461) + if (/^R ( [!-~]+ ) \z/xs) { $recip->dsn_orcpt($1) } + # RFC 3461 NOTIFY value + if (/^N ( [FSDN]+ ) \z/xs) { + my %notify_values = ( F => 'FAILURE', S => 'SUCCESS', D => 'DELAY', N => 'NEVER' ); + $recip->dsn_notify([ map { $notify_values{$_} } split(m//, $1) ]); + } + + # DSN RET parameter (RFC 3461) + if (/^t F \z/xs) { $self->{msginfo}->dsn_ret('FULL') } + if (/^t H \z/xs) { $self->{msginfo}->dsn_ret('HDRS') } + # Envid (RFC 3461) + if (/^e ( [!-~]+ ) \z/xs) { $self->{msginfo}->dsn_envid($1) } + + # Authenticated submitter (RFC 2554) + if (/^i ( [!-~]+ ) \z/xs) { $self->{msginfo}->auth_submitter(xtext_decode($1)) } + + # Received-From-MTA + if (/^f .*? ;\s* ( [A-Za-z0-9\.-]+ | \[ [0-9A-Fa-f\.:]+ \] ) \s* + \( ( [A-Za-z0-9\.-]* ) \s* \[ ( [0-9A-Fa-f\.:]+ ) \] \) + \z/xs) { + $self->{msginfo}->client_helo($1); + $self->{msginfo}->client_name($2); + $self->{msginfo}->client_addr($3); + } + + # Courier queue ID + if (/^M ( [0-9A-Fa-f]+ \. [0-9A-Fa-f]+ \. [0-9A-Fa-f]+ ) + \z/xs) { + $self->{msginfo}->queue_id($1); + } + } +} + +# courier_in_obj->open_mail_text( ) +# Opens the mail text file, whose path has been read into msginfo->mail_text_fn +# The file handle is stored in msginfo->mail_text +sub open_mail_text() { + my($self) = @_; + + # Open the file + my $fh = IO::File->new($self->{msginfo}->mail_text_fn, 'r'); + $fh || die "Can't open ", $self->{msginfo}->mail_text_fn, ": $!"; + + # Disable UTF-8 decoding of input data + if ($unicode_aware) { + binmode($fh, ':bytes') || die "Can't cancel :utf8 mode: $!"; + } + + # Store file handle + $self->{msginfo}->mail_text($fh); + + # Record time + section_time('open text'); +} + +# courier_in_obj->change_policy_bank( ) +# Loads a new policy bank if necessary +# Also enables debug_oneshot if necessary, and sets msginfo->client_addr_mynets +sub change_policy_bank() { + my($self) = @_; + my $cl_ip = $self->{msginfo}->client_addr; + my $sender = $self->{msginfo}->sender; + + # Enable debug_oneshot if set for this sender + debug_oneshot(1) if lookup(0, $sender, @{ ca('debug_sender_maps') }); + + # Load MYNETS policy bank if client IP is local + my $cl_ip_mynets = ($cl_ip eq '' ? undef + : lookup_ip_acl($cl_ip, @{ ca('mynetworks_maps') })); + $self->{msginfo}->client_addr_mynets($cl_ip_mynets); + if (($cl_ip_mynets?1:0) > (c('originating')?1:0)) { + $current_policy_bank{'originating'} = $cl_ip_mynets; + } + if ($cl_ip_mynets && defined($policy_bank{'MYNETS'})) { + Amavis::load_policy_bank('MYNETS'); + } + + # Load MYUSERS policy bank if sender is local + if ($sender ne '' && defined($policy_bank{'MYUSERS'}) + && lookup(0, $sender, @{ ca('local_domains_maps') })) + { + Amavis::load_policy_bank('MYUSERS'); + } +} + +# courier_in_obj->call_check_mail(conn, check_mail) +# Calls the check_mail function to check a message - the properties of msginfo +# must already be set +# Also handles the tempdir and closes the mail_text file afterwards +# Saves the STMP response returned by check_mail in $self->{smtp_resp} +# conn: Amavis::In::Connection object +# check_mail: reference to the function to call +sub call_check_mail($$) { + my($self, $conn, $check_mail) = @_; + + # Initialise variables + Amavis::check_mail_begin_task(); + + # Prepare temporary directory + $self->{tempdir}->prepare(); + $self->{msginfo}->mail_tempdir($self->{tempdir}->path); + + # Courier is responsible for relaying the message, and so for success DSNs + $self->{msginfo}->dsn_passed_on(c('forward_method') eq '' ? 1 : 0); + + # Log the message + do_log(1, 'Courier %s %s: %s -> %s%s', + $self->{msginfo}->queue_id, $self->{tempdir}->path, + $self->{msginfo}->sender_smtp, + join(',', map { $_->recip_addr_smtp } + @{ $self->{msginfo}->per_recip_data }), + join('', + !$self->{msginfo}->auth_submitter || + $self->{msginfo}->auth_submitter eq '<>' ? (): + ' AUTH='.$self->{msginfo}->auth_submitter, + !$self->{msginfo}->dsn_ret ? () : + ' RET='.$self->{msginfo}->dsn_ret, + !$self->{msginfo}->dsn_envid ? () : + ' ENVID='.xtext_decode($self->{msginfo}->dsn_envid), + )); + + # The temporary directory is about to become non-empty + $self->{tempdir}->empty(0); + # Do the work + my ($smtp_resp, $exit_code, $preserve_evidence) + = $check_mail->($conn, $self->{msginfo}, 0); + # Preserve evidence if necessary + $preserve_evidence && $self->{tempdir}->preserve(1); + + # Clean the temporary directory + $self->{tempdir}->clean(); + + # Close the mail text file + $self->{msginfo}->mail_text->close() || die "Can't close temp file: $!"; + $self->{msginfo}->mail_text(undef); + + # Save the SMTP response + $self->{smtp_resp} = $smtp_resp; +} + +# courier_in_obj->process_result( ) +# Processes the result of mail scanning - recipient addition/deletion +# Before calling this, the SMTP response must be stored in $self->{smtp_resp} +# and may be altered +# This does not send the SMTP response back to Courier +sub process_result() { + my($self) = @_; + + if ($self->{smtp_resp} =~ /^25/) { + foreach my $r (@{ $self->{msginfo}->per_recip_data }) { + my ($addr, $newaddr) = ($r->recip_addr, $r->recip_final_addr); + + if ($r->recip_done) { + # Deleted recipient + $self->delete_recipient($r) if defined $addr; + + } elsif (!defined($r->courier_control_file)) { + # Newly added recipient + $self->add_recipient($newaddr, '', $r->dsn_notify); + + } elsif ($newaddr ne $addr) { + # Recipient with address changed + $r->recip_smtp_response("251 2.1.5 Amavisd replaced recip with <$newaddr>"); + $self->delete_recipient($r) if defined $addr; + + my $orcpt = $r->dsn_orcpt || orcpt_encode($r->recip_addr_smtp); + $self->add_recipient($newaddr, $orcpt, $r->dsn_notify); + } + } + } + + # Record time + section_time('process result'); +} + +# delete_recipient(recip) +# Deletes a recipient by marking them as successfully delivered in the control +# file. If the same recipient appears more than once in the control files, +# every instance will be marked as done. +# recip: Amavis::In::Message::PerRecip object +sub delete_recipient($) { + my($self, $recip) = @_; + + do_log(1, "Amavis::In::Courier: Deleting recipient <%s>: %s", + $recip->recip_addr, $recip->recip_smtp_response); + + my $filename = $recip->courier_control_file; + my $control_file = IO::File->new($filename, 'a'); + # Not sure why we do the seek when the file is already opened for append, + # but courier-pythonfilter does it so it's probably a good idea + seek($control_file, 0, 2); + # Courier may still append to the control file after calling the filter, + # so write a long blank line first to ensure that its additional records + # only overwrite the blank line + $control_file->print(" " x 254, "\n"); + + # Tell Courier the message is delivered + $control_file->printf("I%d R %s\n", $recip->courier_recip_index, + $recip->recip_smtp_response); + $control_file->printf("S%d %d\n", $recip->courier_recip_index, time); + + $control_file->close or die "Error closing control file $filename: $!"; +} + +# add_recipient(recip) +# Adds a recipient to the last control file for the message. +# address: recipient address to add +# orig_recip: RFC 3461 original recipient (if any) +# notify: reference to array containing values of the DSN NOTIFY value +sub add_recipient($$$) { + my ($self, $address, $orig_recip, $notify) = @_; + + do_log(1, "Amavis::In::Courier: Adding recipient <%s>", $address); + + # Convert $notify array into character string + my $notify_str = join('', map { substr($_, 0, 1) } @$notify); + + # Open the last control file + my $filename = $self->{control_files}->[-1]; + my $control_file = IO::File->new($filename, 'a'); + # Take care with the control file: see comments in delete_recipient + seek($control_file, 0, 2); + $control_file->print(" " x 254, "\n"); + + # Add recipient to control file + $control_file->print("r$address\n"); + $control_file->print("R$orig_recip\n"); + $control_file->print("N$notify_str\n"); + + $control_file->close or die "Error closing control file $filename: $!"; +} 1; amavisd-new-2.7.1/amavisd-signer000751 000621 000620 00000110376 11425614271 016335 0ustar00markcmi000000 000000 #!/usr/bin/perl -T #------------------------------------------------------------------------------ # This is amavisd-signer, a DKIM signing service daemon for amavisd. # It uses an AM.PDP protocol lookalike to receive a request from amavisd # and provides two services: choosing a signing key, and signing a # message digest with a chosen DKIM private key. # # Using a separate signing service (which may run under a dedicated UID or # GID or as root, having exclusive access to private keys) releaves amavisd # process from needing to have access to private keys. Separating roles can # provide improved protection for DKIM private keys, and/or can provide more # flexibility in choosing a signing key. # # Usage: # amavisd-signer & # # Author: Mark Martinec # Copyright (C) 2010 Mark Martinec, All Rights Reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # * Neither the name of the author, nor the name of the "Jozef Stefan" # Institute, nor the names of contributors may be used to endorse or # promote products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER # OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # #(the license above is the new BSD license, and pertains to this program only) # # Patches and problem reports are welcome. # The latest version of this program is available at: # http://www.ijs.si/software/amavisd/ #------------------------------------------------------------------------------ package AmavisSigner; use strict; use re 'taint'; use warnings FATAL => 'utf8'; no warnings 'uninitialized'; use Sys::Syslog; # used by Net::Server for logging use MIME::Base64; use Mail::DKIM; use Mail::DKIM::PrivateKey; use Net::Server 0.91; use Net::Server::Multiplex; use vars qw(@ISA); @ISA = qw(Net::Server::Multiplex); use vars qw( $VERSION $log_level %dkim_signing_keys_by_domain @dkim_signing_keys_list @dkim_signing_keys_storage @dkim_signature_options_bysender_maps $daemon_chroot_dir $daemon_user $daemon_group $pid_file $daemonize $inet_socket_bind @listen_sockets $listen_queue_size $syslog_ident $syslog_facility ); $VERSION = 1.000; # 20100730 # # Please adjust the following settings as necessary: # $daemon_user = 'vscan'; $daemon_group = 'vscan'; # $daemon_chroot_dir = '/var/amavis'; # chroot directory or undef # $daemonize = 1; $log_level = 2; # 0..5 $syslog_facility = 'mail'; $syslog_ident = 'amavisd-signer'; # the $inet_socket_bind and @listen_sockets should correspond to a # setting $dkim_signing_service in amavisd.conf : $inet_socket_bind = '127.0.0.1'; @listen_sockets = ( 20203 ); $listen_queue_size = undef; # uses a default # Load all available private keys and supply their public key RR constraints. # Arguments are a domain, a selector, a key (a file name of a private key in # PEM format), followed by optional attributes/constraints (tags, represented # here as Perl hash key/value pairs) which are allowed by RFC 4871 in a public # key resource record (v, g, h, k, n, s, t), of which only g, h, k, s and t # are considered to be constraints limiting the choice of a signing key. # # signing domain selector private key options # ------------- -------- ---------------------- ---------- # dkim_key('example.org', 'abc', '/var/db/dkim/a.key.pem'); # dkim_key('example.org', 'yyy', '/var/db/dkim/b.key.pem', t=>'s'); # dkim_key('example.org', 'zzz', '/var/db/dkim/b.key.pem', h=>'sha256'); # dkim_key('example.com', 'sel-2008', '/var/db/dkim/sel-example-com.key.pem', # t=>'s:y', g=>'*', k=>'rsa', h=>'sha256:sha1', s=>'email', # n=>'testing; 1, 2'); # dkim_key('guest.example.com', 'g', '/var/db/dkim/g-guest-ex-com.key.pem'); # dkim_key('mail.example.com', 'notif', '/var/db/dkim/notif-mail.key.pem'); # @dkim_signature_options_bysender_maps maps author/sender addresses or # domains to signature tags/requirements; possible signature tags according # to RFC 4871 are: (v), a, (b), (bh), c, d, (h), i, l, q, s, (t), x, z; # of which the following are determined implicitly: v, b, bh, h, t # (tag h is controlled by %signed_header_fields); currently ignored tags # are l and z; instead of an absolute expiration time (tag x) one may use # a pseudo tag 'ttl' to specify a relative expiration time in seconds, which # is converted to an absolute expiration time prior to signing: x = t + ttl; # a built-in default is provided for each tag if no better match is found # # @dkim_signature_options_bysender_maps = ( { # 'postmaster@mail.example.com' => { a => 'rsa-sha1', ttl => 7*24*3600 }, # 'spam-reporter@example.com' => { a => 'rsa-sha1', ttl => 7*24*3600 }, # 'mail.example.com' => { a => 'rsa-sha1', ttl => 10*24*3600 }, # # explicit 'd' forces a third-party signature on foreign (hosted) domains # 'guest.example' => { d => 'guest.example.com' }, # '.example.com' => { d => 'example.com' }, # # catchall defaults # '.' => { a => 'rsa-sha256', c => 'relaxed/simple', ttl => 30*24*3600 }, # # 'd' defaults to a domain of an author/sender address, # # 's' defaults to whatever selector is offered by a matching key # } ); # # ====================================================================== # No further user-configurable settings below (but feel free # to customize code in choose_key_request() or replace it altogether. # ====================================================================== # sub ll($) { my($level) = @_; $level <= $log_level; } my($server); # a Net::Server object sub do_log($$;@) { my($level, $errmsg, @args) = @_; $errmsg = sprintf($errmsg,@args) if @args; if ($level <= $log_level) { my($prio); # Net::Server logging priority # 0=err, 1=warning, 2=notice, 3=info, 4=debug if ($level >= 3) { $prio = 4 } elsif ($level >= 0) { $prio = 2 } elsif ($level >= -1) { $prio = 1 } else { $prio = 0 } $server->log($prio, sanitize_str($errmsg)); # Net::Server directs STDERR to the log_file # print STDERR sanitize_str($errmsg)."\n" } } sub sanitize_str { my($str, $keep_eol) = @_; my(%map) = ("\r" => '\\r', "\n" => '\\n', "\f" => '\\f', "\t" => '\\t', "\b" => '\\b', "\e" => '\\e', "\\" => '\\\\'); if ($keep_eol) { $str =~ s/([^\012\040-\133\135-\176])/ # and \240-\376 ? exists($map{$1}) ? $map{$1} : sprintf(ord($1)>255 ? '\\x{%04x}' : '\\%03o', ord($1))/eg; } else { $str =~ s/([^\040-\133\135-\176])/ # and \240-\376 ? exists($map{$1}) ? $map{$1} : sprintf(ord($1)>255 ? '\\x{%04x}' : '\\%03o', ord($1))/eg; } $str; } sub split_address($) { my($mailbox) = @_; local($1,$2); $mailbox =~ /^ (.*?) ( \@ (?: \[ (?: \\. | [^\]\\] ){0,999} (?: \] | \z) | [^\[\@] )* ) \z/xs ? ($1, $2) : ($mailbox, ''); } # THE dkim_key IS A DIRECT COPY OF THE SAME ROUTINE FROM amavisd # # Store a private DKIM signing key for a given domain and selector. # The argument $key can be a Mail::DKIM::PrivateKey object or a file # name containing a key in a PEM format (e.g. as generated by openssl). # For compatibility with dkim_milter the signing domain can include a '*' # as a wildcard - this is not recommended as this way amavisd could produce # signatures which have no corresponding public key published in DNS. # The proper way is to have one dkim_key entry for each published DNS RR. # Optional arguments can provide additional information about the resource # record (RR) of a public key, i.e. its options according to RFC 4871. # The subroutine is typically called from a configuration file, once for # each signing key available. # sub dkim_key($$$;@) { my($domain,$selector,$key) = @_; shift; shift; shift; @_%2 == 0 or die "dkim_key: a list of key/value pairs expected as options\n"; my(%key_options) = @_; # remaining args are options from a public key RR defined $domain && $domain ne '' or die "dkim_key: domain must not be empty: ($domain,$selector,$key)"; defined $selector && $selector ne '' or die "dkim_key: selector must not be empty: ($domain,$selector,$key)"; my($key_storage_ind); if (ref $key) { # key already preprocessed and provided as an object push(@dkim_signing_keys_storage, [$key]); $key_storage_ind = $#dkim_signing_keys_storage; } else { # assume a name of a file containing a private key in PEM format my($fname) = $key; my($pem_fh) = IO::File->new; # open a file with a private key $pem_fh->open($fname,'<') or die "Can't open PEM file $fname: $!"; my(@stat_list) = stat($pem_fh); # soft-link friendly @stat_list or warn "Error on accessing $fname: $!"; my($dev,$inode) = @stat_list; if ($dev && $inode) { for my $j (0..$#dkim_signing_keys_storage) { # same file reused? my($k,$dv,$in,$fn) = @{$dkim_signing_keys_storage[$j]}; if ($dv == $dev && $in == $inode) { $key_storage_ind = $j; last } } } if (!defined($key_storage_ind)) { # read file and store its contents as a new entry my($nbytes,$buff); $key = ''; while (($nbytes=$pem_fh->read($buff,16384)) > 0) { $key .= $buff } defined $nbytes or die "Error reading key from file $fname: $!"; push(@dkim_signing_keys_storage, [$key,$dev,$inode,$fname]); $key_storage_ind = $#dkim_signing_keys_storage; } $pem_fh->close or die "Error closing file $fname: $!"; $key_options{k} = 'rsa' if defined $key_options{k}; # force RSA } $domain = lc($domain) if !ref($domain); # possibly a regexp $selector = lc($selector); $key_options{domain} = $domain; $key_options{selector} = $selector; $key_options{key_storage_ind} = $key_storage_ind; if (@dkim_signing_keys_list > 100) { # sorry, skip the test to avoid slow O(n^2) searches } else { !(grep { $_->{domain} eq $domain && $_->{selector} eq $selector } @dkim_signing_keys_list) or die "dkim_key: selector $selector for domain $domain already in use\n"; } $key_options{key_ind} = $#dkim_signing_keys_list + 1; push(@dkim_signing_keys_list, \%key_options); # using a list preserves order } # THE dkim_key_postprocess IS A DIRECT COPY OF THE SAME ROUTINE FROM amavisd # # Convert private keys (as strings in PEM format) into RSA objects # and do some pre-processing on @dkim_signing_keys_list entries # (may run unprivileged) # sub dkim_key_postprocess() { # convert private keys (as strings in PEM format) into RSA objects for my $ks (@dkim_signing_keys_storage) { my($pkcs1,$dev,$inode,$fname) = @$ks; if (ref($pkcs1) && UNIVERSAL::isa($pkcs1,'Crypt::OpenSSL::RSA')) { # it is already a Crypt::OpenSSL::RSA object } else { # assume a string is a private key in PEM format, convert it to RSA obj $ks->[0] = Crypt::OpenSSL::RSA->new_private_key($pkcs1); } } for my $ent (@dkim_signing_keys_list) { my($domain) = $ent->{domain}; $dkim_signing_keys_by_domain{$domain} = [] if !$dkim_signing_keys_by_domain{$domain}; } my($any_wild); my($j) = 0; for my $ent (@dkim_signing_keys_list) { $ent->{v} = 'DKIM1' if !defined $ent->{v}; # provide a default if (defined $ent->{n}) { # encode n as qp-section (rfc4871, rfc2047) $ent->{n} =~ s{([\000-\037\177=;"])}{sprintf('=%02X',ord($1))}egs; } my($domain) = $ent->{domain}; if (ref($domain) eq 'Regexp') { $ent->{domain_re} = $domain; $any_wild = sprintf("key#%d, %s", $j+1, $domain) if !defined $any_wild; } elsif ($domain =~ /\*/) { # wildcarded signing domain in a key declaration, evil, asks for trouble! # support wildcards in signing domain for compatibility with dkim_milter my($regexp) = $domain; $regexp =~ s/\*{2,}/*/gs; # collapse successive wildcards # '*' is a wildcard, quote the rest $regexp =~ s{ ([@#/.^$|*+?(){}\[\]\\]) }{$1 eq '*' ? '.*' : '\\'.$1}gex; $regexp = '^' . $regexp . '\\z'; # implicit anchors $regexp =~ s/^\^\.\*//s; # remove leading anchor if redundant $regexp =~ s/\.\*\\z\z//s; # remove trailing anchor if redundant $regexp = '(?:)' if $regexp eq ''; # just in case, non-empty regexp # presence of {'domain_re'} entry lets get_dkim_key use this regexp # instead of a direct string comparision with {'domain'} $ent->{domain_re} = qr{$regexp}; # compiled regexp object $any_wild = sprintf("key#%d, %s", $j+1, $domain) if !defined $any_wild; } # %dkim_signing_keys_by_domain entries contain lists of indices into # the @dkim_signing_keys_list of all potentially applicable signing keys. # This hash (keyed by domain name) avoids linear searching for signing # keys for all fully-specified domains in @dkim_signing_keys_list. # Wildcarded entries must still be looked up sequentially at run-time # to preserve the declared order and the 'first match wins' paradigm. # Such entries are only supported for compatibility with dkim_milter # and are evil because amavisd has no quick way of verifying that DNS RR # really exists, so signatures generated by amavisd can fail when not all # possible DNS resource records exist for wildcarded signing domains. # if (!defined($ent->{domain_re})) { # no regexp, just plain match on domain push(@{$dkim_signing_keys_by_domain{$domain}}, $j); } else { # a wildcard in a signing domain, compatibility with dkim_milter # wildcarded signing domain potentially matches any _by_domain entry for my $d (keys %dkim_signing_keys_by_domain) { push(@{$dkim_signing_keys_by_domain{$d}}, $j); } # the '*' entry collects only wildcarded signing keys $dkim_signing_keys_by_domain{'*'} = [] if !$dkim_signing_keys_by_domain{'*'}; push(@{$dkim_signing_keys_by_domain{'*'}}, $j); } $j++; } do_log(0,"dkim: wildcard in signing domain (%s), may produce unverifiable ". "signatures with no published public key, avoid!", $any_wild) if $any_wild; } # THE get_dkim_key IS A DIRECT COPY OF THE SAME ROUTINE FROM amavisd # # Fetch a private DKIM signing key for a given signing domain, with its # resource-record (RR) constraints compatible with proposed signature options. # The first such key is returned as a hash; if no key is found an empty hash # is returned. When a selector (s) is given it must match the selector of # a key; when algorithm (a) is given, the key type and a hash algorithm must # match the desired use too; the service type (s) must be 'email' or '*'; # when identity (i) is given it must match the granularity (g) of a key; # # sign.opts. key options # ---------- ----------- # d => domain # s => selector # a => k, h(list) # i => g, t=s # sub get_dkim_key(@) { @_ % 2 == 0 or die "get_dkim_key: a list of pairs is expected as query opts"; my(%options) = @_; # signature options (v, a, c, d, h, i, l, q, s, t, x, z), # of which d is required, while s, a and t are optional but taken into # account in searching for a compatible key - the rest are ignored my(%key_options); my($domain) = $options{d}; defined $domain && $domain ne '' or die "get_dkim_key: domain is required, but tag 'd' is missing"; $domain = lc($domain); my(@indices) = $dkim_signing_keys_by_domain{$domain} ? @{$dkim_signing_keys_by_domain{$domain}} : $dkim_signing_keys_by_domain{'*'} ? @{$dkim_signing_keys_by_domain{'*'}} : (); if (@indices) { my($selector) = $options{s}; $selector = $selector eq '' ? undef : lc($selector) if defined $selector; local($1,$2); my($keytype,$hashalg) = defined $options{a} && $options{a} =~ /^([a-z0-9]+)-(.*)\z/is ? ($1,$2) : ('rsa',undef); my($identity_localpart,$identity_domain) = !defined($options{i}) ? () : split_address($options{i}); $identity_localpart = '' if !defined $identity_localpart; $identity_domain = '' if !defined $identity_domain; # find the first key (associated with a domain) with compatible options for my $j (@indices) { my($ent) = $dkim_signing_keys_list[$j]; next unless defined $ent->{domain_re} ? $domain =~ $ent->{domain_re} : $domain eq $ent->{domain}; next if defined $selector && $ent->{selector} ne $selector; next if $keytype ne (exists $ent->{k} ? $ent->{k} : 'rsa'); next if exists $ent->{s} && !(grep { $_ eq '*' || $_ eq 'email' } split(/:/, $ent->{s}) ); next if defined $hashalg && exists $ent->{'h'} && !(grep { $_ eq $hashalg } split(/:/, $ent->{'h'}) ); if (defined($options{i})) { if (lc($identity_domain) eq $domain) { # ok } elsif (exists $ent->{t} && (grep {$_ eq 's'} split(/:/,$ent->{t}))) { next; # no subdomains allowed } if (!exists($ent->{g}) || $ent->{g} eq '*') { # ok } elsif ($ent->{g} =~ /^ ([^*]*) \* (.*) \z/xs) { next if $identity_localpart !~ /^ \Q$1\E .* \Q$2\E \z/xs; } else { next if $identity_localpart ne $ent->{g}; } } %key_options = %$ent; last; # found a suitable match } } if (defined $key_options{key_storage_ind}) { # obtain actual key from @dkim_signing_keys_storage ($key_options{key}) = @{$dkim_signing_keys_storage[$key_options{key_storage_ind}]}; } %key_options; } sub proto_encode($@) { my($attribute_name,@strings) = @_; local($1); for ($attribute_name,@strings) { # just in case, handle non-octet characters: s/([^\000-\377])/sprintf('\\x{%04x}',ord($1))/eg and do_log(-1,"proto_encode: non-octet character encountered: %s", $_); } $attribute_name =~ # encode all but alfanumerics, . _ + - s/([^0-9a-zA-Z._+-])/sprintf("%%%02x",ord($1))/eg; for (@strings) { # encode % and nonprintables s/([^\041-\044\046-\176])/sprintf("%%%02x",ord($1))/eg; } $attribute_name . '=' . join(' ',@strings); } sub proto_decode($) { my($str) = @_; local($1); $str =~ s/%([0-9a-fA-F]{2})/pack("C",hex($1))/egs; $str; } sub split_localpart($$) { my($localpart, $delimiter) = @_; my($owner_request_special) = 1; # configurable ??? my($extension); local($1,$2); if ($localpart =~ /^(postmaster|mailer-daemon|double-bounce)\z/i) { # do not split these, regardless of what the delimiter is } elsif ($delimiter eq '-' && $owner_request_special && $localpart =~ /^owner-.|.-request\z/si) { # don't split owner-foo or foo-request } elsif ($localpart =~ /^(.+?)(\Q$delimiter\E.*)\z/s) { ($localpart, $extension) = ($1, $2); # extension includes a delimiter # do not split the address if the result would have a null localpart } ($localpart, $extension); } sub unique_ref(@) { my($r) = @_ == 1 && ref($_[0]) ? $_[0] : \@_; # accept list, or a list ref my(%seen); my(@result) = grep { defined($_) && !$seen{$_}++ } @$r; \@result; } sub make_query_keys($$$;$) { my($addr,$at_with_user,$include_bare_user,$append_string) = @_; my($localpart,$domain) = split_address($addr); $domain = lc($domain); my($saved_full_localpart) = $localpart; $localpart = lc($localpart); ### if !c('localpart_is_case_sensitive'); # chop off leading @, and trailing dots local($1); $domain = $1 if $domain =~ /^\@?(.*?)\.*\z/s; my($extension); my($delim) = '+'; ### c('recipient_delimiter'); if ($delim ne '') { ($localpart,$extension) = split_localpart($localpart,$delim); # extension includes a delimiter since amavisd-new-2.5.0! } $extension = '' if !defined $extension; # mute warnings my($append_to_user,$prepend_to_domain) = $at_with_user ? ('@','') : ('','@'); my(@keys); # a list of query keys push(@keys, $addr); # as is push(@keys, $localpart.$extension.'@'.$domain) if $extension ne ''; # user+foo@example.com push(@keys, $localpart.'@'.$domain); # user@example.com if ($include_bare_user) { # typically enabled for local users only push(@keys, $localpart.$extension.$append_to_user) if $extension ne ''; # user+foo(@) push(@keys, $localpart.$append_to_user); # user(@) } push(@keys, $prepend_to_domain.$domain); # (@)sub.example.com if ($domain =~ /\[/) { # don't split address literals push(@keys, $prepend_to_domain.'.'); # (@). } else { my(@dkeys); my($d) = $domain; for (;;) { # (@).sub.example.com (@).example.com (@).com (@). push(@dkeys, $prepend_to_domain.'.'.$d); last if $d eq ''; $d = ($d =~ /^([^.]*)\.(.*)\z/s) ? $2 : ''; } if (@dkeys > 10) { @dkeys = @dkeys[$#dkeys-9 .. $#dkeys] } # sanity limit push(@keys,@dkeys); } if (defined $append_string && $append_string ne '') { $_ .= $append_string for @keys; } my($keys_ref) = unique_ref(\@keys); # remove duplicates ll(5) && do_log(5,"query_keys: %s", join(', ',@$keys_ref)); # the rhs replacement strings are similar to what would be obtained # by lookup_re() given the following regular expression: # /^( ( ( [^\@]*? ) ( \Q$delim\E [^\@]* )? ) (?: \@ (.*) ) )$/xs my($rhs) = [ # a list of right-hand side replacement strings $addr, # $1 = User+Foo@Sub.Example.COM $saved_full_localpart, # $2 = User+Foo $localpart, # $3 = user $extension, # $4 = +foo $domain, # $5 = sub.example.com ]; ($keys_ref, $rhs); } sub lookup_hash($$;$%) { my($addr, $hash_ref,$get_all,%options) = @_; ref($hash_ref) eq 'HASH' or die "lookup_hash: arg2 must be a hash ref: $hash_ref"; local($1,$2,$3,$4); my(@matchingkey,@result); my($append_string); $append_string = $options{AppendStr} if defined $options{AppendStr}; my($keys_ref,$rhs_ref) = make_query_keys($addr,1,1,$append_string); for my $key (@$keys_ref) { # do the search if (exists $$hash_ref{$key}) { # got it push(@result,$$hash_ref{$key}); push(@matchingkey,$key); last if !$get_all; } } # do the right-hand side replacements if any $n, ${n} or $(n) is specified for my $r (@result) { # remember that $r is just an alias to array elements if (defined($r) && !ref($r) && index($r,'$') >= 0) { # plain string with $ my($any) = $r =~ s{ \$ ( (\d+) | \{ (\d+) \} | \( (\d+) \) ) } { my($j)=$2+$3+$4; $j<1 ? '' : $rhs_ref->[$j-1] }gxse; # bring taintedness of input to the result $r .= substr($addr,0,0) if $any; } } if (!$get_all) { ($result[0], $matchingkey[0]) } else { (\@result, \@matchingkey) } } sub lookup2($$$%) { my($get_all, $addr, $tables_ref, %options) = @_; (@_ - 3) % 2 == 0 or die "lookup2: options argument not in pairs (not hash)"; my($label, @result,@matchingkey); for my $tb (!$tables_ref ? () : @$tables_ref) { my($t) = ref($tb) eq 'REF' ? $$tb : $tb; # allow one level of indirection if (!ref($t) || ref($t) eq 'SCALAR') { # a scalar always matches my($r) = ref($t) ? $$t : $t; # allow direct or indirect reference if (defined $r) { do_log(5,'lookup: (scalar) matches, result="%s"', $r); push(@result,$r); push(@matchingkey,"(constant:$r)"); } } elsif (ref($t) eq 'HASH') { my($r,$mk); ($r,$mk) = lookup_hash($addr,$t,$get_all,%options) if %$t; if (!defined $r) {} elsif (!$get_all) { push(@result,$r); push(@matchingkey,$mk) } elsif (@$r) { push(@result,@$r); push(@matchingkey,@$mk) } } else { die "TROUBLE: lookup table not implemented for object: " . ref($t); } last if @result && !$get_all; } if (!$get_all) { ($result[0], $matchingkey[0]) } else { (\@result, \@matchingkey) } } sub parse_quoted_rfc2821($$) { my($addr,$unquote) = @_; # the angle-bracket stripping is not really a duty of this subroutine, # as it should have been already done elsewhere, but we allow it here anyway: $addr =~ s/^\s*\s*\z//s; # tolerate unmatched angle brkts local($1,$2); my($source_route,$localpart,$domain) = ('','',''); # RFC 2821: so-called "source route" MUST BE accepted, # SHOULD NOT be generated, and SHOULD be ignored. # Path = "<" [ A-d-l ":" ] Mailbox ">" # A-d-l = At-domain *( "," A-d-l ) # At-domain = "@" domain if (index($addr,':') >= 0 && # triage before more testing for source route $addr =~ m{^ ( [ \t]* \@ (?: [0-9A-Za-z.!\#\$%&*/^{}=_+-]* | \[ (?: \\. | [^\]\\] ){0,999} \] ) [ \t]* (?: , [ \t]* \@ (?: [0-9A-Za-z.!\#\$%&*/^{}=_+-]* | \[ (?: \\. | [^\]\\] ){0,999} \] ) [ \t]* )* : [ \t]* ) (.*) \z }xs) { # NOTE: we are quite liberal on allowing whitespace around , and : here, # and liberal in allowed character set and syntax of domain names, # we mainly avoid stop-characters in the domain names of source route $source_route = $1; $addr = $2; } if ($addr =~ m{^ ( .*? ) ( \@ (?: [^\@\[\]]+ | \[ (?: \\. | [^\]\\] ){0,999} \] | [^\@] )* ) \z}xs) { ($localpart,$domain) = ($1,$2); } else { ($localpart,$domain) = ($addr,''); } $localpart =~ s/ " | \\ (.) | \\ \z /$1/xsg if $unquote; # undo quoted-pairs ($source_route, $localpart, $domain); } sub unquote_rfc2821_local($) { my($mailbox) = @_; my($source_route,$localpart,$domain) = parse_quoted_rfc2821($mailbox,1); # make address with '@' in the localpart but no domain (like <"aa@bb.com"> ) # distinguishable from by representing it as aa@bb.com@ in # unquoted form; (it still obeys all regular rules, it is not a dirty trick) $domain = '@' if $domain eq '' && $localpart ne '' && $localpart =~ /\@/; $localpart . $domain; } # # ====================================================================== # Code above is copied from amavisd; some day it should be factored out. # Code from here on is specific to amavisd-signer. # ====================================================================== # # process a request to choose a signing key; # sub choose_key_request($) { my($attr) = @_; my(@results); my(%sig_options); # signature options, and constraints for choosing a key my(%key_options); # options associated with a signing key my(@tried_domains); # used for logging a failure my($chosen_addr,$chosen_addr_src); my($cand) = $attr->{candidate}; my(@candidates) = !defined $cand ? () : !ref $cand ? $cand : @$cand; my($sobm) = \@dkim_signature_options_bysender_maps; for my $pair (@candidates) { my($addr_src,$addr) = split(' ',$pair,2); $addr = unquote_rfc2821_local($addr); my($addr_localpart,$addr_domain) = split_address($addr); $addr_domain = lc($addr_domain); my($dkim_options_ref,$mk_ref) = lookup2(1,$addr,$sobm); $dkim_options_ref = [] if !defined $dkim_options_ref; #***? # place catchall default(s) at the end of the list of options; push(@$dkim_options_ref, { c => 'relaxed/simple', a => 'rsa-sha256' }); %sig_options = (); # signature options: # (v), a, (b), (bh), c, d, (h), i, (l), q, s, (t), x, (z) # traverse from specific to general, first match wins for my $opts_hash_ref (@$dkim_options_ref) { while (my($k,$v) = each(%$opts_hash_ref)) { $sig_options{$k} = $v if !exists($sig_options{$k}) } } # a default for a signing domain is a domain of each tried address if (!exists($sig_options{d})) { my($d) = $addr_domain; $d =~ s/^\@//; $sig_options{d} = $d } push(@tried_domains, $sig_options{d}); ll(5) && do_log(5, "signature options for %s(%s): %s", $addr,$addr_src, join('; ', map { $_.'='.$sig_options{$_} } keys %sig_options)); # find a private key associated with a signing domain and selector, # and meeting constraints %key_options = get_dkim_key(%sig_options) if defined $sig_options{d} && $sig_options{d} ne ''; my($key) = $key_options{key}; if (defined $key && $key ne '') { # found; copy the key and its options $sig_options{key} = $key; $sig_options{s} = $key_options{selector}; $chosen_addr = $addr; $chosen_addr_src = $addr_src; last; } } # if any signature options were specified in the request and not overruled # by more specific ones here, copy them to the resulting set of sig options for my $opt (keys %$attr) { if ($opt =~ /^sig\.(.+)\z/) { $sig_options{$1} = $attr->{$opt} if !exists($sig_options{$1}); } } ll(5) && do_log(5, "sig options: %s", join('; ', map { $_.'='.$sig_options{$_} } keys %sig_options)); my(%key_options); if (defined $sig_options{d} && $sig_options{d} ne '') { %key_options = get_dkim_key(%sig_options); } do_log(5, "key options: %s is %s", $_, $key_options{$_}) for keys %key_options; my($s) = $key_options{'selector'}; my($d) = $key_options{'domain'}; $sig_options{'s'} = $s; $sig_options{'d'} = $d; delete $sig_options{'key'}; # no use of key ref in the protocol for my $opt (sort keys %sig_options) { if (defined $sig_options{$opt}) { push(@results, proto_encode('sig.'.$opt, $sig_options{$opt})); } } # optional information if available: client may log it, or use for debugging if (defined $chosen_addr_src && defined $chosen_addr) { push(@results, proto_encode('chosen_candidate', $chosen_addr_src, $chosen_addr)); } \@results; } # sign a digest code using the specified algorithm and a private signing key # sub dkim_rsa_sign($$$) { my($digest,$alg_name,$key) = @_; my($result); $digest = '' if !defined $digest; $alg_name = '' if !defined $alg_name; if (defined $key && $key ne '') { my($key) = Mail::DKIM::PrivateKey->load(Cork => $key); $key or die "no key available\n"; $result = $key->sign_digest($alg_name,$digest); } $result; } # process a request to sign the supplied digest with a selected key # # presence of the 'b' attribute in the result indicates success, # otherwise the result is treated as signature unavailable # sub sign_request($) { my($attr) = @_; my(@results, $reason, $sig); my($digest, $digest_alg, $selector, $domain) = @$attr{qw(digest digest_alg s d)}; if (!defined $digest || $digest eq '') { $reason = 'cannot sign, digest not provided, nothing to sign'; } elsif (!defined $digest_alg || $digest_alg eq '') { $reason = 'cannot sign, digest algorithm name not provided'; } elsif (!defined $domain || $domain eq '') { $reason = 'cannot sign, signing domain not provided'; } elsif (!defined $selector || $selector eq '') { $reason = 'cannot sign, selector not provided'; } else { my(%sig_options); # signature options: v, a, c, d, h, i, l, q, s, t, x, z $sig_options{s} = $selector; $sig_options{d} = $domain; my(%key_options) = get_dkim_key(%sig_options); if (!defined $key_options{key}) { $reason = 'cannot sign, signing key not available'; } else { do_log(5, "key options: %s is %s", $_, $key_options{$_}) for keys %key_options; eval { $sig = dkim_rsa_sign(decode_base64($digest), $digest_alg, $key_options{key}); 1; } or do { my($eval_stat) = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; do_log(0, "signing failed: %s", $eval_stat); $reason = 'cannot sign: ' . $eval_stat; }; push(@results, proto_encode('d', $key_options{'domain'})); push(@results, proto_encode('s', $key_options{'selector'})); } } if (defined $sig && $sig ne '') { push(@results, proto_encode('b', encode_base64($sig,''))); } else { $reason = 'cannot sign: signing failed' if !defined $reason; push(@results, proto_encode('reason', $reason)); } \@results; } # process the request received from amavisd # sub do_the_request($) { my($attr) = @_; ll(2) && do_log(2, "got: %s", join('; ', map { my($k) = $_; my($v) = $attr->{$k}; map { $k.'='.$_ } (!ref $v ? $v : @$v) } keys %$attr)); my(@results); my($req_id) = $attr->{request_id}; my($log_id) = $attr->{log_id}; push(@results, proto_encode('request_id', $req_id)) if defined $req_id; push(@results, proto_encode('log_id', $log_id)) if defined $log_id; my($request_type) = $attr->{request}; $request_type = '' if !defined $request_type; if ($request_type eq 'choose_key') { push(@results, @{choose_key_request($attr)}); } elsif ($request_type eq 'sign') { push(@results, @{sign_request($attr)}); } else { push(@results, proto_encode('reason', 'unknown request type')); do_log(2, "got: ignoring request: %s", $request_type); } ll(1) && do_log(1, "response: %s", join('; ', @results)); do_log(5, ""); \@results; } # IO::Multiplex -style callback hook # sub mux_connection { my($self,$mux,$fh) = @_; do_log(3, "client %s just connected", $self->{peeraddr}); $self->{attr} = {}; } # the mux_connection callback is guaranteed to have already been run once # sub mux_input { my($self,$mux,$fh,$in_ref) = @_; my $attr = $self->{attr}; do_log(5, "input from %s ready", $self->{peeraddr}); # process each line in the input, leaving partial lines in the input buffer local($1,$2); my($quit) = 0; while ($$in_ref =~ s/^(.*?)\015?\012//) { my($ln) = $1; if ($ln eq '') { # empty line indicates end of a request my($results_ref) = do_the_request($attr); print(join('', map { $_."\015\012" } (@$results_ref,''))) or do_log(0,"mux_input: error writing a response to socket" ); %$attr = (); # reset, awaiting next request in the same session } elsif ($ln =~ /^ ([^=\000\012]*?) (?: = | : [ \t]* ) (.*) \z/xsi) { my($attr_name) = proto_decode($1); my($attr_val) = proto_decode($2); if (!exists $attr->{$attr_name}) { $attr->{$attr_name} = $attr_val; # simple scalar for one-time attrs } elsif (!ref($attr->{$attr_name})) { # multiple, convert to a list $attr->{$attr_name} = [ $attr->{$attr_name}, $attr_val ]; } else { # append to a list of same-name attributes push(@{$attr->{$attr_name}}, $attr_val); } } else { do_log(0, "mux_input: ignored line: %s", $ln); } } close(STDOUT) if $quit; } # # Main program starts here (after initializations near the top of this file) # dkim_key_postprocess(); # set up a Net::Server configuration $server = AmavisSigner->new({ # limit socket bind (e.g. to the loopback interface) host => (!defined $inet_socket_bind || $inet_socket_bind eq '' ? '*' : $inet_socket_bind), port => \@listen_sockets, # listen on these sockets (Unix or inet) listen => $listen_queue_size, # undef for a default user => ($> == 0 || $< == 0) ? $daemon_user : undef, group => ($> == 0 || $< == 0) ? $daemon_group : undef, background => $daemonize ? 1 : undef, setsid => $daemonize ? 1 : undef, chroot => $daemon_chroot_dir ne '' ? $daemon_chroot_dir : undef, pid_file => $pid_file, log_file => $daemonize ? 'Sys::Syslog' : undef, syslog_ident => $syslog_ident, syslog_facility => $syslog_facility, syslog_logsock => 'native', # 0=err, 1=warning, 2=notice, 3=info, 4=debug log_level => $log_level >= 5 ? 4 : 2, }); $server->run; # transferring control to Net::Server exit 1; # shouldn't get here amavisd-new-2.7.1/MANIFEST000640 000621 000620 00000005537 11424276727 014642 0ustar00markcmi000000 000000 MANIFEST this file LICENSE GPL license text AAAREADME.first start reading here... RELEASE_NOTES ...followed by reading/browsing this file INSTALL installation instructions README_FILES/ MTA-specific instructions and other documentation README_FILES/amavisd-new-docs.html a more detailed documentation on selected topics (a work in progress), the most recent version is at: http://www.ijs.si/software/amavisd/amavisd-new-docs.html amavisd the amavisd-new daemon (should go into /usr/local/sbin/) amavisd.conf its configuration file (should go into /etc/) amavisd.conf-default lists all configuration variables with their defaults amavisd-custom.conf example custom hooks, to be invoked from amavisd.conf amavisd-agent a demo program to access and display SNMP-like counters being updated and made available as a Berkeley DB by amavisd amavisd-nanny a program to show the status and keep an eye on the health of child processes in amavisd-new (experimental) amavisd-release a program to request releasing a message from a quarantine amavisd-submit a simple program to pass an email message to amavisd daemon and to adjust its exit status according to a response received p0f-analyzer.pl a program to interface amavisd with a p0f utility amavisd-snmp-subagent a SNMP AgentX program, exporting the amavisd statistical counters and gauges database as well as a process health database to a snmpd daemon supporting AgentX protocol (RFC 2741), such a NET-SNMP; AMAVIS-MIB.txt The MIB module (SNMP Management information base) describing amavisd-new statistics and health information. Useful to a SNMP client program such as snmpwalk or Cacti; amavisd-signer A DKIM signing service daemon for amavisd. It uses an AM.PDP protocol lookalike to receive a request from amavisd and provides two services: choosing a signing key, and signing a message digest with a chosen DKIM private key. Amavisd uses this signing service configured through a $dkim_signing_service setting if it is nonempty; JpegTester.pm a Perl module needed if 'check-jpeg' AV checker entry is enabled; to be placed in Perl include paths if needed; test-messages/ contains sample/test mail messages TODO missing features, wish list, ... CONTRIBUTED WORK: LDAP.schema amavisd-new LDAP schema for LDAP lookups LDAP.ldif amavisd-new LDAP schema for LDAP lookups (in ldif format) amavisd-new-courier.patch adds support for the Courier MTA amavisd-new-qmqpqq.patch adds support for the qmail MTA amavisd_init.sh sample init shell script amavisd-new.spec rpm spec file amavisd-new-2.7.1/JpegTester.pm000640 000621 000620 00000010101 10233022772 016064 0ustar00markcmi000000 000000 package JpegTester; # Author: Mark Martinec , 2004-10; # The (new)BSD license applies to this package JpegTester; use strict; use re 'taint'; use vars qw($buf $buf_l $buf_ofs); sub makeTwo($) { # ensure at least two characters in $buf, except near eof my($fh) = @_; $buf_l>=0 or die "jpeg: Panic, program error1, pos=$buf_ofs"; if ($buf_l<2) { my($len) = sysread($fh,$buf,2048,$buf_l); # 2k is about the optimum size defined $len or die "jpeg: Can't read: $!\n"; $buf_l += $len; } } sub takeN($$) { # swallow n characters my($fh,$n) = @_; my($err) = undef; for ($buf_l>=2||makeTwo($fh); $n>0; ) { if ($buf_l<=0) { $err = "Truncated by $n bytes or more"; last } if ($n >= $buf_l) { $n -= $buf_l; $buf_ofs += $buf_l; $buf = '' } else { $buf = substr($buf,$n); $buf_ofs += $n; $n = 0 } $buf_l = length($buf); $buf_l>=2 || makeTwo($fh); } $err; } sub takeECS($) { # quickly swallow entropy-coded data segment my($fh) = @_; for ($buf_l>=2||makeTwo($fh); $buf_l>0; ) { if ($buf =~ s/^([^\xff]+)//) { $buf_ofs += length($1) } elsif ($buf =~ s/^(\xff\x00)+//) { $buf_ofs += length($1) } else { last } # last unless $buf =~ s/^(?: [^\xff] | \xff \x00 )+//x; # Perl Bus error $buf_l = length($buf); $buf_l>=2 || makeTwo($fh); } } sub takeFill($) { # swallow fill bytes before marker my($fh) = @_; for (makeTwo($fh); $buf_l>0; $buf_l=length($buf),makeTwo($fh)) { if ($buf =~ s/^ \xff+ (?= \xff )//x) { $buf_ofs += length($1) } else { last } } } sub takeTail($) { # swallow common junk after EOI my($fh) = @_; for (makeTwo($fh); $buf_l>0; $buf_l=length($buf),makeTwo($fh)) { if ($buf =~ s/^[\x00\xff]+//) { $buf_ofs += length($1) } else { last } } } # exit status: 0:clean; 1:exploit; 2:corrupted sub test_jpeg($;@) { my($fn) = @_; # file name to be checked local(*F); my($fh) = \*F; open($fh,"<$fn") or die "jpeg: Can't open file $fn for reading: $!"; binmode($fh) or die "jpeg: Can't set binmode on $fn: $!"; $buf = ''; $buf_l = 0; $buf_ofs = 0; makeTwo($fh); my(@r) = (0,"jpeg ok"); if ($buf !~ /^\xff\xd8/) { @r = (0,"not jpeg") } else { takeN($fh,2); if ($buf !~ /^\xff/) { @r = (0,"not jpeg") } } if ($r[1] eq "jpeg ok") { my($ecs_ok) = 0; local($1); for (;;) { # keep at least 2 chars in buff except near eof if ($buf_l<=0 || $buf eq "\xff") { @r = (2,"Truncated, no EOI, pos=$buf_ofs") if !$r[0]; last; } elsif ($buf =~ /^( [^\xff] | \xff \x00 )/x) { # ecs @r = (2,"Unexpected entropy-coded data segment, pos=$buf_ofs") if !$ecs_ok && !$r[0]; takeECS($fh); $ecs_ok = 0; } elsif ($buf =~ /^ \xff+ (?= \xff ) /x) { # fill bytes before marker takeFill($fh); $ecs_ok = 0; } elsif ($buf =~ /^ \xff ([^\x00\xff]) /x) { # marker my($m) = $1; takeN($fh,2); if ($m =~ /[\xd0-\xd7]/) { # RSTi # printf("marker segm, pos=%d, marker=0x%02X\n", $buf_ofs,ord($m)); $ecs_ok = 1; } elsif ($m =~ /[\x01\xd8]/) { # TEM, SOI # printf("marker segm, pos=%d, marker=0x%02X\n", $buf_ofs,ord($m)); $ecs_ok = 0; } elsif ($m eq "\xd9") { # EOI # printf("marker segm, pos=%d, marker=0x%02X\n", $buf_ofs,ord($m)); takeFill($fh); $ecs_ok = 0; @r = (2,"Trailing garbage, pos=$buf_ofs") if $buf_l>0 && !$r[0]; last; } else { # marker segment $ecs_ok = $m eq "\xda"; # SOS my($len) = unpack("n",substr($buf,0,2)); # printf("marker segm len %d, pos=%d, marker=0x%02X\n", $len,$buf_ofs,ord($m)); @r = (1,sprintf("Invalid marker segm len %d, pos=%d, marker=0x%02X", $len,$buf_ofs,ord($m)) ) if $len<2; my($err) = takeN($fh,$len); @r = (2,"$err, pos=$buf_ofs") if defined $err && !$r[0]; } } else { die "jpeg: Panic, program error2, pos=$buf_ofs" } $buf_l>=2 || makeTwo($fh); } } close($fh) or die "jpeg: Can't close $fn: $!"; $r[1] = "bad jpeg: ".$r[1] if $r[0]; @r; } 1; # insure a defined return amavisd-new-2.7.1/amavisd_init.sh000640 000621 000620 00000002570 07565344027 016506 0ustar00markcmi000000 000000 #!/bin/sh # # amavisd This script controls the amavisd-new daemon. # (to be used with version amavisd-new-20020630 or later) # # chkconfig: 2345 79 31 # description: amavisd is an interface between MTA and content checkers # processname: amavisd # pidfile: /var/amavis/amavisd.pid # Source function library. . /etc/rc.d/init.d/functions # Source networking configuration. . /etc/sysconfig/network #prog="/opt/amavisd-new/sbin/amavisd" prog="/usr/sbin/amavisd" prog_base="$(basename ${prog})" prog_config_file="/etc/amavisd.conf" # Source configuration. [ -e /etc/sysconfig/${prog_base} ] && . /etc/sysconfig/${prog_base} ## Check that networking is up. #[ ${NETWORKING} = "no" ] && exit 0 RETVAL=0 # See how we were called. case "$1" in start) action $"Starting ${prog_base}:" ${prog} -c ${prog_config_file} RETVAL=$? [ $RETVAL -eq 0 ] && touch /var/lock/subsys/${prog_base} echo ;; stop) action $"Shutting down ${prog_base}:" ${prog} -c ${prog_config_file} stop RETVAL=$? if [ $RETVAL -eq 0 ] ; then echo "${prog_base} stopped" rm -f /var/lock/subsys/${prog_base} else echo fi ;; status) status ${prog_base} RETVAL=$? ;; restart) $0 stop $0 start RETVAL=$? ;; reload) action $"Reloading ${prog_base}:" ${prog} -c ${prog_config_file} reload RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|restart|reload}" exit 1 esac exit $RETVAL amavisd-new-2.7.1/amavisd.conf-default000640 000621 000620 00000111105 11744021577 017407 0ustar00markcmi000000 000000 use strict; ## A CONFIGURATION FILE FOR AMAVISD-NEW, LISTING ALL CONFIGURATION VARIABLES ## WITH THEIR DEFAULT VALUES (FOR REFERENCE ONLY, NON-AUTHORITATIVE) ## This software is licensed under the GNU General Public License (GPL). ## See comments at the start of file amavisd for the whole license text. ## Copyright (C) 2002-2011 Mark Martinec, All Rights Reserved. ## The 'after-default' comment indicates that these variables obtain their ## default value if the config file left them undefined. It means these values ## are not yet available during processing of the configuration file, but that ## they can derive their value from other configurations variables no matter ## where in the configuration file they appear. ## GENERAL # $myhostname = ... predefined default from uname(3), must be a FQDN # $mydomain = ... no useful default, should be set if used in expressions # $snmp_contact = ''; # $snmp_location = ''; # $daemon_user = undef; # $daemon_group = undef; # $MYHOME = '/var/amavis'; # $TEMPBASE = $MYHOME; # after-default # $db_home = "$MYHOME/db"; # after-default # $pid_file = "$MYHOME/amavisd.pid"; # after-default # $lock_file = undef; # $daemon_chroot_dir = undef; # $max_requests = 20; # retire a child after that many accepts # $max_servers = 2; # number of pre-forked children # $min_servers = undef; # see Net::Server::Prefork for semantics # $min_spare_servers = undef; # $max_spare_servers = undef; # $child_timeout = 8*60; # $localpart_is_case_sensitive = 0; # $enable_db = 0; # $nanny_details_level = 1; # verbosity: 0, 1, 2 # @additional_perl_modules = (); # @local_domains_maps=(\%local_domains,\@local_domains_acl,\$local_domains_re); # @mynetworks = qw( 127.0.0.0/8 [::1] [FE80::]/10 [FEC0::]/10 # 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 ); # @mynetworks_maps = (\@mynetworks); # @client_ipaddr_policy = map { $_ => 'MYNETS' } @mynetworks_maps; ## LOGGING AND DEBUGGING # $log_level = 0; # $logfile = undef; # $do_syslog = undef; # same as 0 # $syslog_ident = 'amavis'; # $syslog_facility = 'mail'; # $logline_maxlen = 980; # $log_short_templ ... built-in default at the end of file amavisd # $log_verbose_templ ... built-in default at the end of file amavisd # $log_recip_templ = ... built-in default at the end of file amavisd # $log_templ = $log_short_templ; # @debug_sender_acl = (); # @debug_sender_maps = (\@debug_sender_acl); # @debug_recipient_maps = (); # $sa_debug = undef; # $allow_preserving_evidence = 1; ## DKIM VERIFICATION # $enable_dkim_verification = undef; # $reputation_factor = 0.2; # @signer_reputation_maps = (); # $myauthservid = $myhostname; # after-default (RFC 5451) ## DKIM SIGNING # $enable_dkim_signing = undef; # %dkim_signing_keys = (); # @dkim_signature_options_bysender_maps = (); # $dkim_signing_service = undef; ## MTA INTERFACE - INPUT # @listen_sockets = ... $unix_socketname and $inet_socket_port are added here # $unix_socketname = undef; # Unix socket to accept amavis helper protocol # $unix_socket_mode = undef; # sets sockets protection (numeric mode), or undef # $inet_socket_port = undef; # accept connections on this TCP port(s) (SMTP...) # $inet_socket_bind = '127.0.0.1'; # @inet_acl = qw( 127.0.0.1 [::1] ); # $listen_queue_size = undef; # $protocol = ... defaults to 'SMTP' or 'LMTP' (autodetected) on inet and inet6 # sockets; must be configured explicitly for Unix sockets. # Possible values: 'SMTP', 'LMTP', 'AM.PDP', # and with appropriate patches applied also: 'COURIER' or 'QMQPqq' # $soft_bounce = undef; # $smtpd_timeout = 8*60; # $smtpd_recipient_limit = 1100; # $smtpd_message_size_limit = undef; # site-wide limit # @message_size_limit_maps = (); # per-recipient limits # $smtpd_greeting_banner = '${helo-name} ${protocol} ${product} service ready'; # $smtpd_quit_banner = '${helo-name} ${product} closing transmission channel'; # $auth_required_inp = undef; # $auth_required_release = 1; # @auth_mech_avail=(); # empty list disables incoming AUTH; or: qw(PLAIN LOGIN) # $tls_security_level_in = undef; # undef, 'may', 'encrypt', ... # $smtpd_tls_cert_file = undef; # $smtpd_tls_key_file = undef; # $smtp_connection_cache_on_demand = 1; # $smtp_connection_cache_enable = 1; # $enforce_smtpd_message_size_limit_64kb_min = 1; # @smtpd_discard_ehlo_keywords = (); ## MTA INTERFACE - OUTPUT ## see also $notify_method, $forward_method and $*_quarantine_method # $localhost_name = 'localhost'; # my EHLO name, and inserted in Received # $local_client_bind_address = undef; # my source IP address as a SMTP client # $auth_required_out = undef; # $amavis_auth_user = undef; # for submitting notifications and quarantine # $amavis_auth_pass = undef; # $auth_reauthenticate_forwarded = undef; # our credentials for forwarding too # $tls_security_level_out = undef; # undef, 'may', 'encrypt', ... ## MAIL FORWARDING # $forward_method = 'smtp:[127.0.0.1]:10025'; # may be arrayref # @forward_method_maps = ( sub { Opaque(c('forward_method')) } ); # $resend_method = undef; # falls back to $forward_method # $always_bcc = undef; # $final_virus_destiny = D_DISCARD; # subj to @viruses_that_fake_sender_maps # $final_banned_destiny = D_DISCARD; # $final_spam_destiny = D_PASS; # subject to $sa_dsn_cutoff_level # $final_bad_header_destiny = D_PASS; ## QUARANTINE # $release_method = undef; # falls back to $notify_method # $requeue_method = 'smtp:[127.0.0.1]:25'; # $release_format = 'resend'; # (dsn), (arf), attach, plain, resend # $report_format = 'arf'; # (dsn), arf, attach, plain, resend # $attachment_password = ''; # '': no pwd, undef: PIN, code ref, or static str # $attachment_email_name = 'msg-%m.eml'; # $attachment_outer_name = 'msg-%m.zip'; # $virus_quarantine_method = 'local:virus-%m'; # $spam_quarantine_method = 'local:spam-%m.gz'; # $banned_files_quarantine_method = 'local:banned-%m'; # $bad_header_quarantine_method = 'local:badh-%m'; # $clean_quarantine_method = undef; # $archive_quarantine_method = undef; # $mail_id_size_bits = 72; # $QUARANTINEDIR = undef; # $quarantine_subdir_levels = undef; # 0 or 1 (undef treated as 0) # $sql_quarantine_chunksize_max; # see SQL section # $virus_quarantine_to = 'virus-quarantine'; # via %local_delivery_aliases # $banned_quarantine_to = 'banned-quarantine'; # $bad_header_quarantine_to= 'bad-header-quarantine'; # $spam_quarantine_to = 'spam-quarantine'; # $spam_quarantine_bysender_to = undef; # $clean_quarantine_to = 'clean-quarantine'; # $archive_quarantine_to = 'archive-quarantine'; # @virus_quarantine_to_maps = (\$virus_quarantine_to); # @banned_quarantine_to_maps = (\$banned_quarantine_to); # @bad_header_quarantine_to_maps = (\$bad_header_quarantine_to); # @spam_quarantine_to_maps = (\$spam_quarantine_to); # @spam_quarantine_bysender_to_maps = (\$spam_quarantine_bysender_to); # @clean_quarantine_to_maps = (\$clean_quarantine_to); # @archive_quarantine_to_maps = (\$archive_quarantine_to); # %local_delivery_aliases ... predefined, used by a delivery method 'local:' # $mailfrom_to_quarantine = undef; # undef keeps original sender ## NOTIFICATIONS (DSN, admin, recip) # $notify_method = 'smtp:[127.0.0.1]:10025'; # $propagate_dsn_if_possible = 1; # $terminate_dsn_on_notify_success = 0; # $newvirus_admin = undef; # $virus_admin = undef; # $spam_admin = undef; # $banned_admin = undef; # $bad_header_admin = undef; # $dsn_bcc = undef; # @newvirus_admin_maps = (\$newvirus_admin); # @virus_admin_maps = (\%virus_admin, \$virus_admin); # @banned_admin_maps = (\$banned_admin, \%virus_admin, \$virus_admin); # @spam_admin_maps = (\%spam_admin, \$spam_admin); # @bad_header_admin_maps= (\$bad_header_admin); # $hdr_encoding = 'UTF-8'; # header field bodies charset # $bdy_encoding = 'UTF-8'; # notification body text charset # $hdr_encoding_qb = 'Q'; # quoted-printable (Q or B) # $notify_sender_templ = ... built-in default at the end of file amavisd # $notify_virus_sender_templ = ... built-in default at the end of file amavisd # $notify_spam_sender_templ = ... built-in default at the end of file amavisd # $notify_virus_admin_templ = ... built-in default at the end of file amavisd # $notify_spam_admin_templ = ... built-in default at the end of file amavisd # $notify_virus_recips_templ = ... built-in default at the end of file amavisd # $notify_spam_recips_templ = ... built-in default at the end of file amavisd # $notify_release_templ = ... built-in default at the end of file amavisd # $notify_report_templ = ... built-in default at the end of file amavisd # $mailfrom_notify_admin = undef; # $mailfrom_notify_recip = undef; # $mailfrom_notify_spamadmin = undef; ## these are after-defaults: # $hdrfrom_notify_sender = "\"Content-filter at $myhostname\" "; # $hdrfrom_notify_recip = ... derived from $mailfrom_notify_recip # $hdrfrom_notify_admin = ... derived from $mailfrom_notify_admin # $hdrfrom_notify_spamadmin = ... derived from $mailfrom_notify_spamadmin # $hdrfrom_notify_release = $hdrfrom_notify_sender; # $hdrfrom_notify_report = $hdrfrom_notify_sender; # $warnbannedsender = undef; # $warnbadhsender = undef; # $warn_offsite = undef; # $warnvirusrecip = undef; # $warnbannedrecip = undef; # $warnbadhrecip = undef; # @warnvirusrecip_maps = (\$warnvirusrecip); # @warnbannedrecip_maps = (\$warnbannedrecip); # @warnbadhrecip_maps = (\$warnbadhrecip); ## MODIFICATIONS TO PASSED MAIL # %allowed_added_header_fields = ...; # built-in default # %prefer_our_added_header_fields = ...; # built-in default # $remove_existing_x_scanned_headers = 0; # $remove_existing_spam_headers = 1; # @remove_existing_spam_headers_maps = (\$remove_existing_spam_headers); # $allow_fixing_improper_header = 1; # all-white folding lines and long lines # $allow_fixing_improper_header_folding = 1; # $allow_fixing_long_header_lines = 1; # $prepend_header_fields_hdridx = 0; # $X_HEADER_TAG = 'X-Virus-Scanned'; # after-default # $X_HEADER_LINE = "$myproduct_name at $mydomain"; # after-default # $defang_virus = undef; # $defang_banned = undef; # $defang_spam = undef; # $defang_bad_header = undef; # $defang_undecipherable = undef; # $defang_all = undef; # mostly for testing # $allow_disclaimers = undef; # $enable_anomy_sanitizer = 0; # @anomy_sanitizer_args = (); # a config file or list of var=value pairs # $altermime = 'altermime'; # a path to the program # @altermime_args_defang = qw(--verbose --removeall); # @altermime_args_disclaimer = qw(--disclaimer=/etc/altermime-disclaimer.txt); # @disclaimer_options_bysender_maps = (); # $undecipherable_subject_tag = '***UNCHECKED*** '; # $sa_spam_subject_tag = undef; # $sa_spam_level_char = '*'; # @spam_subject_tag_maps = (\$sa_spam_subject_tag1); # N.B.: inconsistent name # @spam_subject_tag2_maps = (\$sa_spam_subject_tag); # N.B.: inconsistent name # @spam_subject_tag3_maps = (); ## ADDING ADDRESS EXTENSIONS TO RECIPIENTS - 'plus addressing' # $recipient_delimiter = undef; # $replace_existing_extension = 1; # $addr_extension_virus = undef; # $addr_extension_banned = undef; # $addr_extension_spam = undef; # $addr_extension_bad_header = undef; # @addr_extension_virus_maps = (\$addr_extension_virus); # @addr_extension_banned_maps = (\$addr_extension_banned); # @addr_extension_spam_maps = (\$addr_extension_spam); # @addr_extension_bad_header_maps = (\$addr_extension_bad_header); ## MAIL DECODING # $bypass_decode_parts = undef; # $keep_decoded_original_re = undef; # @keep_decoded_original_maps = (\$keep_decoded_original_re); # $map_full_type_to_short_type_re = ... predefined regexp lookup table # @map_full_type_to_short_type_maps = (\$map_full_type_to_short_type_re); # $MAXLEVELS = undef; # $MAXFILES = undef; # $MIN_EXPANSION_QUOTA = undef; # $MAX_EXPANSION_QUOTA = undef; # $MIN_EXPANSION_FACTOR = 5; # times original mail size # $MAX_EXPANSION_FACTOR = 500; # times original mail size # $path = undef; # $file = 'file'; # For backwards compatibility the @decoders list defaults to use of legacy # variables $gzip, $bzip2, $lzop, ... It is cleaner to explicitly assign # a list to @decoders in amavisd.conf and directly specify program paths, # without indirections through legacy variables $gzip, etc. # # $gzip = $bzip2 = $lzop = $rpm2cpio = undef; # $uncompress = $unfreeze = $arc = $unarj = $unrar = undef; # $zoo = $lha = $pax = $cpio = $cabextract = undef; # # @decoders = ( # ['mail', \&do_mime_decode], ### ['asc', \&do_ascii], ### ['uue', \&do_ascii], ### ['hqx', \&do_ascii], ### ['ync', \&do_ascii], # ['F', \&do_uncompress, \$unfreeze], # ['Z', \&do_uncompress, \$uncompress], # ['gz', \&do_gunzip], # ['gz', \&do_uncompress, \$gunzip], # ['bz2', \&do_uncompress, \$bunzip2], # ['xz', \&do_uncompress, # ['xzdec', 'xz -dc', 'unxz -c', 'xzcat'] ], # ['lzma', \&do_uncompress, # ['lzmadec', 'xz -dc --format=lzma', # 'lzma -dc', 'unlzma -c', 'lzcat', 'lzmadec'] ], # ['lzo', \&do_uncompress, \$unlzop], # ['rpm', \&do_uncompress, \$rpm2cpio], # ['cpio', \&do_pax_cpio, \$pax], # ['cpio', \&do_pax_cpio, \$cpio], # ['tar', \&do_pax_cpio, \$pax], # ['tar', \&do_pax_cpio, \$cpio], ### ['tar', \&do_tar], # no longer supported # ['deb', \&do_ar, \$ar], ### ['a', \&do_ar, \$ar], # unpacking .a seems an overkill # ['zip', \&do_unzip], # ['kmz', \&do_unzip], # ['7z', \&do_7zip, ['7zr','7za','7z'] ], # ['rar', \&do_unrar, \$unrar], # ['arj', \&do_unarj, \$unarj], # ['arc', \&do_arc, \$arc], # ['zoo', \&do_zoo, \$zoo], ### ['lha', \&do_lha, \$lha], # not safe # ['doc', \&do_ole, \$ripole], # ['cab', \&do_cabextract, \$cabextract], # ['tnef', \&do_tnef_ext, \$tnef], # ['tnef', \&do_tnef], ### ['sit', \&do_unstuff, \$unstuff], # broken/unsafe decoder # ['exe', \&do_executable, \$unrar,\$lha,\$unarj], # ); ## ANTI-VIRUS AND INVALID/FORBIDDEN CONTENTS CONTROLS # @av_scanners = (); # @av_scanners_backup = (); # $first_infected_stops_scan = undef; # $virus_scanners_failure_is_fatal = undef; # $viruses_that_fake_sender_re = undef; # @viruses_that_fake_sender_maps = (\$viruses_that_fake_sender_re, 1); # @virus_name_to_policy_bank_maps = (); # # @virus_name_to_spam_score_maps = # (new_RE( # the order matters! # [ qr'^Structured\.(SSN|CreditCardNumber)\b' => 0.1 ], # [ qr'^(Heuristics\.)?Phishing\.' => 0.1 ], # [ qr'^(Email|HTML)\.Phishing\.(?!.*Sanesecurity)' => 0.1 ], # [ qr'^Sanesecurity\.(Malware|Rogue|Trojan)\.' => undef ],# keep as infected # [ qr'^Sanesecurity\.' => 0.1 ], # [ qr'^Sanesecurity_PhishBar_' => 0 ], # [ qr'^Sanesecurity.TestSig_' => 0 ], # [ qr'^Email\.Spam\.Bounce(\.[^., ]*)*\.Sanesecurity\.' => 0 ], # [ qr'^Email\.Spammail\b' => 0.1 ], # [ qr'^MSRBL-(Images|SPAM)\b' => 0.1 ], # [ qr'^VX\.Honeypot-SecuriteInfo\.com\.Joke' => 0.1 ], # [ qr'^VX\.not-virus_(Hoax|Joke)\..*-SecuriteInfo\.com(\.|\z)' => 0.1 ], # [ qr'^Email\.Spam.*-SecuriteInfo\.com(\.|\z)' => 0.1 ], # [ qr'^Safebrowsing\.' => 0.1 ], # [ qr'^winnow\.(phish|spam)\.' => 0.1 ], # [ qr'^INetMsg\.SpamDomain' => 0.1 ], # [ qr'^Doppelstern\.(Scam4|Phishing)' => 0.1 ], # [ qr'^ScamNailer\.' => 0.1 ], # [ qr'^HTML/Bankish' => 0.1 ], # F-Prot # [ qr'-SecuriteInfo\.com(\.|\z)' => undef ], # keep as infected # [ qr'^MBL_NA\.UNOFFICIAL' => 0.1 ], # false positives # [ qr'^MBL_' => undef ], # keep as infected # )); # $banned_namepath_re = undef; # new-style # $banned_filename_re = undef; # traditional # @banned_filename_maps = (\$banned_filename_re); # traditional # @bypass_virus_checks_maps = (\%bypass_virus_checks, \@bypass_virus_checks_acl, \$bypass_virus_checks_re); # @bypass_banned_checks_maps = (\%bypass_banned_checks, \@bypass_banned_checks_acl, \$bypass_banned_checks_re); # @bypass_header_checks_maps = (\%bypass_header_checks, \@bypass_header_checks_acl, \$bypass_header_checks_re); # @virus_lovers_maps = (\%virus_lovers, \@virus_lovers_acl, \$virus_lovers_re); # @banned_files_lovers_maps = (\%banned_files_lovers, \@banned_files_lovers_acl, \$banned_files_lovers_re); # @bad_header_lovers_maps = (\%bad_header_lovers, \@bad_header_lovers_acl, \$bad_header_lovers_re); # @unchecked_lovers_maps = (); # $allowed_header_tests{$_} = 1 for qw(other mime 8bit control empty long # syntax missing multiple); ## ANTI-Spam CONTROLS # @spam_scanners = ( ['SpamAssassin', 'Amavis::SpamControl::SpamAssassin'] ); # $helpers_home = $MYHOME; # after-default # $sa_configpath = undef; # $sa_siteconfigpath = undef; # $sa_num_instances = 1; # @sa_userconf_maps = (); # @sa_username_maps = (); # $sa_mail_body_size_limit = undef; # $sa_local_tests_only = 0; # $sa_spawned = 0; # $dspam = undef; # $sa_timeout = 30; # @bypass_spam_checks_maps = (\%bypass_spam_checks, \@bypass_spam_checks_acl, \$bypass_spam_checks_re); # @spam_lovers_maps = (\%spam_lovers, \@spam_lovers_acl, \$spam_lovers_re); # $sa_tag_level_deflt = undef; # $sa_tag2_level_deflt = undef; # $sa_tag3_level_deflt = undef; # $sa_kill_level_deflt = undef; # $sa_dsn_cutoff_level = undef; # $sa_crediblefrom_dsn_cutoff_level = undef; # $sa_quarantine_cutoff_level = undef; # @spam_tag_level_maps = (\$sa_tag_level_deflt); # @spam_tag2_level_maps = (\$sa_tag2_level_deflt); # @spam_tag3_level_maps = (\$sa_tag3_level_deflt); # @spam_kill_level_maps = (\$sa_kill_level_deflt); # @spam_quarantine_cutoff_level_maps = (\$sa_quarantine_cutoff_level); # @spam_notifyadmin_cutoff_level_maps = (); # @spam_dsn_cutoff_level_maps = (\$sa_dsn_cutoff_level); # @spam_dsn_cutoff_level_bysender_maps = (\$sa_dsn_cutoff_level); # @spam_crediblefrom_dsn_cutoff_level_maps = # (\$sa_crediblefrom_dsn_cutoff_level); # @spam_crediblefrom_dsn_cutoff_level_bysender_maps = # (\$sa_crediblefrom_dsn_cutoff_level); # $bounce_killer_score = 0; # $penpals_bonus_score = undef; # $penpals_halflife = 7*24*60*60; # $penpals_threshold_low = 1.0; # $penpals_threshold_high = undef; # $reputation_factor = 0.2; # @score_sender_maps = (); # @signer_reputation_maps = (); # @blacklist_sender_maps = (\%blacklist_sender, \@blacklist_sender_acl, \$blacklist_sender_re); # @whitelist_sender_maps = (\%whitelist_sender, \@whitelist_sender_acl, \$whitelist_sender_re); # $per_recip_blacklist_sender_lookup_tables = undef; # $per_recip_whitelist_sender_lookup_tables = undef; # deprecated # $os_fingerprint_method = undef; # $os_fingerprint_dst_ip_and_port = undef; ## SQL & LDAP # $trim_trailing_space_in_lookup_result_fields = 0; # $lookup_maps_imply_sql_and_ldap = 1; # @lookup_sql_dsn = (); # SQL data source name for lookups, or empty # @storage_sql_dsn = (); # SQL data source name for log/quarantine, or empty # $sql_store_info_for_all_msgs = 1; # $sql_schema_version = $myversion_id_numeric; # $timestamp_fmt_mysql = undef; # $sql_partition_tag = undef; # $sql_allow_8bit_address = 0; # VARCHAR (0), VARBINARY/BYTEA (1) # $sql_lookups_no_at_means_domain = 0; # $sql_quarantine_chunksize_max = 16384; # $sql_select_policy = # 'SELECT *,users.id'. # ' FROM users LEFT JOIN policy ON users.policy_id=policy.id'. # ' WHERE users.email IN (%k) ORDER BY users.priority DESC'; # $sql_select_white_black_list = # 'SELECT wb'. # ' FROM wblist JOIN mailaddr ON wblist.sid=mailaddr.id'. # ' WHERE wblist.rid=? AND mailaddr.email IN (%k)'. # ' ORDER BY mailaddr.priority DESC'; # %sql_clause = ( # 'sel_policy' => \$sql_select_policy, # 'sel_wblist' => \$sql_select_white_black_list, # 'sel_adr' => # 'SELECT id FROM maddr WHERE partition_tag=? AND email=?', # 'ins_adr' => # 'INSERT INTO maddr (partition_tag, email, domain) VALUES (?,?,?)', # 'ins_msg' => # 'INSERT INTO msgs (partition_tag, mail_id, secret_id, am_id,'. # ' time_num, time_iso, sid, policy, client_addr, size, host)'. # ' VALUES (?,?,?,?,?,?,?,?,?,?,?)', # 'upd_msg' => # 'UPDATE msgs SET content=?, quar_type=?, quar_loc=?, dsn_sent=?,'. # ' spam_level=?, message_id=?, from_addr=?, subject=?, client_addr=?,'. # ' originating=?'. # ' WHERE partition_tag=? AND mail_id=?', # 'ins_rcp' => # 'INSERT INTO msgrcpt (partition_tag, mail_id, rseqnum, rid, is_local,'. # ' content, ds, rs, bl, wl, bspam_level, smtp_resp)'. # ' VALUES (?,?,?,?,?,?,?,?,?,?,?,?)', # 'ins_quar' => # 'INSERT INTO quarantine (partition_tag, mail_id, chunk_ind, mail_text)'. # ' VALUES (?,?,?,?)', # 'sel_msg' => # obtains partition_tag if missing in a release request # 'SELECT partition_tag FROM msgs WHERE mail_id=?', # 'sel_quar' => # 'SELECT mail_text FROM quarantine'. # ' WHERE partition_tag=? AND mail_id=?'. # ' ORDER BY chunk_ind', # 'sel_penpals' => # no message-id references list # "SELECT msgs.time_num, msgs.mail_id, subject". # " FROM msgs JOIN msgrcpt USING (partition_tag,mail_id)". # " WHERE sid=? AND rid=? AND msgs.content!='V' AND ds='P'". # " ORDER BY msgs.time_num DESC", # LIMIT 1 # 'sel_penpals_msgid' => # with a nonempty list of message-id references # "SELECT msgs.time_num, msgs.mail_id, subject, message_id, rid". # " FROM msgs JOIN msgrcpt USING (partition_tag,mail_id)". # " WHERE sid=? AND msgs.content!='V' AND ds='P' AND message_id IN (%m)". # " AND rid!=sid". # " ORDER BY rid=? DESC, msgs.time_num DESC", # LIMIT 1 # ); ## LDAP, Please see file README.lookups for more info. # $enable_ldap = 0; # $ldap_lookups_no_at_means_domain = 0; # # $default_ldap = { # hostname => 'localhost', # localaddr => undef, # port => undef, # 389 or 636, default provided by Net::LDAP # scheme => undef, # 'ldaps' or 'ldap', depending on hostname # inet6 => $have_inet6 ? 1 : 0, # version => 3, # timeout => 120, # deref => 'find', # bind_dn => undef, # bind_password => undef, # tls => 0, # verify => 'none', # sslversion => 'tlsv1', # clientcert => undef, # clientkey => undef, # cafile => undef, # capath => undef, # sasl => 0, # sasl_mech => undef, # space-separated list of mech names # sasl_auth_id => undef, # }; ## hierarchy by which a final setting is chosen: ## policy bank (based on port or IP address) -> *_by_ccat ## *_by_ccat (based on mail contents) -> *_maps ## *_maps (based on recipient address) -> final configuration value ## MAPPING A CONTENTS CATEGORY TO A SETTING CHOSEN # %final_destiny_by_ccat = ( # CC_VIRUS, sub { c('final_virus_destiny') }, # CC_BANNED, sub { c('final_banned_destiny') }, # CC_UNCHECKED, sub { c('final_unchecked_destiny') }, # CC_SPAM, sub { c('final_spam_destiny') }, # CC_BADH, sub { c('final_bad_header_destiny') }, # CC_MTA.',1', D_TEMPFAIL, # CC_MTA.',2', D_REJECT, # CC_OVERSIZED, D_BOUNCE, # CC_CATCHALL, D_PASS, # ); # %forward_method_maps_by_ccat = ( # CC_CATCHALL, sub { ca('forward_method_maps') }, # ); # %smtp_reason_by_ccat = ( # # currently only used for blocked messages only, status 5xx # # a multiline message will produce a valid multiline SMTP response # CC_VIRUS, 'id=%n - INFECTED: %V', # CC_BANNED, 'id=%n - BANNED: %F', # CC_UNCHECKED, 'id=%n - UNCHECKED', # CC_SPAM, 'id=%n - spam', # CC_SPAMMY.',1', 'id=%n - spammy (tag3)', # CC_SPAMMY, 'id=%n - spammy', # CC_BADH.',1', 'id=%n - BAD HEADER: MIME error', # CC_BADH.',2', 'id=%n - BAD HEADER: nonencoded 8-bit character', # CC_BADH.',3', 'id=%n - BAD HEADER: contains invalid control character', # CC_BADH.',4', 'id=%n - BAD HEADER: line made up entirely of whitespace', # CC_BADH.',5', 'id=%n - BAD HEADER: line longer than RFC 5322 limit', # CC_BADH.',6', 'id=%n - BAD HEADER: syntax error', # CC_BADH.',7', 'id=%n - BAD HEADER: missing required header field', # CC_BADH.',8', 'id=%n - BAD HEADER: duplicate header field', # CC_BADH, 'id=%n - BAD HEADER', # CC_OVERSIZED, 'id=%n - Message size exceeds recipient\'s size limit', # CC_MTA.',1', 'id=%n - Temporary MTA failure on relaying', # CC_MTA.',2', 'id=%n - Rejected by next-hop MTA on relaying', # CC_MTA, 'id=%n - Unable to relay message back to MTA', # CC_CLEAN, 'id=%n - CLEAN', # CC_CATCHALL, 'id=%n - OTHER', # should not happen # ); # %lovers_maps_by_ccat = ( # CC_VIRUS, sub { ca('virus_lovers_maps') }, # CC_BANNED, sub { ca('banned_files_lovers_maps') }, # CC_UNCHECKED, sub { ca('unchecked_lovers_maps') }, # CC_SPAM, sub { ca('spam_lovers_maps') }, # CC_SPAMMY, sub { ca('spam_lovers_maps') }, # CC_BADH, sub { ca('bad_header_lovers_maps') }, # ); # %defang_maps_by_ccat = ( # CC_VIRUS, sub { c('defang_virus') }, # CC_BANNED, sub { c('defang_banned') }, # CC_UNCHECKED, sub { c('defang_undecipherable') }, # CC_SPAM, sub { c('defang_spam') }, # CC_SPAMMY, sub { c('defang_spam') }, # # CC_BADH.',3', 1, # NUL or CR character in header section # # CC_BADH.',5', 1, # header line longer than 998 characters # # CC_BADH.',6', 1, # header field syntax error # CC_BADH, sub { c('defang_bad_header') }, # ); # %subject_tag_maps_by_ccat = ( # CC_VIRUS, [ '***INFECTED*** ' ], # CC_BANNED, undef, # CC_UNCHECKED, sub { [ c('undecipherable_subject_tag') ] }, # not by-recip # CC_SPAM, undef, # CC_SPAMMY.',1', sub { ca('spam_subject_tag3_maps') }, # CC_SPAMMY, sub { ca('spam_subject_tag2_maps') }, # CC_CLEAN.',1', sub { ca('spam_subject_tag_maps') }, # ); # %quarantine_method_by_ccat = ( # CC_VIRUS, sub { c('virus_quarantine_method') }, # CC_BANNED, sub { c('banned_files_quarantine_method') }, # CC_UNCHECKED, sub { c('unchecked_quarantine_method') }, # CC_SPAM, sub { c('spam_quarantine_method') }, # CC_BADH, sub { c('bad_header_quarantine_method') }, # CC_CLEAN, sub { c('clean_quarantine_method') }, # ); # %quarantine_to_maps_by_ccat = ( # CC_VIRUS, sub { ca('virus_quarantine_to_maps') }, # CC_BANNED, sub { ca('banned_quarantine_to_maps') }, # CC_UNCHECKED, sub { ca('unchecked_quarantine_to_maps') }, # CC_SPAM, sub { ca('spam_quarantine_to_maps') }, # CC_BADH, sub { ca('bad_header_quarantine_to_maps') }, # CC_CLEAN, sub { ca('clean_quarantine_to_maps') }, # ); # %admin_maps_by_ccat = ( # CC_VIRUS, sub { ca('virus_admin_maps') }, # CC_BANNED, sub { ca('banned_admin_maps') }, # CC_UNCHECKED, sub { ca('virus_admin_maps') }, # CC_SPAM, sub { ca('spam_admin_maps') }, # CC_BADH, sub { ca('bad_header_admin_maps') }, # ); # %always_bcc_by_ccat = ( # CC_CATCHALL, sub { c('always_bcc') }, # ); # %dsn_bcc_by_ccat = ( # CC_CATCHALL, sub { c('dsn_bcc') }, # ); # %mailfrom_notify_admin_by_ccat = ( # CC_SPAM, sub { c('mailfrom_notify_spamadmin') }, # CC_CATCHALL, sub { c('mailfrom_notify_admin') }, # ); # %hdrfrom_notify_admin_by_ccat = ( # CC_SPAM, sub { c('hdrfrom_notify_spamadmin') }, # CC_CATCHALL, sub { c('hdrfrom_notify_admin') }, # ); # %mailfrom_notify_recip_by_ccat = ( # CC_CATCHALL, sub { c('mailfrom_notify_recip') }, # ); # %hdrfrom_notify_recip_by_ccat = ( # CC_CATCHALL, sub { c('hdrfrom_notify_recip') }, # ); # %hdrfrom_notify_sender_by_ccat = ( # CC_CATCHALL, sub { c('hdrfrom_notify_sender') }, # ); # %hdrfrom_notify_release_by_ccat = ( # CC_CATCHALL, sub { c('hdrfrom_notify_release') }, # ); # %hdrfrom_notify_report_by_ccat = ( # CC_CATCHALL, sub { c('hdrfrom_notify_report') }, # ); # %notify_admin_templ_by_ccat = ( # CC_SPAM, sub { cr('notify_spam_admin_templ') }, # CC_CATCHALL, sub { cr('notify_virus_admin_templ') }, # ); # %notify_recips_templ_by_ccat = ( # CC_SPAM, sub { cr('notify_spam_recips_templ') }, #usualy empty # CC_CATCHALL, sub { cr('notify_virus_recips_templ') }, # ); # %notify_sender_templ_by_ccat = ( # bounce templates # CC_VIRUS, sub { cr('notify_virus_sender_templ') }, # CC_BANNED, sub { cr('notify_virus_sender_templ') }, #historical reason # CC_SPAM, sub { cr('notify_spam_sender_templ') }, # CC_CATCHALL, sub { cr('notify_sender_templ') }, # ); # %notify_release_templ_by_ccat = ( # CC_CATCHALL, sub { cr('notify_release_templ') }, # ); # %notify_report_templ_by_ccat = ( # CC_CATCHALL, sub { cr('notify_report_templ') }, # ); # %notify_autoresp_templ_by_ccat = ( # CC_CATCHALL, sub { cr('notify_autoresp_templ') }, # ); # %warnsender_by_ccat = ( # deprecated use, except perhaps for CC_BADH # CC_VIRUS, undef, # CC_BANNED, sub { c('warnbannedsender') }, # CC_SPAM, undef, # CC_BADH, sub { c('warnbadhsender') }, # ); # %warnrecip_maps_by_ccat = ( # CC_VIRUS, sub { ca('warnvirusrecip_maps') }, # CC_BANNED, sub { ca('warnbannedrecip_maps') }, # CC_SPAM, undef, # CC_BADH, sub { ca('warnbadhrecip_maps') }, # ); # %addr_extension_maps_by_ccat = ( # CC_VIRUS, sub { ca('addr_extension_virus_maps') }, # CC_BANNED, sub { ca('addr_extension_banned_maps') }, # CC_SPAM, sub { ca('addr_extension_spam_maps') }, # CC_SPAMMY, sub { ca('addr_extension_spam_maps') }, # CC_BADH, sub { ca('addr_extension_bad_header_maps') }, # # CC_OVERSIZED, 'oversized'; # ); # %addr_rewrite_maps_by_ccat = ( ); ## POLICY BANKS # %interface_policy = (); # maps input interface/port to policy bank name # $policy_bank{''} = { ...predefined... }; ## the built-in policy bank (empty name) is predefined, and includes ## references to most other variables listed above (the dynamic config ## variables), which are accessed only indirectly through the currently ## installed policy bank. Overlaying a policy bank with another policy ## bank may bring-in references to entirely different variables, ## possibly unnamed. Here is a list of configuration variables ## referenced from the built-in policy bank by keys of the same name ## (e.g. { log_level => \$log_level, inet_acl => \@inet_acl, ...} ) ## ## $child_timeout $smtpd_timeout ## $policy_bank_name $protocol @inet_acl ## $myhostname $myauthservid $snmp_contact $snmp_location ## $myprogram_name $syslog_ident $syslog_facility ## $log_level $log_templ $log_recip_templ ## $forward_method $notify_method $resend_method $report_format ## $release_method $requeue_method $release_format ## $attachment_password $attachment_email_name $attachment_outer_name ## $os_fingerprint_method $os_fingerprint_dst_ip_and_port ## $originating @smtpd_discard_ehlo_keywords $soft_bounce ## $propagate_dsn_if_possible $terminate_dsn_on_notify_success ## $amavis_auth_user $amavis_auth_pass $auth_reauthenticate_forwarded ## $auth_required_out $auth_required_inp $auth_required_release ## @auth_mech_avail $tls_security_level_in $tls_security_level_out ## $local_client_bind_address $smtpd_message_size_limit ## $localhost_name $smtpd_greeting_banner $smtpd_quit_banner ## $mailfrom_to_quarantine $warn_offsite $bypass_decode_parts @decoders ## @av_scanners @av_scanners_backup @spam_scanners ## $first_infected_stops_scan $virus_scanners_failure_is_fatal ## $sa_spam_level_char $sa_mail_body_size_limit ## $penpals_bonus_score $penpals_halflife $bounce_killer_score ## $reputation_factor ## $undecipherable_subject_tag $localpart_is_case_sensitive ## $recipient_delimiter $replace_existing_extension ## $hdr_encoding $bdy_encoding $hdr_encoding_qb ## $allow_disclaimers ## $prepend_header_fields_hdridx ## $allow_fixing_improper_header ## $allow_fixing_improper_header_folding $allow_fixing_long_header_lines ## %allowed_added_header_fields %prefer_our_added_header_fields ## %allowed_header_tests ## $X_HEADER_TAG $X_HEADER_LINE ## $remove_existing_x_scanned_headers $remove_existing_spam_headers ## %sql_clause $partition_tag ## %local_delivery_aliases $banned_namepath_re ## $per_recip_whitelist_sender_lookup_tables ## $per_recip_blacklist_sender_lookup_tables ## @anomy_sanitizer_args @altermime_args_defang ## @altermime_args_disclaimer @disclaimer_options_bysender_maps ## %signed_header_fields @dkim_signature_options_bysender_maps ## $enable_dkim_verification $enable_dkim_signing $dkim_signing_service ## ## @local_domains_maps @mynetworks_maps @client_ipaddr_policy ## @forward_method_maps @newvirus_admin_maps @banned_filename_maps ## @spam_quarantine_bysender_to_maps ## @spam_tag_level_maps @spam_tag2_level_maps @spam_tag3_level_maps ## @spam_kill_level_maps ## @spam_subject_tag_maps @spam_subject_tag2_maps @spam_subject_tag3_maps ## @spam_dsn_cutoff_level_maps @spam_dsn_cutoff_level_bysender_maps ## @spam_crediblefrom_dsn_cutoff_level_maps ## @spam_crediblefrom_dsn_cutoff_level_bysender_maps ## @spam_quarantine_cutoff_level_maps @spam_notifyadmin_cutoff_level_maps ## @whitelist_sender_maps @blacklist_sender_maps @score_sender_maps ## @author_to_policy_bank_maps @signer_reputation_maps ## @message_size_limit_maps @debug_sender_maps @debug_recipient_maps ## @bypass_virus_checks_maps @bypass_spam_checks_maps ## @bypass_banned_checks_maps @bypass_header_checks_maps ## @viruses_that_fake_sender_maps ## @virus_name_to_spam_score_maps @virus_name_to_policy_bank_maps ## @remove_existing_spam_headers_maps ## @sa_userconf_maps @sa_username_maps ## ## %final_destiny_by_ccat %forward_method_maps_by_ccat ## %lovers_maps_by_ccat %defang_maps_by_ccat %subject_tag_maps_by_ccat ## %quarantine_method_by_ccat %quarantine_to_maps_by_ccat ## %notify_admin_templ_by_ccat %notify_recips_templ_by_ccat ## %notify_sender_templ_by_ccat %notify_autoresp_templ_by_ccat ## %notify_release_templ_by_ccat %notify_report_templ_by_ccat ## %warnsender_by_ccat ## %hdrfrom_notify_admin_by_ccat %mailfrom_notify_admin_by_ccat ## %hdrfrom_notify_recip_by_ccat %mailfrom_notify_recip_by_ccat ## %hdrfrom_notify_sender_by_ccat ## %hdrfrom_notify_release_by_ccat %hdrfrom_notify_report_by_ccat ## %admin_maps_by_ccat %warnrecip_maps_by_ccat ## %always_bcc_by_ccat %dsn_bcc_by_ccat ## %addr_extension_maps_by_ccat %addr_rewrite_maps_by_ccat ## %smtp_reason_by_ccat # legacy dynamic configuration variables: ## $final_virus_destiny $final_banned_destiny $final_unchecked_destiny ## $final_spam_destiny $final_bad_header_destiny ## @virus_lovers_maps @spam_lovers_maps @unchecked_lovers_maps ## @banned_files_lovers_maps @bad_header_lovers_maps ## $always_bcc $dsn_bcc ## $mailfrom_notify_sender $mailfrom_notify_recip ## $mailfrom_notify_admin $mailfrom_notify_spamadmin ## $hdrfrom_notify_sender $hdrfrom_notify_recip ## $hdrfrom_notify_admin $hdrfrom_notify_spamadmin ## $hdrfrom_notify_release $hdrfrom_notify_report ## $notify_virus_admin_templ $notify_spam_admin_templ ## $notify_virus_recips_templ $notify_spam_recips_templ ## $notify_virus_sender_templ $notify_spam_sender_templ ## $notify_sender_templ $notify_release_templ ## $notify_report_templ $notify_autoresp_templ ## $warnbannedsender $warnbadhsender ## $defang_virus $defang_banned $defang_spam ## $defang_bad_header $defang_undecipherable $defang_all ## $virus_quarantine_method $banned_files_quarantine_method ## $unchecked_quarantine_method $spam_quarantine_method ## $bad_header_quarantine_method $clean_quarantine_method ## $archive_quarantine_method ## @virus_quarantine_to_maps @banned_quarantine_to_maps ## @unchecked_quarantine_to_maps @spam_quarantine_to_maps ## @bad_header_quarantine_to_maps @clean_quarantine_to_maps ## @archive_quarantine_to_maps ## @virus_admin_maps @banned_admin_maps ## @spam_admin_maps @bad_header_admin_maps @spam_modifies_subj_maps ## @warnvirusrecip_maps @warnbannedrecip_maps @warnbadhrecip_maps ## @addr_extension_virus_maps @addr_extension_spam_maps ## @addr_extension_banned_maps @addr_extension_bad_header_maps 1; # insure a defined return value amavisd-new-2.7.1/README_FILES/README.sql-mysql000644 000621 000620 00000063327 11733141214 020177 0ustar00markcmi000000 000000 USING SQL FOR LOOKUPS, LOG/REPORTING AND QUARANTINE =================================================== This text only describes SQL specifics for a MySQL database and provides a schema. For general aspects of lookups, please see README.lookups. For general SQL notes and further examples please see README.sql. For PostgreSQL-specific notes and schema please see README.sql-pg (which in most respects also applies to a SQLite database). SERIAL can be used instead of INT UNSIGNED NOT NULL AUTO_INCREMENT with databases which do not recognize AUTO_INCREMENT; The attribute SERIAL was introduced with MySQL 4.1.0, but it implicitly creates an additional UNIQUE index, which is redundant. Instead of declaring a time_iso field in table msgs as a string: time_iso char(16) NOT NULL, one may want to declare is as: time_iso TIMESTAMP NOT NULL DEFAULT 0, in which case $timestamp_fmt_mysql *MUST* be set to 1 in amavisd.conf to avoid MySQL inability to accept ISO 8601 timestamps with zone Z and ISO date/time delimiter T; failing to set $timestamp_fmt_mysql makes MySQL store zero time on INSERT and write current local time on UPDATE if auto-update is allowed, which is different from the intended mail timestamp (localtime vs. UTC, off by seconds) Field quarantine.mail_text should be of data type 'blob' and not 'text' as suggested in earlier documentation; this is to prevent it from being unjustifiably associated with a character set, and to be able to store any byte value; to convert existing field from type 'text' to type 'blob' the following clause may be used: ALTER TABLE quarantine CHANGE mail_text mail_text blob; Although MySQL is not particularly picky in checking and enforcing data types, it is appropriate to declare fields users.email, mailaddr.email, and maddr.email as byte strings with no associated character set (which is what these fields are, according to RFC 2821) instead of CHAR or VARCHAR. The following clauses convert pre-2.6.0 tables into the now recommended form: ALTER TABLE users CHANGE email email varbinary(255); ALTER TABLE mailaddr CHANGE email email varbinary(255); ALTER TABLE maddr CHANGE email email varbinary(255); If VARBINARY data type is chosen for these three fields, the setting $sql_allow_8bit_address should be set to true to let the amavisd program use the appropriate data type in SQL commands: $sql_allow_8bit_address = 1; # maddr.email: VARCHAR (0), VARBINARY/BYTEA (1) Similarly, semantics of some other fields is strings of octets too, with no associated character set (which is a characteristic of char and varchar data types), so the following alternations to pre-2.6.2 schema may be beneficial to prevent SQL server from checking validity of octets data against non-applicable arbitrary character set constraints: ALTER table msgs CHANGE mail_id mail_id varbinary(16); ALTER table msgs CHANGE secret_id secret_id varbinary(16); ALTER table msgs CHANGE quar_loc quar_loc varbinary(255); ALTER table msgrcpt CHANGE mail_id mail_id varbinary(16); ALTER table quarantine CHANGE mail_id mail_id varbinary(16); and for good measure: ALTER table msgrcpt CHANGE rid rid bigint unsigned; ALTER table msgs CHANGE sid sid bigint unsigned; Starting with amavisd-new-2.7.0, three fields need to be added to table msgrcpt and one to table msgs: ALTER TABLE msgrcpt ADD rseqnum integer DEFAULT 0 NOT NULL; ALTER TABLE msgrcpt ADD content char(1) DEFAULT ' ' NOT NULL; ALTER TABLE msgrcpt ADD is_local char(1) DEFAULT ' ' NOT NULL; ALTER TABLE msgs ADD originating char(1) DEFAULT ' ' NOT NULL; If you need to create a primary key on table msgrcpt for some reason (clustering?), try something like: UPDATE msgrcpt SET rseqnum=1+floor(999999999*rand()) WHERE rseqnum=0; ALTER TABLE msgrcpt ADD PRIMARY KEY (partition_tag,mail_id,rseqnum); -- local users CREATE TABLE users ( id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, -- unique id priority integer NOT NULL DEFAULT '7', -- sort field, 0 is low prior. policy_id integer unsigned NOT NULL DEFAULT '1', -- JOINs with policy.id email varbinary(255) NOT NULL UNIQUE, fullname varchar(255) DEFAULT NULL -- not used by amavisd-new -- local char(1) -- Y/N (optional field, see note further down) ); -- any e-mail address (non- rfc2822-quoted), external or local, -- used as senders in wblist CREATE TABLE mailaddr ( id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, priority integer NOT NULL DEFAULT '7', -- 0 is low priority email varbinary(255) NOT NULL UNIQUE ); -- per-recipient whitelist and/or blacklist, -- puts sender and recipient in relation wb (white or blacklisted sender) CREATE TABLE wblist ( rid integer unsigned NOT NULL, -- recipient: users.id sid integer unsigned NOT NULL, -- sender: mailaddr.id wb varchar(10) NOT NULL, -- W or Y / B or N / space=neutral / score PRIMARY KEY (rid,sid) ); CREATE TABLE policy ( id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, -- 'id' this is the _only_ required field policy_name varchar(32), -- not used by amavisd-new, a comment virus_lover char(1) default NULL, -- Y/N spam_lover char(1) default NULL, -- Y/N unchecked_lover char(1) default NULL, -- Y/N banned_files_lover char(1) default NULL, -- Y/N bad_header_lover char(1) default NULL, -- Y/N bypass_virus_checks char(1) default NULL, -- Y/N bypass_spam_checks char(1) default NULL, -- Y/N bypass_banned_checks char(1) default NULL, -- Y/N bypass_header_checks char(1) default NULL, -- Y/N spam_modifies_subj char(1) default NULL, -- Y/N virus_quarantine_to varchar(64) default NULL, spam_quarantine_to varchar(64) default NULL, banned_quarantine_to varchar(64) default NULL, unchecked_quarantine_to varchar(64) default NULL, bad_header_quarantine_to varchar(64) default NULL, clean_quarantine_to varchar(64) default NULL, archive_quarantine_to varchar(64) default NULL, spam_tag_level float default NULL, -- higher score inserts spam info headers spam_tag2_level float default NULL, -- inserts 'declared spam' header fields spam_tag3_level float default NULL, -- inserts 'blatant spam' header fields spam_kill_level float default NULL, -- higher score triggers evasive actions -- e.g. reject/drop, quarantine, ... -- (subject to final_spam_destiny setting) spam_dsn_cutoff_level float default NULL, spam_quarantine_cutoff_level float default NULL, addr_extension_virus varchar(64) default NULL, addr_extension_spam varchar(64) default NULL, addr_extension_banned varchar(64) default NULL, addr_extension_bad_header varchar(64) default NULL, warnvirusrecip char(1) default NULL, -- Y/N warnbannedrecip char(1) default NULL, -- Y/N warnbadhrecip char(1) default NULL, -- Y/N newvirus_admin varchar(64) default NULL, virus_admin varchar(64) default NULL, banned_admin varchar(64) default NULL, bad_header_admin varchar(64) default NULL, spam_admin varchar(64) default NULL, spam_subject_tag varchar(64) default NULL, spam_subject_tag2 varchar(64) default NULL, spam_subject_tag3 varchar(64) default NULL, message_size_limit integer default NULL, -- max size in bytes, 0 disable banned_rulenames varchar(64) default NULL, -- comma-separated list of ... -- names mapped through %banned_rules to actual banned_filename tables disclaimer_options varchar(64) default NULL, forward_method varchar(64) default NULL, sa_userconf varchar(64) default NULL, sa_username varchar(64) default NULL ); -- R/W part of the dataset (optional) -- May reside in the same or in a separate database as lookups database; -- REQUIRES SUPPORT FOR TRANSACTIONS; specified in @storage_sql_dsn -- -- MySQL note ( http://dev.mysql.com/doc/mysql/en/storage-engines.html ): -- ENGINE is the preferred term, but cannot be used before MySQL 4.0.18. -- TYPE is available beginning with MySQL 3.23.0, the first version of -- MySQL for which multiple storage engines were available. If you omit -- the ENGINE or TYPE option, the default storage engine is used. -- By default this is MyISAM. -- -- Please create additional indexes on keys when needed, or drop suggested -- ones as appropriate to optimize queries needed by a management application. -- See your database documentation for further optimization hints. With MySQL -- see Chapter 15 of the reference manual. For example the chapter 15.17 says: -- InnoDB does not keep an internal count of rows in a table. To process a -- SELECT COUNT(*) FROM T statement, InnoDB must scan an index of the table, -- which takes some time if the index is not entirely in the buffer pool. -- -- Wayne Smith adds: When using MySQL with InnoDB one might want to -- increase buffer size for both pool and log, and might also want -- to change flush settings for a little better performance. Example: -- innodb_buffer_pool_size = 384M -- innodb_log_buffer_size = 8M -- innodb_flush_log_at_trx_commit = 0 -- The big performance increase is the first two, the third just helps with -- lowering disk activity. Consider also adjusting the key_buffer_size. -- provide unique id for each e-mail address, avoids storing copies CREATE TABLE maddr ( partition_tag integer DEFAULT 0, -- see $partition_tag id bigint unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, email varbinary(255) NOT NULL, -- full mail address domain varchar(255) NOT NULL, -- only domain part of the email address -- with subdomain fields in reverse CONSTRAINT part_email UNIQUE (partition_tag,email) ) ENGINE=InnoDB; -- information pertaining to each processed message as a whole; -- NOTE: records with NULL msgs.content should be ignored by utilities, -- as such records correspond to messages just being processes, or were lost -- NOTE: instead of a character field time_iso, one might prefer: -- time_iso TIMESTAMP NOT NULL DEFAULT 0, -- but the following MUST then be set in amavisd.conf: $timestamp_fmt_mysql=1 CREATE TABLE msgs ( partition_tag integer DEFAULT 0, -- see $partition_tag mail_id varbinary(16) NOT NULL, -- long-term unique mail id, dflt 12 ch secret_id varbinary(16) DEFAULT '', -- authorizes release of mail_id, 12 ch am_id varchar(20) NOT NULL, -- id used in the log time_num integer unsigned NOT NULL, -- rx_time: seconds since Unix epoch time_iso char(16) NOT NULL, -- rx_time: ISO8601 UTC ascii time sid bigint unsigned NOT NULL, -- sender: maddr.id policy varchar(255) DEFAULT '', -- policy bank path (like macro %p) client_addr varchar(255) DEFAULT '', -- SMTP client IP address (IPv4 or v6) size integer unsigned NOT NULL, -- message size in bytes originating char(1) DEFAULT ' ' NOT NULL, -- sender from inside or auth'd content char(1), -- content type: V/B/U/S/Y/M/H/O/T/C -- virus/banned/unchecked/spam(kill)/spammy(tag2)/ -- /bad-mime/bad-header/oversized/mta-err/clean -- is NULL on partially processed mail -- (prior to 2.7.0 the CC_SPAMMY was logged as 's', now 'Y' is used; -- to avoid a need for case-insenstivity in queries) quar_type char(1), -- quarantined as: ' '/F/Z/B/Q/M/L -- none/file/zipfile/bsmtp/sql/ -- /mailbox(smtp)/mailbox(lmtp) quar_loc varbinary(255) DEFAULT '', -- quarantine location (e.g. file) dsn_sent char(1), -- was DSN sent? Y/N/q (q=quenched) spam_level float, -- SA spam level (no boosts) message_id varchar(255) DEFAULT '', -- mail Message-ID header field from_addr varchar(255) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '', -- mail From header field, UTF8 subject varchar(255) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT '', -- mail Subject header field, UTF8 host varchar(255) NOT NULL, -- hostname where amavisd is running PRIMARY KEY (partition_tag,mail_id) -- FOREIGN KEY (sid) REFERENCES maddr(id) ON DELETE RESTRICT ) ENGINE=InnoDB; CREATE INDEX msgs_idx_sid ON msgs (sid); CREATE INDEX msgs_idx_mess_id ON msgs (message_id); -- useful with pen pals CREATE INDEX msgs_idx_time_num ON msgs (time_num); -- alternatively when purging based on time_iso (instead of msgs_idx_time_num): -- CREATE INDEX msgs_idx_time_iso ON msgs (time_iso); -- per-recipient information related to each processed message; -- NOTE: records in msgrcpt without corresponding msgs.mail_id record are -- orphaned and should be ignored and eventually deleted by external utilities CREATE TABLE msgrcpt ( partition_tag integer DEFAULT 0, -- see $partition_tag mail_id varbinary(16) NOT NULL, -- (must allow duplicates) rseqnum integer DEFAULT 0 NOT NULL, -- recip's enumeration within msg rid bigint unsigned NOT NULL, -- recipient: maddr.id (dupl. allowed) is_local char(1) DEFAULT ' ' NOT NULL, -- recip is: Y=local, N=foreign content char(1) DEFAULT ' ' NOT NULL, -- content type V/B/U/S/Y/M/H/O/T/C ds char(1) NOT NULL, -- delivery status: P/R/B/D/T -- pass/reject/bounce/discard/tempfail rs char(1) NOT NULL, -- release status: initialized to ' ' bl char(1) DEFAULT ' ', -- sender blacklisted by this recip wl char(1) DEFAULT ' ', -- sender whitelisted by this recip bspam_level float, -- per-recipient (total) spam level smtp_resp varchar(255) DEFAULT '', -- SMTP response given to MTA PRIMARY KEY (partition_tag,mail_id,rseqnum) -- FOREIGN KEY (rid) REFERENCES maddr(id) ON DELETE RESTRICT, -- FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE ) ENGINE=InnoDB; CREATE INDEX msgrcpt_idx_mail_id ON msgrcpt (mail_id); CREATE INDEX msgrcpt_idx_rid ON msgrcpt (rid); -- mail quarantine in SQL, enabled by $*_quarantine_method='sql:' -- NOTE: records in quarantine without corresponding msgs.mail_id record are -- orphaned and should be ignored and eventually deleted by external utilities CREATE TABLE quarantine ( partition_tag integer DEFAULT 0, -- see $partition_tag mail_id varbinary(16) NOT NULL, -- long-term unique mail id chunk_ind integer unsigned NOT NULL, -- chunk number, starting with 1 mail_text blob NOT NULL, -- store mail as chunks of octets PRIMARY KEY (partition_tag,mail_id,chunk_ind) -- FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE ) ENGINE=InnoDB; -- field msgrcpt.rs is primarily intended for use by quarantine management -- software; the value assigned by amavisd is a space; -- a short _preliminary_ list of possible values: -- 'V' => viewed (marked as read) -- 'R' => released (delivered) to this recipient -- 'p' => pending (a status given to messages when the admin received the -- request but not yet released; targeted to banned parts) -- 'D' => marked for deletion; a cleanup script may delete it -- ===================== -- Example data follows: -- ===================== INSERT INTO users VALUES ( 1, 9, 5, 'user1+foo@y.example.com','Name1 Surname1', 'Y'); INSERT INTO users VALUES ( 2, 7, 5, 'user1@y.example.com', 'Name1 Surname1', 'Y'); INSERT INTO users VALUES ( 3, 7, 2, 'user2@y.example.com', 'Name2 Surname2', 'Y'); INSERT INTO users VALUES ( 4, 7, 7, 'user3@z.example.com', 'Name3 Surname3', 'Y'); INSERT INTO users VALUES ( 5, 7, 7, 'user4@example.com', 'Name4 Surname4', 'Y'); INSERT INTO users VALUES ( 6, 7, 1, 'user5@example.com', 'Name5 Surname5', 'Y'); INSERT INTO users VALUES ( 7, 5, 0, '@sub1.example.com', NULL, 'Y'); INSERT INTO users VALUES ( 8, 5, 7, '@sub2.example.com', NULL, 'Y'); INSERT INTO users VALUES ( 9, 5, 5, '@example.com', NULL, 'Y'); INSERT INTO users VALUES (10, 3, 8, 'userA', 'NameA SurnameA anywhere', 'Y'); INSERT INTO users VALUES (11, 3, 9, 'userB', 'NameB SurnameB', 'Y'); INSERT INTO users VALUES (12, 3,10, 'userC', 'NameC SurnameC', 'Y'); INSERT INTO users VALUES (13, 3,11, 'userD', 'NameD SurnameD', 'Y'); INSERT INTO users VALUES (14, 3, 0, '@sub1.example.net', NULL, 'Y'); INSERT INTO users VALUES (15, 3, 7, '@sub2.example.net', NULL, 'Y'); INSERT INTO users VALUES (16, 3, 5, '@example.net', NULL, 'Y'); INSERT INTO users VALUES (17, 7, 5, 'u1@example.org', 'u1', 'Y'); INSERT INTO users VALUES (18, 7, 6, 'u2@example.org', 'u2', 'Y'); INSERT INTO users VALUES (19, 7, 3, 'u3@example.org', 'u3', 'Y'); -- INSERT INTO users VALUES (20, 0, 5, '@.', NULL, 'N'); -- catchall INSERT INTO policy (id, policy_name, virus_lover, spam_lover, banned_files_lover, bad_header_lover, bypass_virus_checks, bypass_spam_checks, bypass_banned_checks, bypass_header_checks, spam_modifies_subj, spam_tag_level, spam_tag2_level, spam_kill_level) VALUES (1, 'Non-paying', 'N','N','N','N', 'Y','Y','Y','N', 'Y', 3.0, 7, 10), (2, 'Uncensored', 'Y','Y','Y','Y', 'N','N','N','N', 'N', 3.0, 999, 999), (3, 'Wants all spam','N','Y','N','N', 'N','N','N','N', 'Y', 3.0, 999, 999), (4, 'Wants viruses', 'Y','N','Y','Y', 'N','N','N','N', 'Y', 3.0, 6.9, 6.9), (5, 'Normal', 'N','N','N','N', 'N','N','N','N', 'Y', 3.0, 6.9, 6.9), (6, 'Trigger happy', 'N','N','N','N', 'N','N','N','N', 'Y', 3.0, 5, 5), (7, 'Permissive', 'N','N','N','Y', 'N','N','N','N', 'Y', 3.0, 10, 20), (8, '6.5/7.8', 'N','N','N','N', 'N','N','N','N', 'N', 3.0, 6.5, 7.8), (9, 'userB', 'N','N','N','Y', 'N','N','N','N', 'Y', 3.0, 6.3, 6.3), (10,'userC', 'N','N','N','N', 'N','N','N','N', 'N', 3.0, 6.0, 6.0), (11,'userD', 'Y','N','Y','Y', 'N','N','N','N', 'N', 3.0, 7, 7); -- sender envelope addresses needed for white/blacklisting INSERT INTO mailaddr VALUES (1, 5, '@example.com'); INSERT INTO mailaddr VALUES (2, 9, 'owner-postfix-users@postfix.org'); INSERT INTO mailaddr VALUES (3, 9, 'amavis-user-admin@lists.sourceforge.net'); INSERT INTO mailaddr VALUES (4, 9, 'makemoney@example.com'); INSERT INTO mailaddr VALUES (5, 5, '@example.net'); INSERT INTO mailaddr VALUES (6, 9, 'spamassassin-talk-admin@lists.sourceforge.net'); INSERT INTO mailaddr VALUES (7, 9, 'spambayes-bounces@python.org'); -- whitelist for user 14, i.e. default for recipients in domain sub1.example.net INSERT INTO wblist VALUES (14, 1, 'W'); INSERT INTO wblist VALUES (14, 3, 'W'); -- whitelist and blacklist for user 17, i.e. u1@example.org INSERT INTO wblist VALUES (17, 2, 'W'); INSERT INTO wblist VALUES (17, 3, 'W'); INSERT INTO wblist VALUES (17, 6, 'W'); INSERT INTO wblist VALUES (17, 7, 'W'); INSERT INTO wblist VALUES (17, 5, 'B'); INSERT INTO wblist VALUES (17, 4, 'B'); -- $sql_select_policy setting in amavisd.conf tells amavisd -- how to fetch per-recipient policy settings. -- See comments there. Example: -- -- SELECT *,users.id FROM users,policy -- WHERE (users.policy_id=policy.id) AND (users.email IN (%k)) -- ORDER BY users.priority DESC; -- -- $sql_select_white_black_list in amavisd.conf tells amavisd -- how to check sender in per-recipient whitelist/blacklist. -- See comments there. Example: -- -- SELECT wb FROM wblist,mailaddr -- WHERE (wblist.rid=?) AND (wblist.sid=mailaddr.id) AND (mailaddr.email IN (%k)) -- ORDER BY mailaddr.priority DESC; -- NOTE: the SELECT, INSERT and UPDATE clauses as used by the amavisd-new -- program are configurable through %sql_clause; see amavisd.conf-default Some examples of a query: -- mail from last two minutes: SELECT UNIX_TIMESTAMP()-msgs.time_num AS age, SUBSTRING(policy,1,2) as pb, msgrcpt.content AS c, dsn_sent as dsn, ds, bspam_level AS level, size, SUBSTRING(sender.email,1,18) AS s, SUBSTRING(recip.email,1,18) AS r, SUBSTRING(msgs.subject,1,10) AS subj FROM msgs LEFT JOIN msgrcpt ON msgs.mail_id=msgrcpt.mail_id LEFT JOIN maddr AS sender ON msgs.sid=sender.id LEFT JOIN maddr AS recip ON msgrcpt.rid=recip.id WHERE msgrcpt.content IS NOT NULL AND UNIX_TIMESTAMP()-msgs.time_num < 120 ORDER BY msgs.time_num DESC; -- clean messages ordered by count, grouped by domain: SELECT count(*) as cnt, avg(bspam_level), sender.domain FROM msgs LEFT JOIN msgrcpt ON msgs.mail_id=msgrcpt.mail_id LEFT JOIN maddr AS sender ON msgs.sid=sender.id LEFT JOIN maddr AS recip ON msgrcpt.rid=recip.id WHERE msgrcpt.content='C' GROUP BY sender.domain ORDER BY cnt DESC LIMIT 50; -- top spamy domains with >10 messages, sorted by spam average, -- grouped by domain: SELECT count(*) as cnt, avg(bspam_level) as spam_avg, sender.domain FROM msgs LEFT JOIN msgrcpt ON msgs.mail_id=msgrcpt.mail_id LEFT JOIN maddr AS sender ON msgs.sid=sender.id LEFT JOIN maddr AS recip ON msgrcpt.rid=recip.id WHERE bspam_level IS NOT NULL GROUP BY sender.domain HAVING count(*) > 10 ORDER BY spam_avg DESC LIMIT 50; -- sender domains with >100 messages, sorted on sender.domain: SELECT count(*) as cnt, avg(bspam_level) as spam_avg, sender.domain FROM msgs LEFT JOIN msgrcpt ON msgs.mail_id=msgrcpt.mail_id LEFT JOIN maddr AS sender ON msgs.sid=sender.id LEFT JOIN maddr AS recip ON msgrcpt.rid=recip.id GROUP BY sender.domain HAVING count(*) > 100 ORDER BY sender.domain DESC LIMIT 100; Upgrading from pre 2.4.0 amavisd-new SQL schema to the 2.4.0 schema requires adding column 'quar_loc' to table msgs, and creating FOREIGN KEY constraint to facilitate deletion of expired records. The following clauses should be executed for upgrading pre-2.4.0 amavisd-new SQL schema to the 2.4.0 schema. Creating a FOREIGN KEY ... ON DELETE CASCADE constraint may (or may not) facilitate deletion of expired records. -- mandatory change: ALTER TABLE msgs ADD quar_loc varchar(255) DEFAULT ''; -- optional, avoids need to purge tables msgrcpt and quarantine explicitly: ALTER TABLE quarantine ADD FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE; ALTER TABLE msgrcpt ADD FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE; -- the following two ALTERs are not essential; if data type of maddr.id is -- incompatible with msgs.sid and msgs.rid (e.g. BIGINT vs. INT) and MySQL -- complains, don't bother to apply the constraint: ALTER TABLE msgs ADD FOREIGN KEY (sid) REFERENCES maddr(id) ON DELETE RESTRICT; ALTER TABLE msgrcpt ADD FOREIGN KEY (rid) REFERENCES maddr(id) ON DELETE RESTRICT; EXAMPLE of a log/report/quarantine database housekeeping ======================================================== Using a changing partition_tag, perhaps by using an ISO 8601 week number (value 1 to 53) as a partition_tag: $partition_tag = sub { my($msginfo)=@_; sprintf("%02d",iso8601_week($msginfo->rx_time)) }; allows for probably the fastest method of purging old records, e.g.: DELETE FROM msgs WHERE partition_tag >= 13 AND partition_tag <= 23; DELETE FROM msgrcpt WHERE partition_tag >= 13 AND partition_tag <= 23; DELETE FROM quarantine WHERE partition_tag >= 13 AND partition_tag <= 23; DELETE FROM maddr WHERE partition_tag >= 13 AND partition_tag <= 23; Note: using native SQL table partitioning as offered by MySQL, the above may be even faster by dropping entire partitions. Not documented here. Alternatively, purge records from table msgs by their creation time: DELETE FROM msgs WHERE time_num < UNIX_TIMESTAMP() - 21*24*3600; DELETE FROM msgs WHERE time_num < UNIX_TIMESTAMP() - 3600 AND content IS NULL; Optionally certain content types may be given shorter lifetime: DELETE FROM msgs WHERE time_num < UNIX_TIMESTAMP() - 7*24*3600 AND (content='V' OR (content='S' AND spam_level > 20)); ( or equivalently, if a data type of msgs.time_iso is TIMESTAMPS and in amavisd.conf the $timestamp_fmt_mysql is set to true: DELETE FROM msgs WHERE time_iso < UTC_TIMESTAMP() - INTERVAL 21 day; DELETE FROM msgs WHERE time_iso < UTC_TIMESTAMP() - INTERVAL 1 hour AND content IS NULL; DELETE FROM msgs WHERE time_iso < UTC_TIMESTAMP() - INTERVAL 7 day AND (content='V' OR (content='S' AND spam_level > 20)); ) Then delete unreferenced records from tables msgrcpt and quarantine, unless they were already automatically deleted while purging the msgs table and FOREIGN KEY ... ON DELETE CASCADE is in place: DELETE FROM msgrcpt WHERE NOT EXISTS (SELECT 1 FROM msgs WHERE mail_id=msgrcpt.mail_id); DELETE FROM quarantine WHERE NOT EXISTS (SELECT 1 FROM msgs WHERE mail_id=quarantine.mail_id); ( or equivalently: DELETE msgrcpt FROM msgrcpt LEFT JOIN msgs USING(mail_id) WHERE msgs.mail_id IS NULL; DELETE quarantine FROM quarantine LEFT JOIN msgs USING(mail_id) WHERE msgs.mail_id IS NULL; ) Finally delete unreferenced records from table maddr: DELETE FROM maddr WHERE NOT EXISTS (SELECT 1 FROM msgs WHERE sid=id) AND NOT EXISTS (SELECT 1 FROM msgrcpt WHERE rid=id); SOME FURTHER THOUGHTS on a log/report/quarantine database housekeeping ====================================================================== Discarding indexes makes deletion faster; if we expect a large proportion of records to be deleted it may be quicker to discard index, do deletions, and re-create index; for daily maintenance this does not pay off DROP INDEX msgs_idx_sid ON msgs; DROP INDEX msgrcpt_idx_rid ON msgrcpt; DROP INDEX msgrcpt_idx_mail_id ON msgrcpt; Re-create indexes (if they were removed in the first step): CREATE INDEX msgs_idx_sid ON msgs (sid); CREATE INDEX msgrcpt_idx_rid ON msgrcpt (rid); CREATE INDEX msgrcpt_idx_mail_id ON msgrcpt (mail_id); Optionally, optimize tables once in a while: OPTIMIZE TABLE msgs, msgrcpt, quarantine, maddr; amavisd-new-2.7.1/README_FILES/amavisd-new-docs.html000640 000621 000620 00000340560 11505137353 021404 0ustar00markcmi000000 000000 amavisd-new documentation bits and pieces

amavisd-new documentation bits and pieces

The most recent version of this document is available at http://www.ijs.si/software/amavisd/amavisd-new-docs.html

Performing mail checks

The following checks on mail are available

  • mail header validity checks
  • banned names and types checks
  • virus checks
  • spam checks
  • is sender white- or blacklisted (regarding spam)

Although checks are presently not performed in parallel, it is best to consider the order of their evaluation unspecified (unknown). Besides possible future parallel implementation, another reason is the caching of results, where subsequent mail with the same contents may benefit from earlier checks if validity of these check results has not yet expired -- so a check result may be instantly available, regardless of whether it has been asked for or not.

Using configuration variables @bypass_virus_checks_maps, @bypass_banned_checks_maps, @bypass_header_checks_maps and @bypass_spam_checks_maps each recipient (or administrator on their behalf) may suggest that certain tests are not needed, primarily for performance reasons. Although the @bypass_*_checks_maps pertain to individual recipients, a mail check is an operation done on the whole message, regardless of the number of recipients and their individual preferences. Suggestion by some of the recipients that certain check is not needed (is to be bypassed) does not guarantee the test will not be performed.

Similarly the (hard) blacklisting or whitelisting of sender address may make running spam check unnecessary, but it does not guarantee the spam check result will not be available for subsequent decisions.

There are two primary reasons why a check result may still be available despite the bypass hint or a sender being black- or whitelisted:

  • a check result from some previous mail with the same contents has been cached and is still valid;
  • when mail has multiple recipients and not all of them agree that a check should be bypassed.

The amavisd-new program is allowed to skip some check for performance reasons if all recipients agree that a check is not necessary (that it may be bypassed), or if the outcome of a check to be skipped could not influence further mail processing and delivery/non-delivery of the message (as is the case of a sender being black- or whitelisted regarding spam check).

For example spam checks may be skipped if it is already known that a mail is infected. This is an implementation and optimization issue, and no guarantee is given about interdependency of checks. Future version may use a different strategy of performing checks (e.g. some checks may be performed in parallel), as long as a change does not affect the final outcome.

Acting on mail checks results

Based on the outcome of mail checks performed during mail analysis or cached from previous mail with the same contents, and based on global settings and individual recipient preferences, the program now decides what action to perform next. As described in the previous section, not all results of checks are necessarily known (e.g. if all recipients voted for some check to be bypassed). For the purpose of deciding further actions, unknown results of a check are considered equivalent to negative (false) results, i.e. skipped virus check is treated the same as non-infected mail, bypassed spam check is equivalent to low spam score (ham).

The following decisions are made at this stage:

  • whether a mail should be quarantined and how;
  • whether an administrator (and which administrator) should receive a notification (and which notification);
  • whether recipients should receive a notification;

and regarding mail delivery and/or sender (non)delivery notifications:

  • whether a mail should be delivered to each recipient or not;
  • whether delivered mail should be modified (header edits, defanging);
  • whether a sender should receive a (non)delivery notification (bounce);
  • what should be the final status code returned to the mailer (reject/pass).

For the purpose of deciding on these actions, a mail is classified based on all available checks results. It is quite possible that more than one check results would be positive (e.g. virus and banned and bad header, or spam and bad header, or virus and spam), yet a mail is considered to be only in one category. The logic is currently hard-wired into the program and can not be influenced by configuration variables. The following order is used, the first condition met decides the outcome:

  1. a virus is detected: mail is considered infected;
  2. contains banned name or type: mail is considered banned;
  3. spam level is above kill level for at least one recipient, or a sender is blacklisted: mail is considered spam;
  4. bad (invalid) headers: mail is considered as having a bad header.

This decision order explains why amavisd-new is not free to skip (to optimize away) virus checks if a presence of a banned name or a bad header is already known or can easily be determined. The order was chosen with the intention that a more informative or a stronger assertion is the one to base further mail delivery on, and to be quoted in notifications and in the log. Even at the expense of possibly longer processing time, it is more important to declare a mail infected than complain about a bad header, a banned executable or spamy contents.

The determined mail category now governs further action. Administrators are notified if enabled for the category, mail is quarantined if quarantining if enabled for the category, recipients are notified if enabled for the category.

Next a mail delivery is attempted. A decision to deliver depends on mail category and on global and individual recipient preferences. The global setting $final_*_destiny=D_PASS or a per-recipient setting @*_lovers_maps ensure mail delivery for corresponding mail category even if mail would otherwise be blocked for being infected or banned or spam or having a bad header.

A mail that is decided to be passed to an individual recipient undergoes some simple header editing which happens on-the-fly during mail forwarding. Certain mail header fields may be inserted or removed, or an existing header field (e.g. Subject) may be modified. This header editing may be different for each recipient even in multi-recipient messages. If necessary, a multi-recipient mail is split into more than one forwarding transaction, grouping (clustering) recipients with same settings into one SMTP transaction.

Based on decisions to forward or to block mail to each recipient, and on the global setting for the mail category ($final_*_destiny=D_BOUNCE or D_REJECT), the sender (non)delivery notification is now prepared in case of D_BOUNCE, and MTA receives a 2xx status (success); or in case of D_REJECT the MTA receives a 5xx (reject) status and preparing sender notifications is thus delegated to MTA (not recommended in post-queue or dual-MTA content filtering setup).

Even in cases of mail non-delivery when a (non-)delivery status notification (DSN) for the sender should have been prepared and sent, there are certain exceptions where the DSN is suppressed, which makes mail effectively lost as far as the sender and the recipient are concerned (but quarantining is not affected):

  • when $final_*_destiny=D_DISCARD;
  • when mail is infected and the detected virus name matches the @viruses_that_fake_sender_maps (unconditionally true by default);
  • when spam score exceeds level determined by @spam_dsn_cutoff_level_maps for all recipients;
  • when mail is coming from a mailing list, as determined by examining a mail header Precedence: for containing string 'bulk' or 'list' or 'junk';

tag, tag2 and kill levels

When SpamAssassin is called upon to analyze a mail message, it returns a spam score (spam level, hits), which is a numeric representation of spaminess. The higher the number, the more spamy the message is considered. Small numbers near zero or negative indicate a clean message, colloquially called ham. Spam score is a characteristic of the whole message, and does not depend on recipient preferences. SpamAssassin is called only once for each message regardless of the number of recipients.

To determine further course of action, amavisd-new compares the spam score to three numeric values: tag level, tag2 level and kill level. These values may be different for each recipient, and further actions may be different for each recipient. If necessary, the mail forwarding is split into more than one transaction to cater for different recipient preferences.

tag level
if spam score is at or above tag level, spam-related header fields (X-Spam-Status, X-Spam-Level) are inserted for local recipients; undefined (unknown) spam score is interpreted as lower than any spam score;
tag2 level
if spam score is at or above tag2 level, spam-related header fields (X-Spam-Status, X-Spam-Level, X-Spam-Flag and X-Spam-Report) are inserted for local recipients, and X-Spam-Flag and X-Spam-Status bear a YES; also recipient address extension (if enabled) is tacked onto recipient address for local recipients; for these actions to have any effect, mail must be allowed to be delivered to a recipient;
kill level
if spam score is at or above kill level, mail is blocked; and sender receives a nondelivery notification unless spam score exceeds dsn cutoff level.

The general idea is that kill level is what controls the main actions as far as MTA and amavisd-new is concerned (regardless of what recipients' MUA later does with the mail).

Reaching kill level for at least one recipient controls the following:

  • mail gets quarantined (unless disabled)
  • spam administrator gets a notification (unless disabled)
  • ContentSpamMsgs counter is incremented
  • sender gets a notification if warnspamsender is true and $final_spam_destiny is D_PASS
  • if message is not delivered, sender gets a nondelivery notification (suppressed under certain conditions)
  • the main log entry says: Passed/Blocked SPAM.

On the other hand the tag2 level just adds some mark to the passed mail (only for local recipients), which recipient or his MUA may decide to act on or not. Specifically:

  • Subject header field is modified (unless disabled)
  • X-Spam-Flag and X-Spam-Status header field get a Yes
  • address extension for spam gets tacked on the recipient address
  • spam defanging is done (unless disabled)
  • the main log entry says: Passed/Blocked SPAMMY.

For mail below kill level, if a recipient (or his MUA) decides to discard a message based on tag2 marking, there is no way to retrieve it later from a quarantine, the sender is never notified, spam administrator is never notified. As far as the MTA and amavisd-new are concerned, the message was successfully delivered. Whatever MUA does with the mail is entirely the responsibility and jurisdiction of the recipient and his LDA and MUA.

Quarantine

Mail quarantining is attempted when enabled for a given contents category, which usually includes infected, or banned, or spam mail with score for at least one of its recipients at or above his kill level. It is also possible to enable quarantining of clean messages for archiving or troubleshooting purposes. The *quarantine_to for each recipient (when nonempty), along with a corresponding global *_quarantine_method, determines where the quarantine location should be.

quarantine_method

The *_quarantine_method can be considered a static and a site-wide setting, generally controlling a format and location of the quarantine on the system. The *quarantine_to can be considered a dynamic part of the quarantine location, possibly affected by per-recipient settings and the type of malware (contents category). It serves to fully specify the final location, e.g. a file or a mailbox.

Depending on mail contents category (type of malware), the following variables specify the quarantine method: $virus_quarantine_method, $spam_quarantine_method, $banned_files_quarantine_method, and $bad_header_quarantine_method. One way to globally disable quarantine is to specify undef or an empty string as a value of these variables. A nonempty string should follow a syntax:

  • local:filename-template
  • bsmtp:filename-template
  • smtp:[IPv4-or-IPv6-address-or-hostname]:port
  • smtp:hostname:port
  • smtp:/path/to/a/unix/socket
  • lmtp:[IPv4-or-IPv6-address-or-hostname]:port
  • lmtp:hostname:port
  • lmtp:/path/to/a/unix/socket
  • pipe:argv=command args...
  • sql:anything

The local:, bsmtp: and sql: methods are the usual methods for quarantining. The smtp: or lmtp: methods are only useful for quarantining if quarantine location is some dedicated mailbox instead of a local file or directory. The smtp:, lmtp: and pipe: methods are more often used for forwarding and notifications, and only rarely for quarantining. The following features became available with version 2.5.0: the lmtp: method, support for IPv6, and specifying a Unix socket to a smtp: or lmtp: method.

When quarantine method starts with local:, the rest of the string is a filename-template, which serves to specify a file name to store a quarantined message. The template may contain placeholders which are composed of a percent character, followed by exactly one character. The following expansions are recognized:

  • %b is replaced by $msginfo->body_digest
  • %P is replaced by $msginfo->partition_tag
  • %m is replaced by $msginfo->mail_id
  • %n is replaced by $msginfo->log_id
  • %i is replaced by ISO 8601 timestamp of a mail reception time
  • %% is replaced by a single %

If a filename-template ends up in .gz, the resulting file will be gzip-compressed.

quarantine_to

Depending on the method specified (local/bsmtp/smtp/sql) a per-recipient setting *quarantine_to adopts different semantics and syntax, possibly modified by the configuration variable $QUARANTINEDIR.

method quarantine_to $QUARANTINEDIR effect
anything empty or undef anything not quarantined
empty or undef anything anything not quarantined
local: pseudo-alias mapped through %local_delivery_aliases directory stored as an individual file below the directory $QUARANTINEDIR, file name comes from the template specified in the *_quarantine_method; if a template file name ends in .gz the message will be gzip-compressed
local: pseudo-alias mapped through %local_delivery_aliases filename of a mailbox appended to a file $QUARANTINEDIR in mbox format
local: pseudo-alias mapped through %local_delivery_aliases empty or undef not quarantined
local: e-mail address containing '@'-sign anything sent via SMTP to a mailer for storage, uses $notify_method to specify how to deliver to MTA; much like a newer 'smtp:' entry below
smtp: e-mail address anything sent via SMTP to a mailer for storage, uses the specified IP address and port, or a Unix socket for delivery; formerly a 'local:' method was used for this purpose
lmtp: e-mail address anything sent via LMTP to a mailer for storage, uses the specified IP address and port, or a Unix socket for delivery
bsmtp: anything (nonempty) anything stored in a file specified in the *_quarantine_method in BSMTP format (if file name is absolute, i.e. starts with a "/")
bsmtp: anything (nonempty) directory stored in a file specified in the *_quarantine_method in BSMTP format (file name relative to $QUARANTINEDIR)
sql: anything (nonempty) anything stored into SQL database specified by @storage_sql_dsn

The *quarantine_to is currently quite limited in functionality, it is often used only to turn off the quarantining for some user or local subdomain. The reason for this limited functionality is a more vulnerable nature of this value, as it may come from SQL or LDAP lookups where non-careful access controls to these databases might permit users to enter any value in the *quarantine_to field, which is why we do not let it control the directory or the exact file name of the quarantine file. This may be somewhat relaxed in the future.

In common setups the quarantine location (e.g. a directory or a dedicated mailbox) is the same for all recipients. If at least one recipient specifies a nonempty *quarantine_to specifying this location, the message is quarantined (stored) there once, regardless of the number of recipients.

The general algorithm is: the *quarantine_to value associated with each recipient is looked up. Empty or undef values are ignored and duplicates are discarded. A mail to be quarantined is then stored/sent to each unique location remaining on the list.

The "bsmtp:" quarantine method is somewhat special in that the quarantine file location is entirely determined by the *_quarantine_method setting, and the value of per-recipient *quarantine_to settings do not influence the quarantine location, as long as this value is nonempty.

When using the "bsmtp:" quarantine method and versions of amavisd-new earlier than 2.2.0, the *_quarantine_to was completely ignored, which made it impossible to turn off quarantining selectively for certain users by specifying an empty or undef value. Since 2.2.0, an empty *_quarantine_to turns off quarantine for a recipient regardless of the quarantine method. A nonempty string in *_quarantine_to (the exact value is ignored) must now be used even with "bsmtp:" to enable quarantining.

Releasing from a quarantine

The utility amavisd-release tells the amavisd daemon to fetch a mail from a local quarantine, and send it to MTA through its regular channels ($notify_method), bypassing re-checking.

By default it connects to socket /var/amavis/amavisd.sock, on which amavisd should be listening for AM.PDP protocol, but one can use inet socket instead of a Unix socket if there is a need to run amavisd-release from a remote host.

In the amavisd.conf the following should be added:

$unix_socketname = "$MYHOME/amavisd.sock";  # listen on Unix socket

# alternatively (less common):
# $inet_socket_port = [10024, 9998];  # listen on listed inet tcp ports

# apply policy bank AM.PDP-SOCK on a Unix socket:
#  (note that this precludes the use of old amavis-milter
#   helper program (with sendmail) on the same socket)
$interface_policy{'SOCK'} = 'AM.PDP-SOCK';

# apply policy bank AM.PDP-INET to some inet tcp socket, e.g. tcp port 9998:
$interface_policy{'9998'} = 'AM.PDP-INET';

$policy_bank{'AM.PDP-SOCK'} = {
  protocol => 'AM.PDP',  # select Amavis policy delegation protocol
  auth_required_release => 0,  # don't require secret_id for amavisd-release
};
$policy_bank{'AM.PDP-INET'} = {
  protocol => 'AM.PDP',  # select Amavis policy delegation protocol
  inet_acl => [qw( 127.0.0.1 [::1] )],  # restrict access to these IP addresses
# auth_required_release => 0,  # don't require secret_id for amavisd-release
};

Setting of $auth_required_release decides whether the requestor needs to specify secret_id in addition to mail_id to authorize a mail release. The secret_id is stored in SQL table msgs when logging to SQL is enabled, otherwise this information is not accessible.

Note that turning off $auth_required_release check is safe as long as access to the socket is restricted, like with file protections on a Unix socket, or restricted with inet_acl to specific IP addresses. Enabling or disabling $auth_required_release is a management / setup decision and convenience.

To release a mail message an exact quarantine location from a log file should be specified as an argument to amavisd-release, e.g.:

amavis[29297]: (29297-01-6) Blocked SPAM,
  ... <xxx> -> <yyy>,
  quarantine: spam/U/UM3XM3XDbN52.gz,
  Message-ID:<...>, mail_id: UM3XM3XDbN52, Hits: 13.365,

$ amavisd-release spam/U/UM3XM3XDbN52.gz
250 2.6.0 Ok, id=rel-UM3XM3XDbN52,
  from MTA([193.2.4.66]:10025): 250 2.0.0 Ok: queued as F137717B88B

The amavisd-release utility also accepts mail_id from STDIN if releasing more than one message in one go is more convenient:

$ amavisd-release -
spam/U/UM3XM3XDbN52.gz
spam/g/gnwKVFKiuey3.gz
spam/X/Xpkj9mLLBHTR.gz

Redirecting malware to a different mailbox -- plus addressing

Amavisd-new can tag passed malware by appending an address extension to a recipient address. An address extension is usually a short string (such as 'spam') appended to the local part of the recipient address, delimited from it by a single character delimiter, often a '+' (or sometimes a '-'). This is why address extensions are also known as "plus addressing". Examples of such mail addresses belonging to user jim@example.com are: jim+spam@example.com, jim+cooking@example.com, jim+health@example.com, jim+postfix@example.com.

Most mailers (MTA), including Postfix and sendmail, have some provision to put address extensions to good use. Similarly, local delivery agents (LDA) such as Cyrus or LDAs that come with MTA, can be configured to recognize and make use of address extensions.

The most common application for address extensions is to provide additional information to LDA to store mail into a separate mail folder. Users may for example choose to use this feature to let LDA automatically file messages from mailing lists to a dedicated subfolder, or to file spam to a spam folder, just by letting LDA simply and quickly examine the envelope recipient address, without having to parse mail header or having to configure and run filters such as procmail or Sieve.

Mailers (MTA and LDA) usually attempt first to examine (to check for validity, to lookup in virtual or aliases maps) a full unmodified recipient address. If the attempt is unsuccessful, they strip away the extension part, and try again. This way a presence of some unknown address extension is simply ignored. For example, a delivery for jim+health@example.com would deliver the mail to the main Jim's inbox if he hasn't provided a subfolder health in his mailbox.

For this fallback to work (to ignore unknown extensions), it is important that all components that need to deal with address extensions (MTA, LDA, content filters) have the same notion of the delimiter in use on the system. For Postfix the configuration option is recipient_delimiter=+ (see also propagate_unmatched_extensions), for amavisd-new the option is $recipient_delimiter='+'; for Cyrus the delimiter is hardcoded as '+', see Cyrus IMAP FAQ -> plus addressing.

The amavisd-new configuration options for adding address extensions are @addr_extension_virus_maps, @addr_extension_spam_maps, @addr_extension_banned_maps, @addr_extension_bad_header_maps. The configuration must also ensure the malware mail is to be delivered, otherwise there is nothing to tack an address extension on -- either by setting kill level sufficiently high, or by declaring spam lovers, or by $final_spam_destiny=D_PASS; an example:

$recipient_delimiter = '+';
@addr_extension_spam_maps = ('spam');
$sa_tag2_level_deflt = 6.7 ;    # score above which spam extension is added
$sa_kill_level_deflt = 15;      # block higher score entirely
$final_spam_destiny=D_DISCARD;  # junk all above kill level

or provide extension string more selectively for certain users or subdomains:

@addr_extension_spam_maps = (
  { '.sub1.example.com' => 'spam',     # an entire subdomain
    'user1@example.com' => 'spam',     # a particular user
    'user2@example.com' => 'malware',  # another user wants a different ext.
    '.'                 => '' }  # all the rest do not receive an extension
);

If one is considering using a quarantine mechanism but wants a per-user (or perhaps per-subdomain) quarantines, this is not such a good idea, because quarantined files are not supposed to be directly visible or handled by recipients: to protect the privacy of the sender, some header pre-processing must be performed on a quarantined file before handing it over to a recipient.

The cleanest way to achieve per-user quarantine which may be directly accessible and/or manipulated by recipients is to turn on adding address extensions, and configure MTA and/or LDA to store such mail wherever necessary, either to a user's dedicated subfolder, or perhaps to some centralized dedicated set of malware mailboxes (per-user or perhaps per-subdomain).

If it is desired to reroute extension-tagged mail to some mailbox away from the usual LDA, the virtual alias mapping by MTA is the tool for the job. With Postfix, a pcre-based virtual map can specify for example:

/^(.*)\+spam@([^@]*)\.example\.com$/   spam-$2-box@example.com

which will collect all spam into one mailbox for each subdomain.

For the Postfix local(8) LDA, a presence of a file $HOME/.forward+spam can redirect mail for user+spam to some dedicated file. For the Postfix virtual(8) LDA, a virtual_mailbox_maps may contain entries like:

user1         mbxfile1
user1+spam    mbxspamfile1
user2         mbxfile2
user2+spam    mbxspamfile2

Hard black- and whitelisting senders regarding spam

The blacklisting and the whitelisting are ways of telling that we already know that a message is spam or is ham (non-spam) just by examining the envelope sender address and comparing it to lists of known spammers or to lists of known legitimate senders of ham. It is a quick check, potentially saving us the trouble of examining the mail contents. It has a big drawback however in that the sender mail address can be (and often is) faked and there is no guarantee that the claimed sender address represents the actual sender.

The sender address is usually faked for spam messages, so whitelisting some sender address is a of questionable value, and often lets in far more spam than it does good by approving legitimate mail. For a reliable way of permitting certain sending clients to send spamy mail see policy banks.

Blacklisting however is still useful: spammer has no desire to pretend to be some blacklisted sending address, when he can choose any other address. Genuine sender that is intentionally blacklisted can only avoid being blocked by falsifying his address (joining spammers in his methods) and sending non-spamy mail, the later being our objective anyway. Although amavisd-new does provide blacklisting, it is functionally equivalent but more effective to blacklist senders at the MTA, preventing such mail from even entering the mail system.

It should be emphasized that whitelisting (and blacklisting) only affects spam checks. It has no influence on other checks such as virus, banned or header checks. Infected mail from whitelisted sender would still be blocked if our policy is to block viruses.

Another point to bear in mind is that the sender address examined is the one from the SMTP protocol, exactly as provide by MTA to amavisd-new. It is known as the envelope sender address or return path. This address does not necessarily match the mail author's address from the mail header (From:) or the sender's address from the header (Sender:). This is most obvious with mail from mailing lists, where the envelope sender address is usually the address of a mailing list management service, while the author's address (From:) is the address of a person sending the message. Using the envelope sender address in most cases makes it easier to black- or whitelist mail from mailing lists, compared to guessing a sender address by parsing mail header.

To avoid surprises, whitelisted sender suppresses inserting/editing the tag2-level header fields (X-Spam-*, Subject), appending spam address extension, and quarantining, even if we know the message is spam (e.g. because the spam check result on the same mail contents has been cached from some earlier mail or known from check on behalf of another recipient).

For mail from blacklisted senders, the effect is as if the spam level were artificially pushed high, resulting in 'X-Spam-Flag: YES', high 'X-Spam-Level' bar and other usual reactions to spam, including possible rejection. If the message nevertheless still passes (e.g. for spam loving recipients), it is tagged as BLACKLISTED in the 'X-Spam-Status' header field, but the reported spam value and set of tests in this report header field is not adjusted (if available from SpamAssassin, which may or may not have been called)

If all recipients of a message either white- or blacklist the sender, amavisd is free to skip spam scanning (calling the SpamAssassin), saving on time. There is no guarantee however that spam scanning will actually and always be skipped.

The following variables (lists of lookup tables) are available, with the semantics and syntax as specified in README.lookups: @whitelist_sender_maps, @blacklist_sender_maps, which implement global policy applicable to all recipients. Similarly there are $per_recip_blacklist_sender_lookup_tables and $per_recip_whitelist_sender_lookup_tables, which make possible for each recipient or subdomain to specify its own set of black- or whitelisted senders. The per-recipient tables take precedence over global tables.

For SQL lookups, amavisd-new will first lookup the recipient in table users in order of descending priority, e.g. user@sub.domain.org, user, @.sub.domain.org, @.domain.org, @.org, and @. (which can be considered a catchall). Each matching recipient record may have a list of senders associated (through join on field users.id and wblist.rid). The sender address is then looked up in the associated list of senders (wblist) in order of descending priority, e.g. sender@sub.example.com, @.sub.example.com, @.example.com, @.com, and @. . This search stops at the first matching sender record with a non-NULL field wblist.wb. The value of a field wblist.wb from the matched record determines if the sender is considered whitelisted ('W'), blacklisted ('B') or neutral (' ') for this recipient.

The neutral value is there just as a way to explicitly stop the search, which may be used by a recipient to overrule site-wide or static white- or blacklisting defaults for some specific sender, and to explicitly neither whitelist nor blacklist the sender, letting the normal spam check determine the spaminess of a mail.

For recipient user@sub.domain.com and sender sender@sub.example.com the following search is performed:

user@sub.domain.org
  sender@sub.example.com @.sub.example.com @.example.com @.com @.

user
  sender@sub.example.com @.sub.example.com @.example.com @.com @.

@.sub.domain.org
  sender@sub.example.com @.sub.example.com @.example.com @.com @.

@.domain.org
  sender@sub.example.com @.sub.example.com @.example.com @.com @.

@.org
  sender@sub.example.com @.sub.example.com @.example.com @.com @.

@.
  sender@sub.example.com @.sub.example.com @.example.com @.com @.

Soft black- and whitelisting senders regarding spam -- @score_sender_maps

Instead of hard black- or whitelisting a sender address (unconditionally considering mail spam or ham solely based on sender address regardless of mail contents), a more gentle approach is to add score points (penalties) to the spam score for mail from certain senders or sending domains. Positive points lean towards blacklisting, negative towards whitelisting. This is much like adding SpamAssassin rules or using its white/blacklisting, except that here only envelope sender addresses are considered (not addresses in a mail header), and that score points can be assigned per-recipient (or per-domain or globally), and that the assigned penalties are customarily much lower than the default SpamAssassin white/blacklisting score.

The table structure of @score_sender_maps is similar to $per_recip_blacklist_sender_lookup_tables i.e. the first level key is recipient address, pointing to by-sender lookup tables. The essential difference is that scores from all matching by-recipient lookups (not just the first that matches) are summed to give the final score boost. That means that both the site and domain administrators, as well as the recipient can have a say on the final score.

For SQL lookups, the mechanism is much like the one described for hard black- or whitelisting, with the following differences:

  • the field wblist.wb is numeric, representing score points, instead of containing a character W or B or space;
  • the search through matching recipients does not stop at the first match, but traverses all matching recipients, summing up the corresponding wblist.wb field values.

Namely, amavisd will lookup the recipient, e.g. user@sub.domain.org, user, @.sub.domain.org, @.domain.org, @.org, and @. . Since the search will not stop at the first recipient match, the search order in this case is unimportant, although it is actually the same descending-priority order as with hard b/w listing. Each matching recipient record may have a list of senders associated (through join on field users.id and wblist.rid). The sender address is then looked up in the associated list of senders (wblist) in order of descending priority, e.g. sender@sub.example.com, @.sub.example.com, @.example.com, @.com, and @. . This search stops at the first matching sender record with a non-NULL field wblist.wb, but this does not terminate the outer recipients search. Numeric values of a field wblist.wb from matched records are summed up across all matching recipients tables, and the result is added to the spam score as produced by SpamAssassin.

Unlike static tables, where hard and soft w/b-listing use separate tables, the SQL-based hard and soft w/b-listing uses the same SQL tables and the same field wblist.wb. Mixing the 'W', 'B' with numeric values is somewhat frowned upon, but is supported to facilitate transition. The search goes like described above as long as only numeric field values are encountered, summing up the values and adding the accumulated sum to the final score. If a non-numeric value of field wblist.wb is encountered during this search, its value (W or B or space) is interpreted as described for hard w/b listing, and the search stops at this point.

Configuration variables

The behaviour of the amavisd-new is controlled by a set of configuration variables, which are just normal module-global Perl variables (in package Amavis::Conf). At daemon startup time these variables are first assigned an initial value (often just an undefined value, the undef). The default values of configuration variables are documented in file amavisd.conf-defaults, which lists all configuration variables.

Next a configuration file amavisd.conf (or other file as specified by option -c) is read and interpreted by the Perl interpreter itself. The amavisd.conf is just a normal Perl program, and can in principle do whatever and however it pleases, but its main purpose is to assign values to configuration variables.

After execution of amavisd.conf is done, the daemon may correct some configuration variable values (mainly to maintain backwards compatibility with earlier version of configuration file), and may assign a default value to certain variables which are still undefined -- these variables and their default values are marked "after-defaults" in the documentation file amavisd.conf-defaults. The main reason for existence of the "after-defaults" concept is that some default values depend on other configuration variables and can not be computed before the amavisd.conf is finished. To force such variables to an off/false/disabled state, one needs to assign some false but defined value to them, such as '' (an empty string) or a 0 for booleans.

Perl variables always start with a character $, @ or % to indicate a type of variable. This leading character is part of the variable name for all practical purposes.

$ (dollar character)
indicates a scalar variable (a string, a number, a reference)
@ (at sign)
indicates an array variable (a list)
% (percent character)
indicates an associative array (also known as hash), which maps keys to values

A couple of Perl syntactical elements deserve mention at this point, as they are often used in the amavisd.conf configuration file.

"...", a double-quoted string
is a string; variables within are evaluated, e.g. "$MYHOME/tmp"
'...', a single-quoted string
is a string; variables within are not evaluated, the $ and @ loose their special meaning, e.g. 'user@example.com'
(...)
is a list of comma-separated expressions, e.g. (1,2,"test"); a list is normally assigned to an array variable
qw(string)
is an operator that interprets its argument as a single string, splits it on whitespace to words, and returns a list of words (strings); it is a convenience to avoid some typing, e.g. qw(user@example.com .example.net .org) is exactly equivalent to ('user@example.com', '.example.net', '.org');
[...]
is a reference to an anonymous list of comma-separated expressions, e.g. [1,2,"test"]; (note: a reference is a scalar)
{...}
is a reference to an anonymous associative array, e.g. {'alfa'=>1, 'beta'=>99, 'other'=>'test'}; (note: a reference is a scalar)
\variable
is a reference to a variable, e.g. \$virus_admin, \@mynetworks, \%whitelist_sender; (note: a reference is a scalar)

Historically amavisd-new accessed all configuration variables directly with their name, e.g. %spam_lovers, @spam_lovers_acl, $spam_lovers_re. Later it became apparent that certain groups of variables (lookups) are always used together in the same way, so new array variables like @spam_lovers_maps were introduced. The program now never accesses old lookup table variables directly, but always through higher level lists. The solution is fully backwards compatible, as the default value for the new lists references the old variables, e.g.:

@spam_lovers_maps = (\%spam_lovers, \@spam_lovers_acl, \$spam_lovers_re);

Administrator is free to modify or replace the lists in variables like @spam_lovers_maps, perhaps rearranging the order or loosing all references to legacy variables, and replacing them with other variables, often anonymous arrays/lists or anonymous associative maps (hashes), or constants which can serve as a convenient catchall default value when used last in the list.

Since amavisd-new version 2.0, there is one further generalization step in the way a program accesses configuration variables. More than a hundred configuration variables which control amavisd-new operation on a by-message level (as opposed to by-recipient and truly global settings) are now grouped in associative array called a policy bank. These configuration variables are no longer accessed directly by their variable name by the program, but always through a currently installed policy bank. Administrator is free to modify the policy bank, normally by providing replacement policy banks and specifying under what conditions the replacement policy bank is to be automatically installed.

Policy banks

Policy banks hold sets of configuration variables controlling most of per-message settings, including: static lookup tables, IP interface access rules, forwarding address, log level, templates, administrator addresses, spam trigger levels, quarantine rules, lists of anti-virus scanner entries (or just a subset), banned names rules, defang settings, etc. The whole set of these settings may be replaced with another predefined set based on incoming port number, making it possible for one amavisd daemon to cope with more diverse needs of served user communities which could so far only be implemented by running more than one instance of the amavisd daemon, each with its own configuration file.

This mechanism brings new potentials for the future: in principle policy banks could be swapped not only based on port number or SMTP client IP address, but on any characteristics pertaining to a mail message as a whole (not specific to each of its recipients), or to characteristics of a connection from a mailer (e.g. the interface address or protocol);

Until a better mechanism is available, a policy bank named 'MYNETS' has special semantics: this policy bank is loaded (if it exists) whenever MTA supplies a SMTP client's IP address (through Postfix XFORWARD extension to the SMTP protocol, or via a new AM.PDP protocol) and that address matches the @mynetworks list (actually: the list referenced by 'mynetworks_maps' key in the currently installed policy map).

An associative array %interface_policy is a current mechanism of assigning a policy bank to an incoming TCP port number (port must be in the list @$inet_socket_port, otherwise amavisd will not listen on that port). Whenever a connection from MTA is received, first a built-in policy bank with an empty name -- the $policy_bank{''} gets loaded, which brings in all the global/legacy settings. Then it is overlaid by whatever configuration settings are in the bank named in the $interface_policy{$port} if any, and finally the policy bank named 'MYNETS' (i.e. settings from $policy_bank{'MYNETS'}) is overlaid if such policy bank exists and the SMTP client IP address is known (by XFORWARD SMTP extension command from MTA) and it matches the current mynetworks_maps.

When a new policy bank is overlaid over an existing set of configuration variables, the variables not present in the new policy bank retain their value. This makes it possible to specify new policy banks which carry only a minimal set of settings that need to be changed.

The built-in policy bank (with empty name) is predefined, and includes references to most other variables (the dynamic config variables), which are accessed only indirectly through the currently installed policy bank. Overlaying a policy bank with another policy bank may bring in references to entirely different variables, possibly unnamed, and may remove references to legacy variables if it so chooses.

Configuration variables are referenced from a policy bank (which is implemented as a perl associative array, i.e. a hash) by keys of the same name, e.g. { log_level => \$log_level, inet_acl => \@inet_acl, ...}. For scalars one level of indirection is allowed, e.g. a policy bank { log_level => \$log_level }; $log_level=2; is equivalent to { log_level => $log_level } or to { log_level => 2 }, but in the first example with an indirect reference, the $log_level may be assigned to even _after_ the policy bank has already been formed.

A word of caution: the syntax of entries within a policy bank hash is slightly different from assignments to configuration variables. This is because entries within policy bank are not assignments, but key=>value pairs as in any Perl associative array. And these pairs are delimited by commas, unlike statements, which are delimited by semicolons. Value is separated from its key by '=>' (or by a comma), whereas the assignment operator is '='. Keys of a policy bank are without leading $ or @ or %, unlike variable names. Values of an associative array can only be scalars (e.g. strings or numbers or references to arrays or references to associative array).

Compare:

  • value of a policy bank is a reference to a Perl associative array, e.g.: 
        { log_level => 3,
      forward_method => 'smtp:[127.0.0.1]:10025',
      spam_admin_maps => ["spamalert\@$mydomain"],
      blacklist_sender_maps => [ [qw(.example.org .example.net)] ],
    }
  • normal assignments look like: 
          $log_level = 3;
      $forward_method = 'smtp:[127.0.0.1]:10025';
      @spam_admin_maps = ("spamalert\@$mydomain");
      @blacklist_sender_maps = ( [qw(.example.org .example.net)] );

And a final note: Perl can detect and report typing mistakes in variable names, but mistyped key is just some unused associative array entry lurking in a hash, never used and never reported as mistyped/useless.

Putting policy banks to good use -- examples

The sender address can be faked, so comparing envelope sender address to @local_domains_maps or some other lookup table to base some important decisions on would not be trustworthy. The only reliable information is the recipient's e-mail address and information about client SMTP session, such as the IP address of the sending SMTP client and the server port number or the interface address. Such information can be made available by MTA to amavisd-new through a feeding protocol (e.g. XFORWARD extension or via AM.PDP), or separate MTA paths can be set up for mail that needs to be treated differently, such as internally originating and externally originating mail, or perhaps separating authenticated mail from the rest.

Amavisd-new has two ways of receiving such extra information from MTA:

  • it can listen on more than one TCP port and apply different policy banks to each port (applicable to any dual-MTA setup including Postfix), and/or
  • it can accept SMTP client's IP address from MTA by XFORWARD extension to the SMTP protocol (available in Postfix only), which can control loading of policy bank MYNETS. This is in addition to loading policy banks based on TCP port number. The MYNETS policy bank is loaded (if applicable) after (on top of the) the port-assigned policy bank.

The following examples illustrate several ways of distinguishing between different mail origins. For most common purposes the only distinction that really matters is separating internally originating mail from the rest, and for this purpose the use of policy bank MYNETS and a sufficiently recent version of Postfix supporting XFORWARD suffices -- the complication with multiple ports and multiple interfaces is needed only for more demanding sites which prefer maximum flexibility.

Example 1

As stated earlier, a policy bank named 'MYNETS' is loaded (if it exists) whenever MTA supplies an original SMTP client's IP address (e.g. via the Postfix XFORWARD extension) and that address matches the @mynetworks list. This covers most common needs to distinguish internally-originating mail from the rest, and allows them to be treated differently, as illustrated by the following example:

$policy_bank{'MYNETS'} = {  # mail originating from @mynetworks
  virus_admin_maps => ["security\@$mydomain"], # alert of infected local hosts
  spam_admin_maps  => ["abuse\@$mydomain"],    # alert of internal spam
  spam_kill_level_maps => [7.0],  # slightly more permissive spam kill level
  spam_dsn_cutoff_level_maps => [15],
  banned_filename_maps => [
    new_RE(
    # block double extensions in names:
      qr'\.[^./]*\.(exe|vbs|pif|scr|bat|cmd|com|cpl|dll)\.?$'i,
    # allow any name or type (except viruses) within an archive:
      [ qr'^\.(Z|gz|bz2|rpm|cpio|tar|zip|rar|arc|arj|zoo)$' => 0],
    # blocks MS executable file(1) types, unless allowed above:
      qr'^\.(exe-ms)$',
    ),
  ],
};

Example 2

In the following example some of the external mail is coming in via fetchmail, the rest of the externally originating mail is coming in via normal SMTP at tcp port 25, and all internally originating mail is coming to MTA via mail submission port 587 reserved for that purpose, or via dedicated IP address accessible only from inside, or through a Postfix pickup service. We'll use Postfix in this example, although it does not rely on any particular Postfix capability that wouldn't be available in any general purpose MTA in some form or another.

Only the specifics of this setup are described here. Missing bits like the MTA re-entry port 10025 and other options are described in README.postfix and are assumed here. Specifying additional smtpd restrictions and options may be desired, and is omitted here for brevity.

To let amavisd-new be able to distinguish between all four mail entry routes, we let amavisd listen on four TCP ports (the fifth is for good measure, to be used in the next example): $inet_socket_port = [10040,10041,10042,10043,10044]; (any unused non-privileged TCP ports can be used)

In Postfix configuration file master.cf we attach different content_filter options to each of the Postfix services receiving mail. We'll assume the MTA host has two IP addresses 192.0.2.1 and 192.0.2.2 assigned (IP aliases or separate physical interfaces), which makes it easier to distinguish between internally originating mail and the rest even if XFORWARD can not be used (older Postfix versions or some other MTA):

# regular incoming mail, originating from anywhere (usually from outside)
# the MX record (or backup mailers) should point to this IP address
192.0.2.1:smtp inet  n  -  n  -  -  smtpd
  -o content_filter=amavisfeed:[127.0.0.1]:10040

# incoming mail from fetchmail, considered externally originating
# (add 'smtphost localhost/2345' to the poll section in .fetchmailrc)
127.0.0.1:2345 inet  n  -  n  -  -  smtpd
  -o content_filter=amavisfeed:[127.0.0.1]:10041
  -o smtpd_client_restrictions=permit_mynetworks,reject
  -o mynetworks=127.0.0.0/8

# IP address to be used by internal hosts for mail submission
192.0.2.2:smtp inet  n  -  n  -  -  smtpd
  -o content_filter=amavisfeed:[127.0.0.1]:10042
  -o smtpd_client_restrictions=permit_mynetworks,reject

# or, tcp port 587 to be used by internal hosts for mail submission
submission inet  n  -  n  -  -  smtpd
  -o content_filter=amavisfeed:[127.0.0.1]:10042
  -o smtpd_client_restrictions=permit_mynetworks,reject

# locally originating mail submitted on this host through a sendmail binary
pickup     fifo  n  -  n  60  1  pickup
  -o content_filter=amavisfeed:[127.0.0.1]:10043

A global option content_filter in file main.cf could provide a convenient default, only services that need a different setting would then need to override it.

Now let's make up names for policy banks which will cover all four cases. We'll pick names EXT, EXT-FM, INT, INT-HOST for policy banks. The amavisd needs to be told to load corresponding policy when a request comes in on each of the listening ports:

  $interface_policy{'10040'} = 'EXT';
  $interface_policy{'10041'} = 'EXT-FM';
  $interface_policy{'10042'} = 'INT';
  $interface_policy{'10043'} = 'INT-HOST';
  $interface_policy{'10044'} = 'AUTH';  # to be used in the next example

Next we'll prepare each policy and specify there the options which should be different from global options. Note that the following policies serve mostly as an example and to provide ideas -- they should not be considered a recommendation. For example:

# regular incoming mail, originating from anywhere (usually from outside)
$policy_bank{'EXT'} = {
  # just use global settings, no special overrides
};

# incoming mail from fetchmail, considered externally originating
$policy_bank{'EXT-FM'} = {
  log_level => 2,
    # no bounces for spam, not even for score below spam_dsn_cutoff_level_maps:
  final_spam_destiny => D_DISCARD,
};

# locally originating mail guaranteed to be from inside
$policy_bank{'INT'} = {
    # enable/redirect admin notifications for locally originating malware:
  virus_admin_maps => ["virusalert\@$mydomain"],
  spam_admin_maps  => ["virusalert\@$mydomain"],
    # be slightly more permissive on spam levels for mail from our hosts:
  spam_kill_level_maps => [7.0],
  spam_dsn_cutoff_level_maps => [15],
  final_virus_destiny => D_BOUNCE,  # (unless in viruses_that_fake_sender_maps)
  final_spam_destiny  => D_BOUNCE,  # (unless above spam_dsn_cutoff_level_maps)
  bypass_banned_checks_maps => [ 1 ],  # allow sending any file type or name
    # provide customized sender notifications for spam from our users:
  notify_spam_sender_templ => read_text("$MYHOME/notify_spam_sender.txt"),
};

# mail locally submitted on the host on which MTA runs
$policy_bank{'INT-HOST'} = {
    # NOTE: this is just an example; ignoring internally generated spam
    # may not be such a good idea, consider zombified infected local PCs
  bypass_spam_checks_maps   => [ 1 ],
  bypass_banned_checks_maps => [ 1 ],
  final_spam_destiny   => D_PASS,
  final_banned_destiny => D_PASS,
};

# authenticated mail (used by the next example)
$policy_bank{'AUTH'} = {
    # enable admin notifications for malware originating from our users:
  virus_admin_maps => ["virusalert\@$mydomain"],
  spam_admin_maps  => ["virusalert\@$mydomain"],
    # be slightly more permissive on spam levels for mail from our users:
  spam_kill_level_maps => 7.0,
  spam_dsn_cutoff_level_maps => 15,
  bypass_banned_checks_maps => 1,  # allow sending any file type or name
  final_bad_header_destiny => D_BOUNCE;  # block invalid headers
};

If not all four cases need to be distinguished, the same policy bank name (or none at all) can be assigned to more than one port. Also the MTA configuration can use the same amavisd port for more than one of its incoming services if there is no need for different settings.

Example 3

Besides setting different content_filter options for different Postfix services, one may use the option FILTER in Postfix lookup tables, as described in Postfix man pages access(5) and header_checks(5), to specify different content_filter settings based on various conditions, such as sender domain name or IP address, mail header fields, etc.

Consider the next example which uses the FILTER settings to distinguish from internally originating, authenticated external mail and the rest.

# global default:
content_filter=amavisfeed:[127.0.0.1]:10044

# note that permit_mynetworks only checks for key presence and ignores rhs
mynetworks = cidr:/etc/postfix/mynetworks-filter.cidr

smtpd_sender_restrictions =
  ... the usual rejects if any ...
  check_client_access cidr:/etc/postfix/mynetworks-filter.cidr
  permit_mynetworks
  permit_sasl_authenticated
  permit_tls_clientcerts
  check_sender_access regexp:/etc/postfix/filter-catchall.regexp

The check_client_access cidr:/etc/postfix/mynetworks-filter.cidr preceeds the permit_mynetworks (which uses the same cidr table, but ignores the righthand side), and it serves to override the global content_filter setting by the use of FILTER for each of the networks (presumably internal) listed in mynetworks-filter.cidr. The final effect is that mail matching networks listed in mynetworks-filter.cidr will be sent for content filtering to tcp port 10042 (the FILTER setting in access map), authenticated non-local mail will be sent for content filtering to port 10044 (the global setting), while all the rest will be sent to port 10040 (as specified in catchall filter). If there are any other overrides in master.cf like in the previous example, they take precedence over the global settings, but the FILTER rules take the ultimate precedence.

/etc/postfix/mynetworks-filter.cidr :

127.0.0.0/8    FILTER amavisfeed:[127.0.0.1]:10042
10.0.0.0/8     FILTER amavisfeed:[127.0.0.1]:10042
172.16.0.0/12  FILTER amavisfeed:[127.0.0.1]:10042
192.168.0.0/16 FILTER amavisfeed:[127.0.0.1]:10042

/etc/postfix/filter-catchall.regexp:

/^/            FILTER amavisfeed:[127.0.0.1]:10040

Note that in place of the last catchall entry: check_sender_access regexp:/etc/postfix/filter-catchall.regexp one would be tempted to do: check_sender_access static:FILTER amavisfeed:[127.0.0.1]:10040, but unfortunately spaces are not allowed within an option value in master.cf, so we have to resort to a lookup table.

$max_requests

Amavisd-new runs under process control of Net::Server. This is a pre-forked environment where $max_servers child processes are constantly kept alive and ready to accept new tasks (mail messages to be checked). Each amavisd child process is able to handle several tasks in a row, which helps to reduce startup (fork) costs. In case of SMTP or LMTP protocol, each session may consist of several SMTP/LMTP transactions. Each SMTP/LMTP transaction is counted a one task, regardless of whether it came in from the same SMTP/LMTP client in a multi-transaction session, or as separate sessions, possibly from different SMTP/LMTP clients.

A configuration variable $max_requests (default value 20) controls the approximate number of tasks each child process is willing to handle. After that the child process terminates and Net::Server provides a new child process to take its place.

The exact value of $max_requests is not critical. There are two opposing needs, and some in-between value should be chosen.

On the low side, the number should not be too small in order for the startup cost to be averaged out / sufficiently diluted over an entire child lifetime. A value above 5 or 10 meets this goal in most amavisd-new configurations.

On the high side, the value depends on the amavisd-new configuration. The amavisd daemon itself is conservative in its use of dynamically allocated memory and does not load mail into memory, but keeps mail being processed and its components on files. Similarly, most of the called external virus scanners and decoders are rational in their use of memory (a notable exception was Archive::Tar which was used if a pax or cpio command was not available, but is no longer supported). Unfortunately this is not true for Perl module Mail::SpamAssassin, which expects to have an entire decoded mail in memory in order to be able to run its large set of rules on it in reasonable time. This is a design decision of SpamAssassin.

When amavisd-new is not configured to use SpamAssassin, the value of $max_requests can be quite high without any known or expected problems. For general sanity reasons, an upper limit could be a 100 for example, although anything above 20 or so would not bring measurable benefit to the maximum sustained mail throughput.

When amavisd-new is configured to use SpamAssassin however, the slurping of entire mail in memory and decoding it may have implications, depending on the $sa_mail_body_size_limit value, on the maximum mail size allowed at the MTA (e.g. Postfix setting for message_size_limit) and on the mail compression factor. Even though the allocated memory is reclaimed by Perl after mail processing, and is reused for subsequent processing, the process virtual memory footprint never shrinks, it can only expand as needed.

The $sa_mail_body_size_limit sets a limit on a mail size beyond which SpamAssassin is not called, so it can not contribute to memory usage much beyond this limit, times a small factor (2-5?, due to multiple internal representations of a message). If the $sa_mail_body_size_limit is large, and MTA mail size is not limited, or if mail has a huge mail header, the memory footprint can become noticable. For the rest of a lifetime the child process that processed the mail stays at its high virtual memory size. If this happens frequently, host resources may become scarce. Limiting the number of tasks is very much desirable in this case.

The default value of 20 for $max_servers was chosen as a good compromise between averaging-out the startup costs and not wasting too much resources on hosts with high message size limit and SpamAssassin enabled.

In the setup with Postfix where its lmtp client is chosen to feed amavisd-new, this client tries to keep LMTP session open and submit several mail messages in multiple transactions. With recent Postfix versions its SMTP client is capable and willing of using multiple transaction sessions as well, although it seems to be less persistent than the LMTP client.

According to SMTP and LMTP protocol specifications, dropping the session on the server side is considered rude and should be used only as a last resort. In order to respect the $max_requests setting (which is not strictly enforced by amavisd, and is considered an advisory value), the client side should preferably be configured with a comparable limit. Starting with amavisd-new-2.2.0 the amavisd daemon is more strict in enforcing the limit and drops the SMTP or LMTP session after $max_servers is exceeded by one. This was a recommendation from the Postfix community, as the option of reducing Postfix max_use setting is considered less appropriate.

Nevertheless, Postfix doesn't take session dropping lightly, it backs off a while after content filter forcibly drops the session, which is undesired. Better behaviour is achieved when Postfix voluntarily terminates a SMTP session before amavisd would reach its $max_requests limit. This can be achieved by applying max_use to the Postfix smtp service feeding a content filter (typically this entry in master.cf is named 'amavisfeed').

Setting up DKIM mail signing and verification

A DKIM standard (RFC 4871) states the following, which applies to its predecessor DomainKeys (historical: RFC 4870) as well:

DomainKeys Identified Mail (DKIM) defines a mechanism by which email messages can be cryptographically signed, permitting a signing domain to claim responsibility for the introduction of a message into the mail stream. Message recipients can verify the signature by querying the signer's domain directly to retrieve the appropriate public key, and thereby confirm that the message was attested to by a party in possession of the private key for the signing domain.

The DomainKeys specification was a primary source from which the DomainKeys Identified Mail [DKIM] specification has been derived. The purpose in submitting the RFC 4870 document is as an historical reference for deployed implementations written prior to the DKIM specification.

The main advantage of DKIM signing to sending domains is that it allows recipients to reliably validate mail origin for purposes of whitelisting on spam checks and whitelisting reception of otherwise banned mail contents. By signing outbound mail you give your correspondents a chance to distinguish between your genuine mail, and fraud or spam mail which may happen to carry your domain name as a sender address. Signing outbound mail is a kind gesture towards recipients, making it much easier for them to treat your mail as important or desirable if they choose so.

The main advantage of DKIM signature verification to recipients is that it allows them to reliably distinguish genuine mail originating from a claimed sending domain from other (possibly faked) mail. It makes signature-based whitelisting a reliable mechanism. It also makes it possible to recognize and automatically discard fake mail claiming to be from domains which are known to always sign their outbound mail and to always send mail directly. Coupled with reputation schemes (mostly manual/static at present, or dynamic in the future) makes it possible to assign score points (positive or negative) based on merit and past experience with each signing domain. A valid signature also offers non-repudiation: a domain which signed a message can not disclaim message origin, which offers recipient a strong argument when reporting abuse to the signing domain.

For the impatient - signing from scratch

Here is a quick Spartanic setup of DKIM signing and DKIM/DK verification by amavisd for the impatient, without much explanation, assuming all originating mail comes from internal networks (not from authenticated roaming clients), only one domain needs signing, using default signature tags, no milters are in use and no mailing list manager needs signing. No changes in Postfix configuration is necessary for this simple setup. For more information and more complex setups please see sections further on.

Generate a signing key:

  $ amavisd genrsa /var/db/dkim/example-foo.key.pem

add to amavisd.conf:

  $enable_dkim_verification = 1;
  $enable_dkim_signing = 1;
  dkim_key('example.com', 'foo', '/var/db/dkim/example-foo.key.pem');
  @dkim_signature_options_bysender_maps = (
    { '.' => { ttl => 21*24*3600, c => 'relaxed/simple' } } );
  @mynetworks = qw(0.0.0.0/8 127.0.0.0/8 10.0.0.0/8 172.16.0.0/12
                   192.168.0.0/16);  # list your internal networks

run:

  $ amavisd showkeys

add the public key (as displayed) to your DNS zone, increment SOA sequence number and reload DNS; then test signing and a published key:

  $ amavisd testkeys

if all went well:

  $ amavisd reload

For the impatient - replacing signing by dkim-milter with signing by amavisd

For sites already signing their mail by dkim-milter, most work of preparing signing keys and publishing public keys in DNS has already been done. All it needs to be done is to declare these signing keys in amavisd.conf and turn on $enable_dkim_signing.

To facilitate transition of DKIM signing from dkim-milter to amavisd-new, a new command-line tool is available with amavisd-new-2.6.2 (the extra utility code is not loaded during normal operation), taking a file name as its argument, e.g.:

  $ amavisd convert_keysfile /var/db/dkim/keysfile.txt

and writing to stdout a set of lines that may be directly included into amavisd.conf configurations file, matching semantics of a dkim-filter keys file. It can be useful during transition, or for those who prefer to specify signing keys and sender-to-key mappings as a file in a syntax compatible with options -K -k of dkim-filter, and can live with limitations of such syntax. See dkim-filter(8) man page for details on the syntax.

The produced output consists of signing key declarations (calls to a procedure dkim_key), where each call normally corresponds to exactly one DNS resource record publishing a corresponding DKIM public key. When necessary output also produces an assignment to a list of lookup tables @dkim_signature_options_bysender_maps, which supplies non-default mappings of sender domains to signing keys, e.g. when third-party signatures are desired.

Implementation and mail flow

Signing of originating mail (or mail being redistributed by our domain), and verifying signatures of incoming mail are two tasks that can be performed by the same program, or they can be performed by separate entities. Traditionally with sendmail, both tasks are performed by one milter, which may be easier to maintain, but has certain disadvantages.

Verifying signatures should be performed early, before any local mail transformations get a chance of invalidating a signature, e.g. by performing MIME conversions to quote-printable, by fixing syntactically invalid mail header section, by reformatting or reordering some header fields (some MTAs do it frivolously), by modifying/inserting/removing certain header fields, or by a local mailing list modifying mail text, e.g. by appending footers.

Signing outgoing mail should be performed late, after mail sanitation, after conversion to 7-bit characters (to avoid later uncontrollable changes by a relaying or receiving MTA), and after editing header section by a content filter. Similar applies to local mailing lists, which may be rewriting messages, requiring them to be re-signed by the domain hosting a mailing list, just before being sent out.

Starting with amavisd-new version 2.6.0, DKIM signing can be performed directly by amavisd (using a Perl module Mail::DKIM, which is the same module as used by DKIMproxy and by SpamAssassin). Signing directly by amavisd reduces setup complexity using a milter or DKIMproxy, and avoids additional data transfers. Regarding mail flow through the system there are similarities between signing in amavisd and signing by dkim-milter, which is why the diagram below shows both possibilities.

For verification there are three choices: either amavisd itself can do it by calling Mail::DKIM directly, or a SpamAssassin plugin can do it by calling the same Perl module, or a milter in verification-only mode can be invoked by an incoming Postfix smtpd service.

Advantage of invoking signature verification by amavisd is that all mail is checked for signatures, regardless of whether SpamAssassin is called or not. Typically messages beyond a certain size are not passed to SpamAssassin, and neither are infected message or identified bounces. Amavisd also offers loading of policy banks based on valid DKIM/DK signatures (e.g. allowing some domains to send-in otherwise banned files, or whitelisting on spam), offers to add score points based on signing domain reputation, and adds Authentication-Results header field (like a dkim-milter does).

Invoking signature verification by SpamAssassin has an advantage that DKIM-based or DomainKeys-based whitelisting or scoring can be used, but has a disadvantage that possibly not all mail is checked (e.g. large mail and infected mail may be exempt from spam checks). Performing the same signature validation task twice (by amavisd and by SA) may seem wasteful, but in practice it is not too bad: thanks to DNS server caching a network lookup for a public signing key is only done once, and as SpamAssassin does not receive large mail for processing, its signature verification is very quick: few milliseconds for non-signed mail, and of the order of a tenth of a second for signed mail.

Invoking signature verification by calling a milter from incoming smtpd service has an advantage that it has the best chance of seeing mail in its pristine form (before canonical and virtual mapping or masquerading by MTA, regardless of their settings). Because it is poorly integrated with the rest of the chain (e.g. with SpamAssassin rules and amavisd policy banks), and because it adds one extra data transfer, it is mainly still useful as a way to double-check the correctness of DKIM validation by having two independent implementations in use, each inserting its independently derived Authentication-Results header field into passed mail.

To sign as late as possible with a dkim-milter, the signing milter can be invoked by a Postfix smtpd service which is receiving content-checked mail from a content filter such as amavisd-new. As this second-stage smtpd service does not reliably know how a given message came into a mail system and whether it is supposed to be signed or not, a clean solution is to provide two (or more) parallel paths through MTA and through a content filter, one used for mail that is eligible for being signed (originating mail), the other for all the rest. This same dual path approach through amavisd is beneficial for signing by amavisd too, for the same reason of providing a reliable source of information on mail origin to a signature choosing code:

              +------+
              |verify|          (verify)
              +--+---+              | (by amavisd and/or SA)
                ^^^ milter          |
incoming:       |||             +---v-------+
  MX ---->  25 smtpd ---> 10024 >           >---> 10025 smtpd -->
                 ||             |           |
  SASL -->  25 smtpd \          |  amavisd  | (notifications)
submission        |   +->       |           >--->_
  mynets->  25 smtpd ---> 10026 >ORIGINATING>---> 10027 smtpd -->
submission            +->       +-------^---+            |
       --> 587 smtpd /  :               |                v milter
                       (convert         |             +------+
                       to 7-bit)      (sign)          | sign |
                                                      +------+

There are other benefits to providing two parallel paths: a content filter may be configured to apply different rules and settings to mail that is known to be originating from our users. Some suggestions: apply less strict banning rules, enable spam administrator notifications for internally originating spam and viruses, letting SpamAssassin rules be conditionalized based on amavisd-new policy banks loaded, etc.

Configuring multiple mail paths in Postfix

Here is one way of configuring Postfix for providing two paths through a content filter. Locally submitted or authenticated mail will go to a content filter to its port 10026 and will be signed on its way out (either by amavisd or by a signing milter). All other mail (incoming) will be diverted to port 10024 for normal content filtering, and will not be eligible for signing.

main.cf:

  # on re-queueing of a message smtpd_*_restrictions do not apply,
  # so we'd better provide a safe default for a content_filter,
  # even at an expense of later flipping the choice twice
  # (which adds a bit to log clutter, but never mind)
  #
  content_filter = amavisfeed:[127.0.0.1]:10024

  # each triggered FILTER deposits its argument into a
  # content_filter setting, the last deposited value applies
  #
  smtpd_sender_restrictions =
    check_sender_access regexp:/etc/postfix/tag_as_originating.re
    permit_mynetworks
    permit_sasl_authenticated
    permit_tls_clientcerts
    check_sender_access regexp:/etc/postfix/tag_as_foreign.re

  # Make sure to assign FILTER tags in restrictions which
  # are only invoked once per message, e.g. client or sender
  # restrictions, but NOT on smtpd_recipient_restrictions,
  # as a message may have multiple recipients, so multiple
  # passes through FILTER tag assignments can yield a
  # surprising (and incorrect) result.

/etc/postfix/tag_as_originating.re:

  /^/  FILTER amavisfeed:[127.0.0.1]:10026

/etc/postfix/tag_as_foreign.re:

  /^/  FILTER amavisfeed:[127.0.0.1]:10024

In master.cf set up two listening smtpd services for receiving filtered mail from amavisd (as per README.postfix), one on tcp port 10025 (for inbound mail) and the other on port 10027 (for originating mail). If a signing milter is in use it will be attached to a smtpd service on 10027 only. If no milters are in use and signing is done by amavisd, both smtpd services can have exactly the same settings, and in fact only one suffices, in which case redirecting $forward_method and $notify_method to 'smtp:[127.0.0.1]:10027' in later example can be disregarded.

Configuring multiple mail paths in amavisd

In amavisd.conf two parallel paths need to be provided, one receiving on port 10024 and forwarding to 10025, the other receiving on port 10026 and forwarding to 10027.

  $inet_socket_port = [10024,10026];  # listen on two ports

The 10024>10025 path will be controlled by a default policy bank, the other (10026>10027), dedicated to mail intended to be signed, will use a policy bank (arbitrarily) named ORIGINATING:

  $forward_method = 'smtp:[127.0.0.1]:10025';  # MTA with non-signing service
  $notify_method  = 'smtp:[127.0.0.1]:10027';  # MTA with signing service

  # switch policy bank to 'ORIGINATING' for mail received on port 10026:
  $interface_policy{'10026'} = 'ORIGINATING';

  $policy_bank{'ORIGINATING'} = {  # mail originating from our users
    originating => 1,  # indicates client is ours, allows signing
    #
    # force MTA to convert mail to 7-bit before DKIM signing
    # to avoid later conversions which could destroy signature:
    smtpd_discard_ehlo_keywords => ['8BITMIME'],
    #
    # forward to a smtpd service providing DKIM signing service
    # (if using a signing milter instead of signing by amavisd):
    forward_method => 'smtp:[127.0.0.1]:10027',
    #
    # other special treatment of locally originating mail,
    # just some suggestions here:
    spam_admin_maps  => ["spamalert\@$mydomain"],  # warn of spam from us
    virus_admin_maps => ["virusalert\@$mydomain"],
    banned_filename_maps => ['ALT-RULES'],         # more relaxed rules
    spam_quarantine_cutoff_level_maps => undef,    # quarantine all spam
    spam_dsn_cutoff_level_maps => undef,
    spam_dsn_cutoff_level_bysender_maps => # bounce to local senders only
      [ { lc(".$mydomain") => undef,  '.' => 15 } ],
  };

The smtpd_discard_ehlo_keywords=>['8BITMIME'] serves to persuade Postfix to convert mail to 7-bit quoted-printable before submitting it to content filtering and signing. Avoiding 8-bit characters in mail body makes signatures less susceptible to breaking by some relaying or receiving MTA over which we have no control. The same effect (making Postfix convert outgoing mail to 7-bits before DKIM signing) could be achieved by a Postfix setting smtp_discard_ehlo_keywords=8bitmime on a smtp service feeding mail-to-be-signed to amavisd, but this would require setting up two such services, one with the option and one without.

Note that 8-bit to 7-bit conversion may break a S/MIME or PGP signature, so if mail signing is in use, it may not be desirable to let Postfix do the conversion, and it may be acceptable to take a risk that a remote MTA will clobber signatures if it decides the mail text is to be converted to 7-bits QP. The only reliable solution in this case is to configure MUA clients to stick to 7-bit characters/encodings before generating S/MIME or PGP signatures.

The following text from the Postfix documentation file MILTER_README should be disregarded -- amavisd is 8-bit clean, and we do want Postfix to convert to 7-bits on the signing path but not on the other path: Content filters may break domain key etc. signatures. If you use an SMTP-based content filter, then you should add a line to master.cf with "-o disable_mime_output_conversion=yes", as described in the advanced content filter example.

While testing how the configured system plays with some mailing lists (such as postfix-users or SpamAssassin users list), one has to keep in mind that amavisd-new caches spam checking results of recently seen message bodies: a mail going out to a mailing list is not yet signed as it reaches a content filter, but the SpamAssassin verdict is remembered at that point (claiming the message is not signed). When this message with unchanged body comes back from a mailing list, this time signed in the header section by our domain, the signature should prove correct, yet the cached result from a minute ago still claims the message is not signed. If this is of concern, one can turn off caching of spam checking results for ham by setting: $spam_check_negative_ttl = 0;

While on the topic of providing multiple paths through amavisd, when one has to deal with a mailing list manager (e.g. Mailman) in the same setup, and re-signing of its fan-out mail is desired, it may be useful to add a third path through amavisd, this one stripped down to bare bones, providing only DKIM signing and nothing else (no virus or spam checks, no decoding), as these checks were already done once on mail before it reached a mailing list manager. Here is one possibility, accepting mail on port 10028 and sending it to 10025:

  $inet_socket_port = [10024,10026,10028];

  $interface_policy{'10028'} = 'NOCHECKS';

  $policy_bank{'NOCHECKS'} = {  # no checks, just DKIM signing
    originating => 1,  # allows signing
    forward_method => 'smtp:[127.0.0.1]:10025',
    smtpd_greeting_banner =>
      '${helo-name} ${protocol} ${product} NOCHECKS service ready',
    mynetworks_maps => [],  # avoids loading MYNETS policy unnecessarily
    os_fingerprint_method => undef,
    penpals_bonus_score => undef,
    bounce_killer_score => 0,
    bypass_decode_parts => 1,
    bypass_header_checks_maps => [1],
    bypass_virus_checks_maps  => [1],
    bypass_spam_checks_maps   => [1],
    bypass_banned_checks_maps => [1],
    spam_lovers_maps          => [1],
    banned_files_lovers_maps  => [1],
    archive_quarantine_to_maps => [],
    remove_existing_x_scanned_headers => undef,
    remove_existing_spam_headers => undef,
    signed_header_fields => { 'Sender' => 1 },
  };

Hooking-in dkim-milter (optional)

This section can be ignored when all DKIM signing and verification is to be done by amavisd, and dkim-milter will not be used. It is mainly provided for compatibility reasons, retaining the old documentation section.

Let's begin by starting a dkim milter in two instances, one dedicated to signing, the other to verification. For security reasons all milters should run under a dedicated username, certainly not as root, not as user amavis and not as user postfix or mail:

verifying:

  dkim-filter -u dkfilter -b v \
    -l -p inet:4443@127.0.0.1 -P /var/run/dkim-filter-v.pid

signing:

  dkim-filter -u dkimfilter -b s -m ORIGINATING \
    -c relaxed/simple -S rsa-sha1 \
    -d example.com -s myselector -k /var/db/dkim/mykey.pem \
    -l -p inet:4445@127.0.0.1 -P /var/run/dkim-filter-s.pid

Generating a public and a private pair of keys and publishing a public key in DNS is described in the dkim milter documentation and also in the DKIM RFC document.

We are not specifying option -i to milters, the default of -i 127.0.0.1 suits our setup just fine, as mail to be signed is coming from a content filter, usually on a loopback interface from the IP address 127.0.0.1.

Now we can tie the verifying milter to a Postfix smtpd service listening for incoming mail:

master.cf:

  smtp inet n - n - 300 smtpd
    -o milter_default_action=accept
    -o milter_macro_daemon_name=MTA
    -o smtpd_milters=inet:127.0.0.1:4443

and tie the signing milter to a Postfix smtpd service that is receiving checked mail from amavisd, intended to be signed:

master.cf:

  # mail return from a content filter (non-signing)
  10025 inet n - n - - smtpd
    -o content_filter=
    ... (other options, mail not to be signed) ...

  # mail from our users returning from a content filter (DKIM signing)
  10027 inet n - n - - smtpd
    -o content_filter=
    ... (other options, mail intended to be signed) ...
    -o milter_default_action=accept
    -o milter_macro_daemon_name=ORIGINATING
    -o smtpd_milters=inet:127.0.0.1:4445

As a sidenote, attaching milters to sendmail would use the same order of invocations: signature verifying milter first, content filters next, and signing milter last, for example:

  dnl Verifiers:
  INPUT_MAIL_FILTER(`dkim-filter-v', `S=inet:4443@127.0.0.1, T=R:2m')

  dnl Content filter:
  INPUT_MAIL_FILTER(`amavisd-milter',
    `S=unix:/var/amavis/amavisd-milter.sock, F=T, T=S:10m;R:10m;E:10m')

  dnl Signers:
  INPUT_MAIL_FILTER(`dkim-filter-s', `S=inet:4445@127.0.0.1, T=R:2m')

Setting up DKIM signature verification in amavisd

Starting with 2.6.0, verification of DKIM signatures (and historical DomainKeys signatures) is provided directly by amavisd (not only by a SpamAssassin plugin DKIM). A required version of a perl module Mail::DKIM is 0.31 or later, but recommended is 0.33 or later. Signature verification is sufficiently fast so there is no need for concern about extra processing load (see TIMING breakdown in your log, level 2). To turn on DKIM (and historical DomainKeys) signature verification, please add the following line to amavisd.conf (if not already there):

  $enable_dkim_verification = 1;

Benefits:

  • Whitelisting of banned checks or spam checks on messages carrying valid DKIM or DomainKeys signatures from trustworthy signers is possible through the @author_to_policy_bank_maps list of lookup tables. The mechanism uses loading of policy banks based on author's e-mail address (addresses in a 'From:' header field) and a signing domain, so a full flexibility of per-policy-bank settings is available. See description of a new configuration variable @author_to_policy_bank_maps in release notes.
  • To each message passed to local recipients amavisd inserts a header field Authentication-Results (according to RFC 5451) for each signature found in a message, reporting a verification result. These header fields can reliably tell a recipient or his MUA what domains claimed responsibility for a message, or can be used for troubleshooting DKIM signing, verification and tracking mail transformations.
  • Can adjust spam score based on signing domain's reputation for valid signatures found in a message. A useful reputation metric is an average long term spam score for past messages signed by a domain, which can currently be provided manually by @signer_reputation_maps in a configuration file (see example in release notes). A spam score is shifted towards this reputation score by a configurable factor $reputation_factor (value between 0 and 1, default is 0.2) using a formula: adjusted_spam_score = f * reputation + (1-f) * spam_score . Semantics of a $reputation_factor is equivalent to auto_whitelist_factor in a SpamAssassin's AWL plugin, which shifts spam score towards a long term spam score average of a sender.
  • Notifications and bounces show a "(dkim:AUTHOR)" next to a From address, and a "(dkim:SENDER)" next to a Sender address if these header fields were signed and their domain corresponds to a signer's domain identity.
  • A valid DKIM or DomainKeys signature turns on a 'sender_credible' attribute which serves to choose one of the two DSN cutoff levels, so that delivery status notifications can be restricted to or preferred for likely-to-be-valid sending addresses, and bounces to possibly fake addresses can be minimized. More information on the 'sender_credible' attribute can be found in release notes.

Currently the ADSP (RFC 5617, Author Domain Signing Practices, formerly SSP) is not implemented by amavisd, but is implemented in the SpamAssassin's plugin DKIM as of version 3.3.0.

Setting up DKIM signing in amavisd

A recommended version of a perl module Mail::DKIM is 0.33 or later when signing.

1. Generate one or more keys to be used for signing, and enable signing code by adding the following line to amavisd.conf (if not already there):

  $enable_dkim_signing = 1;  # loads DKIM signing code

Signing keys must be made available to amavisd, each private key in a separate file in PEM format. Customarily such keys would be generated and kept in a dedicated directory such as /var/db/dkim or /var/lib/dkim, preferably owned by root.

Private keys can be generated by a 'openssl genrsa' command (see RFC 4871 Appendix C), or by an amavisd equivalent. Commonly one key per signing domain or one key per signing host is used, but other choices are possible. If such keys were already prepared for some other DKIM-signing solution, they can be reused by amavisd.

  # amavisd genrsa /var/db/dkim/a.key.pem
  # amavisd genrsa /var/db/dkim/b.key.pem 786
  # amavisd genrsa /var/db/dkim/sel-example-com.key.pem
  # amavisd genrsa /var/db/dkim/g-guest-ex-com.key.pem
  # amavisd genrsa /var/db/dkim/notif-mail.key.pem 512

Amavisd already ensures the generated files are only readable by owner, but a manual procedure may require explicitly setting file permissions. Private keys must be protected from unauthorized access, only the signing software such as amavisd should have access. Amavisd loads these files on startup before dropping privileges, so if amavisd is started as root it is not necessary that these key files are readable by uid under which amavisd is running.

2. Add commands to amavisd.conf to load private keys, associate them with signing domains and selectors, and describe constraints (tags) to be published with public keys.

Calls to dkim_key() load all available private keys and supply their public key RR constraints. Arguments are a domain, a selector, a key (a file name of a private key in PEM format), followed by optional attributes/constraints (tags, represented here as Perl hash key/value pairs) which are allowed by RFC 4871 in a public key resource record (v, g, h, k, n, s, t), of which only g, h, k, s and t are considered to be constraints limiting the choice of a signing key. A command 'amavisd showkeys' can be used for displaying corresponding public keys in a format directly suitable for inclusion into DNS zone files.

For example:

#        signing domain  selector     private key              options
#        -------------   --------     ----------------------   ----------
dkim_key('example.org', 'abc',       '/var/db/dkim/a.key.pem');
dkim_key('example.org', 'yyy',       '/var/db/dkim/b.key.pem', t=>'s');
dkim_key('example.org', 'zzz',       '/var/db/dkim/b.key.pem', h=>'sha256');
dkim_key('example.com', 'sel-2008',  '/var/db/dkim/sel-example-com.key.pem',
         t=>'s:y', g=>'*', k=>'rsa', h=>'sha256:sha1', s=>'email',
         n=>'testing; 1, 2');
dkim_key('guest.example.com', 'g',     '/var/db/dkim/g-guest-ex-com.key.pem');
dkim_key('mail.example.com',  'notif', '/var/db/dkim/notif-mail.key.pem');

A selector paired with a domain name uniquely identifies a key, both for a signer as well as for a recipient. There may be multiple keys for each domain as long as each one has its own selector.

A selector along with a domain name will be used by a receiving mailer in assembling a DNS query (selector._domainkey.signingdomain) to fetch a public key from a signing domain's DNS server when verifying signature validity.

A selector paired with a domain name will also be used by a signing amavisd when choosing a key applicable to signing, meeting constraints on its public key (tags, RFC 4871 section 3.6) as given by optional arguments. Optional arguments serve as site documentation, may help amavisd choose between multiple choices (ruling out keys with incompatible tags), and supply additional information for step 3.

For a list of options (tags) see RFC 4871 section 3.6. Amavisd does not check the syntax of tag values, except for performing qp-section encoding of a tag 'n'. Note the Perl syntax of key/value pairs, e.g. t => 's:y' will end up as "t=s:y", and n => 'testing; 1, 2' will end up encoded as "n=testing=3B 1, 2".

3. Prepare and publish public keys.

Public keys can be extracted from generated key files (which contain both a private and a public key). To publish public keys they need to be edited into a format suitable for inclusion in a DNS server's zone file for each signing domain, either by following a procedure in RFC 4871 Appendix C, or if step 2 was completed, by asking amavisd to do so:

  # amavisd showkeys

or more selectively, e.g.:

  # amavisd showkeys  .org example.com

This step is not needed if public keys were already prepared and published earlier for some other DKIM-signing solution.

4. Edit zone files in master DNS server(s) for each signing domain, adding the just prepared TXT resource records, not forgetting to bump up the serial number in a SOA record. Optionally add a TXT record with ADSP information (formerly SSP) if a default Author Domain Signing Practices is not appropriate. Then reload zone(s) or restart DNS server(s).

5. Test published public keys.

Similar to 'showkeys', a 'testkeys' command walks through available signing keys (as declared by calls to dkim_key), generates test messages each signed with one key, and validates them by fetching a corresponding public key from a DNS server.

  # amavisd testkeys

or more selectively, e.g.:

  # amavisd testkeys  .org example.com

(btw, if testkeys fails and you believe your DNS is correctly serving your DKIM public keys, you may need to upgrade Perl module Mail-DKIM to version 0.33)

6. Restart amavisd, watch the log at log level 2, searching for " dkim: ".

Note that signing could be started (amavisd reload) right after completing step 2, but mail recipients would not be able to verify validity of signatures until public keys are made available by a signing domain through its DNS. Recipients are supposed to treat mail with signatures which fail verification exactly the same as mail with no signatures, so there is usually no harm done with a premature start of signing, but there is no benefit either.

7. Optional: to override default values for signature tags, one may specify by-sender signature tags through @dkim_signature_options_bysender_maps.

@dkim_signature_options_bysender_maps maps author/sender addresses or domains to signature tags/requirements. Possible signature tags according to RFC 4871 are: (v), a, (b), (bh), c, d, (h), i, l, q, s, (t), x, z; of which the following are determined automatically: v, b, bh, h, t (tag h is controlled by %signed_header_fields). Currently ignored tags are l and z. Instead of an absolute expiration time (tag x) one may use a pseudo tag 'ttl' to specify a relative expiration time in seconds, which is converted to an absolute expiration time prior to signing: x = t + ttl. A built-in default is provided for each tag if no better match is found.

For example:

@dkim_signature_options_bysender_maps = ( {
  'postmaster@mail.example.com' => { a => 'rsa-sha1', ttl =>  7*24*3600 },
  'spam-reporter@example.com'   => { a => 'rsa-sha1', ttl =>  7*24*3600 },
  'mail.example.com'            => { a => 'rsa-sha1', ttl => 10*24*3600 },
  # explicit 'd' forces a third-party signature on foreign (hosted) domains
  'ggg.example.net'             => { d => 'guest.example.com' },
  '.example.com'                => { d => 'example.com' },
  # catchall defaults
  '.' => { a => 'rsa-sha256', c => 'relaxed/simple', ttl => 30*24*3600 },
  # 'd' defaults to a domain of an author/sender address,
  # 's' defaults to whatever selector is offered by a matching key
} );

The result of a by-sender lookup into @dkim_signature_options_bysender_maps is a hash (a set) of DKIM signing requirements (tags), i.e. canonicalization method, hashing algorithm, domain, identity, selector and expiration time. All matching entries can participate in the result: for each tag individually the first setting (the most specific) is chosen from all matching entries. Resulting tags are then used to choose the most appropriate signing key from a set of keys as declared by calls to dkim_key. Main selection criterium is a match on tags d (domain) and s (selector), but other signature requirements must also meet the constraints of a public key (e.g. subdomain matching flag, granularity, hashing algorithm, key type). If a lookup does not find a signing key which meets requirements, no signing takes place. Also, only mail with 'originating' flag is eligible for signing. A lookup is based on either the From header field, the Sender header field, the Resent-From and Resent-Sender header field, or on a mail_from address from the envelope, whichever yields a useful result first. Note that neither the Sender header field, nor the Resent-* header fields, nor a mail_from address has any special meaning in the standard (RFC 4871). This results either in an author signature (i.e. a first-party signature, when based on a From header field), or in a third-party signature (when signing domain does not match the From, regardless of what other header field (or forced through a 'd' tag) it was based on.

An associative array %signed_header_fields controls which header fields are to be signed. By default it contains a standard (RFC 4871) set of header field names, augmented by some additional header field names considered appropriate at the time of a release (RFC 4021, RFC 3834). In addition a 'Sender' header field is excluded because it is frequently replaced by a mailing list, and as the RFC 2821 mandates there can only be one such header field the original one is dropped, invalidating a signature. Also the 'To' and 'Cc' are excluded from a default set because sendmail mailers are known to gratuitously reformat the list, invalidating a signature.

The default set of header fields to be signed can be controlled by setting %signed_header_fields elements to true (to sign) or to false (not to sign). Keys must be in lowercase, e.g.:

  $signed_header_fields{'received'} = 0;  # turn off signing of Received
  $signed_header_fields{'sender'} = 1;    # turn on signing of Sender
  $signed_header_fields{'to'} = 1;        # turn on signing of To
  $signed_header_fields{'cc'} = 1;        # turn on signing of Cc
  $signed_header_fields{lc('X-MySpecialFlag')} = 1;

Putting DKIM verification to good use in SpamAssassin

In SpamAssassin all that is necessary is to add (or uncomment) a line in any of the .pre files (e.g. in local.pre, or in init.pre and v320.pre):

  loadplugin Mail::SpamAssassin::Plugin::DKIM

Perl module Mail::DKIM needs to be installed. Note that Mail::DKIM starting with version 0.20 also recognizes DomainKeys signatures, so that Plugin::DomainKeys is not needed any longer, and in fact its underlying module is not supported any longer. It is advisable to stick to the most recent version of Mail::DKIM, at least 0.32.

The following SpamAssassin rules (in local.cf) work quite well.

  score DKIM_VERIFIED -0.1
  score DKIM_SIGNED    0

  # don't waste time on fetching ASP record, hardly anyone publishes it
  score DKIM_POLICY_SIGNALL  0
  score DKIM_POLICY_SIGNSOME 0
  score DKIM_POLICY_TESTING  0

  # DKIM-based whitelisting of domains with good reputation:
  score USER_IN_DKIM_WHITELIST -8.0

  whitelist_from_dkim  *@ebay.com
  whitelist_from_dkim  *@*.ebay.com
  whitelist_from_dkim  *@ebay.co.uk
  whitelist_from_dkim  *@*.ebay.co.uk
  whitelist_from_dkim  *@ebay.at
  whitelist_from_dkim  *@ebay.ca
  whitelist_from_dkim  *@ebay.de
  whitelist_from_dkim  *@ebay.fr
  whitelist_from_dkim  *@*.paypal.com
  whitelist_from_dkim  *@paypal.com
  whitelist_from_dkim  *@*                paypal.com
  whitelist_from_dkim  *@*.paypal.be

  whitelist_from_dkim  *@cern.ch
  whitelist_from_dkim  *@amazon.com
  whitelist_from_dkim  *@springer.delivery.net
  whitelist_from_dkim  *@cisco.com
  whitelist_from_dkim  *@alert.bankofamerica.com
  whitelist_from_dkim  *@bankofamerica.com
  whitelist_from_dkim  *@cnn.com
  whitelist_from_dkim  *@*.cnn.com
  whitelist_from_dkim  *@skype.net
  whitelist_from_dkim  service@youtube.com
  whitelist_from_dkim  *@welcome.skype.com
  whitelist_from_dkim  *@cc.yahoo-inc.com  yahoo-inc.com
  whitelist_from_dkim  *@cc.yahoo-inc.com
  whitelist_from_dkim  rcapotenoy@yahoo.com
  whitelist_from_dkim  googlealerts-noreply@google.com

  # DKIM-based whitelisting of domains with less then perfect
  # reputation can be given fewer negative score points:
  score USER_IN_DEF_DKIM_WL -1.5
  def_whitelist_from_dkim   *@google.com
  def_whitelist_from_dkim   *@googlemail.com
  def_whitelist_from_dkim   *@*  googlegroups.com
  def_whitelist_from_dkim   *@*  yahoogroups.com
  def_whitelist_from_dkim   *@*  yahoogroups.co.uk
  def_whitelist_from_dkim   *@*  yahoogroupes.fr
  def_whitelist_from_dkim   *@yousendit.com
  def_whitelist_from_dkim   *@meetup.com
  def_whitelist_from_dkim   dailyhoroscope@astrology.com

  # reduce default scores, which are being abused
  score ENV_AND_HDR_DKIM_MATCH -0.1
  score ENV_AND_HDR_SPF_MATCH  -0.5

Another suggestions - penalize mail claiming to be from PayPal, eBay, Yahoo or Gmail but was not signed by their official mailers:

  header   __ML1        Precedence =~ m{\b(list|bulk)\b}i
  header   __ML2        exists:List-Id
  header   __ML3        exists:List-Post
  header   __ML4        exists:Mailing-List
  header   __ML5        Return-Path:addr =~ m{^([^\@]+-(request|bounces|admin|owner)|owner-[^\@]+)(\@|\z)}mi
  meta     __VIA_ML     __ML1 || __ML2 || __ML3 || __ML4 || __ML5
  describe __VIA_ML     Mail from a mailing list

  header   __AUTH_YAHOO1  From:addr =~ m{[\@.]yahoo\.com$}mi
  header   __AUTH_YAHOO2  From:addr =~ m{\@yahoo\.com\.(ar|au|br|cn|hk|mx|my|ph|sg|tw)$}mi
  header   __AUTH_YAHOO3  From:addr =~ m{\@yahoo\.co\.(id|in|jp|nz|th|uk)$}mi
  header   __AUTH_YAHOO4  From:addr =~ m{\@yahoo\.(ca|cn|de|dk|es|fr|gr|ie|it|no|pl|se)$}mi
  meta     __AUTH_YAHOO   __AUTH_YAHOO1 || __AUTH_YAHOO2 || __AUTH_YAHOO3 || __AUTH_YAHOO4
  describe __AUTH_YAHOO   Author claims to be from Yahoo

  header   __AUTH_GMAIL   From:addr =~ m{\@gmail\.com$}mi
  describe __AUTH_GMAIL   Author claims to be from gmail.com

  header   __AUTH_PAYPAL  From:addr =~ /[\@.]paypal\.(com|co\.uk)$/mi
  describe __AUTH_PAYPAL  Author claims to be from PayPal

  header   __AUTH_EBAY    From:addr =~ /[\@.]ebay\.(com|at|be|ca|ch|de|ee|es|fr|hu|ie|in|it|nl|ph|pl|pt|se|co\.(kr|uk)|com\.(au|cn|hk|mx|my|sg))$/mi
  describe __AUTH_EBAY    Author claims to be from eBay

  meta     NOTVALID_YAHOO !DKIM_VERIFIED && __AUTH_YAHOO && !__VIA_ML
  priority NOTVALID_YAHOO 500
  describe NOTVALID_YAHOO Claims to be from Yahoo but is not

  meta     NOTVALID_GMAIL !DKIM_VERIFIED && __AUTH_GMAIL && !__VIA_ML
  priority NOTVALID_GMAIL 500
  describe NOTVALID_GMAIL Claims to be from gmail.com but is not

  meta     NOTVALID_PAY   !DKIM_VERIFIED && (__AUTH_PAYPAL || __AUTH_EBAY)
  priority NOTVALID_PAY   500
  describe NOTVALID_PAY   Claims to be from PayPal or eBay, but is not

  score    NOTVALID_YAHOO  2.8
  score    NOTVALID_GMAIL  2.8
  score    NOTVALID_PAY    6

  # accept replies from abuse@yahoo.com even if not dkim/dk-signed:
  whitelist_from_rcvd abuse@yahoo.com          yahoo.com
  whitelist_from_rcvd MAILER-DAEMON@yahoo.com  yahoo.com

Some experience with DKIM and DomainKeys

Recent versions of software components must be used to avoid bugs and known interoperability problems:

  • if using Postfix with milters, use Postfix versions 2.3.12 or later, or 2.4.5 or later, or 2.5 (or later);
  • amavisd-new 2.6.0 introduced direct support for DKIM signing and verification by calling a perl module Mail::DKIM directly; version amavisd-new-2.6.2 (or later) is recommended;
  • Mail::DKIM is solid; use the latest version, currently 0.33;
  • SpamAssassin 3.2.5 or later;

Several big players are already signing mail from their customers or employees: Yahoo! (worldwide), Gmail, eBay, Earthlink, google.com, Amazon, Springer, CNN, Skype, YouTube, Cisco, many universities, etc.

Mail transformations as performed by some mailing lists are probably the most challenging problem facing DKIM deployment (and to other schemes as well). Nevertheless, mailing lists can be configured to either avoid transformations which invalidate mail signatures, or can re-sign fan-out mail. Examples of mailing lists which work very well with DKIM (and DomainKeys), preserving existing signatures provided by posters, are the postfix-users ( postfix-users@postfix.org ) and the SpamAssassin users list ( users@spamassassin.apache.org ). Example of re-signing mailing lists are Yahoo groups. A representative of another type of mailing lists is Mailman, which often modifies mail body and strips out original signatures, unless explicitly configured not to.

When signatures are missing on mail from domains which are known to be signing all their mail (yahoo.com, gmail.com), the most common reason is that a sender submitted his mail through some other provider, but supplied his Yahoo or gmail e-mail address in the From header field. Similar to other schemes designed to prevent faking of sending address, the DKIM (and the DomainKeys) encourages mail submission only through a domain which is used in the From address.

People need to become aware that their best choice is to submit mail through their native domain to prevent their messages from being treated as second-class. With a widespread support for authorized mail submission for roaming users (SASL, TLS) through a mail submission port (tcp port 587, RFC 4409), supported by practically all modern clients and mailers, there is no longer any good excuse for submitting mail through foreign mail submission agents.

Note that some spam is also being signed by DomainKeys or DKIM lately, which is a good thing -- it indicates the sender owns (or ownz) a domain they are sending mail from. This either shows sender's sincere desire of not hiding behind a faked sender mail address (in which case such mail can be easily filtered if necessary), or they are using a short-lived temporary domain (domain kiting), which can be counteracted by black lists of few-days old freshly registered domains (such as http://support-intelligence.com/dob/), spameatingmonkey.net or other reputation schemes. Signing and verifying mail is a good mechanism for companies to reliably whitelist mail from their partner companies or frequent clients.

Links

mm
Last updated: 2010-10-20

Valid XHTML 1.0!

amavisd-new-2.7.1/README_FILES/README.exim_v4000640 000621 000620 00000005652 10403115217 017576 0ustar00markcmi000000 000000 UPDATED: 2002-05-19, 2004-03-15 How To Use AMaViS With Exim 4.x ******************************* Scanning incoming/outgoing and relayed mail by Jochen Erwied (mbs GmbH), edited by Mark Martinec (based on Postfix guidelines and previous work on Exim setup by John Burnham and Patrice Fournier) [routers] # instruct Exim to pass all mail using SMTP to amavisd, # except the mail that just came-in back to Exim from amavisd # through the local port 10025, as these messages were already # checked and approved by amavisd amavis: driver = manualroute condition = "${if eq {$interface_port}{10025} {0}{1}}" # if scanning incoming mails, uncomment the following line and # change local_domains accordingly # domains = +local_domains transport = amavis route_list = "* localhost byname" self = send (from Patrice Fournier): This router must be the first router if you want to scan all mail. If you only want to scan mail destined to local domains, you either add a "Domain = local_domains" line to it or put it after your remote_smtp router. (from Ralf G. R. Bergs - suggestion to elaborate the above 'condition' line): # Do NOT run if received via 10025/tcp or if already spam-scanned condition = "${if or {{eq {$interface_port}{10025}} \ {eq {$received_protocol}{spam-scanned}} \ }{0}{1}}" (2005-10-28: removed {eq {$sender_address}{}} from the above as recommended by Igor D'Astolfo, as it would let through junk messages with null sender address) (from Aaron Lehmann): The example router configuration in README.exim_v4 should include the no_verify option. Without this, exim will accept mail addressed to invalid users and then complain to postmaster@ about the frozen messages. Better yet, don't accept mail for unknown users in the first place: http://www.webhostgear.com/50.html [transports] # SMTP transport to be used for the Exim -> amavisd path; # by default amavisd listens on the loopback interface on port 10024 # (amavisd.conf: $inet_socket_port = "10024") amavis: driver = smtp port = 10024 allow_localhost The default input configuration needs to be changed to: # Tell Exim to accept SMTP also (besides the usual port 25) on the # loopback interface (localhost) on port 10025, which is where # the checked messaged come from amavisd back to Exim # (amavisd.conf: $relayhost = "127.0.0.1"; $relayhost_port = "10025") local_interfaces = 0.0.0.0.25 : 127.0.0.1.10025 NOTE: There is a problem in the Net::SMTP Perl module (version 2.17 and earlier), which as far as I know only affects the Exim setup with amavisd-new: MTA-generated notifications from <> appear to be comming from <<>> and are rejected by MTA. Please upgrade the Perl module libnet (which contains Net::SMTP) to version libnet-1.12. Upgrading this module is probably a good idea even if you use other MTA. amavisd-new-2.7.1/README_FILES/README.banned000640 000621 000620 00000020125 10123400453 017440 0ustar00markcmi000000 000000 p004 1 Content-type: multipart/related p005 1/1 Content-type: multipart/alternative p001 1/1/1 Content-type: text/plain p002 1/1/2 Content-type: text/html p003 1/2 Content-type: image/gif, name: image001.gif p.path: "P=p004,L=1,M=multipart/related | P=p003,L=1/2,M=image/gif,T=gif,N=image001.gif" p.path: "P=p004,L=1,M=multipart/related | P=p005,L=1/1,M=multipart/alternative | P=p001,L=1/1/1,M=text/plain,T=asc" p.path: "P=p004,L=1,M=multipart/related | P=p005,L=1/1,M=multipart/alternative | P=p002,L=1/1/2,M=text/html,T=html" p006 1 Content-type: multipart/mixed p001 1/1 Content-type: text/plain p002 1/2 Content-type: application/vnd.ms-excel, name: kal_ftr1.xls p003 1/3 Content-type: application/vnd.ms-excel, name: Prikaz-projekt-04-NPD-NET.xls p004 1/4 Content-type: application/vnd.ms-excel, name: Prikaz-ur-04-NPD-Net.xls p005 1/5 Content-type: application/vnd.ms-excel, name: NPD_plan.xls p.path: "P=p006,L=1,M=multipart/mixed | P=p001,L=1/1,M=text/plain,T=txt" p.path: "P=p006,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/vnd.ms-excel,T=doc,N=kal_ftr1.xls" p.path: "P=p006,L=1,M=multipart/mixed | P=p003,L=1/3,M=application/vnd.ms-excel,T=doc,N=Prikaz-projekt-04-NPD-NET.xls" p.path: "P=p006,L=1,M=multipart/mixed | P=p004,L=1/4,M=application/vnd.ms-excel,T=doc,N=Prikaz-ur-04-NPD-Net.xls" p.path: "P=p006,L=1,M=multipart/mixed | P=p005,L=1/5,M=application/vnd.ms-excel,T=doc,N=NPD_plan.xls" p004 1 Content-type: multipart/mixed p005 1/1 Content-type: multipart/alternative p001 1/1/1 Content-type: text/plain p002 1/1/2 Content-type: text/html p003 1/2 Content-type: application/msword, name: Bilten 2004 - 09.doc p.path: "P=p004,L=1,M=multipart/mixed | P=p003,L=1/2,M=application/msword,T=doc,N=Bilten 2004 - 09.doc" p.path: "P=p004,L=1,M=multipart/mixed | P=p005,L=1/1,M=multipart/alternative | P=p001,L=1/1/1,M=text/plain,T=txt" p.path: "P=p004,L=1,M=multipart/mixed | P=p005,L=1/1,M=multipart/alternative | P=p002,L=1/1/2,M=text/html,T=html" p003 1 Content-type: multipart/mixed p001 1/1 Content-type: text/plain p002 1/2 Content-type: application/octet-stream, name: transcript.zip p.path: "P=p003,L=1,M=multipart/mixed | P=p001,L=1/1,M=text/plain,T=asc" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/octet-stream,T=zip,N=transcript.zip | P=p004,L=1/2/1,T=exe,T=exe-ms,N=transcript.exe" p005 1/1 Content-type: multipart/alternative p001 1/1/1 Content-type: text/plain p002 1/1/2 Content-type: text/html p003 1/2 Content-type: audio/x-wav, name: message.scr p.path: "P=p004,L=1,M=multipart/related | P=p003,L=1/2,M=audio/x-wav,T=exe,T=exe-ms,N=message.scr" p.path: "P=p004,L=1,M=multipart/related | P=p005,L=1/1,M=multipart/alternative | P=p001,L=1/1/1,M=text/plain" p.path: "P=p004,L=1,M=multipart/related | P=p005,L=1/1,M=multipart/alternative | P=p002,L=1/1/2,M=text/html,T=html" p003 1 Content-type: multipart/mixed p001 1/1 Content-type: text/plain p002 1/2 Content-type: application/octet-stream, name: message_details.pif p.path: "P=p003,L=1,M=multipart/mixed | P=p001,L=1/1,M=text/plain,T=asc" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/octet-stream,T=exe,T=exe-ms,N=message_details.pif" W32/Bagle-AA p003 1 Content-type: multipart/mixed p001 1/1 Content-type: text/html p002 1/2 Content-type: application/octet-stream, name: MoreInfo.hta p.path: "P=p003,L=1,M=multipart/mixed | P=p001,L=1/1,M=text/html,T=html" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/octet-stream,T=html,N=MoreInfo.hta" Troj/BagleDl-A p003 1 Content-type: multipart/mixed p001 1/1 Content-type: text/plain p002 1/2 Content-type: application/x-zip-compressed, name: =?koi8-r?B?Zm90by56aXA=?= p.path: "P=p003,L=1,M=multipart/mixed | P=p001,L=1/1,M=text/plain,T=asc" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-zip-compressed,T=zip,N==?koi8-r?B?Zm90by56aXA=?=,N=foto.zip | P=p004,L=1/2/1,T=html,N=foto.html" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-zip-compressed,T=zip,N==?koi8-r?B?Zm90by56aXA=?=,N=foto.zip | P=p005,L=1/2/2,N=foto/,A=D" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-zip-compressed,T=zip,N==?koi8-r?B?Zm90by56aXA=?=,N=foto.zip | P=p006,L=1/2/3,T=exe,T=exe-ms,N=foto/expander.exe" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-zip-compressed,T=zip,N==?koi8-r?B?Zm90by56aXA=?=,N=foto.zip | P=p007,L=1/2/4,T=doc,N=foto/Thumbs.db" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-zip-compressed,T=zip,N==?koi8-r?B?Zm90by56aXA=?=,N=foto.zip | P=p008,L=1/2/5,T=jpg,N=foto/photo.jpg" W32/Netsky-N p007 1 Content-type: multipart/report p001 1/1 Content-type: text/plain p002 1/2 Content-type: message/delivery-status p008 1/3 Content-type: message/rfc822 p009 1/3/1 Content-type: multipart/mixed p010 1/3/1/1 Content-type: multipart/related p011 1/3/1/1/1 Content-type: multipart/alternative p003 1/3/1/1/1/1 Content-type: text/plain p004 1/3/1/1/1/2 Content-type: text/html p005 1/3/1/1/2 Content-type: image/gif, name: symantec.gif p006 1/3/1/2 Content-type: application/octet-stream, name: document_all_elson.zip p.path: "P=p007,L=1,M=multipart/report | P=p001,L=1/1,M=text/plain,T=asc" p.path: "P=p007,L=1,M=multipart/report | P=p002,L=1/2,M=message/delivery-status,T=asc" p.path: "P=p007,L=1,M=multipart/report | P=p008,L=1/3,M=message/rfc822 | P=p009,L=1/3/1,M=multipart/mixed | P=p010,L=1/3/1/1,M=multipart/related | P=p005,L=1/3/1/1/2,M=image/gif,T=gif,N=symantec.gif" p.path: "P=p007,L=1,M=multipart/report | P=p008,L=1/3,M=message/rfc822 | P=p009,L=1/3/1,M=multipart/mixed | P=p006,L=1/3/1/2,M=application/octet-stream,T=zip,N=document_all_elson.zip | P=p012,L=1/3/1/2/1,T=exe,T=exe-ms,N=your_details.scr" p.path: "P=p007,L=1,M=multipart/report | P=p008,L=1/3,M=message/rfc822 | P=p009,L=1/3/1,M=multipart/mixed | P=p010,L=1/3/1/1,M=multipart/related | P=p011,L=1/3/1/1/1,M=multipart/alternative | P=p003,L=1/3/1/1/1/1,M=text/plain,T=asc" p.path: "P=p007,L=1,M=multipart/report | P=p008,L=1/3,M=message/rfc822 | P=p009,L=1/3/1,M=multipart/mixed | P=p010,L=1/3/1/1,M=multipart/related | P=p011,L=1/3/1/1/1,M=multipart/alternative | P=p004,L=1/3/1/1/1/2,M=text/html,T=html" W32/Mabutu-A p003 1 Content-type: multipart/mixed p001 1/1 Content-type: text/plain p002 1/2 Content-type: application/x-zip-compressed, name: photo.zip p.path: "P=p003,L=1,M=multipart/mixed | P=p001,L=1/1,M=text/plain" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-zip-compressed,T=zip,N=photo.zip | P=p004,L=1/2/1,T=exe,T=exe-ms,N=photo.jpg .scr" p003 1 Content-type: multipart/mixed p001 1/1 Content-type: text/plain p002 1/2 Content-type: application/x-tgz, name: 00.tar.gz p.path: "P=p003,L=1,M=multipart/mixed | P=p001,L=1/1,M=text/plain" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p005,L=1/2/1/1,N=aa" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p006,L=1/2/1/2,N=alpha,A=D" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p007,L=1/2/1/3,N=one" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p008,L=1/2/1/4,N=two" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p009,L=1/2/1/5,N=three" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p010,L=1/2/1/6,N=bravo,A=D" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p011,L=1/2/1/7,N=first" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p012,L=1/2/1/8,N=second" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p013,L=1/2/1/9,N=charlie,A=D" p.path: "P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/x-tgz,T=gz,N=00.tar.gz | P=p004,L=1/2/1,T=tar,N=00.tar | P=p014,L=1/2/1/10,N=last" amavisd-new-2.7.1/README_FILES/README.postfix.html000640 000621 000620 00000206745 10766020557 020706 0ustar00markcmi000000 000000 Integrating amavisd-new in Postfix

Integrating amavisd-new in Postfix

License: GNU GENERAL PUBLIC LICENSE, Version 2, June 1991

Revision History
Revision 14111. Mar 2008PK
Updated parameters that override Postfix defaults for the amavisdfeed and the reentry smtpd servers
Revision 13911. Mar 2008PK
Added corrections sent in from Chris Pepper
Revision 12215. Jun 2007PK
Added Section on Advanced Configuration
Revision 10822. Apr 2007PK
Initial publication

Table of Contents

1. Requirements
1.1. Which Postfix version is required?
1.2. Catching errors during integration
2. Basic Postfix and amavisd-new configuration
2.1. Configuring amavisd-new for Postfix
2.2. Configuring the transport from Postfix to amavisd-new
2.3. Configuring a dedicated SMTP-server for message reinjection
2.4. Testing basic configuration
3. Message filtering examples
3.1. Filtering E-mail globally
3.2. Filtering E-mail by Postfix service
3.3. Filtering E-Mails per Recipient Domain
3.4. Filtering E-Mails by Sender-Domain
3.5. Filtering E-mail per Content
4. Advanced Postfix and amavisd-new configuration
4.1. Multiple cleanup service architecture
4.2. Configuring two cleanup services
5. Tuning
5.1. Maximum Number of Concurrent Processes
5.2. Additional Tips for Tuning

Abstract

This document describes how amavisd-new can be integrated into the Postfix SMTP delivery process. It lists the necessary requirements, explains how Postfix and amavisd-new need to be configured to basically work together and it gives filter-examples to show how amavisd-new can be called from Postfix.

1. Requirements

The following requirements must be met before integration can begin:

  1. amavisd-new has already been installed and successfully tested.

  2. Postfix has been installed, configured for basic operations and tested successfully.

[Tip]Tip

Independently of the configuration examples shown in this document, it is advisable to set strict_rfc821_envelopes = yes in /etc/postfix/main.cf. Postfix will reject any message from envelope-senders, whose address can't be used to send a reply to.

This avoids accepting e-mails from erroneous envelope-senders that can't be informed of problems, which finally would result in deleting the message - even if Postfix claimed successful delivery in the first.

1.1. Which Postfix version is required?

Integrating amavisd-new into the Postfix delivery process requires that Postfix is able to delegate messages to external content filters. The minimum version that provides content filtering is Postfix release-20010228.

1.2. Catching errors during integration

Chances are that configuration errors during implementation cause Postfix to bounce legitimate messages. Setting the soft_bounce parameter during integration and reloading the Postfix configuration afterwards prevents Postfix from bouncing legitimate mail during that time:

# postconf -e "soft_bounce = yes"
# postfix reload

As soon as soft_bounce has been activated Postfix will treat all delivery errors as temporary errors - any client that wants to send messages to Postfix will keep mail in the mailqueue and it will suspend delivery until the soft_bounce parameter has been removed or set to no.

Once the integration of amavisd-new into the Postfix delivery process has been completed successfully soft_bounce must be removed or Postfix will not generate bounce messages for legitimate mail.

2. Basic Postfix and amavisd-new configuration

There are several moments at which Postfix can hand messages over to amavisd-new (before it accepts a message from a client or after) and there are different filter approaches (globally, per recipient (domain), per network interface, etc.) that can trigger Postfix to transport a message to amavisd-new.

The transport methods - transporting a message from Postfix to amavisd-new and back - however always remain the same. They will be described in this section first. The section that follows will deal with different filtering approaches.

[Tip]Integration procedure

The following examples have been structured to cause minimum trouble on an online mail system. The order of steps ensures that filtering will be enabled at the very last moment. Several tests will have been conducted to verify the delivery chain works before the filter is enabled. Once enabled the complete system should work at once.

2.1. Configuring amavisd-new for Postfix

Configuring amavisd-new to work with Postfix answers the following two questions:

  1. Which port should the amavisd-new daemon listen to for incoming connections from Postfix?

  2. Which IP-address and port should the amavisd-new SMTP client use to (re)inject filtered messages (and notifications about message status) into the Postfix SMTP delivery system?

2.1.1. Configuring amavisd-new for incoming connections

The $inet_socket_port parameter in /etc/amavisd.conf sets the port number where amavisd-new will listen for incoming (E)SMTP connections. The following example explicitly configures amavisd-new to bind to port 10024 (default setting undef):

$inet_socket_port = 10024;

2.1.2. Configuring the reinjection path

Two parameters, $forward_method and $notify_method, need to be configured (usually identically) to reinject messages into the Postfix mail system.

The first parameter, $forward_method, specifies where amavisd-new should transport scanned messages to, while the second parameter, $notify_method, specifies where notifications about scanned messages should be transported to.

By default amavisd uses 127.0.0.1 on port 10025 to contact a SMTP server for reinjection of filtered messages. Unless a different IP address or port should be used, no modifications must be applied and this section can be skipped.

In case a different IP address or port should be used, the parameters $notify_method and $forward_method need to be adjusted to reflect these requirements. The following example edits these parameters in /etc/amavisd.conf and uses 192.0.2.1 as IP address and port 20025:

$notify_method  = 'smtp:[192.0.2.1]:20025';
$forward_method = 'smtp:[192.0.2.1]:20025';

2.2. Configuring the transport from Postfix to amavisd-new

Both amavisd-new and Postfix are able to use either SMTP- or LMTP-communication to transport a message from Postfix to amavisd-new. Both variants will be described in this section.

Why configure a dedicated service?

Theoretically it's possible to transport messages from Postfix to amavisd-new using the existing smtp-, lmtp, or even the relay-service in /etc/postfix/master.cf.

In practice transporting messages to amavisd-new requires imposing transport limits on the transporting service. Imposing such limits on a globally available service would impose these limits on the complete Postfix mail system - it would slow down the system significantly and should be avoided.

[Note]Note

The number of Postfix clients that may connect simultaneously to amavisd-new instances must be limited to the maximum number of daemon child processes amavisd-new starts.

If the Postfix transport client was allowed to open more connections than amavisd-new can handle, amavisd-new would start to queue incoming Postfix connections. Postfix in turn would interpret such behaviour as “unresponsive remote MTA” and would itself begin to queue mail that should be filtered. All this would possibly throttle down the complete system and all further filtering attempts would suffer.

2.2.1. Configuring a dedicated lmtp-client

The following example creates a new, dedicated lmtp-transport named amavisfeed in /etc/postfix/master.cf. Its configuration details are explained following the listing:

# ==========================================================================
# service type  private unpriv  chroot  wakeup  maxproc command + args
#               (yes)   (yes)   (yes)   (never) (100)
# ==========================================================================

...

amavisfeed unix    -       -       n        -      2     lmtp
    -o lmtp_data_done_timeout=1200
    -o lmtp_send_xforward_command=yes
    -o lmtp_tls_note_starttls_offer=no
[Important]Important

A noteworthy quote from the Postfix documentation: “...do not specify whitespace around the ‘=’. In parameter values, either avoid whitespace altogether, ...”. Further details on master.cf configuration syntax can be found in master.cf or master(5).

Here's a quick rundown on the settings that differ from other services defaults:

maxproc

The maximum number of concurrent Postfix amavis-service processes has been limited to 2 (default: default_process_limit = 100). This value reflects the default of 2 amavisd-daemon children processes and is a good setting to start from. The value may be raised later, when the system works stable and still can take a higher load. It should not exceed the number of simultaneous amavisd child processes.

lmtp_data_done_timeout

Setting lmtp_data_done_timeout to 1200 (seconds) doubles the default time span a regular Postfix client waits after message delivery for the server to reply DONE to claim successful delivery. It must be larger than amavisd setting $child_timeout (default 8*60 seconds) and should add a sufficient safety margin, for example to cater for periods of automatic database maintenance (e.g. bayes database on non-SQL database types) which can take a long time in some cases.

If the server does not reply within the configured time span, the Postfix client will quit the connection, put the message into the deferred queue, log a delivery failure and retry later to transport the message to amavisd-new.

[Note]Note

Raising this value serves a trick amavisd uses to avoid message loss in case of power outage etc. The trick consists in keeping the incoming connection as long open as it takes to filter the message and take appropriate action (reinjection, notification, quarantine, etc.).

Only when the message (or notifications etc.) has been reinjected amavisd will send DONE to the client and the client will close the connection. This way Postfix will always keep the message in its own mail queue, where it can be reactivated after a system failure.

lmtp_send_xforward_command

Enabling lmtp_send_xforward_command configures the Postfix lmtp-client to forward the original clients HELO name and IP address to amavisd-new. amavisd-new in turn can use this information for

  • logging and notifications (macro %a)

  • switching policy banks (MYNETS, @mynetworks_maps)

  • pen pals functionality

  • p0f fingerprinting

lmtp_tls_note_starttls_offer

Starting with version 2.6 amavisd-new can offer TLS to a smtp- or lmtp-client. This option cuts down unnecessary logging by Postfix, just in case logging TLS session offers has been enabled globally setting lmtp_tls_note_starttls_offer in Postfix main.cf configuration file.

2.2.2. Configuring a dedicated smtp-client

Configuring a dedicated smtp-client is almost identical to configuring a dedicated lmtp-client. The syntax differences in detail are that the names of parameters start with smtp_ instead of lmtp_ and that the command at the end of the service invokes the smtp- and not lmtp-client. The same reasons given for differing lmtp client options apply to the dedicated smtp client configuration.

Here's an example of a dedicated smtp client given the service name amavisfeed:

# ==========================================================================
# service type  private unpriv  chroot  wakeup  maxproc command + args
#               (yes)   (yes)   (yes)   (never) (100)
# ==========================================================================

...

amavisfeed unix    -       -       n       -       2     smtp
    -o smtp_data_done_timeout=1200
    -o smtp_send_xforward_command=yes
    -o smtp_tls_note_starttls_offer=no

2.3. Configuring a dedicated SMTP-server for message reinjection

The second service that needs to be added to the Postfix mail system is a dedicated SMTP-server. It will exist only to accept filtered messages and notifications from amavisd-new to transported them closer to their final destination.

This dedicated smtpd server will differ in many aspects from the default smtpd daemon. The most important difference is that it configures an empty content_filter parameter, thus overriding any global external content filtering settings in Postfix.

[Note]Note

Delegating messages to an external content filter in Postfix is done using the content_filter parameter. If the dedicated smtpd-daemon would not override any global content_filter settings, the reinjected message would be sent of to the external content filter again - the mail would end in an endless loop.

The following Postfix example uses amavisd-new default settings taken from the $forward_method and $notify_method parameters. These settings configure amavisd-new to forward filtered messages and notifications to 127.0.0.1 on port 10025; the Postfix smtpd daemon will be configured to bind to that IP address and listen on the specified port for incoming connections:

# ==========================================================================
# service type  private unpriv  chroot  wakeup  maxproc command + args
#               (yes)   (yes)   (yes)   (never) (100)
# ==========================================================================

...

127.0.0.1:10025 inet n    -       n       -       -     smtpd
    -o content_filter=
    -o smtpd_delay_reject=no
    -o smtpd_client_restrictions=permit_mynetworks,reject
    -o smtpd_helo_restrictions=
    -o smtpd_sender_restrictions=
    -o smtpd_recipient_restrictions=permit_mynetworks,reject
    -o smtpd_data_restrictions=reject_unauth_pipelining
    -o smtpd_end_of_data_restrictions=
    -o smtpd_restriction_classes=
    -o mynetworks=127.0.0.0/8
    -o smtpd_error_sleep_time=0
    -o smtpd_soft_error_limit=1001
    -o smtpd_hard_error_limit=1000
    -o smtpd_client_connection_count_limit=0
    -o smtpd_client_connection_rate_limit=0
    -o receive_override_options=no_header_body_checks,no_unknown_recipient_checks,no_milters
    -o local_header_rewrite_clients=
    -o smtpd_milters=
    -o local_recipient_maps=
    -o relay_recipient_maps=

Here's a quick rundown on the settings that differ from smtpd defaults:

content_filter

The empty content_filter overrides other, globally set content_filter delegations.

..._maps

Empty ..._maps override any other globally set map lookups. Procedures to enforce settings specified in such maps have already taken place when Postfix accepted the message from the external client. Doing them again will not produce new results but only waste resources.

..._restrictions_...

There's no need to apply any already enforced ..._restrictions_... another time. It would also only waste resources.

mynetworks

To avoid abuse from remote hosts, the dedicated smtpd-daemon will only allow clients from 127.0.0.0/8 to relay messages.

local_header_rewrite_clients

By default this option would “rewrite message header addresses in mail from these clients and update incomplete addresses with the domain name”. If such action has already been taken by Postfix before the message went off to amavis, it should not be done a second time when it reenters the Postfix mail system. Leaving this option empty disables local header rewrites and saves resources.

remaining options

All remaining options either configure the dedicated smtpd-daemon to be more failure tolerant or exist to avoid unnecessary use of resources.

Running the postfix reload will activate the new transports (Postfix will not yet send regular mail to amavisd). Combined with the tail command problems can easily be detected:

# postfix reload && tail -f /var/log/maillog

If there are no problems reported, basic configuration can be tested.

2.4. Testing basic configuration

Testing basic configuration consists of three separate tests, starting at the end of the new delivery chain and working to it's beginning. Their goal is to answer the following questions:

  1. Will amavisd-new accept connections at the specified IP address and port?

  2. Will the new dedicated smtpd-daemon accept connections at the specified IP address and port?

  3. Will a test message, injected into amavisd-new, be filtered, sent to Postfix and delivered into a mailbox?

2.4.1. Testing amavisd's host and port

A test, using the telnet command, serves to verify that amavisd listens on the specified IP address and port. A successful connection looks like this:

$ telnet localhost 10024
220 [127.0.0.1] ESMTP amavisd-new service ready
EHLO localhost
250-[127.0.0.1]
250-VRFY
250-PIPELINING
250-SIZE
250-ENHANCEDSTATUSCODES
250-8BITMIME
250-DSN
250 XFORWARD NAME ADDR PROTO HELO
QUIT
221 2.0.0 [127.0.0.1] amavisd-new closing transmission channel

If the test fails, the following questions may help to debug the problem:

  • Is the amavisd-new daemon running?

  • Does amavisd-new write an error to the log?

  • Do the IP address and port number specified in the amavisd-new configuration match the values used during the test?

  • Does a firewall intercept connections?

2.4.2. Testing the dedicated Postfix smtpd-daemon

When Postfix was reloaded, the new, dedicated smtpd-daemon (127.0.0.1:10025) should have been activated. A successful connection looks like this:

$ telnet 127.0.0.1 10025
220 mail.example.com ESMTP Postfix (2.3.2)
EHLO localhost
250-mail.example.com
250-PIPELINING
250-SIZE 40960000
250-ETRN
250-STARTTLS
250-AUTH PLAIN CRAM-MD5 LOGIN DIGEST-MD5
250-AUTH=PLAIN CRAM-MD5 LOGIN DIGEST-MD5
250-ENHANCEDSTATUSCODES
250-8BITMIME
250 DSN
QUIT
221 2.0.0 Bye

If the test fails, the following questions may help to debug the problem:

  • Is the Postfix master daemon running?

  • Does Postfix write an error to the log?

  • Do the IP address and port number specified in the new services configuration match the values used during the test?

  • Does a firewall intercept connections?

2.4.3. Testing the new transport chain

This test proves amavisd accepts e-mail as specified in Section 2.1, “Configuring amavisd-new for Postfix”, filters it and finally hands it over to Postfix' dedicated smtpd-daemon as specified in Section 2.3, “Configuring a dedicated SMTP-server for message reinjection”.

The following example uses the content of test-messages/sample-nonspam.txt from the amavisd test-messages to send an e-mail:

$ telnet localhost 10024
220 [127.0.0.1] ESMTP amavisd-new service ready
HELO localhost
250 [127.0.0.1]
MAIL FROM: <>
250 2.1.0 Sender  OK
RCPT TO: <postmaster>
250 2.1.5 Recipient postmaster OK
DATA
354 End data with <CR><LF>.<CR><LF>
From: virus-tester
To: undisclosed-recipients:;
Subject: amavisd test - simple - no spam test pattern

This is a simple test message from the amavisd-new test-messages.
.
250 2.6.0 Ok, id=30897-02, from MTA([127.0.0.1]:10025): 250 2.0.0 Ok: queued as 079474CE44
QUIT
221 2.0.0 [127.0.0.1] amavisd-new closing transmission channel

The maillog shows the delivery path. Here's an excerpt from a successful delivery process:

Nov  1 11:28:10 mail postfix/smtpd[30986]: connect from localhost[127.0.0.1] 1
Nov  1 11:28:10 mail postfix/smtpd[30986]: 079474CE44: client=localhost[127.0.0.1]
Nov  1 11:28:10 mail postfix/cleanup[30980]: 079474CE44: message-id=<20061101102810.079474CE44@mail.example.com>
Nov  1 11:28:10 mail postfix/qmgr[20432]: 079474CE44: from=<>, size=822, nrcpt=1 (queue active)
Nov  1 11:28:10 mail amavis[30897]: (30897-02) Passed BAD-HEADER, <> -> <postmaster>, quarantine: badh-le5gjszxowBk, mail_id: le5gjszxowBk, Hits: -1.76, queued_as: 079474CE44, 39505 ms 2
Nov  1 11:28:10 mail postfix/smtpd[30986]: disconnect from localhost[127.0.0.1]
Nov  1 11:28:10 mail postfix/local[30987]: 079474CE44: to=<postmaster@example.com>, relay=local, delay=0.27, delays=0.14/0.05/0/0.08, dsn=2.0.0, status=sent (delivered to mailbox: postmaster) 3
Nov  1 11:28:10 mail postfix/qmgr[20432]: 079474CE44: removed
1

amavisd connects with Postfix dedicated smtpd-daemon and hands over the e-mail that had been sent during the telnet session. smtpd gives a queue-id of 079474CE44 that can be tracked throughout the maillog.

2

amavisd notices it has checked and sent an e-mail to <postmaster>.

3

Postfix' local-service logs it successfully delivered an e-mail with queue-id 079474CE44 to the mailbox of postmaster.

If the test fails, the following questions may help to debug the problem:

  • Does amavisd-new log errors?

  • Does running amavisd-new in debug-mode report errors?

3. Message filtering examples

Postfix can use various criteria to decide whether a message should be sent to amavisd-new for examination. Combinations of criteria may serve to create different configurations. The following section describes the following configurations:

  • Filtering e-mail globally

  • Filtering e-mail globally by service

  • Filtering e-mail per recipient domain

  • Filtering e-mail per sender domain

  • Filtering e-mail by content

3.1. Filtering E-mail globally

In most cases email policies require global filtering - every inbound and every outbound e-mail must be filtered by amavisd-new - before it may be sent closer to its final destination.

[Note]Why check outgoing mail traffic?

Some reasons for checking mail coming from internal networks or from authenticated roaming users are:

  • detect an internal infected PC which is sending viruses

  • detect an internal zombiized PC (or an internal open relay or proxy) which is sending or relaying spam

  • let the SpamAssassin Bayes autolearning feature see a balanced view of all mail, including useful samples of non-spam originating from inside

  • make it possible for pen pals feature to function (if enabled)

In Postfix global settings for its services are written to main.cf. The content_filter parameter, the parameter configuring that messages are sent to amavisd-new, must therefore be placed in main.cf.

The content_filter parameter requires a triplet, consisting of the transport service's name (here: amavisfeed, given in Section 2.2.1, “Configuring a dedicated lmtp-client”), the target hosts IP address and the port where amavisd-new listens for incoming connections. Following the values used in this documents examples the content_filter configuration results in this:

content_filter=amavisfeed:[127.0.0.1]:10024

The new external content filter will be activated once Postfix has been reloaded. Sending a test-mail verifies the system works.

3.2. Filtering E-mail by Postfix service

Postfix is able to filter messages per service. Such configuration requires the content_filter not to be applied globally to all services in main.cf (see: Section 3.1, “Filtering E-mail globally”), but selectively, per service in master.cf.

The following example presumes Postfix runs on a system offering three IP addresses. In this example these are: 192.0.2.1 (WAN), 127.0.0.1 (localhost) and 10.0.0.254 (LAN). The goal is to filter only e-mail that enters from the WAN interface.

This requires to create three dedicated smtpd-daemon instances, each binding to one of the given IP addresses and deactivating the global smtp service calling the smtpd command.

Additionally the WAN interface (here: 192.0.2.1:25) is configured to use content_filter =amavisfeed:[127.0.0.1]:10024 - it will delegate any message that enters the Postfix mail system at this service to the external amavisd content filter.

# ==========================================================================
# service type  private unpriv  chroot  wakeup  maxproc command + args
#               (yes)   (yes)   (yes)   (never) (100)
# ==========================================================================
# smtp      inet  n       -       n       -       -       smtpd

...

192.0.2.1:25 inet n    -       n       -       -     smtpd
    -o content_filter=amavisfeed:[127.0.0.1]:10024
    -o receive_override_options=no_address_mappings

10.0.0.254:25   inet n    -       n       -       -     smtpd

127.0.0.1:10025 inet n    -       n       -       -     smtpd
    -o content_filter=
    -o smtpd_delay_reject=no
    -o smtpd_client_restrictions=permit_mynetworks,reject
    -o smtpd_helo_restrictions=
    -o smtpd_sender_restrictions=
    -o smtpd_recipient_restrictions=permit_mynetworks,reject
    -o smtpd_data_restrictions=reject_unauth_pipelining
    -o smtpd_end_of_data_restrictions=
    -o smtpd_restriction_classes=
    -o mynetworks=127.0.0.0/8
    -o smtpd_error_sleep_time=0
    -o smtpd_soft_error_limit=1001
    -o smtpd_hard_error_limit=1000
    -o smtpd_client_connection_count_limit=0
    -o smtpd_client_connection_rate_limit=0
    -o receive_override_options=no_header_body_checks,no_unknown_recipient_checks,no_milters
    -o local_header_rewrite_clients=
    -o smtpd_milters=
    -o local_recipient_maps=
    -o relay_recipient_maps=

3.3. Filtering E-Mails per Recipient Domain

Postfix is able to filter e-mails per recipient domain. In order to do this the content_filter parameter must not be set globally (see: Section 3.1, “Filtering E-mail globally”). Instead the content_filter parameter has to be associated with one or more recipient domains listed in a lookup table (map).

[Caution]Caution

This filter method is not selective! It will send any mail with a recipient domain listed in the lookup table to amavis even if the mail contains another recipient that should not be examined by the amavis framework.

If fully selective rules are required all mail should be sent to amavis and amavis' own rule sets should be configured to decide whether a message for a given recipient should be examined or not.

When Postfix searches the lookup table and finds the recipients domain listed as key, it will take the action associated with that domain. The action will send the message to a FILTER amavisfeed:[127.0.0.1]:10024.

The following map /etc/postfix/filter_recipient_domains specifies to send messages to the FILTER amavisfeed whenever a message for any recipient at example.com enters the Postfix mailqueues:

example.com               FILTER amavisfeed:[127.0.0.1]:10024

Once the table has been created the postmap command must be used to create an indexed map Postfix can read:

# postmap /etc/postfix/filter_recipient_domains

Once the map has been indexed, the postmap command is used to test the map. In the following example the postmap command queries for the domain example.com and returns the associated action:

# postmap -q "example.com" /etc/postfix/filter_recipient_domains
FILTER amavisfeed:[127.0.0.1]:10024

The tested map must be added to main.cf, before Postfix can make use of the new filter policy. Setting the check_recipient_access parameter in the list of smtpd_recipient_restrictions triggers evaluation of entries in the map - check_recipient_access is triggered by the envelope-recipient(s) given by a SMTP-client in a SMTP-session with Postfix.

The following example puts the check_recipient_access rule before permit_mynetworks - all clients envelope-recipient(s) will be filtered:

smtpd_recipient_restrictions =
    ...
    check_recipient_access hash:/etc/postfix/filter_recipient_domains
    ...
    permit_mynetworks
    reject_unauth_destination
    ...

Filtering E-Mails per Recipient Domain only from External Clients

This example puts the check_recipient_access rule after permit_mynetworks - only messages sent from clients that are not in Postfix $mynetworks list (external or untrusted clients) will be filtered:

smtpd_recipient_restrictions =
    ...
    permit_mynetworks
    reject_unauth_destination
    check_recipient_access hash:/etc/postfix/filter_recipient_domains
    ...

3.4. Filtering E-Mails by Sender-Domain

In general it doesn't make sense to filter e-mails by sender-domain, as anyone can fake a sender-domain during a SMTP-session. Filtering by sender-domain will probably only make sense, if messages are not filtered globally, but e-mails from ones own domain should be checked for spam or viruses before they leave the network.

Most of the configuration steps are identical with the ones noted in Section 3.3, “Filtering E-Mails per Recipient Domain”, except for the parameter that triggers evaluation of the indexed map. In this scenario envelope-senders should trigger map evaluation. The map, named /etc/postfix/filter_sender_domains this time, contains the sender domain (example.com) and associates it with the required FILTER:

example.com               FILTER amavisfeed:[127.0.0.1]:10024

Once the map has been converted and tested with the postmap command (see: Section 3.3, “Filtering E-Mails per Recipient Domain”) it must be added to the list of smtpd_recipient_restrictions using the check_sender_access parameter:

smtpd_recipient_restrictions =
    ...
    check_sender_access hash:/etc/postfix/filter_sender_domains
    ...
    permit_mynetworks
    reject_unauth_destination
    ...
[Important]Important

The map must be listed before permit_mynetworks, because only then it will be applied to all clients - even the ones Postfix trusts, which are very likely the ones from example.com.

3.5. Filtering E-mail per Content

Postfix is able - with deliberate limitations (see: BUILTIN_FILTER_README) - to search for strings in headers, the body and MIME-headers. If a string matches, Postfix may call appropriate action.

The following example configures Postfix to look for the string offer in Subject:-headers and delegate the message to an external content filter if if finds a matching string.

A map, consisting of the search string noted as regexp-expression, associates the search pattern with a FILTER action:

/^Subject:.*offer/   FILTER amavisfeed:[127.0.0.1]:10024
[Note]Indexing regexp- or pcre-maps?

regexp- or pcre-maps are and must be plaintext files. They must not and cannot be converted to an indexed map using the postmap command. They can be tested using the postmap command using the -q command line option.

Once the map has been created, Postfix must be configured to use it. The following example uses the header_checks parameter (not body_checks or mime_header_checks as they apply to other message parts) to implement the map into the Postfix delivery process:

header_checks = regexp:/etc/postfix/filter_header

Once Postfix has been reloaded it will send every e-mail that contains the word offer in the Subject:-header off to the external amavisd content filter.

4. Advanced Postfix and amavisd-new configuration

In a post-queue content filtering setup, a mail message passes through smtpd and cleanup Postfix services twice, once before a content filter, and the second time when an approved message is reinjected from a content filter into the Postfix mail system. This is because checks and transformations that have been configured in main.cf are globally active and will be loaded and run by any instance of these two services. To avoid wasting resources, options that control runtime behavior of these services should not be applied globally in main.cf, but selectively to separate instances of these services in master.cf.

Checks and transformations which are performed by a smtpd Postfix service itself, e.g. access controls, recipient validation, milters etc., can be controlled by adding options (-o) to appropriate smtpd services. This has been shown in the basic configuration examples (see: Section 2.3, “Configuring a dedicated SMTP-server for message reinjection”).

Checks and transformations which are performed by a cleanup Postfix service are trickier because in a normal Postfix setup there is only one cleanup service, unlike smtpd services of which there are many. Some of the more important cleanup settings are dynamically controllable by a smtpd service through the use of its receive_override_options option.

[Tip]Transformations and checks

Any transformation should preferably only be performed once, either before or after content filtering. When to transform depends on the desired effect, for example whether a content filter should see unchanged or modified mail messages. Typical transformations are:

  • rewrite addresses

  • add BCC recipients

  • modify mail header.

Most checks should also be performed only once, preferably only on the first passage, when the mail enters the Postfix mail system the first time. This way messages can be rejected early - if needed - and will not tie up downstream resources. Checking early also avoids bounces in case of negative check results on a second passage after content filtering.

4.1. Multiple cleanup service architecture

To gain more control over a cleanup service than offered by receive_override_options, two (or more) cleanup services, each with its own set of options, must be run. A Postfix setup with more than one cleanup service is possible either with two separate Postfix instances, or through a specification of services and their options in master.cf file of a single Postfix instance.

The following diagram illustrates a setup with two cleanup services in a single Postfix instance:

      .......................................
      :                Postfix              :
   ----->smtpd \                            :
      :         -pre-cleanup-\       /local---->
   ---->pickup /              -queue-       :
      :             -cleanup-/   |   \smtp----->
      :     bounces/    ^        v          :
      : and locally     |        v          :
      :   forwarded   smtpd  amavisfeed     :
      :    messages   10025      |          :
      ...........................|...........
                        ^        |
                        |        v
            ............|...............................
            :           |   $inet_socket_port=10024    :
            :           |                              :
            : $forward_method='smtp:[127.0.0.1]:10025' :
            : $notify_method ='smtp:[127.0.0.1]:10025' :
            :                                          :
            :    amavisd-new                           :
            ............................................

Procedure 1. Message flow with two cleanup services

  1. Messages enter the Postfix system at the regular smtpd or pickup service.

  2. The pre-cleanup cleanup service performs transformations and checks on these messages.

  3. The qmgr service schedules the messages to be sent to the amavisd-new content filter.

  4. amavisd-new performs various tests on the messages.

  5. Messages are re-injected into the Postfix mail system, sending them to a dedicated, local smtpd service.

  6. The cleanup cleanup service performs transformations and checks that must be done at this stage, but omits the ones that have already been carried out in step 2.

4.2. Configuring two cleanup services

Configuring Postfix smtpd services to use two separate, dedicated cleanup services requires the following steps:

  1. Create a second cleanup instance

  2. Modify the existing cleanup service

  3. Configure smtpd services to use either of the two cleanup services.

4.2.1. Creating a second cleanup instance

The following example adds a cleanup daemon named pre-cleanup. It will handle messages before a content filter.

# ==========================================================================
# service type  private unpriv  chroot  wakeup  maxproc command + args
#               (yes)   (yes)   (yes)   (never) (100)
# ==========================================================================
# smtp      inet  n       -       n       -       -       smtpd

...

pre-cleanup unix    n       -       n       -       0       cleanup
    -o virtual_alias_maps=

The above leaves canonicalization address rewriting enabled so that a content filter will see canonicalized (external) sender mail addresses, but it disables globally configured virtual alias transformations.

Such transformations will be done later by the second cleanup service, so that a content filter will see original (external) recipient mail addresses. Other options may also be used as needed.

4.2.2. Modifying the existing cleanup service

The already existing cleanup service - having the service name cleanup - will be used to process messages that re-enter the Postfix mail system (also for delivery notifications and forwarding as generated internally by Postfix).

Cleanup jobs that already have been performed by the pre-cleanup service should not be run again. The following example disables typical checks that have been run before or are not needed for internally generated notifications:

# ==========================================================================
# service type  private unpriv  chroot  wakeup  maxproc command + args
#               (yes)   (yes)   (yes)   (never) (100)
# ==========================================================================
# smtp      inet  n       -       n       -       -       smtpd

...

cleanup unix    n       -       n       -       0       cleanup
    -o mime_header_checks= 1
    -o nested_header_checks= 2
    -o body_checks= 3
    -o header_checks= 4
1 2 3 4

The specified options disable header and body checks as these would already be performed by a pre-cleanup service.

[Note]always_bcc

This cleanup service would also be the appropriate one for specifying always_bcc option - doing it globally would apply to both cleanup services and would result in two copies of each message to be sent to the specified address.

4.2.3. Configuring smtpd services

Finally existing smtpd services on ports 25 and 587 (submission), and the pickup service must be configured to send messages to the new pre-cleanup service instead of a default cleanup service:

# ==========================================================================
# service type  private unpriv  chroot  wakeup  maxproc command + args
#               (yes)   (yes)   (yes)   (never) (100)
# ==========================================================================
# smtp      inet  n       -       n       -       -       smtpd

...

pickup    fifo  n       -       n       60      1       pickup
    -o cleanup_service_name=pre-cleanup
smtp      inet  n       -       n       -       -       smtpd
    -o cleanup_service_name=pre-cleanup
submission inet n       -       n       -       -       smtpd
    -o cleanup_service_name=pre-cleanup

5. Tuning

5.1. Maximum Number of Concurrent Processes

The most important settings to tune and optimize in Postfix and amavisd workflow are the maximum number of concurrent processes. The maximum number of concurrent processes on both sides must be chosen with care.

If the number is too low, hardware resources aren't used efficiently and delivery time will be unnecessarily prolonged. Experience tells that raising the number of processes a little, will not raise the overall throughput in the same proportion.

As the system resources are nearing saturation with each increase of the number of processes, an increase in throughput becomes marginal, and eventually even negative when the number of processes exceeds its near-optimum value. E-mail throughput will decrease, because processes need to wait for each other. At worst e-mail delivery stalls.

Best practice is to start with a (conservative) maximum number of 2 concurrent processes. Everyday use has shown that this value may be raised to a value between 10 and 30 concurrent Postfix client and amavisd server processes. This also depends on the overall resources the system may provide, how amavisd has been integrated into the Postfix delivery process and on the anti-virus and anti-spam software being loaded and used by amavisd-new.

Regardless of the maximum number of concurrent processes, both sides - Postfix and amavisd - should be synchronized. To synchronize both sides edit, the $max_servers parameter for amavisd-new (see: amavisd.conf) and the number of processes in master.cf listed in the dedicated transports maxproc column for Postfix.

Both values should be identical for two reasons: If amavisd-new offers more processes than Postfix will ever use, amavisd-new wastes resources. On the other hand, if Postfix starts more dedicated transports than amavisd can handle simultaneously, e-mail transport will be refused and logged as error.

[Note]Controlling the maximum number of concurrent processes in main.cf

Instead of controlling the maximum number of concurrent processes of Postfix' dedicated transport in master.cf it is also possible to keep the default setting - in master.cf and set the following parameter and option in main.cf:

amavisfeed_destination_concurrency_limit = 2

The name of the parameter starts with the service in master.cf (here: amavisfeed) that should be controlled and goes on with the suffix _destination_concurrency_limit. Here also 2 is set as initial (conservative) value.

5.2. Additional Tips for Tuning

Further Tuning-Tips can be found in README.performance and the slides from amavisd-new, advanced configuration and management.

amavisd-new-2.7.1/README_FILES/README.old.scanners000640 000621 000620 00000073103 10233024304 020605 0ustar00markcmi000000 000000 --------------------------------------- This file is old and not up-to-date !!! --------------------------------------- AMaViS & virus scanners *********************** Contents: 1 List of supported antivirus products 2 Setting up the commandline options 3 Antivirus product information 3.1 Specific Antivirus product information 3.1.1 How to use Kaspersky Anti-Virus AVPDaemon 3.1.2 Kaspersky Anti-Virus 3.1.3 VirusBuster (Daemon / Client) 3.2 Return codes 4 Updates 4.1 Update scripts 4.1.1 Script for Sophos Sweep 4.2.2 Script for NAI uvscan 4.2.3 Script for Kaspersky Anti-Virus 5 Why AMaViS will never stop all viruses 5.1 Blocking certain file(s) / file type(s) 1 List of supported antivirus products AMaViS currently supports the following antivirus products (mostly for Linux) * CyberSoft VFind * F-Secure Inc. (former DataFellows) F-Secure AV * H+BEDV AntiVir/X * Kaspersky Anti-Virus (kavscanner and kavdaemon) * Network Associates Virus Scan for Linux * Sophos Sweep * Trend Micro FileScanner * CAI InoculateIT (currently only the old 4.x version is supported!) * GeCAD RAV AntiVirus 8 (engine version 8.5 or better required!) * ESET Software NOD32 (command line scanner and daemon/client) * Command AntiVirus for Linux * VirusBuster * Sophie, using Sophos AntiVirus Interface * Trophie, using Trend Micro API * FRISK F-Prot / F-Prot Daemon * OpenAntiVirus ScannerDaemon * DrWeb Antivirus for Linux/FreeBSD/Solaris (no support for DrWeb Daemon yet) * MkS_Vir for Linux * CentralCommand Vexira * Norman Virus Control for Linux If you miss support for a specific product, please write to Rainer Link . For an up-to-date product list, see http://www.openantivirus.org/ 2 Setting up the commandline option I advise you to look at the commandline parameters for the scanner(s) you use with AMaViS. Each scanner has its own section at the beginning of the scanmails script and the commandline options can be set with _cmdl, i.e. antvir_cmdl. Please read the documentation of your antivirus software carefully and add (or remove) specific options. If an antivirus product provides the functionally to scan inside (run-time) compressed files (i.e. Diet, LzExe, PkLite, UPX) and archived files (i.e. PkZIP, RAR), I would advise to switch this on, if it's not on by default. 3 Antivirus product information 3.1 Specific Antivirus product information 3.1.1 How to use Kaspersky Anti-Virus AVPDaemon Two possible setups exist: a) AVPDaemon and AVPDaemonClient (in new package renamed to AvpDaemonTst) switch into AVPDaemon/DaemonClients and compile AvpDaemonClient.cpp (new location seems to be Sample) with a simple "make". Then copy this file to the location where AVPDaemon is installed (i.e. /usr/local/avp or /opt/AVP). Run configure, make and make install. b) AVPDaemon alone (AVPDaemon works in daemon mode and client mode) symlink AvpDaemonClient to AvpDaemon, as configure searches for AvpDaemonClient (and AvpDaemonTst). In amavis/av/avpdc, change the line $output = `$avpdc $TEMPDIR/parts`; to $output = `$avpdc -o{$TEMPDIR/parts/}`; run ./configure, make and make install. Well, AVPDaemon (in client mode) shows no output and it can not be switched to verbose mode. Therefore setup a) is the one I currently recommend, otherwise your logfiles don't show which file(s) is/are infected. NOTE: AvpDaemon must be running as a daemon, so it should be started at boot time via an init script (or whatver) as /AvpDaemon -* /var/amavis 3.1.2 Kaspersky Anti-Virus AvpLinx fills the log with a lot of trash because of a simple progress bar by loading the AVC files. If you do not want to have "log flooding", you may set LongStrings=Yes in file defUnix.prf, section Options. This will reduce the output when AvpLinux is loading the AVC files. 3.1.3 Virus Buster (Daemon + Client) Please keep in mind the VirusBuster Daemon has to run under the same user id AMaViS runs as. Moreover, VirusBuster returns 3 for an infection (which is not in sync with the man page). 3.1.4 Sophie / Trophie By default, Sophie/Trophie creates a socket in /var/run, owned by root, group uucp (read/writeable by owner and group). As AMaViS runs as user amavis, it cannot connect to the socket. Please change the group accordingly in sophie.h/trophie.h and re-compile. If Sophie/Trophie is installed, but configure doesn't detect it, you need to upgrade to version 1.15/1.03, resp., or better. 3.1.5 GeCAD RAV AntiVirus 8 The command line options changed with a new version of the virus scanning engine. Therefore, you need at least engine version 8.5. If your engine is too old, please update it (i.e. "ravav -UPDATE"). Just as a side note, with the new engine, an update is later done with -u. 3.1.6 MkS_Vir for Linux MkS expects its config file mks_vir.cfg in /etc. 3.2 Return codes ----------------------------------------------------------------------- NAI VirusScan (uvscan) return codes: ----------------------------------------------------------------------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - as of version 4.x documentation "uvscan.pdf" or "unix403.pdf": - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0 No errors occured; no viruses were found. 2 Driver integrity check failed. 6 A general problem. 8 Could not find a driver. 10 A virus was found in memory. 13 One or more viruses or hostile objects were found. 15 VirusScan self-check failed; it may be infected or damaged. 102 User quit via ESC-X, ^C or Exit button. Exit code 102 occurs where the scan encounters an unespected error, such as denied access or memory shortage. On these occasions, the scan exits immediately and does not finish the scan. ----------------------------------------------------------------------- Sophos Sweep Return Codes: ----------------------------------------------------------------------- Bernhard Nowotny writes: Error codes returned by SWEEP (thanks to christian.weber@sophos.com): SWEEP returns error codes if there is an error or if a virus is found SWEEP returns: 0 If no errors are encountered and no viruses are found 1 If the user interrupts the execution by pressing ESC 2 If some error preventing further execution is discovered, or if compressed files have been found when using the -WC command line qualifier 3 If viruses or virus fragments are discovered A different set of error codes will be returned if SWEEP is run with the -eec command line qualifier. 0 If no errors are encountered and no viruses are found 8 If survivable errors have occured 12 If compressed files have been found and decompressed 16 If compressed files have been found and not decompressed 20 If viruses have been found and disinfected 24 If viruses have been found and not disinfected 28 If viruses have been found in memory 32 If there has been an integrity check failure 36 If unsurvivable errors have occured 40 If execution has been interrupted ------------------------------------------------------------------------- Kaspersky Anti-Virus (formerly AntiViral Toolkit Pro): ------------------------------------------------------------------------- return codes of AvpLinux and AvpDaemonClient according to Readme.txt 0 No viruses were found 1 Virus scan was not complete 3 Suspicious objects were found 4 Known viruses were detected 5 All detected viruses have been deleted 7 File AvpLinux is corrupted -------------------------------------------------------------------------- DataFellows F-Secure AntiVirus: -------------------------------------------------------------------------- return codes of F-Secure AV according to fsav_lin.pdf documentation 0 Normal exit; no viruses or suspicious files found. 1 Abnormal termination; unrecoverable error. (Usually a missing or corrupted file.) 2 Self-test failed; program has been modified. 3 A boot virus or file virus found. 5 Program was terminated by pressing CTRL-C, or by a sigterm or suspend event. 6 At least one virus was removed. 7 Out of memory. 8 Suspicious files found; these are not necessarily infected by a virus. ------------------------------------------------------------------------ H+BEDV AntiVir/X ------------------------------------------------------------------------- NOTE: Since AntiVir 6.12.x you must have a (valid) license key! Either a free license for private use or a commercial license. Otherwise AntiVir/X returns always 214 - regardless if a virus was found or not and this is quite useless for AMaViS. AntiVir/X return codes according to antivir --help 0: Normales Programmende, kein Virus, kein Fehler 0: normal program termination, no virus, no error 1: Virus in Datei (oder Bootsektor) gefunden 1: found virus in file (or bootsector) 2: Virus (evtl. aktiv) im Speicher gefunden 2: found virus (active?) in memory 100: AntiVir hat nur den Hilfetext angezeigt 100: AntiVir displays only help text 101: Es wurde ein Makro in einer Datei gefunden 101: macro found in a file 102: Der Parameter -once war angegeben und AntiVir lief bereits 102: parameter -once used, but AntiVir runs already before 200: Programmabbruch wegen Speichermangel 200: not enough memory - program termination 201: Die angegeben Responsedatei wurde nicht gefunden 201: response file not found 202: Innerhalb einer Responsedatei wurde @ angegeben 202: a respons file contains @ 203: Ungueltiger Parameter angegeben 203: unknown option 204: Ungueltiges Verzeichnis angegeben 204: directory not found 205: Die angegebene Reportdatei konnte nicht erzeugt werden 205: could not generate a report file 210: AntiVir hat eine benoetigte DLL nicht gefunden 210: AntiVir could not found a required lib 211: Programm abgebrochen, da Selbstpruefung fehlgeschlagen 211: Program termination - self check failed 212: Die Datei antivir.vdf nicht gefunden oder Lesefehler 212: File antivir.vdf not found or read error 213: Initialisierungsfehler 213: program init failed 214: Lizenzdatei nicht gefunden 214: License key not found ----------------------------------------------------------------------- Trend Micro FileScanner (vscan) return codes: ----------------------------------------------------------------------- 0: no virus found 1: virus found 2: virus found I do not have a list of return codes. Consider three files a, b and c. a and b are infected, c is not infected: /etc/iscan/vscan /tmp/test/a - return code: 1 /etc/iscan/vscan -a /tmp/test/* - return code: 2 /etc/iscan/vscan -a /tmp/test/ - return code: 0 (although two viruses were detected) ----------------------------------------------------------------------- Cybersoft VFind Return Codes: ----------------------------------------------------------------------- 0 If no errors are encountered and no viruses are found 23 If viruses or virus fragments are discovered 138 License expired or invalid. 255 A general error. ----------------------------------------------------------------------- CAI InoculateIT - inocucmd command line utility 4.0: ----------------------------------------------------------------------- 100 - A virus was detected. >2 - Some type of scan failure. 1 - User pressed cntrl-C. 0 - The scan has completed. No viruses were detected. ----------------------------------------------------------------------- Command AntiVirus for Linux Return Codes: ----------------------------------------------------------------------- Code Description --- ----------- 0-13: Fatal exceptions occurred. Abnormal termination. 5: Break signaled. The user interrupted the scan process via the Break key. 13: The program performed GPF (General Protection Fault). 50: Nothing found. 51: At least one infection found. 52: At least one suspicious file found. 53: At least one virus was disinfected. 100: Scan engine shared library is incorrect or incompatible. No scan was performed. 101: Scan engine failed to initialize. Insufficient memory or critical condition. No scan was performed. 102: sign.def is either missing or is corrupt. 103: macro.def is either missing or is corrupt. 104: -virlist or -virno specified on the command line 105: -today has been specified and a scan has already been made this day. 106: english.tx1 is either missing or is corrupt. NOTE: This applies only to CSAV versions 4.57 or higher. ----------------------------------------------------------------------- Virus Buster for Linux Return Codes: ----------------------------------------------------------------------- Error codes according man page OK (0) = everything is ok, no viruses. VIRKILLED (1) = Virus found and killed. VIRNOTKILLED (2) = Virus found not killed. HEFOUND (3) = heuristically Suspicious HEUDOCFOUND (4) = heuristically suspicious DOC file=20 PACKER (5) = Packed file IMMUNIZER (6) = Immunizing hit VSKMSG (7) = VSK message SCANERROR (64)= Error during scanning ENGERROR (65)= Engine error EMPTYFNAME (66)= There is no filename to scan NOSUCCDMSTOP (67)= Unable to stop the daemon NOSUCCSTART (68)= Unable to start the daemon STATUSFAIL (69)= Unable to ask the status NOENARG (70)= Too less orr wrong parameters UNKNCOMM (71)= Unknown command UNKNOPT (72)= Unknown option DMTIMEOUT (73)= Unable to connect to the daemon (timeout) NOTREGISPRG (74)= The program is not registered. You can't start the client. ----------------------------------------------------------------------- FRISK F-Prot for Linux Return Codes: ----------------------------------------------------------------------- 0 Normal exit. Nothing found, nothing done. 1 Unrecoverable error (for example, missing SIGN.DEF). 2 Selftest failed (program has been modified). 3 At least one virus-infected object was found. 4 5 Abnormal termination (scanning did not finish). 6 At least one virus was removed. 7 Error, out of memory (should never happen, but well...) 8 Something suspicious was found, but no recognized virus. ----------------------------------------------------------------------- GECAD RAV AntiVirus for Linux Return Codes: ----------------------------------------------------------------------- #FILE_OK 1 #FILE_INFECTED 2 #FILE_SUSPICIOUS 3 #FILE_CLEANED 4 #FILE_CLEAN_FAIL 5 #FILE_DELETED 6 #FILE_DELETE_FAIL 7 #FILE_COPIED 8 #FILE_COPY_FAIL 9 #FILE_MOVED 10 #FILE_MOVE_FAIL 11 #FILE_RENAMED 12 #FILE_RENAMED_FAIL 13 #NO_FILES 20 #ENG_ERROR 30 #SINTAX_ERR 31 #HELP_MSG 32 #VIR_LIST 33 ----------------------------------------------------------------------- ESET Software NOD32 for Linux Return Codes: ----------------------------------------------------------------------- NOD32_EXIT_CODE_OK 0 NOD32_EXIT_CODE_VIRUS 1 NOD32_EXIT_CODE_CLEANED 2 NOD32_EXIT_INTERNAL_ERROR 10 ----------------------------------------------------------------------- CentralCommand Vexira/Linux Return Codes: ----------------------------------------------------------------------- Vexira is based on H+BEDV AntiVir/Linux, therefore the command line parameters and return values seem to be completly identical 0: Normal program termination, no virus, no error 1: Virus found in a file or boot sector 2: A virus signature was found in memory 100: Vexira Antivirus only has displayed this help text 101: A macro was found in a document file 102: The option -once was gven and Vexira Antivirus already ran today 200: Program aborted, not enough memory available 201: The given response file could not be found 202: Within a response file another @ directive was found 203: Invalid option 204: Invalid (non-existent) directory given at command line 205: The log file could not be created 210: Vexira Antivirus could not find a necessary dll file 211: Programm aborted, because the self check failed 212: The file vexira.vdf could not be read 213: An error occured during initialisation 214: License key not found -------------------------------------------------------------------------- Norman Virus Control for Linux: -------------------------------------------------------------------------- return codes of Norman Virus Control according to man page 0 - No error 1 - File or boot sector virus found 2 - Virus detected in memory 3 - No scan area given 4 - Configuration file changed 5 - Bad argument 6 - I/O error 8 - Program error 10 - Files skipped 14 - virus detected and removed 4 Updates Some antivirus companies provide updates for the virus definition files (pattern files) for the latest virus/latest viruses in (a) small extra file(s), i.e. Sophos Anti-Virus virus identities (IDE). See http://www.sophos.com/downloads/ide/ for more information about IDE files. For versions of sweep older than 3.37, these files are located in the directory ide/ below your Sophos tree, i.e. /opt/sophos/ide and the environment variable SAV_IDE should therefore be set to SAV_IDE=/opt/sophos/ide in the AMaViS script. From sweep version 3.37 on, this is no longer necessary, as sweep reads the ide directory location from /etc/sav.conf. The default is /usr/local/sav. NAI provides an extra driver, which has to be specified on the command line via --extra /path-to/EXTRA.DAT Please keep in mind that your antivirus software needs regular updates. Set up a cron job with the appropriate ftp/ncftp/wget commands for automatic updates. NAI provides a script in their PDF manual. F-Secure AV comes with their own update program. I would also strongly recommand to subscribe to an alert mailinglist, which most AV companies offer, to get information about the latest virus outbreaks. Note: please keep in mind an update process may fail. So, your script should do first a backup, download the file(s) and after that starting the virus scanner to check the eicar test file virus. If the virus scanner does not exit with exit code "virus found" then your script should do a roll-back and send an alert message to virusalert indication update process failed. 4.1 Update scripts The scripts are provided by users without any warranty. Use them on your own risk. For Sophos, see also http://www.sophos.com/support/faqs/autodown.html ("How to automate the downloading of IDE files"). 4.1.1 Script for Sophos Sweep by Reiner Keller #!/bin/bash #cd $SAV_IDE cd /usr/local/lib/sweep-IDE /usr/bin/wget -q -N `/usr/local/bin/sweep -v |/usr/bin/grep "Product version" |/usr/bin/sed -e "s/.*: \(.\)\.\(..\)$/ http:\/\/www.sophos.com\/downloads\/ide\/\1\2_ides.zip/"` /usr/bin/unzip -q -n "???_ides.zip" chmod 644 * 4.1.2.1 Script for NAI (McAfee) uvscan by Matt Burke #!/bin/bash rm -f .listing* datdir="ftp://ftp.mcafee.com/pub/datfiles/english/" uvdir=/usr/local/mcafee wget -q -O $uvdir/latest-dat.tar $datdir/`wget -qnr $datdir && grep tar .listing | awk {'print $4'}` tar --overwrite --directory=$uvdir -xf $uvdir/latest-dat.tar 4.1.2.2 Script for NAI uvscan by Brian K. West #!/usr/bin/perl # dailyupdate.pl # Auto Update Daily DAT files from NAI uvscan for *nix # By: Brian K. West # Version 1.0.3 # # This is used for Daily Dat file from NAI for early prevention. # This version will email the admin when the DAT files are updated! # I have also done some touchups to make the code cleaner. # Also: $adminemail = "user\@domain.com"; you must escape the "@" # use LWP::Simple; use Archive::Zip; # Settings $location = "http://download.nai.com/products/mcafee-avert/daily_dats/DAILYDAT.ZIP"; $tmpdir = "/tmp"; $uvscandir = "/usr/local/uvscan"; $mailprog = "/bin/mail"; $adminemail = "brian\@bkw.org"; $check = head("$location"); if($check) { # Lets grab the next version if its ready! print "Downloading DAILYDAT.ZIP ...\n"; $datfile = mirror("$location", "$tmpdir/DAILYDAT.ZIP"); if($datfile == "404") { print "No Daily Dat Update avaliable!\n"; exit; } if($datfile == "304") { print "You have the latest Daily Dat file installed!\n"; exit; } } else { print "No Daily Dat Updates avaliable!\n"; exit; } my $zip = Archive::Zip->new("$tmpdir/DAILYDAT.ZIP") || die("error"); my @list = $zip->memberNames(); my $file; print "Extracting DAILYDAT.ZIP to $uvscandir ...\n"; foreach $file (@list) { if (!($file =~ /.*\/$/)) { my $data = $zip->contents($file); $file = lc($file); my $newpart = "$tmpdir/$file"; print "Installing: $file\n"; open(OUTPART, ">$uvscandir/$file"); print(OUTPART $data); close(OUTPART); } } #unlink("$tmpdir/DAILYDAT.ZIP"); $check = `$uvscandir/uvscan --version | $mailprog -s \"Virus Scan Daily DAT Updated\" $adminemail`; print "Daily Dat Installed!\n"; #!/usr/bin/perl # # Auto Update DAT files from NAI uvscan for *nix # By: Brian K. West # Version 1.0.1 # use LWP::Simple; use Archive::Tar; # Settings $location = "http://download.nai.com/products/datfiles/4.x/nai"; $tmpdir = "/tmp"; $uvscandir = "/usr/local/uvscan"; # Get Current Version of dat file. $current = `$uvscandir/uvscan --version | grep \"Virus data file\" | awk '{ print substr(\$4,2,4) }'`; print "Current version installed: $current"; #$current = 4085; # Increase version number by 1 $needed = $current + 1; $check = head("$location/dat-$needed.tar"); if($check) { # Lets grab the next version if its ready! print "Downloading dat-$needed.tar ...\n"; $datfile = mirror("$location/dat-$needed.tar", "$tmpdir/dat-$needed.tar"); if($datfile == "404") { print "No updates avaliable!\n"; exit; } } else { print "No updates avaliable!\n"; exit; } my $tar = Archive::Tar->new("$tmpdir/dat-$needed.tar") || die("error"); my @list = $tar->list_files(); my $file; print "Extracting dat-$needed.tar to $uvscandir ...\n"; foreach $file (@list) { if (!($file =~ /.*\/$/)) { my $data = $tar->get_content($file); my $newpart = "$tmpdir/$file"; print "Installing: $file\n"; open(OUTPART, ">$uvscandir/$file"); print(OUTPART $data); close(OUTPART); } } unlink("$tmpdir/dat-$needed.tar"); $new = `$uvscandir/uvscan --version | grep \"Virus data file\" | awk '{ print substr(\$4,2,4) }'`; if($new == $current) { print "Update Failed!\n"; print "You may have to do it manually!\n"; exit; } print "New installed version: $new"; 4.1.2.3 Script for NAI DAT-files by Julio Cesar Covolato (please have a look at http://www.psi.com.br/~julio/uvscan/ for the latest version) #!/bin/sh ################################################################### ################# UVUPDATE-1.2 ####################### ################################################################### # Script to automate downloading and install new dat files # from ftp.nai.com for the uvscan 4.x virus scanner. ################################################################### # $date Fri Mar 16 01:12:43 EST 2001 ################################################################### # Written by Julio Cesar Covolato ################################################################### # Read the files README, INSTALL and CHANGES before install ################################################################### # # # ################################################################### # MAKE THE CHANGES BELOW TO SUIT YOUR SISTEM ################################################################### # ################################################ # Where are your binary uvscan and datfiles ??? ################################################ uvscan_dir=/usr/local/uvscan/ #################################### # setup our commonly used programs #################################### grep=/bin/grep mail=/bin/mail wget=/usr/bin/wget cut=/usr/bin/cut tar=/bin/tar rm=/bin/rm ls=/bin/ls chmod=/bin/chmod sed=/bin/sed ################################################################# # Setup email and subject to notify news versions, or problems :( ################################################################# mail_to="root@localhost" subject_ok=" UVSCAN - We got a new dat-file" subject_bad=" UVSCAN - Something goes wrong :(( " subject_nonew=" UVSCAN - No new dat-file for today" ############################################################ # Setup wget flags ( see "man 1 wget" ). # If you are behind a firewall, you can add " --passive-ftp" # Thanks to Viraj Alankar ############################################################ wget_opt="-N -q -t 30" ################################################################### # You don't need make changes below ################################################################### cd ${uvscan_dir} # Get the actual running version of the datfile DATVERSION=$(./uvscan --version|grep "Virus data file"|${cut} -c 18-21) # Get the latest txt file info (delta.ini) from NAI, if there are a new one. ${wget} ${wget_opt} ftp://ftp.nai.com/pub/antivirus/datfiles/4.x/delta.ini # Extract the dat-version from the file delta.ini DATVERSIONEW=$(${grep} CurrentVersion delta.ini|${cut} -c 16-19) if [ ${DATVERSION} = ${DATVERSIONEW} ]; then echo -e "\n\n\n\tThe uvscan has the latest version yet!"|${mail} -s "${subject_nonew}" ${mail_to} exit # No new version! :(( Maybe tomorrow! ) else # Get and Install it!!! ${wget} ${wget_opt} ftp://ftp.nai.com/pub/antivirus/datfiles/4.x/dat-${DATVERSIONEW}.tar ${tar} xf dat-${DATVERSIONEW}.tar ${chmod} 744 *.dat fi # We got the new version installed! Test it... NEWDAT=$(./uvscan --version|grep "Virus data file"|${cut} -c 18-21) if [ ${NEWDAT} = ${DATVERSIONEW} ]; then # Send an email to me, notifying the new version! echo -e "\n\n\n\tNew dat file is: ${NEWDAT}\n\n\n" > newvirus.txt $(sed) -n '/\* DV2/,/\* DV3/p' readme.txt >> newvirus.txt cat newvirus.txt|${mail} -s "${subject_ok}" ${mail_to} ${rm} -f dat-$DATVERSION.tar # we don't need anymore the old version else # Send an email to me, notifying that anything goes wrong... :(( echo "Go there: ftp://ftp.nai.com/pub/antivirus/datfiles/4.x/"|\ ${mail} -s "${subject_bad}" ${mail_to} fi exit 4.1.3 Script for KasperskyLab AVP by Andy Wallace #!/usr/bin/perl use Net::FTP; # in the libnet package - you may have to get it from CPAN - I did. # Directory to download into $DIR="/usr/local/AvpLinux"; # Get current time and date ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime(time); # I just want this stuff so I can save each daily.zip as a different filename with a date attached, so I know I haven't missed any. Format is dailyddmmyy.zip (yes I'm British), so I need to make a few changes. # Jan = 0, so add 1 to $mon $mon++; if ($mon<10) { $mon="0$mon"; } # Days of month are 1-31, so that's OK if($mday<10) { $mday="0$mday"; } # gmtime thinks this year is 100! At least in my version of Perl...so don't use this script after 2099 :-) $year -= 100; if($year<10) { $year="0$year"; } # Connect to FTP server and download daily.zip $ftp = NET::FTP->new("ftp.kasperskylab.ru", Passive, 1); $ftp->login("ftp", someone\@somewhere.com"); $ftp->cwd("/bases"); $ftp->binary; $ftp->get("daily.zip", "$DIR/daily$mday$mon$year.zip"); $ftp->quit; # Check it turned up OK, if so unzip it, if not send an email if (-e "$DIR/daily$mday$mon$year.zip") { system("/usr/bin/unzip -o -qq $DIR/daily$mday$mon$year.zip -d $DIR"); } else { system("/bin/mail -s \"Antivirus daily update failure!\" root"); } # Now restart AVP daemon to load updated virus library system("/usr/local/AvpLinux/AvpDaemon -k"); system("/usr/local/AvpLinux/AvpDaemon -* /var/amavis"); # End of perl script Put a call to this in your root crontab to run it every day. e.g. 00 20 * * * /usr/local/bin/getupdate.pl 5 Why AMaViS will never stop all viruses AMaViS is not an antivirus scanner, it's only an "interface" for virus scanning at the eMail gateway in combination with one (or even) more of the virus scanners listed above. Virus detection and stopping depends therefore on the quality of the virus scanner. To get an impression about the detection rate of antivirus products, please have a look at Virus Bulletin (www.virusbtn.com), Virus Test Center (http://agn-www.informatik.uni-hamburg.de/) or AV-Test (www.av-test.com). Please keep in mind that viruses in encrypted eMails/attachments cannot be detected! Also, if an infected attachment file is compressed with a compression format for which AMaViS is not configured (we believe that the most important formats are covered, though), it gets through, unless the virus scanner(s) used is/are able to decode/uncompress it. If this happens, it's the job of your client-side anti-virus software to detect and stop the virus from spreading when the attachment gets decrypted or uncompressed. 5.1 Blocking certain file(s) / file type(s) AMaViS does not currently support blocking certain files by type or extension, e.g. .vbs or .exe. Such a capability may be added in the future. But please keep in mind that the file extension can be forged as easily as the MIME-type. I advise you to read a posting to NTBugTraq from Nick FitzGerald, online at http://www.ntbugtraq.com/default.asp?pid=36&sid=1&A2=ind0005&L=ntbugtraq&F=&S=&P=11152. amavisd-new-2.7.1/README_FILES/README.exim_v3000640 000621 000620 00000015567 10233024267 017610 0ustar00markcmi000000 000000 +======================================================================+ | (Please see instructions in README.exim_v4. The setup described here | | is not recommended with amavisd-new + Exim v4) | +======================================================================+ See also: http://ente.limmat.ch/linux/exim_v3_-_amavisd-new.html http://bugs.debian.org/213422 How To Use AMaViS With exim *************************** Exim 3.x ******** Scanning incoming mail only =========================== First of all, create the group "amavis" and the user "amavis", which is in this group. Add the user amavis to your trusted users, so in the MAIN CONFIGURATION SETTINGS add the following line trusted_users = amavis (or, if this line already exists simply add amavis, separated with a colon). In the TRANSPORT CONFIGURATION, add the following: amavis: driver = smtp port = 10024 hosts = 127.0.0.1 allow_localhost Here below is an old variant of a transport which uses a pipe to a helper program amavis(.c). As feeding to amavisd-new with SMTP is easier and possibly faster, the pipe via helper setup is not recommended - it is included here for the curious: # amavis: # driver = pipe # command = "/usr/sbin/amavis <${sender_address}> ${pipe_addresses}" # prefix = # suffix = # check_string = # escape_string = # # for debugging change return_output to true # return_output = false # return_path_add = false # user = amavis # group = amavis # path = "/bin:/sbin:/usr/bin:/usr/sbin" # current_directory = "/var/amavis" At the *beginning* of the DIRECTORS CONFIGURATION, add the following: amavis_director: condition = "${if eq {$received_protocol}{scanned-ok} {0}{1}}" driver = smartuser transport = amavis # verify must be set to false, to avoid troubles with receiver and/or # sender verify verify = false NOTE: Please keep in mind that the ORDER matters! Scanning incoming/outgoing and relayed mail =========================================== Scanning of outgoing mail is also possible (based on a contribution by John Burnham). Additionally to the steps mentioned at the "Scanning incoming mail only" section, do the following: At the *beginning* of the ROUTERS CONFIGURATION, add the following amavis_router: condition = "${if eq {$received_protocol}{scanned-ok} {0}{1}}" driver = domainlist route_list = "*" transport = amavis # verify must be set to false, to avoid troubles with receiver and/or # sender verify verify = false The configure script should be called with the option --enable-exim. Well, it is enabled by default if the MTA qmail is not installed on the system. NOTE: if the MTA qmail is installed on your system, too, you *MUST* add --disable-qmail NOTE: do not forget to set up an (eMail) alias for "amavis" to a real user account. If you used --with-notifyreceivers, and a user sends a mail to an non-existent remote user, the bounce message from the remote MTA will be send to pseudo-user amavis. Note: If you want to add a "X-Virus-Scanned" header you can do this in your exim configuration, too. Use the headers_add option in the amavis TRANSPORT CONFIGURATION, i.e. headers_add = "X-Virus-Scanned: by AMaViS (http://amavis.org/)" Exim 4.x ******** +======================================================================+ | (Please see instructions in README.exim_v4. The setup described here | | is not recommended with amavisd-new + Exim v4) | +======================================================================+ The setup is very similar to the Exim 3.x one, so you should read this section first. # trusted users trusted_users = amavis Routers configuration: # in routers: amavis_router: condition = "${if eq {$received_protocol}{scanned-ok} {0}{1}}" driver = accept transport = amavis # verify must be set to false, to avoid troubles with receiver and/or # sender verify verify = false Transport configuration: After the remote_smtp: driver = smtp entry, add the following entry amavis: driver = smtp port = 10024 hosts = 127.0.0.1 allow_localhost Or the old variant via pipe to amavis(.c) helper program (not recommended): # amavis: # driver = pipe # command = "/usr/sbin/amavis <${sender_address}> ${pipe_addresses}" # # for debugging change return_output to true # return_output = false # return_path_add = false # user = amavis # group = amavis # path = "/bin:/sbin:/usr/bin:/usr/sbin" # current_directory = "/var/amavis" (based on a mail to exim-users by Niels Dettenbach) AMaViS via exim system filter ***************************** This setup has not been tested by the AMaViS team yet. Feedback on this is very welcome! Based on a mail from Dariusz Sznajder to the amavis- user mailing list (http://marc.theaimsgroup.com/?l=amavis-user&m=102861263206738&w=2) In exim/configure I have: [...] system_filter = /etc/exim.filter system_filter_pipe_transport = amavis_pipe [...] amavis_pipe: driver = pipe user = amavis return_output headers_add = ${if def:authenticated_id {X-Authenticated-Sender: $authenticated_id}{}} [...] /etc/exim.filter # Exim filter # drop out error messages here if $received_protocol is "scanned-ok" then finish endif pipe "/usr/sbin/amavis \"$sender_address\" \"$recipients\"" AMaViS has to be configured as ./configure --enable-exim FAQ --- Q: Whenever I am receiving many mails in large batches (i.e. via UUCP or fetchmail), the system load on the machine on which AMaViS runs climbs to a level that renders the machine virtually unusable. How do I prevent such behaviour? A: The reason for the high load is that one AMaViS process (and in turn AMaViS' children processes) are started for every message that is received. The obvious solution is to limit the number of AMaViS processes that are started simultaniously. AMaViS does not currently support this itself, but every sensible MTA does. Moreover, it makes sense to limit the number of messages that an MTA can deliver at one time, to prevent mail server death by spamming. For Exim, the configuration keywords to look for are deliver_load_max, deliver_queue_load_max, and queue_only_load which control whether mails are delivered (read: passed to AMaViS) or queued only under high load. More detailed information can be found in the Exim Specification. Credits ------- Philip Hazel, Marc Haber, John Burnham, Jeffrey C. Ollie, Hilko Bengen, Niels Dettenbach, Dariusz Sznajder TODO ---- * whatever you whish to add here :-) Author ------ Rainer Link eMail: Rainer.Link@suse.de WWW: http://rainer.w3.to Please send bug reports, questions, suggestions and more directly to the author. Thanks :-) Updated by Mark Martinec, based on suggestion in http://bugs.debian.org/213422 from Enrico amavisd-new-2.7.1/README_FILES/images/000751 000621 000620 00000000000 11003707000 016573 5ustar00markcmi000000 000000 amavisd-new-2.7.1/README_FILES/README.contributed000640 000621 000620 00000002270 10233023671 020541 0ustar00markcmi000000 000000 Please see: http://www.ijs.si/software/amavisd/#contrib for a list of contributed and related software. ------------------------------------------------------------------------ The information below seems stale and the program no longer available: From: Atif Ghaffar Date: Thu, 16 Jan 2003 06:23:35 +0100 I wrote a couple of scripts to be able to search the quarantine files to check for false-positives, get some spam/virus stats etc. There are two scripts. 1) logspam. A perl script that. for each spam-*.gz, virus-* file extract the headers store To, From, Date, Subject, X-Spam-Status, Quarantine-Id, X-AMaVIS-Alert, reject_reason etc in a database. It then moved the spam-* or virus-* file in a backup directory for archival purposes. 2) A PHP script to give a web interface to search the database. This prints some data about the spam/virus/banned messages and lets you see the headers. For demo: see http://newns.marshallnet.com/spamlog/ try recipient aghaffar@developer.ch Sources at http://atifghaffar.com/sections/softwares/spamlog/ copy spamlog.phps to somewhere in your http directory and rename to spamlog.php run logspam every hour or so from cron amavisd-new-2.7.1/README_FILES/README.ldap000640 000621 000620 00000021352 11603341302 017135 0ustar00markcmi000000 000000 LDAP LOOKUPS ------------ This text contains general LDAP-related documentation. Please also see README.lookups for additional lookup information. LDAP lookups are enabled in amavisd.conf with: $enable_ldap = 1; Definitions and default values of LDAP parameters. hostname : The hostname or IP address or a socket path of the LDAP server to connect to. A TCP port may be specified after the host name followed by a colon (ex. localhost:389). You can also specify a URI, such as: 'ldaps://127.0.0.1:636' or 'ldapi://%2Fvar%2Frun%2Fopenldap%2Fldapi/'. May also be a reference to an array of hosts, host:port pairs, or URI's, each will be tried in order until a connection is made. (Default = 'localhost') port : The port where LDAP sends queries. May be overridden by 'hostname'. (Default supplied by Net::LDAP: 389, or 636 if using TLS/SSL) localaddr : Will be passed to IO::Socket as the "LocalAddr" parameter, which sets the client's IP address (as opposed to the server's IP address.) (Default: undef (i.e. unspecified)) scheme : Connection scheme to use when not using an URI as "HOST", possible values are: 'ldap', 'ldaps', or 'ldapi'. (Default: 'ldap', unless a hostname starts with a string 'ldaps', in which case the default scheme is 'ldaps') inet6 : Try to connect to the server using IPv6 if "HOST" resolves to an IPv6 target address. If it resolves to an IPv4 address, the connection is tried using IPv4. (Default: 1 if a module IO::Socket::INET6 is available, 0 otherwise) version : The protocol version to use. (Default = 3) timeout : Timeout (in sec) passed when connecting the remote server. (Default = 120) base : The DN that is the base object entry relative to which the search is to be performed. The string may also contain a '%d' token that will be replaced by the e-mail address domain. (Default = undef) scope : Scope can be 'base', 'one' or 'sub'. (Default = 'sub') query_filter : The filter used to find the amavis account. The string must contain a '%m' token that will be replaced by the actual e-mail address. (Default = '(&(objectClass=amavisAccount)(mail=%m))') deref : By default aliases are dereferenced to locate the base object for the search, but not when searching subordinates of the base object. This may be changed by specifying a "deref" parameter with one of the following values: 'never', 'search', 'find', 'always'. (Default = 'find') bind_dn : If binding is needed, this specifies the DN to bind as. (Default = undef) bind_password : Binding password. (Default = undef) sasl : Enable SASL if true. (Default = 0) sasl_mech : A space-separated list of SASL mechanisms. (Default = '') sasl_auth_id : SASL authentication id (e.g. a username) (Default = undef) tls : Enable TLS/SSL if true. (Default = 0) sslversion : SSL version: sslv2, sslv3, sslv2/3, tlsv1 (Default = 'tlsv1') clientcert : A filename with a client certificate. (Default = undef) clientkey : A filename with a client certificate private key. (Default = undef) cafile : A filename containing the certificate of the CA who signed the server's certificate. (Default = undef) capath : A pathname of a directory containing CA certificates. (Default = undef) verify : How to verify the server's certificate: none, optional, require (Default = 'none') The desired parameters can be specified in amavisd.conf and defaults will be supplied for any parameters not specified, ex: $default_ldap = { hostname => [ 'localhost', 'ldap2.example.com' ], timeout => 5, tls => 0, base => 'ou=People,dc=example,dc=com', query_filter => '(&(objectClass=amavisAccount)(mail=%m))', }; The amavisd-new LDAP schema is available in file LDAP.schema of the distribution, and at http://www.ijs.si/software/amavisd/LDAP.schema LDAP 'search' requests all available fields from the specified directory and the result is cached (just for the duration of this mail message processing). Individual attributes can be extracted one at a time from this cache very quickly, so there is no penalty in using several calls to lookup for different attributes (for the same key) in different parts of the program. lookup_ldap() performs a lookup for an e-mail address in an LDAP directory. If a match is found it returns whatever the map returns (a reference to a hash containing values of requested attributes), otherwise returns undef. Given an e-mail address the following lookups are done by default: with $ldap_lookups_no_at_means_domain = 0 (default): - lookup for user+foo@example.com - lookup for user@example.com (only if $recipient_delimiter is '+') - lookup for user+foo (only if domain part is local) - lookup for user (only local; only if $recipient_delimiter is '+') - lookup for @example.com - lookup for @.example.com - lookup for @.com - lookup for @. (catchall) With $ldap_lookups_no_at_means_domain = 1 : - lookup for user+foo@example.com - lookup for user@example.com (only if $recipient_delimiter is '+') - lookup for user+foo@ (only if domain part is local) - lookup for user@ (only local; only if $recipient_delimiter is '+') - lookup for example.com - lookup for .example.com - lookup for .com - lookup for . (catchall) NOTE: a null reverse path e-mail address used by MTA for delivery status notifications (DSN) has empty local part and empty domain. As far as the lookup is concerned (which uses raw, i.e. non-quoted and non-bracketed address form), this address is @, i.e. a single character "@". The LDAP lookup for null address goes through the following sequence of keys: "", "@", "@." (double quotes added for clarity, they are not part of the query key). lookup_ldap_attr() also performs a lookup for an e-mail address against an LDAP directory. It first calls lookup_ldap() if it hasn't been called yet for this key, but instead of returning all available attributes, it returns just a value of one particular attribute. This is the subroutine that gets called from lookup() for arguments (objects) of type Amavis::Lookup::LDAPattr. LDAP white/black listing ------------------------ amavisWhitelistSender/amavisBlacklistSender are multivalued attributes containing either full email addresses or domain specifications. The envelope sender address is compared against each attribute value until a match is made. amavisBlacklistSender: user@example.com amavisBlacklistSender: @example.com amavisBlacklistSender: @.example.com A domain specification with a leading '@.' matches a domain as well as its subdomains. LDAP banned rule names ---------------------- amavisBannedRuleNames may contain a comma-separated list of names mapped through %banned_rules to actual banned_filename tables. amavisBannedRuleNames: ALLOW_EXE, DEFAULT %banned_rules = ( 'NO-MS-EXEC'=> new_RE( qr'^\.(exe-ms)$' ), 'PASSALL' => new_RE( [qr'^' => 0] ), 'ALLOW_EXE' => new_RE( qr'.\.(vbs|pif|scr|bat)$'i, [qr'^\.exe$' => 0] ), 'ALLOW_VBS' => new_RE( [qr'.\.vbs$' => 0] ), 'DEFAULT' => $banned_filename_re, ); Special handling of optional LDAP attribute 'amavisLocal' --------------------------------------------------------- A special shorthand is provided when LDAP lookups are used: when a match for a recipient address (or domain) is found in LDAP tables (regardless of attribute values), the recipient is considered local, regardless of static @local_domains_acl or %local_domains lookup tables. This simplifies life when a large number of dynamically changing domains is hosted. To overrule this behaviour, have an explicit boolean attribute 'amavisLocal' with a value of True returned for each local domain, and False for other domains which might be present in LDAP for some reason, but are nonlocal. In general LDAP lookups are similar to SQL lookups except for the low level LDAP/SQL specific code. The overall functionality, lookup rules, etc. are identical. amavisd-new-2.7.1/README_FILES/README.chroot000640 000621 000620 00000024354 11320435755 017534 0ustar00markcmi000000 000000 Setting up amavisd-new to run in a chroot jail ============================================== 2003-03, Mark Martinec Last updated: 2005-08-11 Preparing system services to run under chroot is not for inexperienced Unix administrators! This is not an automated script, but rather a checklist, guidelines and ideas to help set up an environment in which amavisd-new and its external utilities can run in a Unix chroot cage (jail) to reduce the possible security threats and protect the rest of the system. Details vary greatly from one Unix system to another. The following examples are based on FreeBSD and Linux, but should be useful for other Unix systems. Some of the paths are likely to be different. The following is based on setting a SMTP-based amavisd-new (e.g. to be use with Postfix or dual-MTA setup). It was tried out with all external decoding programs, with SpamAssassin and with the following virus scanners: Clam Antivirus clamscan and clamd, Sophos sweep, SAVI-Perl and Sophie. If you have Postfix, check its chroot setup script for further hints: postfix-xxx/examples/chroot-setup/ and BASIC_CONFIGURATION_README. exit # This is NOT an automatic script!!! # Don't execute commands without knowing what they will do!!! # !!! ESSENTIAL !!!, DO NOT FORGET to cd to your new chroot home directory # before running commands below, as most of them use relative paths! # umask 0022 mkdir /var/amavis cd /var/amavis # make directory structure within the current directory (/var/amavis) mkdir -p etc dev var/run var/virusmails mkdir -p usr/bin usr/lib usr/libexec usr/share usr/share/zoneinfo mkdir -p usr/share/misc usr/share/spamassassin etc/mail/spamassassin mkdir -p usr/local/lib/perl5/site_perl # optional, depending on template in $*_quarantine_method : mkdir var/virusmails/spam var/virusmails/virus var/virusmails/banned var/virusmails/badh var/virusmails/archive chown vscan:vscan var/virusmails/spam var/virusmails/virus var/virusmails/banned var/virusmails/badh var/virusmails/archive # make devices - adjust MAJOR/MINOR as appropriate ( see ls -l /dev/* ) mknod dev/null c 2 2 # FreeBSD mknod dev/null c 1 3 # Linux? mknod dev/random c 1 8 # Linux? mknod dev/urandom c 1 9 # Linux? mknod dev/urandom c 45 2 # OpenBSD ? mknod dev/random c 2 3 # FreeBSD ? mknod dev/random c 244 0 # FreeBSD5.4 ln -s dev/random dev/urandom # FreeBSD # some external programs may need these: mknod dev/zero c 2 12 # FreeBSD, OpenBSD mknod dev/stdin c 22 0 # FreeBSD?, OpenBSD mknod dev/stdout c 22 1 # FreeBSD?, OpenBSD mknod dev/stderr c 22 2 # FreeBSD?, OpenBSD # NOTE: the file system where dev/null and other device files will reside # must NOT be mounted with "nodev" option (/etc/fstab) ! # make a symbolic link so that chrooted processes can refer to the # home directory as /var/amavis (same as not-chrooted), and need not have # to handle it differently (i.e. referring to it as / ) ln -s / var/amavis # actually, the following is more general: d=`pwd`; ln -s / $d$d # copy required binaries to /var/amavis/usr/bin for j in \ /usr/bin/file /usr/bin/ar /bin/pax /usr/bin/gzip /usr/bin/bzip2 \ /usr/local/bin/nomarch /usr/local/bin/arc \ /usr/local/bin/unrar /usr/local/bin/rar \ /usr/local/bin/arj /usr/local/bin/unarj \ /usr/local/bin/zoo /usr/local/bin/lha /usr/local/bin/tnef \ /usr/local/bin/lzop /usr/local/bin/freeze \ /usr/local/bin/rpm2cpio /usr/local/bin/ripole /usr/local/bin/cabextract \ /usr/local/bin/clamscan /usr/local/bin/sweep /usr/local/sbin/sophie \ /usr/local/bin/dccproc /usr/local/bin/pyzor do cp -p $j usr/bin/; done # copy needed /etc files to /var/amavis/etc for j in \ /etc/protocols /etc/services /etc/netconfig /etc/hosts \ /etc/group /etc/passwd /etc/resolv.conf /etc/localtime \ /etc/nsswitch.conf /etc/svc.conf /etc/host.conf do cp -p $j etc/; done # SECURITY NOTE: # It is likely that the /etc/passwd file is not even needed in the jail. # Whether it is needed or not depends on external programs used for virus # and spam checks. Even when the /etc/passwd file in jail is needed, it # need not provide most regular system usernames and should not provide # their valid passwords; a heavily stripped down or faked version of the # file in the jail suffices for most purposes; also, there is hardly # any need for UID 0 usernames (e.g. root) in the chrooted /etc/passwd # file and such usernames are to be avoided. # copy time zones data /usr/share/zoneinfo (or perhaps /usr/lib/zoneinfo) cp -pR /usr/share/zoneinfo usr/share/ # FreeBSD # copy shared libraries to /var/amavis/lib # (check: ldd /var/amavis/usr/bin/* to see which ones are needed) ln -s usr/lib . ln -s usr/libexec . #FreeBSD: for j in \ /usr/lib/libc.so* /usr/lib/libc_r.so* /usr/lib/libm.so* \ /usr/lib/libthr.so* /usr/lib/libstdc++.so* \ /usr/lib/libz.so* /usr/lib/libz2.so* \ /usr/lib/libmagic.so* /usr/local/lib/libsavi.so* \ /usr/local/lib/libclamav.so.* /usr/local/lib/libgmp.so.* do cp -p $j usr/lib/; done cp -p /usr/libexec/ld-elf.so.1 usr/libexec/ #Linux: for j in \ /lib/libc.so.6 /lib/libm.so.6 /lib/ld-linux.so.2 \ /lib/libpthread.so.0 /lib/libresolv-2.3.2.so /lib/libnss_*.so* \ /lib/libgcc_s.so.1 /usr/lib/libstdc++.so.5 \ /usr/lib/libz.so.1 /usr/lib/libbz2.so.1 /lib/libdb*.so* \ /usr/local/lib/libsavi.so.3 /usr/local/lib/libclamav.so.1 do cp -p $j lib/; done # UTF8 data files needed by Perl Unicode support: cp -pR /usr/local/lib/perl5/5.10.1/unicore usr/local/lib/perl5/site_perl/ # # on OpenBSD 3.8 that would be something like: # cp -pR /usr/libdata/perl5/unicore/ usr/libdata/perl5/ # needed by SpamAssassin: cp -p /etc/mail/spamassassin/{*.pre,*.cf} etc/mail/spamassassin/ cp -pR /usr/local/share/spamassassin usr/share/ # FreeBSD cp -pR /usr/share/spamassassin usr/share/ # Linux # Razor2 (if called from SpamAssassin): echo 'debuglevel = 0' >>/etc/razor-agent.conf cp /etc/razor-agent.conf etc/ # magic files needed by file(1). Different versions and installations expect # magic files in different locations. Use the most recent version of file(1) # and check its documentation. Some usual locations are: cp -p /usr/local/share/file/* usr/local/share/file/ cp -p /usr/share/misc/magic* usr/share/misc/ cp -p /usr/share/magic usr/share/ # needed by AV scanners (ClamAV) mkdir -p var/db/clamav cp -pR /var/db/clamav var/db/ cp /usr/local/bin/freshclam /usr/local/sbin/clamd usr/bin/ cp /usr/local/etc/clamd.conf etc/ # Start clamd and freshclam: # chroot -u vscan /var/amavis /usr/sbin/clamd # chroot -u vscan /var/amavis /usr/bin/freshclam -d \ # -c 4 --log-verbose --datadir=/usr/local/share/clam \ # -l /var/log/clam-update.log # needed by AV scanners (Sophos) mkdir -p usr/local/sav cp -pR /usr/local/sav usr/local/ mkdir -p home var/db/amavis scratch/tmp-am scratch/tmp-sys # Subdirectory 'scratch' may reside on a volatile file system (tmpfs) # set protection and ownership chown -R root:wheel usr/ dev/ etc/ var/ scratch/ chown root:wheel . chown -R vscan:vscan home scratch/tmp-am var/db/amavis var/virusmails chown -R clamav:clamav var/db/clamav chmod 751 . scratch chmod 1777 scratch/tmp-sys ln -s ../scratch/tmp-sys var/tmp ln -s scratch/tmp-sys tmp ln -s scratch/tmp-am tmp-am ln -s var/db/amavis db # for compatibility with traditional location chmod 666 dev/null chmod 644 dev/*random # /etc/passwd: set home directory of user vscan to /var/amavis/home !!! # Daemonized virus scanners (e.g. Sophie, ClamD) may be # started in the same chroot jail, or not. E.g. # chroot /var/amavis /usr/bin/sophie -D # # If you want, you may now remove /usr/local/sav and make a link instead, # to avoid having two copies of Sophos database: # ln -s /var/amavis/usr/local/sav /usr/local/sav # consider: # ln -s /var/amavis/var/run/sophie /var/run/ # Sophie socket # ln -s /var/amavis/var/run/sophie.pid /var/run/ # Programs may be tested individually to see if they are happy # in the chroot jail: # perl -Te 'use POSIX; $ENV{PATH}="/usr/bin"; $uid=getpwnam("vscan") or die "E1:$!"; chroot "/var/amavis" or die "E2:$!"; chdir "/"; POSIX::setuid($uid) or die "E3:$!"; open(STDIN,"0.lis.gz"; printf("E5: %d, %d=0x%x\n",$!,$?,$?)' # ... open(STDOUT,">0.lis.gz") or die "E5:$!"; # exec qw(gzip -c 0.lis) or die "E6:$!"'; echo $ # ... exec qw(clamscan /etc/resolv.conf) or die "E5:$!"'; echo $? Edit /var/amavis/etc/amavisd.conf, setting: $MYHOME = '/var/amavis'; $ENV{TMPDIR} = $TEMPBASE = "$MYHOME/tmp-am"; $daemon_chroot_dir = $MYHOME; $helpers_home = "$MYHOME/home"; # prefer $MYHOME clean and owned by root? $db_home = "$MYHOME/var/db/amavis"; $pid_file = "$helpers_home/amavisd.pid"; $lock_file = "$helpers_home/amavisd.lock"; $QUARANTINEDIR = "$MYHOME/var/virusmails"; Logging should preferably be directed to syslog. Configure syslogd to provide a socket in the amavis jail (option -l on FreeBSD, option -a on OpenBSD and Linux). Under FreeBSD place something like: syslogd_flags="-l /var/amavis/var/run/log -ss" into /etc/rc.conf . Because the program starts outside the chroot jail and brings-in all Perl modules first, there is fortunately no need to make a copy of Perl modules inside the jail. If Perl complains about missing modules, add them to the list in file amavisd: fetch_modules('REQUIRED BASIC MODULES', qw( Exporter POSIX Fcntl Socket Errno Time::HiRes IO::File IO::Socket IO::Wrap IO::Stringy ... With earlier version of Perl you might need to add autoloaded modules to the list, such as: auto::POSIX::setgid auto::POSIX::setuid As SpamAssassin loads its rules files only after chrooting, these need to be made available in the jail. A common procedure is to tell sa-update the directory that needs updating: # sa-update --updatedir /var/amavis/var/lib/spamassassin/3.003000 and periodically refresh them. NOTE: OpenBSD chroot specifics are described in the document http://www.flakshack.com/anti-spam, by Scott Vintinner. NOTE: See note about Net::Server at: http://www.ijs.si/software/amavisd/#faq-net-server amavisd-new-2.7.1/README_FILES/README.lookups000640 000621 000620 00000056756 11562477255 017756 0ustar00markcmi000000 000000 LOOKUP MAPS (hash, SQL) AND ACCESS LISTS EXPLAINED ================================================== Updated: 2002-04, 2002-06, 2002-11, 2002-12, 2003-03, 2003-05, 2003-06, 2003-09, 2003-12, 2004-01, 2004-03, 2004-12, 2005-01, 2005-03, 2005-05, 2005-08, 2010-10 Mark Martinec (applies to the semantics of amavisd.conf variables such as: %virus_lovers, %bypass_checks, @virus_lovers_acl, @bypass_checks_acl, $virus_lovers_re, $bypass_checks_re, %local_domains, @local_domains_acl, %mailto, ... ) NOTE: All lookups are performed with raw (rfc2821-unquoted and unbracketed) addresses as a key, i.e.: Bob "Funny" Dude@example.com not: "Bob \"Funny\" Dude"@example.com and not: <"Bob \"Funny\" Dude"@example.com> Several configurable settings in amavisd are controlled through the use of table lookups (hash/associative array), access control lists (array), Perl-regexp -based access control lists, SQL or LDAP lookups. The subroutine that does all the lookups is: sub lookup($$@) { my($get_all, $addr, @tables) = @_; It perform a lookup for a key (usually a recipient e-mail envelope address, unless otherwise noted) against one or more lookup tables / maps. The set of maps used to be hard-wired into the program (but no longer is), and the order chosen is: from specific to more general, and from faster to slower, which is usually flexible enough. Thus the default sequence of lookups: SQL, LDAP, hash, ACL, regexp, constant. The first that returns a definitive answer (not undef/NULL) stops the search. The SQL and LDAP are somewhat specific and are always consulted first. There can only be one (or none) SQL and one (or none) LDAP lookup. This is an implementational limitation, and might be lifted some day. The lists of static lookup tables are configurable since 20040701 (amavisd-new-2.0), and is controlled by array variables such as: @virus_admin_maps = (\%virus_admin, \$virus_admin); @viruses_that_fake_sender_maps = (\$viruses_that_fake_sender_re, 1); @spam_kill_level_maps = (\$sa_kill_level_deflt); @local_domains_maps = (\%local_domains, \@local_domains_acl, \$local_domains_re); @bypass_virus_checks_maps = (\%bypass_virus_checks, \@bypass_virus_checks_acl, \$bypass_virus_checks_re); @virus_lovers_maps = (\%virus_lovers, \@virus_lovers_acl, \$virus_lovers_re); See amavisd.conf-default for a complete list of these @*_maps variables. The above example shows that the default value of these arrays exactly corresponds to the formerly hard-wired search order. Users are free to leave these @*_maps variables at their default, referencing the legacy variables, or the list can be replaced entirely. There may be any number of lookup tables of any static type specified in these lists. Some restrain is warranted nevertheless for efficiency reasons - one lookup into larger lookup table is ofter quicker than two lookups into smaller ones. Some lookup maps can only return boolean result (e.g. ACL), other maps may return any value, which can be interpreted as boolean, numeric, string or possibly other. The result of some lookup maps (e.g. regexp) may include pieces of lookup key. If a match is found, the subroutine lookup() returns whatever the map returns; undef is returned if nothing matches (which for Perl is false as well). A CONSTANT Specifying a Perl scalar as an argument to lookup() is a degenerate case of a lookup table: it matches any key, and the value of the scalar is returned as the match value. Specifying a scalar argument in a call to lookup() (e.g. as the last element in @*_maps arrays) is useful as a last-resort (catchall, default) value. One level of indirection is alowed, so the following three cases are equivalent: $sa_kill_level_deflt = 6.0; @spam_kill_level_maps = (\%some_hash, \$sa_kill_level_deflt); and: $sa_kill_level_deflt = 6.0; @spam_kill_level_maps = (\%some_hash, $sa_kill_level_deflt); and: @spam_kill_level_maps = (\%some_hash, 6.0); The first case allows for the value of a scalar variable to be assigned even _after_ the assignment to @*_maps, so this still works as expected: @spam_kill_level_maps = (\%some_hash, \$sa_kill_level_deflt); $sa_kill_level_deflt = 6.0; but the following does not (it uses a value in the scalar variable at the time of assignment to the list, which is most likely not 6.0): @spam_kill_level_maps = (\%some_hash, $sa_kill_level_deflt); $sa_kill_level_deflt = 6.0; HASH LOOKUPS (associative array lookups) For arguments to subroutine lookup() of type hash-ref, the argument is passed to subroutine lookup_hash(), which does a lookup into a Perl hash. Hash lookups (e.g. for User+foo@Sub.Example.com) are performed in the following order: - lookup for user+foo@sub.example.com - lookup for user@sub.example.com (only if $recipient_delimiter is '+') - lookup for user+foo@ - lookup for user@ (only if $recipient_delimiter is '+') - lookup for sub.example.com - lookup for .sub.example.com - lookup for .example.com - lookup for .com - lookup for . The search sequence stops as soon as a match is found, and the value of the matched entry determines the result. The domain part of a key is always lowercased, the localpart is lowercase when $localpart_is_case_sensitive is true (not case-sensitive by default). The keys in a hash should match the case of a key, i.e. should typically be entirely in lowercase. A field value undef implies that the next lookup table (if there are more) is to be tried. In plain words, undef means "this table does not know the answer, try the next one". Further searching in this table (for possibly more general defaults) is terminated. NOTE: a null reverse path e-mail address used by MTA for delivery status notifications (DSN) has empty local part and empty domain. As far as the lookup is concerned (which uses raw, i.e. non-quoted and non-bracketed address form), this address is @, i.e. a single character "@". The lookup_hash for null address goes through the following sequence of keys: "", "@", "." (double quotes added for clarity, they are not part of the key). There is a subroutine read_hash() available for use in amavisd.conf. It can read keys from a plain text file, and load them into a Perl hash. Format of the text file: one address per line, anything from '#' to the end of line is treated as a comment, but '#' within correctly quoted rfc2821 addresses is not treated as a comment (e.g. a hash sign within "strange # \"foo\" address"@example.com is valid). Leading and trailing whitespace is discarded, empty lines (containing only whitespace and comment) are ignored. Addresses are converted from quoted form into internal (raw) form and inserted as keys into a given hash, with a value of 1 (true). Each address can have an associated optional value (also known as the 'righthand side' or RHS) separated from the address by whitespace. An absence of a value implies 1 (true). The $hashref argument is returned for convenience, so that one can say for example: $per_recip_whitelist_sender_lookup_tables = { '.my1.example.com' => read_hash({},'/var/amavis/my1-example-com.wl'), '.my2.example.com' => read_hash({},'/var/amavis/my2-example-com.wl') } LIST LOOKUPS (ACL) For arguments to subroutine lookup() of type array-ref, the argument is passed to subroutine lookup_acl(), which does an access list lookup: sub lookup_acl($$) { my($addr, $acl_ref) = @_; The supplied e-mail address is compared with each member of the lookup list in turn, the first match wins (terminates the search), and its value decides whether the result is true (yes, permit, pass) or false (no, deny, drop). Falling through without a match produces false (undef). Search is case-insensitive. NOTE: lookup_acl is not aware of address extensions and they are not handled specially! If a list element contains a '@', the full e-mail address is compared, otherwise if a list element has a leading dot, the domain name part is matched only, and the domain as well as its subdomains can match. If there is no leading dot, the domain must match exactly (subdomains do not match). The presence of character '!' prepended to a list element decides whether the result will be true (without a '!') or false (with '!') in case this list element matches and terminates the search. Because search stops at the first match, it only makes sense to place more specific patterns before the more general ones. Although not a special case, it is good to remember that '.' always matches, so a '.' would stop the search and return true, whereas '!.' would stop the search and return false (0). Examples: given: @acl = qw( me.ac.uk !.ac.uk .uk ) 'u@me.ac.uk' matches me.ac.uk, returns true and search stops given: @acl = qw( me.ac.uk !.ac.uk .uk ) 'u@you.ac.uk' matches .ac.uk, returns false (because of '!'), search stops given: @acl = qw( me.ac.uk !.ac.uk .uk ) 'u@them.co.uk' matches .uk, returns true and search stops given: @acl = qw( me.ac.uk !.ac.uk .uk ) 'u@some.com' does not match anything, falls through and returns undef given: @acl = qw( me.ac.uk !.ac.uk .uk !. ) 'u@some.com' similar to the previous, except it returns 0 instead of undef, which would only make a difference if this ACL is not the last argument in a call to lookup() given: @acl = qw( me.ac.uk !.ac.uk .uk . ) 'u@some.com' matches catchall ".", and returns true more complex example: @acl = qw( !The.Boss@dept1.xxx.com .dept1.xxx.com .dept2.xxx.com .dept3.xxx.com lab.dept4.xxx.com sub.xxx.com !.sub.xxx.com me.d.aaa.com him.d.aaa.com !.d.aaa.com .aaa.com ); Comparing hash (associative array) and ACL: For smaller sets of keys and if only boolean results are needed, both hash and ACL are appropriate. - hash is still effective for lots of keys, ACL search is linear; - hash can return any value, not just true or false; - hash can strip away address extension, ACL can not; - ACL appears simpler and more obvious for smaller sets; - ACL can accommodate arbitrarily nested if-then-elseif-then-...-else cases whereas hash only follows a fixed order of stripping addresses; ACL FOR IP ADDRESSES A special type of lookup is an IP-matching access list implemented by lookup_ip_acl(). It performs a lookup for an IP address against a list or an asssociative array (a hash) of IPv4 or IPv6 networks. It is used by amavisd for example to check if the SMTP client (normally your MTA) is allowed to connect, which is why it is sometimes called 'access control list' or ACL (the variable is @inet_acl). IP address is compared with each member of an access list in turn, the first match wins (terminates the search), and its value decides whether the result is true (yes, permit) or false (no, deny). Falling through without a match produces false (undef). The presence of character '!' prepended to a list member decides whether the result will be true (without a '!') or false (with '!') in case this list member matches and terminates the search. Because search stops at the first match, it only makes sense to place more specific patterns before the more general ones. Network can be specified in classless notation n.n.n.n/k, or using a mask n.n.n.n/m.m.m.m . Missing mask implies /32, i.e. a host address. Although not a special case, it is good to remember that '::/0' always matches any IPv4 or IPv6 address (even syntactically invalid address). The '0/0' is equivalent to '::FFFF:0:0/96' and matches any syntactically valid IPv4 address (including IPv4-mapped IPv6 addresses), but not other IPv6 addresses! Example given: @acl = qw( !192.168.1.12 172.16.3.3 !172.16.3.0/255.255.255.0 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 !0.0.0.0/8 !:: 127.0.0.0/8 ::1 ); matches rfc1918 private address space except host 192.168.1.12 and net 172.16.3/24 (but host 172.16.3.3 within 172.16.3/24 still matches). In addition, the 'unspecified' (null, i.e. all zeros) IPv4 and IPv6 addresses return false, and IPv4 and IPv6 loopback addresses match and return true. If the supplied lookup table is a hash reference, match a canonical IP address: dot-quad IPv4, or preferred IPv6 form, against hash keys. For IPv4 addresses a simple classful subnet specification is allowed in hash keys by truncating trailing bytes from the looked up IPv4 address. A syntactically invalid IP address can only match a hash entry with an undef key. Besides looking up full CIDR-style IPv4 or IPv6 lists, later versions of lookup_ip_acl() also make possible matching against a hash lookup table, which only allows for full addresses in canonical form (dotted-quad IPv4 addresses without leading zeroes or IPv6 addresses in canonical preferred form: x:x:x:x:x:x:x:x), or classful IPv4 subnets with truncated octets, such as: ('10.11.12.13'=>1, '192.168.1.2'=>0, '192.168'=>1, '127'=>1, '10'=>1) A convenient method of reading CIDR-style IP lists or a hash from a file if by calling provided routines read_array or read_hash, e.g.: @mynetworks_maps = (read_array('/etc/amavisd-mynetworks'), \@mynetworks); or: @mynetworks_maps = (read_hash('/etc/amavisd-mynetworks'), \@mynetworks); REGULAR EXPRESSION LOOKUPS For arguments to subroutine lookup() of type Amavis::Lookup::RE (objects), the object is passed to method lookup_re, which does a lookup into a list of Perl regular expressions (regexp or RE for short). The full unmodified e-mail address is always used, so splitting to localpart and domain or lowercasing is NOT performed. The regexp is powerful enough that this is unnecessary. The routine is useful for other RE tests, such as looking for banned file names. Each element of the list can be a ref to a pair, or directly a regexp ('Regexp' object created by qr operator, or just a (less efficient) string containing a regular expression). If it is a pair, the first element is treated as a regexp, and the second provides a return value in case the regexp matches. If not a pair, the implied result value of a match is 1. The regular expression is taken as-is, no implicit anchoring or setting case insensitivity is done, so do use a qr'(?i)^user@example\.com$', and not a sloppy qr'user@example.com', which can easily backfire. Also, if qr is used with a delimiter other than ' (apostrophe), make sure to quote the @ and $ . The pattern allows for capturing of parenthesized substrings, which can then be referenced from the result string using the $1, $2, ... notation, as with the Perl m// operator. The number after a $ may be a multi-digit decimal number. To avoid possible ambiguity the ${n} or $(n) form may be used. Substring numbering starts with 1. Nonexistent references evaluate to empty strings. If any substitution is done, the result inherits the taintedness of the key. Keep in mind that $ and @ characters needs to be backslash-quoted in qq() strings. Example: $virus_quarantine_to = new_RE( [ qr'^(.*)@example\.com$'i => 'virus-${1}@example.com' ], [ qr'^(.*)(@[^@]*)?$'i => 'virus-${1}${2}' ] ); Example (equivalent to the example in lookup_acl): $acl_re = new_re->new( qr'@me\.ac\.uk$'i, [ qr'[@.]ac\.uk$'i => 0 ], qr'\.uk$'i, ); ($r,$k) = $acl_re->lookup_re('user@me.ac.uk'); or $r = lookup('user@me.ac.uk', $acl_re); 'user@me.ac.uk' matches me.ac.uk, returns true and search stops 'user@you.ac.uk' matches .ac.uk, returns false (because of =>0) and search stops 'user@them.co.uk' matches .uk, returns true and search stops 'user@some.com' does not match anything, falls through and returns false (undef) NOTE: new_RE is a synonym (shorthand) for the internal subroutine Amavis::Lookup::RE::new See Perl documentation (or Google the Internet) for the description of Perl regular expressions. They are just enhanced version of Posix regular expressions, i.e. what egrep, awk and sed thrive on. Here are the most important constructs (simplified): . Match any character inter..t | Alternation alfa|beta|gamma () Grouping (pre|post)fix [] Set of characters (char. class) [Aa]lfa[0-9] ^ Match the beginning of the string ^MakeMoney $ Match the end of the string com$ \ Quote the next metacharacter \.com$ ^\$\$\$\+spam@\[127\.0\.0\.1\]$ most other characters just match themselves quantifiers may be placed after the pattern to modify its meaning from 'match itself exactly once' into: * Match 0 or more times ^alfa.*omega$ + Match 1 or more times alfa +beta ? Match 1 or 0 times (first)?aid {n} Match exactly n times 0{6} {n,} Match at least n times !{3,} {n,m} Match at least n but not more than m times SQL LOOKUPS For general SQL considerations and the interpretation of @lookup_sql_dsn please see documentation in README.sql . The amavisd.conf variable @lookup_sql_dsn controls access to the SQL server (dsn = data source name). If the list @lookup_sql_dsn is empty, no attempts to use SQL for lookups will be made, and no code to use DBI will be loaded or compiled (if @storage_sql_dsn is empty as well). For arguments to subroutine lookup() of type Amavis::Lookup::SQLfield (objects), the object is passed to a method lookup_sql_field, which does a lookup into an SQL table by using Perl module DBI. An SQL 'select' requests all available fields from the specified tables, and the result is cached (just for this mail message processing). Individual fields can be extracted one at a time from this cache very quickly, so there is no penalty in using several calls to lookup for different fields (for the same key) in different parts of the program. lookup_sql() performs a lookup for an e-mail address against an SQL map. If a match is found it returns whatever the map returns (a reference to a hash containing values of requested fields), otherwise returns undef. A match aborts further fetching sequence. lookup_sql_field() also performs a lookup for an e-mail address against an SQL map. It first calls lookup_sql() if it hasn't been called yet for this key, requesting it to return all matching records. Instead of returning the whole record as lookup_sql does, it returns just a value of one particular table field, the first one with a defined (non-NULL) value from the list of matching records (or undef if there are none). The lookup_sql_field() is the subroutine that gets called from lookup() for arguments (objects) of type Amavis::Lookup::SQLfield. A field value NULL is translated to a Perl undef, and is treated the same as a nonexistent field of the specified name. For any field (apart from the only exception field 'local') a record with a NULL/undef/nonexistent field is skipped and search continues with the next, more general, matching record until a defined value is found or all matching records are checked. The only exception is a field named 'local', where a nonexistent field on an otherwise matching record implies a value true - see section "Special handling of optional SQL field 'users.local'" further down. Boolean fields are usually represented as a single character (instead of an integer) to minimize storage. Characters N,n,F,f,0,NUL and SPACE represent false (0), any other character represents true. Trailing blanks are ignored. It is customary to use Y for true and N for false. SQL lookups (e.g. for user+foo@example.com) are performed in order which is usually specified by 'ORDER BY...DESC' in the SELECT statement; otherwise the order is unspecified, which is only useful if just specific entries exist in a database (e.g. full address always, not domain part only or mailbox parts only). The following order (implemented by sorting on the 'priority' field in DESCending order, zero is low priority) is recommended, to follow the same specific-to-general principle as in other lookup tables; the first column is a suggested priority (the exact value does not matter as long as the order is maintained): 9 - lookup for user+foo@sub.example.com 8 - lookup for user@sub.example.com (only if $recipient_delimiter is '+') 7 - lookup for user+foo (only if domain part is local) 6 - lookup for user (only local; only if $recipient_delimiter is '+') 5 - lookup for @sub.example.com 3 - lookup for @.sub.example.com 2 - lookup for @.example.com 1 - lookup for @.com 0 - lookup for @. (catchall) NOTE: this is different from hash and ACL lookups in two important aspects: - key without '@' implies a mailbox name, not a domain name; - naked mailbox name lookups (without an '@', e.g. 'user') are only performed when the mail address matches local_domains lookups. The domain part is always lowercased when constructing a key, the localpart is not lowercased when $localpart_is_case_sensitive is true. With SQL datatypes CHAR and VARCHAR this does not matter anyway, as such fields are matched case-insensitively. NOTE: a null reverse path e-mail address used by MTA for delivery status notifications (DSN) has empty local part and empty domain. As far as the lookup is concerned (which uses raw, i.e. non-quoted and non-bracketed address form), this address is @, i.e. a single character "@". The SQL lookup for null address goes through the following sequence of keys: "", "@", "@." (double quotes added for clarity, they are not part of the key). Table names and field names as used for SQL lookups are hard-wired in the routine prepare_sql_queries(). Please adjust it to will. Field names should be unique even without the table prefix. If they are not, the last one in the SELECT field list prevails. For an example schema that can be used with MySQL or PostgreSQL or SQLite see README.sql. Special handling of optional SQL field 'users.local' A special shorthand is provided when SQL lookups are used: when a match for recipient address (or domain) is found in SQL tables (regardless of field values), the recipient is considered local, regardless of static @local_comains_acl or %local_domains lookup tables. This simplifies life when a large number of dynamically changing domains is hosted. To overrule this behaviour, add an explicit boolean field 'local' to table 'users' (missing field defaults to true, meaning record match implies the recipient is local; a NULL field 'local' is not special, it is interpreted as undef like other NULL fields, causing search to continue into other lookup tables). Since amavisd-new-20030616-p7: changed the default value for local_domains_sql lookup for the catchall key '@.' under conditions: when user record with key '@.' is present in the database and a field 'local' is not present. Previously it surprisingly defaulted to true, now it falls back to static lookup table defaults, the same as if the record '@.' were not present in the table or as if the field value 'local' was NULL. Case sentitivity of string comparison Amavisd-new expects string comparison to be case sensitive (but does not mind if it isn't). When forming a SELECT clause it lowercases parts of keys that are supposed to be case-insensitive, such as the domain name. The local part of the e-mail address in SQL search keys is lowercased if and only if the $localpart_is_case_sensitive variable is false (which is a default). This means that case-insensitive parts of e-mail addresses as kept in the SQL database should be in lower case, otherwise match may fail, depending on SQL server behaviour and the use of BINARY prefix in string data types. Since MySQL version 3.23.0 it is possible to declare a data type of a column as BINARY, forcing string comparision to be case sensitive, as it is in PostgreSQL. This is only required for sites that want to treat localpart as case-sentitive and have $localpart_is_case_sensitive true. amavisd-new-2.7.1/README_FILES/README.exim_v4_app000640 000621 000620 00000013273 10301421465 020436 0ustar00markcmi000000 000000 (slightly edited mail to include the relevant references from previous messages) Date: Wed, 09 Oct 2002 18:42:07 +0200 From: Martijn Grendelman To: Mark Martinec Message-id: <053601c26fb2$ce9135c0$270a010a@ilsead.net> From: Martijn Grendelman > The problem occurs when I send a single message to different addresses that > are handled by the same server. For example, martijn@grendelman.net is my > own, valid address and gekkehenkie@grendelman.net is a non-existent address > in the same domain. > When amavis tries to deliver the message to the second instance of Exim > (at port 10025), the mail _as a whole_ gets rejected, because Exim will not > accept _one of the recipients_. Therefor, the firs Exim (at 25) rejects the > message for all recipients as well. Here are some logs to illustrate: From: Mark Martinec | ... There is the third option. I don't know how this is with Exim, | but in the Postfix setup this comes by default and is the reason | why there are not more complaints like yours: the first MTA instance | should be able to check the validity of recipient address. From: Martijn Grendelman > I _think_ there is a way to do this with Exim too, but it requires > a little more hacking in the config file. ... My idea is to duplicate > each router, add an incoming port condition and change it to send > the message to amavisd instead of the original destination if the > condition is met. ... This way, messages will be sent to amavisd > only after a router is matched, and this is what we want. Well, here it is... I decided to be verbose, I hope you don't mind. Feel free to use this in any README's as you see fit. It's not perfect yet, I will explain below. First let me tell you what I did. The filosophy is, that I only want those routers that perform an actual delivery to pass the message to amavisd. Routers that redirect (like system_aliases and userforward) will eventually route the message through a delivering router later, so scanning will take place when that happens. The only problem (the imperfection) is with pipes and appending to files, because in those cases, no other router will be used. This means if you use a pipe or try to append to a file from the aliases file or from a .forward file (i.e. procmail!), no scanning will take place. At this time, I don't have a solution for this yet, and by reading around the Exim docs a bit, I know there can't be an easy one. I think I should mention that I use Exim for "real" as well as virtual mailboxes, using a MySQL backend. The first router, at the top of the list is: amavis: driver = manualroute domains = ! +local_domains condition = "${if or {{eq {$interface_port}{10025}} \ {eq {$received_protocol}{spam-scanned}} \ }{0}{1}}" route_list = "* localhost byname" transport = amavis self = send This one takes care of all non-local domains, i.e. mail that is routed to a remote server later. Since Exim will always accept and queue messages with remote destinations, the problem of amavisd rejecting valid addresses does not occur. Note that "+local_domains" means the use of a variable of type "domainlist", defined earlier like this: LOCAL_DOMAINS = localhost : mydomain.com : myotherdomain.com VIRTUAL_DOMAINS = mysql;MYSQL_DOMAIN domainlist local_domains = LOCAL_DOMAINS : VIRTUAL_DOMAINS MYSQL_DOMAIN is, in its turn, a variable containing an SQL query for retrieving domain names from the database, but let's not go there ;) The second new router is inserted just before the "localuser" router, that matches for local (unix) mailboxes: localuser_amavis: driver = manualroute domains = LOCAL_DOMAINS condition = "${if or {{eq {$interface_port}{10025}} \ {eq {$received_protocol}{spam-scanned}} \ }{0}{1}}" route_list = "* localhost byname" check_local_user transport = amavis self = send This one matches only the real (non-virtual) local domains, and it checks the local part of the recipient with the "check_local_user" directive. The message only gets passed to amavisd, if the local user exists. The third (and last) new router, is the one that matches virtual local users. It is inserted just before the router that handles the actual delivery to those users (included here just to illustrate, since it's not standard Exim configuration; most people won't need this): virtual_localuser_amavis: driver = manualroute domains = ${lookup mysql {MYSQL_USER}{$value}} condition = "${if or {{eq {$interface_port}{10025}} \ {eq {$received_protocol}{spam-scanned}} \ }{0}{1}}" route_list = "* localhost byname" retry_use_local_part transport = amavis self = send virtual_local_user: driver = accept domains = ${lookup mysql {MYSQL_USER}{$value}} retry_use_local_part transport = virtual_delivery The only difference with the earlier routers is the way the local part gets checked. In this case, a MySQL query is used to determine the matching domains, and the actual query makes sure that no domains match if the local part is invalid. I hope this is helpful in some way. Best regards, Martijn. ------------- Date: Mon, 03 May 2004 14:26:25 +0100 From: Tony Middleton tony at middleton dot name Subject: Use of Amavis with EXIM 4. I have been reading the 2 appandices to the README for EXIM 4 related to problems validating addresses. I may be missing the point entirely but I would have thought a simpler solution would have been to put the amavis router at the top with a "no_verify" parameter. This is what I currently do with amavis-ng and it seems to work OK. Please feel free to ignore this message if I am talking rubbish. Regards Tony Middleton. amavisd-new-2.7.1/README_FILES/README.postfix000640 000621 000620 00000132622 10766020475 017732 0ustar00markcmi000000 000000 Integrating amavisd-new in Postfix Patrick Ben Koetter <[1]patrick.koetter@state-of-mind.de> Mark Martinec <[2]Mark.Martinec+amavis@ijs.si> License: GNU GENERAL PUBLIC LICENSE, Version 2, June 1991 +------------------------------------------------------------------------+ | Revision History | |------------------------------------------------------------------------| | Revision 141 | 11. Mar 2008 | PK | |------------------------------------------------------------------------| | Updated parameters that override Postfix defaults for the amavisdfeed | | and the reentry smtpd servers | |------------------------------------------------------------------------| | Revision 139 | 11. Mar 2008 | PK | |------------------------------------------------------------------------| | Added corrections sent in from Chris Pepper | |------------------------------------------------------------------------| | Revision 122 | 15. Jun 2007 | PK | |------------------------------------------------------------------------| | Added Section on Advanced Configuration | |------------------------------------------------------------------------| | Revision 108 | 22. Apr 2007 | PK | |------------------------------------------------------------------------| | Initial publication | +------------------------------------------------------------------------+ Table of Contents [3]1. Requirements [4]1.1. Which Postfix version is required? [5]1.2. Catching errors during integration [6]2. Basic Postfix and amavisd-new configuration [7]2.1. Configuring amavisd-new for Postfix [8]2.2. Configuring the transport from Postfix to amavisd-new [9]2.3. Configuring a dedicated SMTP-server for message reinjection [10]2.4. Testing basic configuration [11]3. Message filtering examples [12]3.1. Filtering E-mail globally [13]3.2. Filtering E-mail by Postfix service [14]3.3. Filtering E-Mails per Recipient Domain [15]3.4. Filtering E-Mails by Sender-Domain [16]3.5. Filtering E-mail per Content [17]4. Advanced Postfix and amavisd-new configuration [18]4.1. Multiple cleanup service architecture [19]4.2. Configuring two cleanup services [20]5. Tuning [21]5.1. Maximum Number of Concurrent Processes [22]5.2. Additional Tips for Tuning Abstract This document describes how amavisd-new can be integrated into the Postfix SMTP delivery process. It lists the necessary requirements, explains how Postfix and amavisd-new need to be configured to basically work together and it gives filter-examples to show how amavisd-new can be called from Postfix. 1. Requirements The following requirements must be met before integration can begin: 1. amavisd-new has already been installed and successfully tested. 2. Postfix has been installed, configured for basic operations and tested successfully. [23][Tip] Tip Independently of the configuration examples shown in this document, it is advisable to set strict_rfc821_envelopes = yes in /etc/postfix/main.cf. Postfix will reject any message from envelope-senders, whose address can't be used to send a reply to. This avoids accepting e-mails from erroneous envelope-senders that can't be informed of problems, which finally would result in deleting the message - even if Postfix claimed successful delivery in the first. 1.1. Which Postfix version is required? Integrating amavisd-new into the Postfix delivery process requires that Postfix is able to delegate messages to external content filters. The minimum version that provides content filtering is Postfix release-20010228. 1.2. Catching errors during integration Chances are that configuration errors during implementation cause Postfix to bounce legitimate messages. Setting the soft_bounce parameter during integration and reloading the Postfix configuration afterwards prevents Postfix from bouncing legitimate mail during that time: # postconf -e "soft_bounce = yes" # postfix reload As soon as soft_bounce has been activated Postfix will treat all delivery errors as temporary errors - any client that wants to send messages to Postfix will keep mail in the mailqueue and it will suspend delivery until the soft_bounce parameter has been removed or set to no. Once the integration of amavisd-new into the Postfix delivery process has been completed successfully soft_bounce must be removed or Postfix will not generate bounce messages for legitimate mail. 2. Basic Postfix and amavisd-new configuration There are several moments at which Postfix can hand messages over to amavisd-new (before it accepts a message from a client or after) and there are different filter approaches (globally, per recipient (domain), per network interface, etc.) that can trigger Postfix to transport a message to amavisd-new. The transport methods - transporting a message from Postfix to amavisd-new and back - however always remain the same. They will be described in this section first. The section that follows will deal with different filtering approaches. [24][Tip] Integration procedure The following examples have been structured to cause minimum trouble on an online mail system. The order of steps ensures that filtering will be enabled at the very last moment. Several tests will have been conducted to verify the delivery chain works before the filter is enabled. Once enabled the complete system should work at once. 2.1. Configuring amavisd-new for Postfix Configuring amavisd-new to work with Postfix answers the following two questions: 1. Which port should the amavisd-new daemon listen to for incoming connections from Postfix? 2. Which IP-address and port should the amavisd-new SMTP client use to (re)inject filtered messages (and notifications about message status) into the Postfix SMTP delivery system? 2.1.1. Configuring amavisd-new for incoming connections The $inet_socket_port parameter in /etc/amavisd.conf sets the port number where amavisd-new will listen for incoming (E)SMTP connections. The following example explicitly configures amavisd-new to bind to port 10024 (default setting undef): $inet_socket_port = 10024; 2.1.2. Configuring the reinjection path Two parameters, $forward_method and $notify_method, need to be configured (usually identically) to reinject messages into the Postfix mail system. The first parameter, $forward_method, specifies where amavisd-new should transport scanned messages to, while the second parameter, $notify_method, specifies where notifications about scanned messages should be transported to. By default amavisd uses 127.0.0.1 on port 10025 to contact a SMTP server for reinjection of filtered messages. Unless a different IP address or port should be used, no modifications must be applied and this section can be skipped. In case a different IP address or port should be used, the parameters $notify_method and $forward_method need to be adjusted to reflect these requirements. The following example edits these parameters in /etc/amavisd.conf and uses 192.0.2.1 as IP address and port 20025: $notify_method = 'smtp:[192.0.2.1]:20025'; $forward_method = 'smtp:[192.0.2.1]:20025'; 2.2. Configuring the transport from Postfix to amavisd-new Both amavisd-new and Postfix are able to use either SMTP- or LMTP-communication to transport a message from Postfix to amavisd-new. Both variants will be described in this section. Why configure a dedicated service? Theoretically it's possible to transport messages from Postfix to amavisd-new using the existing smtp-, lmtp, or even the relay-service in /etc/postfix/master.cf. In practice transporting messages to amavisd-new requires imposing transport limits on the transporting service. Imposing such limits on a globally available service would impose these limits on the complete Postfix mail system - it would slow down the system significantly and should be avoided. [25][Note] Note The number of Postfix clients that may connect simultaneously to amavisd-new instances must be limited to the maximum number of daemon child processes amavisd-new starts. If the Postfix transport client was allowed to open more connections than amavisd-new can handle, amavisd-new would start to queue incoming Postfix connections. Postfix in turn would interpret such behaviour as "unresponsive remote MTA" and would itself begin to queue mail that should be filtered. All this would possibly throttle down the complete system and all further filtering attempts would suffer. 2.2.1. Configuring a dedicated lmtp-client The following example creates a new, dedicated lmtp-transport named amavisfeed in /etc/postfix/master.cf. Its configuration details are explained following the listing: # ========================================================================== # service type private unpriv chroot wakeup maxproc command + args # (yes) (yes) (yes) (never) (100) # ========================================================================== ... amavisfeed unix - - n - 2 lmtp -o lmtp_data_done_timeout=1200 -o lmtp_send_xforward_command=yes -o lmtp_tls_note_starttls_offer=no [26][Important] Important A noteworthy quote from the Postfix documentation: "...do not specify whitespace around the `='. In parameter values, either avoid whitespace altogether, ...". Further details on master.cf configuration syntax can be found in master.cf or master(5). Here's a quick rundown on the settings that differ from other services defaults: maxproc The maximum number of concurrent Postfix amavis-service processes has been limited to 2 (default: default_process_limit = 100). This value reflects the default of 2 amavisd-daemon children processes and is a good setting to start from. The value may be raised later, when the system works stable and still can take a higher load. It should not exceed the number of simultaneous amavisd child processes. lmtp_data_done_timeout Setting lmtp_data_done_timeout to 1200 (seconds) doubles the default time span a regular Postfix client waits after message delivery for the server to reply DONE to claim successful delivery. It must be larger than amavisd setting $child_timeout (default 8*60 seconds) and should add a sufficient safety margin, for example to cater for periods of automatic database maintenance (e.g. bayes database on non-SQL database types) which can take a long time in some cases. If the server does not reply within the configured time span, the Postfix client will quit the connection, put the message into the deferred queue, log a delivery failure and retry later to transport the message to amavisd-new. [27][Note] Note Raising this value serves a trick amavisd uses to avoid message loss in case of power outage etc. The trick consists in keeping the incoming connection as long open as it takes to filter the message and take appropriate action (reinjection, notification, quarantine, etc.). Only when the message (or notifications etc.) has been reinjected amavisd will send DONE to the client and the client will close the connection. This way Postfix will always keep the message in its own mail queue, where it can be reactivated after a system failure. lmtp_send_xforward_command Enabling lmtp_send_xforward_command configures the Postfix lmtp-client to forward the original clients HELO name and IP address to amavisd-new. amavisd-new in turn can use this information for * logging and notifications (macro %a) * switching policy banks (MYNETS, @mynetworks_maps) * pen pals functionality * p0f fingerprinting lmtp_tls_note_starttls_offer Starting with version 2.6 amavisd-new can offer TLS to a smtp- or lmtp-client. This option cuts down unnecessary logging by Postfix, just in case logging TLS session offers has been enabled globally setting lmtp_tls_note_starttls_offer in Postfix main.cf configuration file. 2.2.2. Configuring a dedicated smtp-client Configuring a dedicated smtp-client is almost identical to configuring a dedicated lmtp-client. The syntax differences in detail are that the names of parameters start with smtp_ instead of lmtp_ and that the command at the end of the service invokes the smtp- and not lmtp-client. The same reasons given for differing lmtp client options apply to the dedicated smtp client configuration. Here's an example of a dedicated smtp client given the service name amavisfeed: # ========================================================================== # service type private unpriv chroot wakeup maxproc command + args # (yes) (yes) (yes) (never) (100) # ========================================================================== ... amavisfeed unix - - n - 2 smtp -o smtp_data_done_timeout=1200 -o smtp_send_xforward_command=yes -o smtp_tls_note_starttls_offer=no 2.3. Configuring a dedicated SMTP-server for message reinjection The second service that needs to be added to the Postfix mail system is a dedicated SMTP-server. It will exist only to accept filtered messages and notifications from amavisd-new to transported them closer to their final destination. This dedicated smtpd server will differ in many aspects from the default smtpd daemon. The most important difference is that it configures an empty content_filter parameter, thus overriding any global external content filtering settings in Postfix. [28][Note] Note Delegating messages to an external content filter in Postfix is done using the content_filter parameter. If the dedicated smtpd-daemon would not override any global content_filter settings, the reinjected message would be sent of to the external content filter again - the mail would end in an endless loop. The following Postfix example uses amavisd-new default settings taken from the $forward_method and $notify_method parameters. These settings configure amavisd-new to forward filtered messages and notifications to 127.0.0.1 on port 10025; the Postfix smtpd daemon will be configured to bind to that IP address and listen on the specified port for incoming connections: # ========================================================================== # service type private unpriv chroot wakeup maxproc command + args # (yes) (yes) (yes) (never) (100) # ========================================================================== ... 127.0.0.1:10025 inet n - n - - smtpd -o content_filter= -o smtpd_delay_reject=no -o smtpd_client_restrictions=permit_mynetworks,reject -o smtpd_helo_restrictions= -o smtpd_sender_restrictions= -o smtpd_recipient_restrictions=permit_mynetworks,reject -o smtpd_data_restrictions=reject_unauth_pipelining -o smtpd_end_of_data_restrictions= -o smtpd_restriction_classes= -o mynetworks=127.0.0.0/8 -o smtpd_error_sleep_time=0 -o smtpd_soft_error_limit=1001 -o smtpd_hard_error_limit=1000 -o smtpd_client_connection_count_limit=0 -o smtpd_client_connection_rate_limit=0 -o receive_override_options=no_header_body_checks,no_unknown_recipient_checks,no_milters -o local_header_rewrite_clients= -o smtpd_milters= -o local_recipient_maps= -o relay_recipient_maps= Here's a quick rundown on the settings that differ from smtpd defaults: content_filter The empty content_filter overrides other, globally set content_filter delegations. ..._maps Empty ..._maps override any other globally set map lookups. Procedures to enforce settings specified in such maps have already taken place when Postfix accepted the message from the external client. Doing them again will not produce new results but only waste resources. ..._restrictions_... There's no need to apply any already enforced ..._restrictions_... another time. It would also only waste resources. mynetworks To avoid abuse from remote hosts, the dedicated smtpd-daemon will only allow clients from 127.0.0.0/8 to relay messages. local_header_rewrite_clients By default this option would "rewrite message header addresses in mail from these clients and update incomplete addresses with the domain name". If such action has already been taken by Postfix before the message went off to amavis, it should not be done a second time when it reenters the Postfix mail system. Leaving this option empty disables local header rewrites and saves resources. remaining options All remaining options either configure the dedicated smtpd-daemon to be more failure tolerant or exist to avoid unnecessary use of resources. Running the postfix reload will activate the new transports (Postfix will not yet send regular mail to amavisd). Combined with the tail command problems can easily be detected: # postfix reload && tail -f /var/log/maillog If there are no problems reported, basic configuration can be tested. 2.4. Testing basic configuration Testing basic configuration consists of three separate tests, starting at the end of the new delivery chain and working to it's beginning. Their goal is to answer the following questions: 1. Will amavisd-new accept connections at the specified IP address and port? 2. Will the new dedicated smtpd-daemon accept connections at the specified IP address and port? 3. Will a test message, injected into amavisd-new, be filtered, sent to Postfix and delivered into a mailbox? 2.4.1. Testing amavisd's host and port A test, using the telnet command, serves to verify that amavisd listens on the specified IP address and port. A successful connection looks like this: $ telnet localhost 10024 220 [127.0.0.1] ESMTP amavisd-new service ready EHLO localhost 250-[127.0.0.1] 250-VRFY 250-PIPELINING 250-SIZE 250-ENHANCEDSTATUSCODES 250-8BITMIME 250-DSN 250 XFORWARD NAME ADDR PROTO HELO QUIT 221 2.0.0 [127.0.0.1] amavisd-new closing transmission channel If the test fails, the following questions may help to debug the problem: * Is the amavisd-new daemon running? * Does amavisd-new write an error to the log? * Do the IP address and port number specified in the amavisd-new configuration match the values used during the test? * Does a firewall intercept connections? 2.4.2. Testing the dedicated Postfix smtpd-daemon When Postfix was reloaded, the new, dedicated smtpd-daemon (127.0.0.1:10025) should have been activated. A successful connection looks like this: $ telnet 127.0.0.1 10025 220 mail.example.com ESMTP Postfix (2.3.2) EHLO localhost 250-mail.example.com 250-PIPELINING 250-SIZE 40960000 250-ETRN 250-STARTTLS 250-AUTH PLAIN CRAM-MD5 LOGIN DIGEST-MD5 250-AUTH=PLAIN CRAM-MD5 LOGIN DIGEST-MD5 250-ENHANCEDSTATUSCODES 250-8BITMIME 250 DSN QUIT 221 2.0.0 Bye If the test fails, the following questions may help to debug the problem: * Is the Postfix master daemon running? * Does Postfix write an error to the log? * Do the IP address and port number specified in the new services configuration match the values used during the test? * Does a firewall intercept connections? 2.4.3. Testing the new transport chain This test proves amavisd accepts e-mail as specified in [29]Section 2.1, "Configuring amavisd-new for Postfix", filters it and finally hands it over to Postfix' dedicated smtpd-daemon as specified in [30]Section 2.3, "Configuring a dedicated SMTP-server for message reinjection". The following example uses the content of test-messages/sample-nonspam.txt from the amavisd test-messages to send an e-mail: $ telnet localhost 10024 220 [127.0.0.1] ESMTP amavisd-new service ready HELO localhost 250 [127.0.0.1] MAIL FROM: <> 250 2.1.0 Sender OK RCPT TO: 250 2.1.5 Recipient postmaster OK DATA 354 End data with . From: virus-tester To: undisclosed-recipients:; Subject: amavisd test - simple - no spam test pattern This is a simple test message from the amavisd-new test-messages. . 250 2.6.0 Ok, id=30897-02, from MTA([127.0.0.1]:10025): 250 2.0.0 Ok: queued as 079474CE44 QUIT 221 2.0.0 [127.0.0.1] amavisd-new closing transmission channel The maillog shows the delivery path. Here's an excerpt from a successful delivery process: Nov 1 11:28:10 mail postfix/smtpd[30986]: connect from localhost[127.0.0.1] [31]1 Nov 1 11:28:10 mail postfix/smtpd[30986]: 079474CE44: client=localhost[127.0.0.1] Nov 1 11:28:10 mail postfix/cleanup[30980]: 079474CE44: message-id=<20061101102810.079474CE44@mail.example.com> Nov 1 11:28:10 mail postfix/qmgr[20432]: 079474CE44: from=<>, size=822, nrcpt=1 (queue active) Nov 1 11:28:10 mail amavis[30897]: (30897-02) Passed BAD-HEADER, <> -> , quarantine: badh-le5gjszxowBk, mail_id: le5gjszxowBk, Hits: -1.76, queued_as: 079474CE44, 39505 ms [32]2 Nov 1 11:28:10 mail postfix/smtpd[30986]: disconnect from localhost[127.0.0.1] Nov 1 11:28:10 mail postfix/local[30987]: 079474CE44: to=, relay=local, delay=0.27, delays=0.14/0.05/0/0.08, dsn=2.0.0, status=sent (delivered to mailbox: postmaster) [33]3 Nov 1 11:28:10 mail postfix/qmgr[20432]: 079474CE44: removed [34]1 amavisd connects with Postfix dedicated smtpd-daemon and hands over the e-mail that had been sent during the telnet session. smtpd gives a queue-id of 079474CE44 that can be tracked throughout the maillog. [35]2 amavisd notices it has checked and sent an e-mail to . [36]3 Postfix' local-service logs it successfully delivered an e-mail with queue-id 079474CE44 to the mailbox of postmaster. If the test fails, the following questions may help to debug the problem: * Does amavisd-new log errors? * Does running amavisd-new in debug-mode report errors? 3. Message filtering examples Postfix can use various criteria to decide whether a message should be sent to amavisd-new for examination. Combinations of criteria may serve to create different configurations. The following section describes the following configurations: * Filtering e-mail globally * Filtering e-mail globally by service * Filtering e-mail per recipient domain * Filtering e-mail per sender domain * Filtering e-mail by content 3.1. Filtering E-mail globally In most cases email policies require global filtering - every inbound and every outbound e-mail must be filtered by amavisd-new - before it may be sent closer to its final destination. [37][Note] Why check outgoing mail traffic? Some reasons for checking mail coming from internal networks or from authenticated roaming users are: * detect an internal infected PC which is sending viruses * detect an internal zombiized PC (or an internal open relay or proxy) which is sending or relaying spam * let the SpamAssassin Bayes autolearning feature see a balanced view of all mail, including useful samples of non-spam originating from inside * make it possible for pen pals feature to function (if enabled) In Postfix global settings for its services are written to main.cf. The content_filter parameter, the parameter configuring that messages are sent to amavisd-new, must therefore be placed in main.cf. The content_filter parameter requires a triplet, consisting of the transport service's name (here: amavisfeed, given in [38]Section 2.2.1, "Configuring a dedicated lmtp-client"), the target hosts IP address and the port where amavisd-new listens for incoming connections. Following the values used in this documents examples the content_filter configuration results in this: content_filter=amavisfeed:[127.0.0.1]:10024 The new external content filter will be activated once Postfix has been reloaded. Sending a test-mail verifies the system works. 3.2. Filtering E-mail by Postfix service Postfix is able to filter messages per service. Such configuration requires the content_filter not to be applied globally to all services in main.cf (see: [39]Section 3.1, "Filtering E-mail globally"), but selectively, per service in master.cf. The following example presumes Postfix runs on a system offering three IP addresses. In this example these are: 192.0.2.1 (WAN), 127.0.0.1 (localhost) and 10.0.0.254 (LAN). The goal is to filter only e-mail that enters from the WAN interface. This requires to create three dedicated smtpd-daemon instances, each binding to one of the given IP addresses and deactivating the global smtp service calling the smtpd command. Additionally the WAN interface (here: 192.0.2.1:25) is configured to use content_filter =amavisfeed:[127.0.0.1]:10024 - it will delegate any message that enters the Postfix mail system at this service to the external amavisd content filter. # ========================================================================== # service type private unpriv chroot wakeup maxproc command + args # (yes) (yes) (yes) (never) (100) # ========================================================================== # smtp inet n - n - - smtpd ... 192.0.2.1:25 inet n - n - - smtpd -o content_filter=amavisfeed:[127.0.0.1]:10024 -o receive_override_options=no_address_mappings 10.0.0.254:25 inet n - n - - smtpd 127.0.0.1:10025 inet n - n - - smtpd -o content_filter= -o smtpd_delay_reject=no -o smtpd_client_restrictions=permit_mynetworks,reject -o smtpd_helo_restrictions= -o smtpd_sender_restrictions= -o smtpd_recipient_restrictions=permit_mynetworks,reject -o smtpd_data_restrictions=reject_unauth_pipelining -o smtpd_end_of_data_restrictions= -o smtpd_restriction_classes= -o mynetworks=127.0.0.0/8 -o smtpd_error_sleep_time=0 -o smtpd_soft_error_limit=1001 -o smtpd_hard_error_limit=1000 -o smtpd_client_connection_count_limit=0 -o smtpd_client_connection_rate_limit=0 -o receive_override_options=no_header_body_checks,no_unknown_recipient_checks,no_milters -o local_header_rewrite_clients= -o smtpd_milters= -o local_recipient_maps= -o relay_recipient_maps= 3.3. Filtering E-Mails per Recipient Domain Postfix is able to filter e-mails per recipient domain. In order to do this the content_filter parameter must not be set globally (see: [40]Section 3.1, "Filtering E-mail globally"). Instead the content_filter parameter has to be associated with one or more recipient domains listed in a lookup table (map). [41][Caution] Caution This filter method is not selective! It will send any mail with a recipient domain listed in the lookup table to amavis even if the mail contains another recipient that should not be examined by the amavis framework. If fully selective rules are required all mail should be sent to amavis and amavis' own rule sets should be configured to decide whether a message for a given recipient should be examined or not. When Postfix searches the lookup table and finds the recipients domain listed as key, it will take the action associated with that domain. The action will send the message to a FILTER amavisfeed:[127.0.0.1]:10024. The following map /etc/postfix/filter_recipient_domains specifies to send messages to the FILTER amavisfeed whenever a message for any recipient at example.com enters the Postfix mailqueues: example.com FILTER amavisfeed:[127.0.0.1]:10024 Once the table has been created the postmap command must be used to create an indexed map Postfix can read: # postmap /etc/postfix/filter_recipient_domains Once the map has been indexed, the postmap command is used to test the map. In the following example the postmap command queries for the domain example.com and returns the associated action: # postmap -q "example.com" /etc/postfix/filter_recipient_domains FILTER amavisfeed:[127.0.0.1]:10024 The tested map must be added to main.cf, before Postfix can make use of the new filter policy. Setting the check_recipient_access parameter in the list of smtpd_recipient_restrictions triggers evaluation of entries in the map - check_recipient_access is triggered by the envelope-recipient(s) given by a SMTP-client in a SMTP-session with Postfix. The following example puts the check_recipient_access rule before permit_mynetworks - all clients envelope-recipient(s) will be filtered: smtpd_recipient_restrictions = ... check_recipient_access hash:/etc/postfix/filter_recipient_domains ... permit_mynetworks reject_unauth_destination ... Filtering E-Mails per Recipient Domain only from External Clients This example puts the check_recipient_access rule after permit_mynetworks - only messages sent from clients that are not in Postfix $mynetworks list (external or untrusted clients) will be filtered: smtpd_recipient_restrictions = ... permit_mynetworks reject_unauth_destination check_recipient_access hash:/etc/postfix/filter_recipient_domains ... 3.4. Filtering E-Mails by Sender-Domain In general it doesn't make sense to filter e-mails by sender-domain, as anyone can fake a sender-domain during a SMTP-session. Filtering by sender-domain will probably only make sense, if messages are not filtered globally, but e-mails from ones own domain should be checked for spam or viruses before they leave the network. Most of the configuration steps are identical with the ones noted in [42]Section 3.3, "Filtering E-Mails per Recipient Domain", except for the parameter that triggers evaluation of the indexed map. In this scenario envelope-senders should trigger map evaluation. The map, named /etc/postfix/filter_sender_domains this time, contains the sender domain (example.com) and associates it with the required FILTER: example.com FILTER amavisfeed:[127.0.0.1]:10024 Once the map has been converted and tested with the postmap command (see: [43]Section 3.3, "Filtering E-Mails per Recipient Domain") it must be added to the list of smtpd_recipient_restrictions using the check_sender_access parameter: smtpd_recipient_restrictions = ... check_sender_access hash:/etc/postfix/filter_sender_domains ... permit_mynetworks reject_unauth_destination ... [44][Important] Important The map must be listed before permit_mynetworks, because only then it will be applied to all clients - even the ones Postfix trusts, which are very likely the ones from example.com. 3.5. Filtering E-mail per Content Postfix is able - with deliberate limitations (see: BUILTIN_FILTER_README) - to search for strings in headers, the body and MIME-headers. If a string matches, Postfix may call appropriate action. The following example configures Postfix to look for the string offer in Subject:-headers and delegate the message to an external content filter if if finds a matching string. A map, consisting of the search string noted as regexp-expression, associates the search pattern with a FILTER action: /^Subject:.*offer/ FILTER amavisfeed:[127.0.0.1]:10024 [45][Note] Indexing regexp- or pcre-maps? regexp- or pcre-maps are and must be plaintext files. They must not and cannot be converted to an indexed map using the postmap command. They can be tested using the postmap command using the -q command line option. Once the map has been created, Postfix must be configured to use it. The following example uses the header_checks parameter (not body_checks or mime_header_checks as they apply to other message parts) to implement the map into the Postfix delivery process: header_checks = regexp:/etc/postfix/filter_header Once Postfix has been reloaded it will send every e-mail that contains the word offer in the Subject:-header off to the external amavisd content filter. 4. Advanced Postfix and amavisd-new configuration In a post-queue content filtering setup, a mail message passes through smtpd and cleanup Postfix services twice, once before a content filter, and the second time when an approved message is reinjected from a content filter into the Postfix mail system. This is because checks and transformations that have been configured in main.cf are globally active and will be loaded and run by any instance of these two services. To avoid wasting resources, options that control runtime behavior of these services should not be applied globally in main.cf, but selectively to separate instances of these services in master.cf. Checks and transformations which are performed by a smtpd Postfix service itself, e.g. access controls, recipient validation, milters etc., can be controlled by adding options (-o) to appropriate smtpd services. This has been shown in the basic configuration examples (see: [46]Section 2.3, "Configuring a dedicated SMTP-server for message reinjection"). Checks and transformations which are performed by a cleanup Postfix service are trickier because in a normal Postfix setup there is only one cleanup service, unlike smtpd services of which there are many. Some of the more important cleanup settings are dynamically controllable by a smtpd service through the use of its receive_override_options option. [47][Tip] Transformations and checks Any transformation should preferably only be performed once, either before or after content filtering. When to transform depends on the desired effect, for example whether a content filter should see unchanged or modified mail messages. Typical transformations are: * rewrite addresses * add BCC recipients * modify mail header. Most checks should also be performed only once, preferably only on the first passage, when the mail enters the Postfix mail system the first time. This way messages can be rejected early - if needed - and will not tie up downstream resources. Checking early also avoids bounces in case of negative check results on a second passage after content filtering. 4.1. Multiple cleanup service architecture To gain more control over a cleanup service than offered by receive_override_options, two (or more) cleanup services, each with its own set of options, must be run. A Postfix setup with more than one cleanup service is possible either with two separate Postfix instances, or through a specification of services and their options in master.cf file of a single Postfix instance. The following diagram illustrates a setup with two cleanup services in a single Postfix instance: ....................................... : Postfix : ----->smtpd \ : : -pre-cleanup-\ /local----> ---->pickup / -queue- : : -cleanup-/ | \smtp-----> : bounces/ ^ v : : and locally | v : : forwarded smtpd amavisfeed : : messages 10025 | : ...........................|........... ^ | | v ............|............................... : | $inet_socket_port=10024 : : | : : $forward_method='smtp:[127.0.0.1]:10025' : : $notify_method ='smtp:[127.0.0.1]:10025' : : : : amavisd-new : ............................................ Procedure 1. Message flow with two cleanup services 1. Messages enter the Postfix system at the regular smtpd or pickup service. 2. The pre-cleanup cleanup service performs transformations and checks on these messages. 3. The qmgr service schedules the messages to be sent to the amavisd-new content filter. 4. amavisd-new performs various tests on the messages. 5. Messages are re-injected into the Postfix mail system, sending them to a dedicated, local smtpd service. 6. The cleanup cleanup service performs transformations and checks that must be done at this stage, but omits the ones that have already been carried out in step 2. 4.2. Configuring two cleanup services Configuring Postfix smtpd services to use two separate, dedicated cleanup services requires the following steps: 1. Create a second cleanup instance 2. Modify the existing cleanup service 3. Configure smtpd services to use either of the two cleanup services. 4.2.1. Creating a second cleanup instance The following example adds a cleanup daemon named pre-cleanup. It will handle messages before a content filter. # ========================================================================== # service type private unpriv chroot wakeup maxproc command + args # (yes) (yes) (yes) (never) (100) # ========================================================================== # smtp inet n - n - - smtpd ... pre-cleanup unix n - n - 0 cleanup -o virtual_alias_maps= The above leaves canonicalization address rewriting enabled so that a content filter will see canonicalized (external) sender mail addresses, but it disables globally configured virtual alias transformations. Such transformations will be done later by the second cleanup service, so that a content filter will see original (external) recipient mail addresses. Other options may also be used as needed. 4.2.2. Modifying the existing cleanup service The already existing cleanup service - having the service name cleanup - will be used to process messages that re-enter the Postfix mail system (also for delivery notifications and forwarding as generated internally by Postfix). Cleanup jobs that already have been performed by the pre-cleanup service should not be run again. The following example disables typical checks that have been run before or are not needed for internally generated notifications: # ========================================================================== # service type private unpriv chroot wakeup maxproc command + args # (yes) (yes) (yes) (never) (100) # ========================================================================== # smtp inet n - n - - smtpd ... cleanup unix n - n - 0 cleanup -o mime_header_checks= [48]1 -o nested_header_checks= [49]2 -o body_checks= [50]3 -o header_checks= [51]4 [52]1 The specified options disable header and body checks as these would [53]2 already be performed by a pre-cleanup service. [54]3 [55]4 [56][Note] always_bcc This cleanup service would also be the appropriate one for specifying always_bcc option - doing it globally would apply to both cleanup services and would result in two copies of each message to be sent to the specified address. 4.2.3. Configuring smtpd services Finally existing smtpd services on ports 25 and 587 (submission), and the pickup service must be configured to send messages to the new pre-cleanup service instead of a default cleanup service: # ========================================================================== # service type private unpriv chroot wakeup maxproc command + args # (yes) (yes) (yes) (never) (100) # ========================================================================== # smtp inet n - n - - smtpd ... pickup fifo n - n 60 1 pickup -o cleanup_service_name=pre-cleanup smtp inet n - n - - smtpd -o cleanup_service_name=pre-cleanup submission inet n - n - - smtpd -o cleanup_service_name=pre-cleanup 5. Tuning 5.1. Maximum Number of Concurrent Processes The most important settings to tune and optimize in Postfix and amavisd workflow are the maximum number of concurrent processes. The maximum number of concurrent processes on both sides must be chosen with care. If the number is too low, hardware resources aren't used efficiently and delivery time will be unnecessarily prolonged. Experience tells that raising the number of processes a little, will not raise the overall throughput in the same proportion. As the system resources are nearing saturation with each increase of the number of processes, an increase in throughput becomes marginal, and eventually even negative when the number of processes exceeds its near-optimum value. E-mail throughput will decrease, because processes need to wait for each other. At worst e-mail delivery stalls. Best practice is to start with a (conservative) maximum number of 2 concurrent processes. Everyday use has shown that this value may be raised to a value between 10 and 30 concurrent Postfix client and amavisd server processes. This also depends on the overall resources the system may provide, how amavisd has been integrated into the Postfix delivery process and on the anti-virus and anti-spam software being loaded and used by amavisd-new. Regardless of the maximum number of concurrent processes, both sides - Postfix and amavisd - should be synchronized. To synchronize both sides edit, the $max_servers parameter for amavisd-new (see: amavisd.conf) and the number of processes in master.cf listed in the dedicated transports maxproc column for Postfix. Both values should be identical for two reasons: If amavisd-new offers more processes than Postfix will ever use, amavisd-new wastes resources. On the other hand, if Postfix starts more dedicated transports than amavisd can handle simultaneously, e-mail transport will be refused and logged as error. [57][Note] Controlling the maximum number of concurrent processes in main.cf Instead of controlling the maximum number of concurrent processes of Postfix' dedicated transport in master.cf it is also possible to keep the default setting - in master.cf and set the following parameter and option in main.cf: amavisfeed_destination_concurrency_limit = 2 The name of the parameter starts with the service in master.cf (here: amavisfeed) that should be controlled and goes on with the suffix _destination_concurrency_limit. Here also 2 is set as initial (conservative) value. 5.2. Additional Tips for Tuning Further Tuning-Tips can be found in README.performance and the slides from [58]amavisd-new, advanced configuration and management. References Visible links 58. http://www.ijs.si/software/amavisd/amavisd-new-magdeburg-20050519.pdf amavisd-new-2.7.1/README_FILES/README.exim_v3_app000640 000621 000620 00000010454 10233024273 020433 0ustar00markcmi000000 000000 Date: Wed, 03 Sep 2003 23:17:12 +0200 From: Andreas Zeidler Subject: [AMaViS-user] smtp only setup with exim 3.x (request for comments) To: AMaViS-user Message-id: <20030903211712.GA12537@kreativkombinat.de> hi, during the last two days i've finally found some time to setup amavisd-new with exim v3, clamav and spamassassin. after looking around the web for a sample configuration, i found some remarks about shortcomings of the amavis.c approach [1]. that is, feeding the mail in question to amavisd by defining a transport like ... amavis: driver = pipe command = "/usr/sbin/amavis ${sender_address} ${pipe_addresses}" also, judging from the comments in amavisd.conf, it seemed to me that returning the now checked mail by... $forward_method = 'pipe:flags=q argv=/usr/sbin/exim -oMr scanned-ok -i -f ${sender} -- ${recipient}'; is not preferable to using regular smtp. while i've read about those problems, i do not know under which exact circumstances the above methods would cause trouble. anyway, putting together several pieces and reading the exim documentation i've come up with a setup that uses smtp both ways and seems to work fine so far (it's been running for some 24 hours on our mail server now). so, the reason i'm writing is that i'd like some comments about this setup, since i wouldn't know enough details to be sure there are no other problems with it. following are the relevant parts from the configuration of amavisd and exim. clamd and spamd are installed with their respective default configuration. the transport is defined as... amavis: driver = smtp hosts = localhost port = 10024 allow_localhost # transport_filter = "/usr/bin/spamc" which causes exim to relay the mail to amavisd listening on the local port 10024 (the default). thanks to a (indeed) neat suggestion [2] the need for a second round-trip through spamassassin and several more (exim) drivers is gone by using spamc as a transport filter. also, the generated spam-headers are conserved nicely. the transport method for the way back is defined (in amavisd.conf), so that the mail is re-injected into exim on the regular smtp port. afaik exim 3.3 doesn't support listening on several ports at once, so port 10025 is not possible... $forward_method = 'smtp:127.0.0.1:25'; $notify_method = $forward_method; $localhost_name = "amavis"; $relayhost_is_client = 0; the 'localhost_name' setting is necessary in order to distinguish amavis from other processes using smtp via localhost. the director is defined accordingly... amavis_director: condition = "${if and {{eq {$sender_host_address}{127.0.0.1}} \ {eq {$sender_helo_name}{amavis}}} {0}{1}}" driver = smartuser transport = amavis verify = false the condition tests against the helo name provided by amavis and set up in amavisd.conf as described above. at the same time this is the only flaw i can see with this setup so far. a local user could use this helo name to prevent the mail from being scanned by amavis, but on the other hand that's also possible with the suggested setup for exim 4.x (re-injecting through port 10025). also, at least on our mail server local users are pretty rare, so that's no problem... for exim to receive a proper sender address it is also necessary to make the user running amavisd trusted... trusted_users = mail:amavis finally, if outgoing mail should be scanned as well, a possible definition for a route could be... amavis_router: condition = "${if and {{eq {$sender_host_address}{127.0.0.1}} \ {eq {$sender_helo_name}{amavis}}} {0}{1}}" driver = domainlist transport = amavis route_list = "* localhost byname" verify = false self = send i think that's about everything relevant. like i said, this setup is working fine here so far. what do you think? would this be an okay way to run things or are there problems to be expected? take care, andi [1] http://marc.theaimsgroup.com/?t=103014542500001&r=1&w=2 [2] http://marc.theaimsgroup.com/?l=exim-users&m=102977722707468&w=2 -- Kreativkombinat GbR Konrad-Adenauer-Allee 25 * 86150 Augsburg Telefon +49 821 4441269 * Fax +49 821 4401310 Web http://www.kreativkombinat.de/ amavisd-new-2.7.1/README_FILES/README.exim_v4_app2000640 000621 000620 00000077505 10271127632 020535 0ustar00markcmi000000 000000 Date: Thu, 22 May 2003 00:59:38 -0700 (PDT) From: Louis Erickson Subject: amavisd-new and Exim 4.x. ... Second, I wanted to share with you an improved way to run amavisd-new with Exim. Exim is my preferred MTA these days, and I've figured out an improvement on the system which Martijn Grendelman has provided. As Mr. Grendelman did and as is typical for me, I'm going to go in great and exhausting detail. In a nutshell, the problem we had to solve was to keep Exim from passing invalid RCPT TO lines to amavisd-new, because the second Exim instance would cause amavisd to reject the whole message instead of just the single recipient. Mr. Grendelman tries to solve that problem by having each router that will perform a local delivery pass the message through amavisd. He admits this has a flaw with routers which can be used to redirect mail to another address or to feed the message to a pipe or append it to a file. The standard redirect router can do this while parsing /etc/aliases, and that made his solution unworkable in my configuration. His work did clarify the problem, and get me thinking about other ways to solve it. I would have never found the solution I did without his suggestions. After thinking about it for a while and reading the Exim documentation, I was able to turn the problem around backwards. I figured out a way to get amavisd in the process for every local delivery, only for validated addresses. Single invalid addresses are ignored, regardless of the router or transport which handles them. Rather than trying to add code to each router that can deliver, I added routers to validate the addresses the same way the real delivery routers would. If one router accepts the address it is given to amavis. If no routers accept the address it will fail before amavis gets it. To do this I put copies of any router that can do local delivery before the amavis router. Those copies are set to verify_only, and will only be used for RCPT checking. As they are verify_only they have no transport set and will never deliver a message, only check RCPTs. They have the identical conditions and drivers as the corresponding delivery routers. Each address check router is set to pass a successfully validated address to the amavis router. Between those and the amavis router, I put in a verify_only router which will always fail. This is what scrubs out invalid addresses before they get to amavis. This router is skipped by the pass_router settings on the address check routers. After that, I have the amavis router, pretty much exactly as suggested in the Exim README, and the normal local delivery routers which will eventually handle the messages. The default configuration file contains the following routers, in the following order - order is important: domain_literal # Deliver directly to [xx.xx.xx.xx] addresses - not local dnslookup # Deliver to remote smtp sites - not local system_aliases # Deliver to /etc/aliases - may be local! userforward # Handle .forward files - may be local! localuser # Deliver to local mailboxes - local! When the configuration I'm using is done, it would have: check_domain_literal # Verify addresses pre-amavisd check_dnslookup # Verify addresses pre-amavisd check_system_aliases # Verify addresses pre-amavisd check_localuser # Verify addresses pre-amavisd failed_address_router # Always fails! amavis # Deliver to amavisd-new on 127.0.0.1:10024 domain_literal # Deliver directly to [xx.xx.xx.xx] addresses dnslookup # Deliver to remote smtp sites system_aliases # Deliver to /etc/aliases userforward # Handle .forward files localuser # Deliver to local mailboxes Note that there is no check_userforward because userforward cannot fail an address, as it is set for no_verify. The configuration I usually run has at least three other routers in it to handle my virtual domains. There is an amavis transport, also pretty much out of the Exim README. It is very simple and just forwards the messages to 127.0.0.1 on port 10024. They say a picture is worth a thousand words, and I've tried to draw one above. I think an example is probably also worth a thousand words, and I've modified the stock Exim configuration file to include amavisd-new. Note that you can't quite just drop this in and go; you'll have to set the primary_hostname value correctly. Look for mail.example.com and change it. Please note that I have NOT tested the sample configuration file I'm sending you! It's a modification of the default file to show what I'm discussing. I made sure Exim would accept it as a valid file, but I can't install it on my local machine here because of the other features I need. I believe it works, but I have NOT actually tested it. My configuration never runs amavisd for remote smtp deliveries, as I'm only using SpamAssassin, and I have several other routers. My customers would be grumpy if I shut off all their virtual domains. =) I'm happy to try and explain any of this that isn't clear, or to help others understand what I've done and how to do it to their configuration. This should be a general solution which can be applied to any Exim configuration. Please feel free to edit my message and/or to distribute it and the sample file as you see fit. Thanks again for all your work! -- Louis Erickson - lerickson@rdwarf.net - http://www.rdwarf.com/~wwonko/ Content-type: TEXT/PLAIN; charset=US-ASCII; name="exim-configure.amavisd" ###################################################################### # Runtime configuration file for Exim # # modified for amavisd-new # ###################################################################### # This is a default configuration file which will operate correctly in # uncomplicated installations. Please see the manual for a complete list # of all the runtime configuration options that can be included in a # configuration file. There are many more than are mentioned here. The # manual is in the file doc/spec.txt in the Exim distribution as a plain # ASCII file. Other formats (PostScript, Texinfo, HTML, PDF) are available # from the Exim ftp sites. The manual is also online at the Exim web sites. # This file is divided into several parts, all but the first of which are # headed by a line starting with the word "begin". Only those parts that # are required need to be present. Blank lines, and lines starting with # # are ignored. ########### IMPORTANT ########## IMPORTANT ########### IMPORTANT ########### # # # Whenever you change Exim's configuration file, you *must* remember to # # HUP the Exim daemon, because it will not pick up the new configuration # # until you do. However, any other Exim processes that are started, for # # example, a process started by an MUA in order to send a message, will # # see the new configuration as soon as it is in place. # # # # You do not need to HUP the daemon for changes in auxiliary files that # # are referenced from this file. They are read every time they are used. # # # # It is usually a good idea to test a new configuration for syntactic # # correctness before installing it (for example, by running the command # # "exim -C /config/file.new -bV"). # # # ########### IMPORTANT ########## IMPORTANT ########### IMPORTANT ########### ###################################################################### # MAIN CONFIGURATION SETTINGS # ###################################################################### # Specify your host's canonical name here. This should normally be the fully # qualified "official" name of your host. If this option is not set, the # uname() function is called to obtain the name. In many cases this does # the right thing and you need not set anything explicitly. # Note that you MUST set the primary hostname when listening on multiple # interfaces. I can't remember why. primary_hostname = mail.example.com # Listen on every interface on the system on the standard port 25, and # also allow connections on port 10025 from just the loopback. local_interfaces = 0.0.0.0.25 : 127.0.0.1.10025 # The next three settings create two lists of domains and one list of hosts. # These lists are referred to later in this configuration using the syntax # +local_domains, +relay_to_domains, and +relay_from_hosts, respectively. They # are all colon-separated lists: domainlist local_domains = @ domainlist relay_to_domains = hostlist relay_from_hosts = 127.0.0.1 # Most straightforward access control requirements can be obtained by # appropriate settings of the above options. In more complicated situations, you # may need to modify the Access Control List (ACL) which appears later in this # file. # The first setting specifies your local domains, for example: # # domainlist local_domains = my.first.domain : my.second.domain # # You can use "@" to mean "the name of the local host", as in the default # setting above. This is the name that is specified by primary_hostname, # as specified above (or defaulted). If you do not want to do any local # deliveries, remove the "@" from the setting above. If you want to accept mail # addressed to your host's literal IP address, for example, mail addressed to # "user@[192.168.23.44]", you can add "@[]" as an item in the local domains # list. You also need to uncomment "allow_domain_literals" below. This is not # recommended for today's Internet. # The second setting specifies domains for which your host is an incoming relay. # If you are not doing any relaying, you should leave the list empty. However, # if your host is an MX backup or gateway of some kind for some domains, you # must set relay_to_domains to match those domains. For example: # # domainlist relay_to_domains = *.myco.com : my.friend.org # # This will allow any host to relay through your host to those domains. # See the section of the manual entitled "Control of relaying" for more # information. # The third setting specifies hosts that can use your host as an outgoing relay # to any other host on the Internet. Such a setting commonly refers to a # complete local network as well as the localhost. For example: # # hostlist relay_from_hosts = 127.0.0.1 : 192.168.0.0/16 # # The "/16" is a bit mask (CIDR notation), not a number of hosts. Note that you # have to include 127.0.0.1 if you want to allow processes on your host to send # SMTP mail by using the loopback address. A number of MUAs use this method of # sending mail. # All three of these lists may contain many different kinds of item, including # wildcarded names, regular expressions, and file lookups. See the reference # manual for details. The lists above are used in the access control list for # incoming messages. The name of this ACL is defined here: acl_smtp_rcpt = acl_check_rcpt # You should not change that setting until you understand how ACLs work. # Specify the domain you want to be added to all unqualified addresses # here. An unqualified address is one that does not contain an "@" character # followed by a domain. For example, "caesar@rome.example" is a fully qualified # address, but the string "caesar" (i.e. just a login name) is an unqualified # email address. Unqualified addresses are accepted only from local callers by # default. See the recipient_unqualified_hosts option if you want to permit # unqualified addresses from remote sources. If this option is not set, the # primary_hostname value is used for qualification. # qualify_domain = # If you want unqualified recipient addresses to be qualified with a different # domain to unqualified sender addresses, specify the recipient domain here. # If this option is not set, the qualify_domain value is used. # qualify_recipient = # The following line must be uncommented if you want Exim to recognize # addresses of the form "user@[10.11.12.13]" that is, with a "domain literal" # (an IP address) instead of a named domain. The RFCs still require this form, # but it makes little sense to permit mail to be sent to specific hosts by # their IP address in the modern Internet. This ancient format has been used # by those seeking to abuse hosts by using them for unwanted relaying. If you # really do want to support domain literals, uncomment the following line, and # see also the "domain_literal" router below. # allow_domain_literals # No deliveries will ever be run under the uids of these users (a colon- # separated list). An attempt to do so causes a panic error to be logged, and # the delivery to be deferred. This is a paranoic safety catch. Note that the # default setting means you cannot deliver mail addressed to root as if it # were a normal user. This isn't usually a problem, as most sites have an alias # for root that redirects such mail to a human administrator. never_users = root # The setting below causes Exim to do a reverse DNS lookup on all incoming # IP calls, in order to get the true host name. If you feel this is too # expensive, you can specify the networks for which a lookup is done, or # remove the setting entirely. host_lookup = * # The settings below, which are actually the same as the defaults in the # code, cause Exim to make RFC 1413 (ident) callbacks for all incoming SMTP # calls. You can limit the hosts to which these calls are made, and/or change # the timeout that is used. If you set the timeout to zero, all RFC 1413 calls # are disabled. RFC 1413 calls are cheap and can provide useful information # for tracing problem messages, but some hosts and firewalls have problems # with them. This can result in a timeout instead of an immediate refused # connection, leading to delays on starting up an SMTP session. rfc1413_hosts = * rfc1413_query_timeout = 30s # By default, Exim expects all envelope addresses to be fully qualified, that # is, they must contain both a local part and a domain. If you want to accept # unqualified addresses (just a local part) from certain hosts, you can specify # these hosts by setting one or both of # # sender_unqualified_hosts = # recipient_unqualified_hosts = # # to control sender and recipient addresses, respectively. When this is done, # unqualified addresses are qualified using the settings of qualify_domain # and/or qualify_recipient (see above). # If you want Exim to support the "percent hack" for certain domains, # uncomment the following line and provide a list of domains. The "percent # hack" is the feature by which mail addressed to x%y@z (where z is one of # the domains listed) is locally rerouted to x@y and sent on. If z is not one # of the "percent hack" domains, x%y is treated as an ordinary local part. This # hack is rarely needed nowadays; you should not enable it unless you are sure # that you really need it. # # percent_hack_domains = # # As well as setting this option you will also need to remove the test # for local parts containing % in the ACL definition below. # When Exim can neither deliver a message nor return it to sender, it "freezes" # the delivery error message (aka "bounce message"). There are also other # circumstances in which messages get frozen. They will stay on the queue for # ever unless one of the following options is set. # This option unfreezes frozen bounce messages after two days, tries # once more to deliver them, and ignores any delivery failures. ignore_bounce_errors_after = 2d # This option cancels (removes) frozen messages that are older than a week. timeout_frozen_after = 7d ###################################################################### # ACL CONFIGURATION # # Specifies access control lists for incoming SMTP mail # ###################################################################### begin acl # This access control list is used for every RCPT command in an incoming # SMTP message. The tests are run in order until the address is either # accepted or denied. acl_check_rcpt: # Accept if the source is local SMTP (i.e. not over TCP/IP). We do this by # testing for an empty sending host field. accept hosts = : # Deny if the local part contains @ or % or / or | or !. These are rarely # found in genuine local parts, but are often tried by people looking to # circumvent relaying restrictions. # Also deny if the local part starts with a dot. Empty components aren't # strictly legal in RFC 2822, but Exim allows them because this is common. # However, actually starting with a dot may cause trouble if the local part # is used as a file name (e.g. for a mailing list). deny local_parts = ^.*[@%!/|] : ^\\. # Accept mail to postmaster in any local domain, regardless of the source, # and without verifying the sender. accept local_parts = postmaster domains = +local_domains # Deny unless the sender address can be verified. require verify = sender ############################################################################# # There are no checks on DNS "black" lists because the domains that contain # these lists are changing all the time. However, here are two examples of # how you could get Exim to perform a DNS black list lookup at this point. # The first one denies, while the second just warns. # # deny message = rejected because $sender_host_address is in a black list at $dnslist_domain\n$dnslist_text # dnslists = black.list.example # # warn message = X-Warning: $sender_host_address is in a black list at $dnslist_domain # log_message = found in $dnslist_domain # dnslists = black.list.example ############################################################################# # Accept if the address is in a local domain, but only if the recipient can # be verified. Otherwise deny. The "endpass" line is the border between # passing on to the next ACL statement (if tests above it fail) or denying # access (if tests below it fail). accept domains = +local_domains endpass message = unknown user verify = recipient # Accept if the address is in a domain for which we are relaying, but again, # only if the recipient can be verified. accept domains = +relay_to_domains endpass message = unrouteable address verify = recipient # If control reaches this point, the domain is neither in +local_domains # nor in +relay_to_domains. # Accept if the message comes from one of the hosts for which we are an # outgoing relay. Recipient verification is omitted here, because in many # cases the clients are dumb MUAs that don't cope well with SMTP error # responses. If you are actually relaying out from MTAs, you should probably # add recipient verification here. accept hosts = +relay_from_hosts # Accept if the message arrived over an authenticated connection, from # any host. Again, these messages are usually from MUAs, so recipient # verification is omitted. accept authenticated = * # Reaching the end of the ACL causes a "deny", but we might as well give # an explicit message. deny message = relay not permitted ###################################################################### # ROUTERS CONFIGURATION # # Specifies how addresses are handled # ###################################################################### # THE ORDER IN WHICH THE ROUTERS ARE DEFINED IS IMPORTANT! # # An address is passed to each router in turn until it is accepted. # ###################################################################### begin routers # Before we'll deliver any message, we want to pass the message # through amavisd-new. amavisd-new causes the rejection of the whole # message if any of the recipients fail when it tries to redeliver the # message later. Since this is not desirable behavior, we need to check # all local recipients before we try to route through amavisd, so it won't # see any invalid users. # To do this, each router which validates users below has a corresponding # entry here, with the same criteria as below. Instead of actually # specifying the transport and delivering the mail the router is marked # for use for address verification only, and simply passes off routing # to the amavis router if it accepts a user. If no routers verify a # user, control will reach a router which always fails, and that user # will be refused prior to delivery to amavis. # If a message would be delivered with the domain literal [xx.yy.zz.nn] # notation, accept it. Note that this is mostly used these days by # junk mailers to send things you don't want. The default configuration # leaves it commented out, despite that not being RFC compliant. # Remove the comments here and on the domain_literal router below to # use it. # check_domain_literal: # driver = ipliteral # domains = ! +local_domains # verify_only # pass_router = amavis # This router routes addresses that are not in local domains by doing a DNS # lookup on the domain name. Any domain that resolves to 0.0.0.0 or to a # loopback interface address (127.0.0.0/8) is treated as if it had no DNS # entry. Note that 0.0.0.0 is the same as 0.0.0.0/32, which is commonly treated # as the local host inside the network stack. It is not 0.0.0.0/0, the default # route. If the DNS lookup fails, no further routers are tried because of # the no_more setting, and consequently the address is unrouteable. check_dnslookup: driver = dnslookup domains = ! +local_domains ignore_target_hosts = 0.0.0.0 : 127.0.0.0/8 verify_only pass_router = amavis no_more # The remaining routers check addresses in the local domain(s). # The system_aliases router allows delivery from a standard aliases file, # often called /etc/aliases. To check it, use the same transports and # flags, but set verify_only and pass_router. No transports are needed. check_system_aliases: driver = redirect allow_fail allow_defer data = ${lookup{$local_part}lsearch{/etc/aliases}} verify_only pass_router = amavis # There is no check_userforward because that router is not used during # address verification, and therefore won't ever refuse an address. # It uses no_verify, which is the opposite of the verify_only we're using # to check for valid users. # The localuser router delivers to local system mailboxes, of various kinds. # To check it, the check_localuser router uses the same settings and driver, # but doesn't use any of the other settings and is verify_only. check_localuser: driver = accept check_local_user verify_only pass_router = amavis # If we've run the gamut of the check routers and gotten here then none of # those routers will deliver this address. To prevent it from trying to # be delivered, this router will fail to verify any address. failed_address_router: driver = accept verify_only fail_verify # The verify routers have passed processing to the amavis router, or been # skipped because they're verify_only. This means we should now actually # try and virus scan a message. If the message has come in on port 10025 # has been scanned already, or is a bounce message, this router will accept # the message and process it with the amavis transport. Otherwise, # routing continues with the 'normal' delivery methods below. amavis: driver = manualroute # Do NOT run if received via 10025/tcp or if already spam-scanned # or if bounce message ($sender_address="") condition = "${if or {{eq {$interface_port}{10025}} \ {eq {$received_protocol}{spam-scanned}} \ {eq {$sender_address}{}} \ }{0}{1}}" transport = amavis route_list = "* localhost byname" self = send # This router routes to remote hosts over SMTP by explicit IP address, # when an email address is given in "domain literal" form, for example, # . The RFCs require this facility. However, it is # little-known these days, and has been exploited by evil people seeking # to abuse SMTP relays. Consequently it is commented out in the default # configuration. If you uncomment this router, you also need to uncomment # allow_domain_literals above, so that Exim can recognize the syntax of # domain literal addresses. # domain_literal: # driver = ipliteral # domains = ! +local_domains # transport = remote_smtp # This router routes addresses that are not in local domains by doing a DNS # lookup on the domain name. Any domain that resolves to 0.0.0.0 or to a # loopback interface address (127.0.0.0/8) is treated as if it had no DNS # entry. Note that 0.0.0.0 is the same as 0.0.0.0/32, which is commonly treated # as the local host inside the network stack. It is not 0.0.0.0/0, the default # route. If the DNS lookup fails, no further routers are tried because of # the no_more setting, and consequently the address is unrouteable. dnslookup: driver = dnslookup domains = ! +local_domains transport = remote_smtp ignore_target_hosts = 0.0.0.0 : 127.0.0.0/8 no_more # The remaining routers deliver to addresses in the local domain(s). # This router handles aliasing using a linearly searched alias file with the # name /etc/aliases. When this configuration is installed automatically, # the name gets inserted into this file from whatever is set in Exim's # build-time configuration. The default path is the traditional /etc/aliases. # If you install this configuration by hand, you need to specify the correct # path in the "data" setting below. # ##### NB You must ensure that the alias file exists. It used to be the case ##### NB that every Unix had that file, because it was the Sendmail default. ##### NB These days, there are systems that don't have it. Your aliases ##### NB file should at least contain an alias for "postmaster". # # If any of your aliases expand to pipes or files, you will need to set # up a user and a group for these deliveries to run under. You can do # this by uncommenting the "user" option below (changing the user name # as appropriate) and adding a "group" option if necessary. Alternatively, you # can specify "user" on the transports that are used. Note that the transports # listed below are the same as are used for .forward files; you might want # to set up different ones for pipe and file deliveries from aliases. system_aliases: driver = redirect allow_fail allow_defer data = ${lookup{$local_part}lsearch{/etc/aliases}} # user = exim file_transport = address_file pipe_transport = address_pipe # This router handles forwarding using traditional .forward files in users' # home directories. If you want it also to allow mail filtering when a forward # file starts with the string "# Exim filter", uncomment the "allow_filter" # option. # The no_verify setting means that this router is skipped when Exim is # verifying addresses. Similarly, no_expn means that this router is skipped if # Exim is processing an EXPN command. # The check_ancestor option means that if the forward file generates an # address that is an ancestor of the current one, the current one gets # passed on instead. This covers the case where A is aliased to B and B # has a .forward file pointing to A. # The three transports specified at the end are those that are used when # forwarding generates a direct delivery to a file, or to a pipe, or sets # up an auto-reply, respectively. userforward: driver = redirect check_local_user file = $home/.forward no_verify no_expn check_ancestor # allow_filter file_transport = address_file pipe_transport = address_pipe reply_transport = address_reply # This router matches local user mailboxes. localuser: driver = accept check_local_user transport = local_delivery ###################################################################### # TRANSPORTS CONFIGURATION # ###################################################################### # ORDER DOES NOT MATTER # # Only one appropriate transport is called for each delivery. # ###################################################################### # A transport is used only when referenced from a router that successfully # handles an address. begin transports # This transport is used for delivering messages over SMTP connections. remote_smtp: driver = smtp # This is the SMTP transport used to deliver messages to amavisd-new. # It is a simple smtp transport, delivering to the localhost on a specific # port. amavis: driver = smtp port = 10024 allow_localhost # This transport is used for local delivery to user mailboxes in traditional # BSD mailbox format. By default it will be run under the uid and gid of the # local user, and requires the sticky bit to be set on the /var/mail directory. # Some systems use the alternative approach of running mail deliveries under a # particular group instead of using the sticky bit. The commented options below # show how this can be done. local_delivery: driver = appendfile file = /var/mail/$local_part delivery_date_add envelope_to_add return_path_add # group = mail # mode = 0660 # This transport is used for handling pipe deliveries generated by alias or # .forward files. If the pipe generates any standard output, it is returned # to the sender of the message as a delivery error. Set return_fail_output # instead of return_output if you want this to happen only when the pipe fails # to complete normally. You can set different transports for aliases and # forwards if you want to - see the references to address_pipe in the routers # section above. address_pipe: driver = pipe return_output # This transport is used for handling deliveries directly to files that are # generated by aliasing or forwarding. address_file: driver = appendfile delivery_date_add envelope_to_add return_path_add # This transport is used for handling autoreplies generated by the filtering # option of the userforward router. address_reply: driver = autoreply ###################################################################### # RETRY CONFIGURATION # ###################################################################### begin retry # This single retry rule applies to all domains and all errors. It specifies # retries every 15 minutes for 2 hours, then increasing retry intervals, # starting at 1 hour and increasing each time by a factor of 1.5, up to 16 # hours, then retries every 6 hours until 4 days have passed since the first # failed delivery. # Domain Error Retries # ------ ----- ------- * * F,2h,15m; G,16h,1h,1.5; F,4d,6h ###################################################################### # REWRITE CONFIGURATION # ###################################################################### # There are no rewriting specifications in this default configuration file. begin rewrite ###################################################################### # AUTHENTICATION CONFIGURATION # ###################################################################### # There are no authenticator specifications in this default configuration file. begin authenticators ###################################################################### # CONFIGURATION FOR local_scan() # ###################################################################### # If you have built Exim to include a local_scan() function that contains # tables for private options, you can define those options here. Remember to # uncomment the "begin" line. It is commented by default because it provokes # an error with Exim binaries that are not built with LOCAL_SCAN_HAS_OPTIONS # set in the Local/Makefile. # begin local_scan # End of Exim configuration file amavisd-new-2.7.1/README_FILES/README.sql-pg000644 000621 000620 00000051542 11733137635 017450 0ustar00markcmi000000 000000 USING SQL FOR LOOKUPS, LOG/REPORTING AND QUARANTINE =================================================== This text describes SQL specifics for a PostgreSQL database, and provides a schema. In most respects it also applies to an SQLite database. For general aspects of lookups, please see README.lookups. For general SQL notes and further examples please see README.sql. For MySQL-specific notes and schema please see README.sql-mysql. Upgrade note: field quarantine.mail_text should be of data type 'bytea' and not 'text' as suggested in earlier documentation; this is to prevent it from being unjustifiably associated with a character set, and to be able to store any byte value; to convert existing field from type 'text' to type 'bytea' the following clause may be used: ALTER TABLE quarantine ALTER mail_text TYPE bytea USING decode(replace(mail_text,'\\','\\\\'),'escape'); Starting with amavisd-new-2.6.0 the fields users.email, mailaddr.email, and maddr.email should preferably be declared as byte strings (bytea) with no associated character set (which is what these fields are, according to RFC 2821) instead of CHAR or VARCHAR. The following clauses convert pre-2.6.0 tables into the now preferred and more universal form: ALTER TABLE users ALTER email TYPE bytea USING decode(email,'escape'); ALTER TABLE mailaddr ALTER email TYPE bytea USING decode(email,'escape'); ALTER TABLE maddr ALTER email TYPE bytea USING decode(email,'escape'); If a data type 'bytea' is chosen for these three fields, the setting $sql_allow_8bit_address MUST be set to true to let the amavisd program use the appropriate data type in SQL commands: $sql_allow_8bit_address = 1; # maddr.email: VARCHAR (0), VARBINARY/BYTEA (1) otherwise PostgreSQL will complain with: 'types bytea and character varying cannot be matched' when amavisd tries to execute SQL commands. Starting with amavisd-new-2.7.0, three fields need to be added to table 'msgrcpt', and one to table 'msgs': ALTER TABLE msgrcpt ADD COLUMN rseqnum integer DEFAULT 0 NOT NULL; ALTER TABLE msgrcpt ADD COLUMN content char(1) DEFAULT ' ' NOT NULL; ALTER TABLE msgrcpt ADD COLUMN is_local char(1) DEFAULT ' ' NOT NULL; ALTER TABLE msgs ADD COLUMN originating char(1) DEFAULT ' ' NOT NULL; Table 'policy' received a couple of new optional fields with 2.7.0, and dropped one field. As all fields in this table are optional and any extra field is just ignored by amavisd, it is not necessary to update this table unless one really needs these new fields. The following should adjust a pre-2.7.0 schema: ALTER TABLE policy ADD COLUMN unchecked_lover char(1) default NULL; ALTER TABLE policy ADD COLUMN spam_tag3_level real default NULL; ALTER TABLE policy ADD COLUMN spam_subject_tag3 varchar(64) default NULL; ALTER TABLE policy ADD COLUMN disclaimer_options varchar(64) default NULL; ALTER TABLE policy ADD COLUMN forward_method varchar(64) default NULL; ALTER TABLE policy ADD COLUMN sa_userconf varchar(64) default NULL; ALTER TABLE policy ADD COLUMN sa_username varchar(64) default NULL; ALTER TABLE policy DROP COLUMN spam_modifies_subj; If you need to create a primary key on table msgrcpt for some reason (clustering perhaps?), try something like: UPDATE msgrcpt SET rseqnum=1+floor(999999999*random()) WHERE rseqnum=0; CREATE UNIQUE INDEX msgrcpt_idx_primary ON msgrcpt (partition_tag,mail_id,rseqnum); Also, fields mail_id and secret_id should be treated case-sensitively, so data types char or varchar (as suggested by versions before 2.7.0) should be avoided - a suitable type is bytea. Moreover, starting with version 2.7.0 the size of mail_id is configurable through a setting $mail_id_size_bits, so the previous varchar(12) may not suffice if the size is increased from its default of 72 bytes (12 characters). The following clauses convert the data type of affected fields from varchar or char to bytea: ALTER TABLE msgs ALTER mail_id TYPE bytea USING decode(mail_id,'escape'), ALTER secret_id DROP DEFAULT, ALTER secret_id TYPE bytea USING decode(secret_id,'escape'), ALTER secret_id SET DEFAULT ''; ALTER TABLE msgrcpt ALTER mail_id TYPE bytea USING decode(mail_id,'escape'); ALTER TABLE quarantine ALTER mail_id TYPE bytea USING decode(mail_id,'escape'); Version of Perl module DBD::Pg 1.48 or higher should be used; Short installation notes for PostgreSQL 8.2 are available at: http://www.postgresql.org/docs/8.2/interactive/install-short.html In short: run: 'initdb -D ...' as user postgres, then edit pg_hba.conf providing restricted access to database, create users and create databases. Something like the following may be placed into pg_hba.conf : # TYPE DATABASE USER CIDR-ADDRESS METHOD # # amavis lookups: local mail_prefs vscan md5 host mail_prefs vscan 127.0.0.1/32 md5 host mail_prefs vscan ::1/128 md5 # # amavis logging and pen pals: local mail_log vscan md5 host mail_log vscan 127.0.0.1/32 md5 host mail_log vscan ::1/128 md5 # # spamassassin Bayes and AWL databases: local mail_bayes vscan md5 host mail_bayes vscan 127.0.0.1/32 md5 host mail_bayes vscan ::1/128 md5 local mail_awl vscan md5 host mail_awl vscan 127.0.0.1/32 md5 host mail_awl vscan ::1/128 md5 Create an SQL username (role) for use by amavisd, e.g. vscan: $ createuser -U pgsql -S -D -R -P -e vscan Create databases for amavisd: $ createdb -U pgsql mail_prefs $ createdb -U pgsql mail_log and optionally databases for SpamAssassin: $ createdb -U pgsql mail_bayes $ createdb -U pgsql mail_awl The provided schema can be cut/pasted or fed directly into the client program to create a database. The '--' introduces comments according to SQL specs. Populate databases using the schema below: $ psql -U vscan mail_prefs <... $ psql -U vscan mail_log <... (for SpamAssassin database schema see its documentation: sql/README*) Something like the following can be placed into amavisd.conf (supplying correct passwords): @lookup_sql_dsn = ([ 'DBI:Pg:database=mail_prefs', 'vscan', 'LK40.gtklkKK' ]); @storage_sql_dsn = ([ 'DBI:Pg:database=mail_log', 'vscan', 'LK40.gtklkKK' ]); Equivalent settings for AWL and Bayes databases belong to a SpamAssassin's configuration file local.cf, according to SpamAssassin documentation. Amavisd and SpamAssassin need not use the same username or password, nor do they need to reside on the same SQL server. SQLite notes: - use INTEGER PRIMARY KEY AUTOINCREMENT instead of SERIAL; - SQLite is well suited for lookups database, but is not appropriate for @storage_sql_dsn due to coarse lock granularity; CREATE TABLE policy ( id serial PRIMARY KEY, -- 'id' is the _only_ required field policy_name varchar(32), -- not used by amavisd-new, a comment virus_lover char(1) default NULL, -- Y/N spam_lover char(1) default NULL, -- Y/N unchecked_lover char(1) default NULL, -- Y/N banned_files_lover char(1) default NULL, -- Y/N bad_header_lover char(1) default NULL, -- Y/N bypass_virus_checks char(1) default NULL, -- Y/N bypass_spam_checks char(1) default NULL, -- Y/N bypass_banned_checks char(1) default NULL, -- Y/N bypass_header_checks char(1) default NULL, -- Y/N virus_quarantine_to varchar(64) default NULL, spam_quarantine_to varchar(64) default NULL, banned_quarantine_to varchar(64) default NULL, unchecked_quarantine_to varchar(64) default NULL, bad_header_quarantine_to varchar(64) default NULL, clean_quarantine_to varchar(64) default NULL, archive_quarantine_to varchar(64) default NULL, spam_tag_level real default NULL, -- higher score inserts spam info headers spam_tag2_level real default NULL, -- inserts 'declared spam' header fields spam_tag3_level real default NULL, -- inserts 'blatant spam' header fields spam_kill_level real default NULL, -- higher score triggers evasive actions -- e.g. reject/drop, quarantine, ... -- (subject to final_spam_destiny setting) spam_dsn_cutoff_level real default NULL, spam_quarantine_cutoff_level real default NULL, addr_extension_virus varchar(64) default NULL, addr_extension_spam varchar(64) default NULL, addr_extension_banned varchar(64) default NULL, addr_extension_bad_header varchar(64) default NULL, warnvirusrecip char(1) default NULL, -- Y/N warnbannedrecip char(1) default NULL, -- Y/N warnbadhrecip char(1) default NULL, -- Y/N newvirus_admin varchar(64) default NULL, virus_admin varchar(64) default NULL, banned_admin varchar(64) default NULL, bad_header_admin varchar(64) default NULL, spam_admin varchar(64) default NULL, spam_subject_tag varchar(64) default NULL, spam_subject_tag2 varchar(64) default NULL, spam_subject_tag3 varchar(64) default NULL, message_size_limit integer default NULL, -- max size in bytes, 0 disable banned_rulenames varchar(64) default NULL, -- comma-separated list of ... -- names mapped through %banned_rules to actual banned_filename tables disclaimer_options varchar(64) default NULL, forward_method varchar(64) default NULL, sa_userconf varchar(64) default NULL, sa_username varchar(64) default NULL ); -- local users CREATE TABLE users ( id serial PRIMARY KEY, -- unique id priority integer NOT NULL DEFAULT 7, -- sort field, 0 is low prior. policy_id integer NOT NULL DEFAULT 1 CHECK (policy_id >= 0) REFERENCES policy(id), email bytea NOT NULL UNIQUE, -- email address, non-rfc2822-quoted fullname varchar(255) DEFAULT NULL -- not used by amavisd-new -- local char(1) -- Y/N (optional, see SQL section in README.lookups) ); -- any e-mail address (non- rfc2822-quoted), external or local, -- used as senders in wblist CREATE TABLE mailaddr ( id serial PRIMARY KEY, priority integer NOT NULL DEFAULT 9, -- 0 is low priority email bytea NOT NULL UNIQUE ); -- per-recipient whitelist and/or blacklist, -- puts sender and recipient in relation wb (white or blacklisted sender) CREATE TABLE wblist ( rid integer NOT NULL CHECK (rid >= 0) REFERENCES users(id), sid integer NOT NULL CHECK (sid >= 0) REFERENCES mailaddr(id), wb varchar(10) NOT NULL, -- W or Y / B or N / space=neutral / score PRIMARY KEY (rid,sid) ); -- R/W part of the dataset (optional) -- May reside in the same or in a separate database as lookups database; -- REQUIRES SUPPORT FOR TRANSACTIONS; specified in @storage_sql_dsn -- -- Please create additional indexes on keys when needed, or drop suggested -- ones as appropriate to optimize queries needed by a management application. -- See your database documentation for further optimization hints. -- provide unique id for each e-mail address, avoids storing copies CREATE TABLE maddr ( id serial PRIMARY KEY, partition_tag integer DEFAULT 0, -- see $partition_tag email bytea NOT NULL, -- full e-mail address domain varchar(255) NOT NULL, -- only domain part of the email address -- with subdomain fields in reverse CONSTRAINT part_email UNIQUE (partition_tag,email) ); -- information pertaining to each processed message as a whole; -- NOTE: records with a NULL msgs.content should be ignored by utilities, -- as such records correspond to messages just being processed, or were lost CREATE TABLE msgs ( partition_tag integer DEFAULT 0, -- see $partition_tag mail_id bytea NOT NULL, -- long-term unique mail id, dflt 12 ch secret_id bytea DEFAULT '', -- authorizes release of mail_id, 12 ch am_id varchar(20) NOT NULL, -- id used in the log time_num integer NOT NULL CHECK (time_num >= 0), -- rx_time: seconds since Unix epoch time_iso timestamp WITH TIME ZONE NOT NULL,-- rx_time: ISO8601 UTC ascii time sid integer NOT NULL CHECK (sid >= 0), -- sender: maddr.id policy varchar(255) DEFAULT '', -- policy bank path (like macro %p) client_addr varchar(255) DEFAULT '', -- SMTP client IP address (IPv4 or v6) size integer NOT NULL CHECK (size >= 0), -- message size in bytes originating char(1) DEFAULT ' ' NOT NULL, -- sender from inside or auth'd content char(1), -- content type: V/B/U/S/Y/M/H/O/T/C -- virus/banned/unchecked/spam(kill)/spammy(tag2)/ -- /bad-mime/bad-header/oversized/mta-err/clean -- is NULL on partially processed mail -- (prior to 2.7.0 the CC_SPAMMY was logged as 's', now 'Y' is used; --- to avoid a need for case-insenstivity in queries) quar_type char(1), -- quarantined as: ' '/F/Z/B/Q/M/L -- none/file/zipfile/bsmtp/sql/ -- /mailbox(smtp)/mailbox(lmtp) quar_loc varchar(255) DEFAULT '', -- quarantine location (e.g. file) dsn_sent char(1), -- was DSN sent? Y/N/q (q=quenched) spam_level real, -- SA spam level (no boosts) message_id varchar(255) DEFAULT '', -- mail Message-ID header field from_addr varchar(255) DEFAULT '', -- mail From header field, UTF8 subject varchar(255) DEFAULT '', -- mail Subject header field, UTF8 host varchar(255) NOT NULL, -- hostname where amavisd is running CONSTRAINT msgs_partition_mail UNIQUE (partition_tag,mail_id), PRIMARY KEY (partition_tag,mail_id) --FOREIGN KEY (sid) REFERENCES maddr(id) ON DELETE RESTRICT ); CREATE INDEX msgs_idx_sid ON msgs (sid); CREATE INDEX msgs_idx_mess_id ON msgs (message_id); -- useful with pen pals CREATE INDEX msgs_idx_time_iso ON msgs (time_iso); CREATE INDEX msgs_idx_time_num ON msgs (time_num); -- optional -- per-recipient information related to each processed message; -- NOTE: records in msgrcpt without corresponding msgs.mail_id record are -- orphaned and should be ignored and eventually deleted by external utilities CREATE TABLE msgrcpt ( partition_tag integer DEFAULT 0, -- see $partition_tag mail_id bytea NOT NULL, -- (must allow duplicates) rseqnum integer DEFAULT 0 NOT NULL, -- recip's enumeration within msg rid integer NOT NULL, -- recipient: maddr.id (duplicates allowed) is_local char(1) DEFAULT ' ' NOT NULL, -- recip is: Y=local, N=foreign content char(1) DEFAULT ' ' NOT NULL, -- content type V/B/U/S/Y/M/H/O/T/C ds char(1) NOT NULL, -- delivery status: P/R/B/D/T -- pass/reject/bounce/discard/tempfail rs char(1) NOT NULL, -- release status: initialized to ' ' bl char(1) DEFAULT ' ', -- sender blacklisted by this recip wl char(1) DEFAULT ' ', -- sender whitelisted by this recip bspam_level real, -- per-recipient (total) spam level smtp_resp varchar(255) DEFAULT '', -- SMTP response given to MTA CONSTRAINT msgrcpt_partition_mail_rseq UNIQUE (partition_tag,mail_id,rseqnum), PRIMARY KEY (partition_tag,mail_id,rseqnum) --FOREIGN KEY (rid) REFERENCES maddr(id) ON DELETE RESTRICT, --FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE ); CREATE INDEX msgrcpt_idx_mail_id ON msgrcpt (mail_id); CREATE INDEX msgrcpt_idx_rid ON msgrcpt (rid); -- mail quarantine in SQL, enabled by $*_quarantine_method='sql:' -- NOTE: records in quarantine without corresponding msgs.mail_id record are -- orphaned and should be ignored and eventually deleted by external utilities CREATE TABLE quarantine ( partition_tag integer DEFAULT 0, -- see $partition_tag mail_id bytea NOT NULL, -- long-term unique mail id chunk_ind integer NOT NULL CHECK (chunk_ind >= 0), -- chunk number, 1.. mail_text bytea NOT NULL, -- store mail as chunks of octects PRIMARY KEY (partition_tag,mail_id,chunk_ind) --FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE ); -- field msgrcpt.rs is primarily intended for use by quarantine management -- software; the value assigned by amavisd is a space; -- a short _preliminary_ list of possible values: -- 'V' => viewed (marked as read) -- 'R' => released (delivered) to this recipient -- 'p' => pending (a status given to messages when the admin received the -- request but not yet released; targeted to banned parts) -- 'D' => marked for deletion; a cleanup script may delete it Some examples of a query: -- mail from last two minutes: SELECT now()-time_iso AS age, SUBSTRING(policy,1,2) as pb, msgrcpt.content AS c, dsn_sent as dsn, ds, bspam_level AS level, size, SUBSTRING(sender.email,1,18) AS s, SUBSTRING(recip.email,1,18) AS r, SUBSTRING(msgs.subject,1,10) AS subj FROM msgs LEFT JOIN msgrcpt ON msgs.mail_id=msgrcpt.mail_id LEFT JOIN maddr AS sender ON msgs.sid=sender.id LEFT JOIN maddr AS recip ON msgrcpt.rid=recip.id WHERE msgrcpt.content IS NOT NULL AND now() - time_iso < INTERVAL '2 minutes' ORDER BY msgs.time_num DESC; -- clean messages ordered by count, grouped by domain: SELECT count(*) as cnt, avg(bspam_level), sender.domain FROM msgs LEFT JOIN msgrcpt ON msgs.mail_id=msgrcpt.mail_id LEFT JOIN maddr AS sender ON msgs.sid=sender.id LEFT JOIN maddr AS recip ON msgrcpt.rid=recip.id WHERE msgrcpt.content='C' GROUP BY sender.domain ORDER BY cnt DESC LIMIT 50; -- top spamy domains with >10 messages, sorted by spam average, -- grouped by domain: SELECT count(*) as cnt, avg(bspam_level) as spam_avg, sender.domain FROM msgs LEFT JOIN msgrcpt ON msgs.mail_id=msgrcpt.mail_id LEFT JOIN maddr AS sender ON msgs.sid=sender.id LEFT JOIN maddr AS recip ON msgrcpt.rid=recip.id WHERE bspam_level IS NOT NULL GROUP BY sender.domain HAVING count(*) > 10 ORDER BY spam_avg DESC LIMIT 50; -- sender domains with >100 messages, sorted on sender.domain: SELECT count(*) as cnt, avg(bspam_level) as spam_avg, sender.domain FROM msgs LEFT JOIN msgrcpt ON msgs.mail_id=msgrcpt.mail_id LEFT JOIN maddr AS sender ON msgs.sid=sender.id LEFT JOIN maddr AS recip ON msgrcpt.rid=recip.id GROUP BY sender.domain HAVING count(*) > 100 ORDER BY sender.domain DESC LIMIT 100; EXAMPLE of a log/report/quarantine database housekeeping ======================================================== Using a changing partition_tag, perhaps by using an ISO 8601 week number (value 1 to 53) as a partition_tag: $partition_tag = sub { my($msginfo)=@_; sprintf("%02d",iso8601_week($msginfo->rx_time)) }; allows for probably the fastest method of purging old records, e.g.: DELETE FROM msgs WHERE partition_tag >= 13 AND partition_tag <= 23; DELETE FROM msgrcpt WHERE partition_tag >= 13 AND partition_tag <= 23; DELETE FROM quarantine WHERE partition_tag >= 13 AND partition_tag <= 23; DELETE FROM maddr WHERE partition_tag >= 13 AND partition_tag <= 23; Alternatively, purge records from table msgs by their creation time: DELETE FROM msgs WHERE time_iso < now() - INTERVAL '3 weeks'; DELETE FROM msgs WHERE time_iso < now() - INTERVAL '1 h' AND content IS NULL; Optionally certain content types may be given shorter lifetime: DELETE FROM msgs WHERE time_iso < now() - INTERVAL '1 week' AND (content='V' OR (content='S' AND spam_level > 20)); and then delete unreferenced records from tables msgrcpt, quarantine, and maddr: DELETE FROM msgrcpt WHERE mail_id IN (SELECT mail_id FROM msgrcpt LEFT JOIN msgs USING(mail_id) WHERE msgs.mail_id IS NULL); DELETE FROM quarantine WHERE mail_id IN (SELECT mail_id FROM quarantine LEFT JOIN msgs USING(mail_id) WHERE msgs.mail_id IS NULL); DELETE FROM maddr WHERE NOT EXISTS (SELECT 1 FROM msgs WHERE sid=id) AND NOT EXISTS (SELECT 1 FROM msgrcpt WHERE rid=id); On more recent testings, the following 'DELETE FROM maddr' seems to be faster from the one above by a factor of 1.5 to 2, and is functionally equivalent: DELETE FROM maddr WHERE id IN ( SELECT id FROM maddr LEFT JOIN ( SELECT sid AS id, 1 AS f FROM msgs UNION ALL SELECT rid AS id, 1 AS f FROM msgrcpt ) AS u USING(id) WHERE u.f IS NULL); Check also a thread 'Faster purging of SQL logging database' (2007-06) on the amavis-user mailing list, archived at: http://marc.info/?t=118190428300003 The third option for purging old records is to use: FOREIGN KEY ... ON DELETE CASCADE on tables msgrcpt and quarantine, in which case these records will be deleted automatically when a corresponding record in table msgs is deleted. This seems to be the slowest method. amavisd-new-2.7.1/README_FILES/README.courier000640 000621 000620 00000012233 10554016220 017665 0ustar00markcmi000000 000000 How to use amavisd-new with Courier *********************************** by Martin Orr There may be additional or more up-to-date information at: http://www.martinorr.name/amavisd-new WARNING: This README applies to the current version of the Courier patch, and requires Net::Server version 0.90 or later. For older versions of Net::Server please use old courier patch and refer to README.courier-old. The design of courierfilter means that amavisd-new must behave somewhat differently from the ways in which it normally behaves. This has two main effects: 1. amavisd is started and stopped by courierfilter whenever Courier starts and stops. You must not start, restart or stop it directly, but instead use filterctl {start|stop} amavisd (It is possible but not recommended to configure amavisd to be manually started and stopped; see below under manual startup/shutdown mode.) 2. It is not possible to modify the headers of messages. This makes amavisd-new/courierfilter of limited use for spam checking. If you want spam checking, you should run spamd separately and pass mail to it either using maildrop or from .courier files. CONFIGURING AMAVISD You will need to make a the following changes to amavisd.conf: 1. Comment out the lines setting $daemon_user, $daemon_group: amavisd will be started as whatever user Courier runs as (not root) so is unable to change to another user. 2. Set $forward_method to undef and $notify_method to 'pipe:flags=q argv=/usr/sbin/sendmail -f ${sender} -- ${recipient}' If you wish to include "local" in your enablefiltering file, or you are using a version of Courier older than 0.49.0, then you must instead use 'pipe:flags=q argv=perl -e $pid=fork();if($pid==-1){exit(75)}elsif($pid==0){exec(@ARGV)}else{exit(0)} /usr/sbin/sendmail -f ${sender} -- ${recipient}' 3. Set $courierfilter_shutdown to 1. 4. Set $unix_socketname to DIR/amavisd where DIR is the appropriate courierfilter directory. You need to decide whether you want a mandatory filter, in which case all mail passing through your server will be filtered, or an optional filter, in which case only mail to local users will be filtered, and your users will have the option of overriding filtering (you will almost certainly want a mandatory filter). You should see the courierfilter manpage for more details, and also for the correct directories on your system (on mine, they are /var/lib/courier/allfilters for mandatory filters and /var/lib/courier/filters for optional filters). 5. Replace any existing setting of $interface_policy{'SOCK'} with $interface_policy{'SOCK'} = 'AM-SOCK'; $policy_bank{'AM-SOCK'} = { protocol => 'COURIER' }; CONFIGURING COURIER Install amavisd or a link to it in /usr/lib/courier/filters (or whatever directory is correct on your system - again, see the courierfilter manpage). The name of this link must match the name of the socket in $unix_socketname. Make sure that the enablefiltering file exists in your Courier configuration directory (/etc/courier or equivalent) and contains "esmtp". If you wish to include "local", to filter mail sent through the sendmail command, then you must use the long forking value of $notify_method in step 2 above. MESSAGE DESTINIES As of amavisd-new 2.4.0, it is possible to use all message destinies with Courier. D_REJECT is probably the best choice as the message (if infected of course) is rejected in the original SMTP session, and never becomes your responsibility to deliver or send a DSN. The client MTA receives the response "550 5.7.1 Message content rejected". RELEASING ITEMS FROM QUARANTINE If you wish to be able to release items from quarantine using the AM.PDP protocol, you must set this to use a TCP port. Include the following in amavisd.conf: $inet_socket_port = 9998; $interface_policy{'9998'} = 'AM.PDP'; $policy_bank{'AM.PDP'} = { protocol => 'AM.PDP', inet_acl => [qw( 127.0.0.1 [::1] )] # restrict access to these IP addresses }; Modify the amavisd-release script to use $socketname = '127.0.0.1:9998'. MANUAL STARTUP/SHUTDOWN MODE You may prefer to start and stop amavisd independently of Courier (perhaps through its own init script). However this means that you are on your own in ensuring that it is started at the right time, namely as soon as possible after Courier is started (or restarted). Messages which arrive after Courier starts but before amavisd is ready will simply not be filtered. However, starting amavisd before Courier will unfortunately not work. If you wish to disable virus checking you must not only stop amavisd but also remove the socket - Courier will refuse to accept mail while the socket exists but amavisd is not running. To use manual startup/shutdown mode, do not put a link to amavisd in /usr/lib/courier/filters and set $courierfilter_shutdown to 0. If you are using manual startup/shutdown mode, you do need to set the $daemon_user and $daemon_group variables. $daemon_user can be whatever you like and $daemon_group must be the same group as Courier runs as. In order to allow amavisd to create its socket you must either start amavisd as root or make the filter directory group-writable. amavisd-new-2.7.1/README_FILES/README.sql000640 000621 000620 00000025065 11460422206 017025 0ustar00markcmi000000 000000 USING SQL FOR LOOKUPS, LOG/REPORTING AND QUARANTINE =================================================== This text contains a general SQL-related documentation. For aspects specific to using SQL database for lookups, please see README.lookups . For general aspects of lookups, please see README.lookups. For MySQL-specific notes and schema please see README.sql-mysql. For PostgreSQL-specific notes and schema please see README.sql-pg (which in most respects applies also to a SQLite database). Since version of amavisd-new-20020630 a SQL is supported for lookups. Since amavisd-new-2.3.0, SQL is also supported for storing information about processed mail (logging/reporting) and optionally for quarantining to a SQL database. The amavisd.conf variables @storage_sql_dsn and @lookup_sql_dsn control access to a SQL server and specify a database (dsn = data source name). The @lookup_sql_dsn enables and specifies a database for lookups, the @storage_sql_dsn enables and specifies a database for reporting and quarantining. Both settings are independent. Interpretation of @lookup_sql_dsn and @storage_sql_dsn lists is as follows: - empty list disables the function and is a default; - if both lists are empty no SQL support code will be compiled-in, reducing the amount of virtual memory needed for each child process; - a list can contain one or more triples: [dsn,user,passw]; more than one triple may be specified to specify multiple (backup) SQL servers - the first that responds will be used as long as it works, then search is retried; - if both lists contain refs to the _same_ triples (not just equal triples), only one connection to a SQL server will be used; otherwise two independent connections to databases will be used, possibly to different SQL servers, which may even be of different type (e.g. SQLlite for lookups (read-only), and PostgreSQL or MySQL for transactional reporting, offering fine lock granularity). Example setting: @lookup_sql_dsn = ( ['DBI:mysql:database=mail;host=127.0.0.1;port=3306', 'user1', 'passwd1'], ['DBI:mysql:database=mail;host=host2', 'username2', 'password2'], ['DBI:Pg:database=mail;host=host1', 'amavis', ''] ["DBI:SQLite:dbname=$MYHOME/sql/mail_prefs.sqlite", '', ''] ); @storage_sql_dsn = @lookup_sql_dsn; # none, same, or separate database See man page for the Perl module DBI, and corresponding DBD modules man pages (DBD::mysql, DBD::Pg, DBD::SQLite, ...) for syntax of the first argument. Since version 2.3.0 amavisd-new also offers quarantining to a SQL database, along with a mechanism to release quarantined messages (either from SQL or from normal files, possibly gzipped). To enable quarantining to SQL, the @storage_sql_dsn must be enabled (facilitating quarantine management), and some or all variables $virus_quarantine_method, $spam_quarantine_method, $banned_files_quarantine_method and $bad_header_quarantine_method should specify the value 'sql:'. Specifying 'sql:' as a quarantine method without also specifying a database in @storage_sql_dsn is an error. When setting up access controls to a database, keep in mind that amavisd-new only needs read-only access to the database used for lookups, the permission to do a SELECT suffices. For security reasons it is undesirable to permit other operations such as INSERT, DELETE or UPDATE to a dataset used for lookups. For managing the lookups database one should preferably use a different username with more privileges. The database specified in @storage_sql_dsn needs to provide read/write access (SELECT, INSERT, UPDATE), and a database server offering transactions must be used. Database schemas are available in README.sql-mysql (for MySQL) and in README.sql-pg (for PostgreSQL and SQLite). There are two parts of a schema, an read-only part used for lookups, and a R/W part used for logging and quarantining. They are completely independent and may reside on different SQL servers (even on different types of SQL server), but may also coexist in a single database if desired. Note that some databases are very well suited for lookups, but less so for highly concurent transactional use in logging/quarantining. Some experience: - SQLite works nicely for lookups, avoiding a need for a separate server process, but its coarse locking granularity makes its unquitable for logging and quarantining; - MySQL and PostgreSQL are both fine for lookups; - PostgreSQL is better suited for SQL logging/quarantining because maintenance operations (cleaning of old records) are much faster than with MySQL; Note that SQL logging is needed for amavisd-new pen-pals feature to work; - if using MySQL for logging/quarantining, a sufficiently recent version must be used, as support for transactions is required for the R/W access; - if using SpamAssassin with its Bayes (and AWL) database on SQL, bayes plugin works faster with MySQL than with PostgreSQL; note that SA databases are independent from amavisd-new databases and may reside on a separate SQL server, possibly of a different type. See SpamAssassin documentation that comes with its distribution, files sql/README* . ===================== Example data follows: ===================== INSERT INTO users VALUES ( 1, 9, 5, 'u1+foo@y.example.com','Name1 Surname1'); INSERT INTO users VALUES ( 2, 8, 5, 'u1@y.example.com', 'Name1 Surname1'); INSERT INTO users VALUES ( 3, 8, 2, 'u2@y.example.com', 'Name2 Surname2'); INSERT INTO users VALUES ( 4, 8, 7, 'u3@z.example.com', 'Name3 Surname3'); INSERT INTO users VALUES ( 5, 8, 7, 'u4@example.com', 'Name4 Surname4'); INSERT INTO users VALUES ( 6, 8, 1, 'u5@example.com', 'Name5 Surname5'); INSERT INTO users VALUES ( 7, 7, 9, 'userB+bar', 'NameB SurnameB'); INSERT INTO users VALUES ( 8, 6, 10, 'userC', 'NameC SurnameC'); INSERT INTO users VALUES ( 9, 6, 11, 'userD', 'NameD SurnameD'); INSERT INTO users VALUES (10, 5, 61, '@.a.b.example.com', NULL); INSERT INTO users VALUES (11, 4, 62, '@.sub2.example.com', NULL); INSERT INTO users VALUES (12, 3, 70, '@example.com', NULL); INSERT INTO users VALUES (12, 2, 70, '@.example.com', NULL); INSERT INTO users VALUES (13, 1, 80, '@.com', NULL); INSERT INTO users VALUES (13, 1, 80, '@.edu', NULL); INSERT INTO users VALUES (14, 0, 99, '@.', NULL); -- catchall INSERT INTO users VALUES (15, 5, 0, '@sub1.example.net', NULL); INSERT INTO users VALUES (16, 5, 7, '@sub2.example.net', NULL); INSERT INTO users VALUES (17, 3, 5, '@example.net', NULL); INSERT INTO users VALUES (18, 8, 5, 'u1@example.org', 'u1'); INSERT INTO users VALUES (19, 8, 6, 'u2@example.org', 'u2'); INSERT INTO users VALUES (20, 8, 3, 'u3@example.org', 'u3'); INSERT INTO policy (id, policy_name, virus_lover, spam_lover, bypass_virus_checks, bypass_spam_checks, spam_modifies_subj, spam_tag2_level, spam_kill_level) VALUES (0, 'none', NULL,NULL, NULL,NULL, NULL, NULL, NULL), (1, 'Non-paying', 'N','N', 'Y','Y', 'N', NULL, NULL), (2, 'Uncensored', 'Y','Y', 'N','N', 'N', NULL, NULL), (3, 'Wants all spam','N','Y', 'N','N', 'Y', NULL, NULL), (4, 'Wants viruses', 'Y','N', 'N','N', 'Y', NULL, NULL), (5, 'Normal', 'N','N', 'N','N', NULL, NULL, NULL), (6, 'Trigger happy', NULL,NULL, NULL,NULL, NULL, 4.9, 4.9), (7, 'Permissive', NULL,NULL, NULL,NULL, NULL, 9, 20), (8, '6.5/7.8', NULL,NULL, NULL,NULL, NULL, 6.5, 7.8), (9, 'userB', NULL,NULL, NULL,NULL, 'N', 6.3, 6.3), (10,'userC', NULL,NULL, NULL,NULL, 'Y', 6.0, 6.0), (11,'userD', NULL,NULL, NULL,NULL, NULL, 7, 7), (61,'our-sub-a', NULL,'Y', NULL,NULL, NULL, NULL, NULL), (62,'our-sub-2', NULL,'Y', NULL,NULL, NULL, NULL, NULL), (70,'our domain', NULL,NULL, NULL,NULL, NULL, NULL, NULL), (80,'our com & edu', NULL,NULL, NULL,NULL, NULL, NULL, 6.6), (99,'catchall', NULL,NULL, NULL,NULL, NULL, 5.6, 6.7); -- sender envelope addresses needed for white/blacklisting INSERT INTO mailaddr VALUES (1, 5, '@example.com'); INSERT INTO mailaddr VALUES (2, 9, 'owner-postfix-users@postfix.org'); INSERT INTO mailaddr VALUES (3, 9, 'amavis-user-admin@lists.sourceforge.net'); INSERT INTO mailaddr VALUES (4, 9, 'makemoney@example.com'); INSERT INTO mailaddr VALUES (5, 5, '@example.net'); INSERT INTO mailaddr VALUES (6, 9, 'spamassassin-talk-admin@lists.sourceforge.net'); INSERT INTO mailaddr VALUES (7, 9, 'spambayes-bounces@python.org'); -- whitelist for user 14, i.e. default for recipients in domain sub1.example.net INSERT INTO wblist VALUES (14, 1, 'W'); INSERT INTO wblist VALUES (14, 3, 'W'); -- whitelist and blacklist for user 17, i.e. u1@example.org INSERT INTO wblist VALUES (17, 2, 'W'); INSERT INTO wblist VALUES (17, 3, 'W'); INSERT INTO wblist VALUES (17, 6, 'W'); INSERT INTO wblist VALUES (17, 7, 'W'); INSERT INTO wblist VALUES (17, 5, 'B'); INSERT INTO wblist VALUES (17, 4, 'B'); -- $sql_select_policy setting in amavisd.conf tells amavisd -- how to fetch per-recipient policy settings. -- See comments there. Example: -- -- SELECT *,users.id FROM users,policy -- WHERE (users.policy_id=policy.id) AND (users.email IN (%k)) -- ORDER BY users.priority DESC; -- -- $sql_select_white_black_list in amavisd.conf tells amavisd -- how to check sender in per-recipient whitelist/blacklist. -- See comments there. Example: -- -- SELECT wb FROM wblist,mailaddr -- WHERE (wblist.rid=?) AND (wblist.sid=mailaddr.id) AND (mailaddr.email IN (%k)) -- ORDER BY mailaddr.priority DESC; NOTE: the SELECT, INSERT and UPDATE clauses as used by the amavisd-new program are configurable through %sql_clause; see amavisd.conf-default Upgrading from pre 2.4.0 amavisd-new SQL schema to the 2.4.0 schema requires adding column 'quar_loc' to table msgs, and creating FOREIGN KEY constraint to facilitate deletion of expired records. The following clauses should be executed for upgrading pre-2.4.0 amavisd-new SQL schema to the 2.4.0 schema: -- mandatory change: ALTER TABLE msgs ADD quar_loc varchar(255) DEFAULT ''; -- optional but highly recommended: ALTER TABLE quarantine ADD FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE; ALTER TABLE msgrcpt ADD FOREIGN KEY (mail_id) REFERENCES msgs(mail_id) ON DELETE CASCADE; -- the following two ALTERs are not essential; if data type of maddr.id is -- incompatible with msgs.sid and msgs.rid (e.g. BIGINT vs. INT) and MySQL -- complains, don't bother to apply the constraint: ALTER TABLE msgs ADD FOREIGN KEY (sid) REFERENCES maddr(id) ON DELETE RESTRICT; ALTER TABLE msgrcpt ADD FOREIGN KEY (rid) REFERENCES maddr(id) ON DELETE RESTRICT; amavisd-new-2.7.1/README_FILES/screen.css000640 000621 000620 00000007610 10640436060 017336 0ustar00markcmi000000 000000 /* $Id: screen.css 11 2005-12-27 17:00:04Z patrick $ */ body { font-family: Verdana,Helvetica,sans-serif; font-size: 100.01%; color: #000000; background-color: #FFFFFF; margin:0; padding:0.5em; } a:link { color: blue; text-decoration: underline; } a:active { color: red; text-decoration: underline; } a:visited { color: darkblue; text-decoration: underline; } a:hover { color: red; text-decoration: underline; } div.author, div.chapter { font-size:0.8em; } table { font-size:1em; } .title { font-size:1em; } h2, h3, h4, h5 { margin:2em 0em 0em 0em; } pre { font-size:1.1em; } .programlisting { font-family: "Courier New", monotype; padding:0.5em; border-left:1px dashed #000000; background-color:#EBEBEB; } .screen { font-family: "Courier New", monotype; padding:0.5em; background-color:#F0F8FF; } .important { -moz-border-radius:15; border:2px solid #FFFF00; padding:0.5em; } .note, .tip { -moz-border-radius:15; border:2px solid #949494; padding:0.5em; } code { font-size:1.2em; } em.parameter { font-family: "Courier New", monotype; font-style:normal; } ol li p { margin:0; } ul li p { margin:0; } p { line-height:1.3em; } dd p { margin:0em 0em 0.5em 0em; padding:0em 0em 0em 0em; } dt { margin:1em 0em 0em 0em; padding:0em 0em 0em 0em; } div.toc a { text-decoration:none; } div.toc { margin:2em 0em 2em 0em; } div.toc dt { margin:0.2em 0em 0em 0em; padding:0em 0em 0em 0em; } div.calloutlist a { color:#000000; text-decoration:none; } div.revhistory table, div.revhistory table td, div.revhistory table th { border:none; } /* h1 { font-size: 0.7em; margin: 0em 0em 0em 0em; } h2 { font-size: 0.7em; margin: 0.5em 0em 0em 0em; } h3 { font-size: 0.7em; margin: 0.5em 0em 0em 0em; } h4 { font-size: 0.7em; margin: 0em 0em 0em 0em; } h5 { font-size: 0.7em; margin: 0em 0em 0em 0em; } p { font-size: 0.7em; margin: 0em 0em 0.5em 0em; } ol { font-size: 0.7em; } tt { font-family: monotype; font-weight: bold; font-size:100%; } em { font-size:120%; } code { } p code { font-size:0.7em; } pre { font-size:0.7em; } hr { display:none; } .authorgroup { margin:1em 0em; } .revhistory table { border: none; padding: 0px 0px 0px 0px; margin: 1em 0em 1em 0em; } .revhistory th { border: none; padding: 0px 0px 0px 0px; } .revhistory td { border: none; padding: 0px 0px 0px 0px; } .note { border: 1px solid #CCCC99; background-color: #F5F5E7; padding: 0.5em 0.5em 0.5em 0.5em; margin: 1em 0em 1em 0em; } .note table { font-size:0.7em; } .caution { border: 1px solid #F6EA00; background-color: #FFFFC9; padding: 0.5em 0.5em 0.5em 0.5em; margin: 1em 0em 1em 0em; } .tipp { background-color: #F5F5DC; } .important { font-family: "Courier New"; border: 1px solid #F95E00; background-color: #F9CDB3; padding: 0.5em 0.5em 0.5em 0.5em; margin: 2em 0em 2em 0em; } .screen { font-size:0.9em; font-family: monospace; border: 1px solid #999999; background-color: #EBEBEB; padding: 0.5em 0.5em 0.5em 0.5em; margin: 1.5em 0em; } .programlisting { font-size:1em; font-family: monospace; border: 1px solid #B2DBFF; background-color: #F0F8FF; padding: 0.5em 0.5em 0.5em 0.5em; margin: 1.5em 0em; } .important .programlisting { font-family: "Courier New"; font-size: 0.7em; border: 1px solid #B2DBFF; background-color: #F0F8FF; padding: 0.5em 0.5em 0.5em 0.5em; margin: 2em 0em 2em 0em; } .important .screen { font-family: "Courier New"; font-size: 0.7em; border: 1px solid #999999; background-color: #EBEBEB; padding: 0.5em 0.5em 0.5em 0.5em; margin: 2em 0em 2em 0em; } .informalexample { font-family: "Courier New"; font-size: 0.7em; border: 1px solid #B2DBFF; background-color: #F0F8FF; padding: 0.5em 0.5em 0.5em 0.5em; margin: 2em 0em 2em 0em; } */ amavisd-new-2.7.1/README_FILES/README.customize000640 000621 000620 00000105110 11741066641 020247 0ustar00markcmi000000 000000 Customization of notification messages and log entries ====================================================== Mark Martinec , 2002, 2004, 2006, 2007, 2008, 2010, 2011, 2012 Since March 2002 amavisd-new provides a way to customize e-mail notification messages that are sent in response to a virus (and spam) detection, without having to resort to modifications of Perl code. Three types of messages are generated: - administrator notifications are sent to administrator; (two types: virus, spam) - sender (non)delivery notifications may be sent to the mail originator; (three types: virus, spam, rejects by outgoing MTA) - recipient warnings may be sent to envelope recipients of the e-mail containing a virus. These notifications are normally disabled since they are more of a nuisance than benefit. (only for viruses) Besides the three types of e-mail notifications, the same customization principle is applied to customize the most common and useful log entry (present even at log level 0) that is generated when an e-mail containing an unwanted content is detected. Default Template texts are glued to the end of the 'amavisd' file, separated one from another by __DATA__ lines. Please see comments in these templates. These template texts are assigned to variables: $log_templ, $notify_sender_templ, $notify_virus_sender_templ, $notify_virus_admin_templ, $notify_virus_recips_templ, $notify_spam_sender_templ, $notify_spam_admin_templ The value of these variables may be overruled by assignment or by reading into them in the amavisd.conf file, which is run at amavisd startup time. If assigning to variables, care must be taken to properly quote certain special characters (like backslash), as required by Perl quoting rules. Text read from amavisd file or from external files is not subject to Perl quoting rules. Template text is subject to simple run-time macro expansion as described in the next section. Macro expansion is performed by the routine expand(), which receives substitution text of simple macros from 'amavisd' program. expand() takes a template string as its argument, performs macro expansion, and returns resulting multiline string back to 'amavisd', which uses it to send mail notifications or to write log entries. The substitution text for the following simple macros is built-in: - to be used in forming a mail header (properly quoted addresses as required by RFC2822): f administrator's e-mail address (typically used in 'From:' header of notification messages); T a list of recipients to be used in 'To:' header of the notification; C a list of recipients to be used in 'Cc:' header of the notification; B a list of recipients to be used in 'Bcc:' header of the notification; (the T, C and B lists are determined by each notification subroutine) - to be used in forming a notification mail body or log entry: p the current policy bank name (or empty if a built-in policy bank is still in place); h dns name of this host, or configurable name (variable $myhostname) HOSTNAME same as h n amavis internal log id (also called task id, am_id) as shown in the log and by amavisd-nanny, e.g. 58725-05-2 b message digest of a mail body: digest calculated by MD5 algorithm, encoded as hex digits, high nybble first; date_unix_utc timestamp of the message reception - Unix time (seconds since 1970-01-01T00:00Z as a decimal integer) date_iso8601_utc timestamp of the message reception - ISO 8601 (EN 28601) UTC date-time date_iso8601_local => sub {iso8601_timestamp($MSGINFO->rx_time)}, date_rfc2822_local timestamp of the message reception - RFC 2822 local date-time format week_iso8601 returns an ISO 8601 week number (between 1 and 53) corresponding to the current message reception time (date_iso8601_local) partition_tag give the current value of a partition_tag attribute which will be stored in SQL records when SQL quarantining or logging is enabled; the value is derived from a setting $partition_tag (global or per-policy bank); a typical usage is to use ISO 8601 week number as a pertition tag; P same as partition_tag u same as date_unix_utc d same as date_rfc2822_local U same as date_iso8601_utc y elapsed time in ms for this message (not including final cleanup) s original envelope sender, rfc2821-quoted and enclosed in angle brackets rfc2822_sender an e-mail address from a Sender header field, or empty rfc2822_from an e-mail address from a From header field (possibly more than one) rfc2822_resent_sender e-mail address from all Resent-Sender header fields rfc2822_resent_from e-mail address from all Resent-From header fields tls_in returns TLS ciphers in use by a SMTP session if mail came to amavisd through a TLS-encrypted session, otherwise result is empty t first entry in the 'Received' trace of the mail header g original SMTP client DNS name as obtained from an XFORWARD NAME field, or from a 'client_name' attribute in an AM.PDP request; empty if unknown; e best guess of the originator IP address: the bottom-most public IP address as obtained by parsing Received trace fields; client_helo client-supplied EHLO/HELO domain name from the original SMTP session as obtained through XFORWARD HELO or from a 'helo_name' attribute in an AM.PDP request; client_addr original SMTP client source IP address, same as %a, as obtained through XFORWARD ADDR or from a 'client_address' attribute in an AM.PDP request, or by parsing the topmost Received header field with a valid IP address as a last resort; a is a synonym for client_addr client_port original SMTP client source TCP port number as obtained through XFORWARD PORT or from a 'client_port' attribute in an AM.PDP request; client_addr_port combines addr and port, similar to: \[%a\]:[:client_port] l (letter ell, suggesting 'local') is true if a variable 'originating' is true, and is an empty string otherwise; the boolean variable 'originating' is under policy bank control, and usually corresponds to a sending host (SMTP client's IP address) matching @mynetworks_maps, or a client being an authenticated roaming user; o best attempt at determining true sender of the virus - normally same as %s S address that will get sender notification; this is normally a one-entry list containing sender address (same as %s), but may be unmangled/reconstructed in an attempt to undo the address forging done by some viruses; in case of unknown (e.g. forged) sender address, the result is empty. R a list of original envelope recipients (for use in notification body, not headers) D a list of recipients with successful delivery status (will get mail) O a list of recipients with unsuccessful delivery status (will not get mail) N a list of recipients with UNsuccessful delivery status (will NOT get mail) with included short per-recipient delivery reports as used in the free format first MIME part of delivery status notifications j 'Subject' header field body m 'Message-ID' header field body r first 'Resent-Message-ID' header field body header_field field body of the header field specified in the argument; the first argument is a header field name (case insensitive); optional second argument truncates the result to n characters; optional third argument j helps choosing the header field in case of multiple fields of the same name: returns a j-th header field with a given field name; search proceeds top-down if j >= 0, or bottom up for negative values (-1=last, -2=next-to-last, ...); unspecified j is equivalent to -1, i.e. the last header field of the specified name; useragent returns 'User-Agent: ...' or 'X-Mailer: ...' header field (whichever is present); an optional argument specifies whether an entire field is to be returned (empty or unrecognized argument), or just a field name (argument: 'name'), e.g. 'X-Mailer'; or just a field body (argument 'body'), e.g. 'Thunderbird_1.5.0.9'; Q MTA queue ID of the message if available (Courier, sendmail milter/AM.PDP) H a list of all header lines (field may be wrapped over more than one line); this does not include the 'Return-Path:' or 'Delivered-To:' header fields, which would have been added (or will be added later) by the local delivery agent if mail would have been delivered to a mailbox. z original mail size (in bytes) i long-term unique mail_id on this system, possibly used in log and in quarantine names (also in releasing from a quarantine), encoded in base64url (RFC 4648), e.g. jaUETfyBMJHG q list of quarantine mailbox names, or empty if not quarantined ccat_maj (deprecated, use [:ccat|major]) a major category number of the blocking or a main contents category, see constants CC_* in file amavisd ccat_min (deprecated, use [:ccat|minor]) a minor category number of the blocking or a main contents category, usually 0 unless more specific information is available (e.g. details about bad header or tag3 spam level) ccat_name (deprecated, use [:ccat|name]) a display name of the blocking or main contents category best describing mail contents ccat a new general-purpose macro providing access to information about a mail contents category. Macro 'ccat' takes two optional fixed-string arguments, which are interpreted case-insensitively. In their absence it expands to a string "(maj,min)" which shows a major and a minor contents category number of a blocking ccat for a blocked message, and of a main contents category for a passed message. The first argument specifies which attribute of a ccat is to be provided, the second argument specifies whether a main or a blocking contents category is to be consulted: The first argument may be any of the following strings: name ... provide a human-readable name of a ccat (%ccat_display_names) major ... provide a number: a major contents category, values correspond to CC_* constants in the program minor ... provide a number: a minor contents category, often a 0 ... empty argument (also a default) results in a string "(maj,min)" is_blocking ... '1' if blocking_ccat is true (message is being blocked), or an empty string when a message is being passed; is_nonblocking .. the opposite: '1' if blocking_ccat false, '' otherwise is_blocked_by_nonmain .. '1' if blocking_ccat is true _and_ is different from a main contents category; The second argument may be any of the following strings: main ... provide information on main contents category when asked for name/major/minor/ blocking.. provide information on blocking contents category if it exists, otherwise it falls back to providing info on main ccat; this is also a default in the absence of this argument; v output of the (last) virus checking program V a list of virus names found; contains at least one entry (possibly empty) if a virus was found, otherwise a null list banning_rule_key a lookup key (a regexp) of a banning rule which matched, e.g.: (?-xism:^\\.(exe-ms|dll)$(?# rule #9)) banning_rule_comment a comment from a regexp in banning_rule_key, or the whole banning_rule_key if it does not contain a comment, e.g.: rule #9 banning_rule_rhs right-hand side of a banning rule which matched, often just a '1', but can be any string which evaluates to true banned_parts a list of banned parts, with their full path in a MIME/archive e.g.: multipart/mixed | application/octet-stream,.exe,.exe-ms,videos.exe banned_parts_as_attr similar to banned_parts, but uses a different syntax using attribute/value pairs; currently known attributes are: P: part's base name, i.e. a file name in a ./parts/ temporary directory L: part's location (path) in a mail tree M: MIME type as declared in MIME header fields of a message T: short part's content type according to a file(1) utility N: declared part names (none, one or more), as declared in MIME header fields or in an archive (tar, zip, ...) A: part's attributes as follows: U=undecodable, C=crypted, D=directory, S=special(device), L=link e.g.: P=p003,L=1,M=multipart/mixed | P=p002,L=1/2,M=application/octet-stream,T=rar,N=Setup1.1.rar | P=p007,L=1/2/4,T=exe,T=exe-ms,N=setup.exe F just the first leaf node from banned_parts, with a prepended rule comment (if any), e.g.: rule #9:application/octet-stream,.exe,.exe-ms,videos.exe W a list of av scanner names detecting a virus X a list of header syntax violations A a list of SpamAssassin report lines for body report, similar to macro SUMMARY, but provides a list instead of a single string c spam level/hits (mnemonic: sCore) as provided by SpamAssassin, including a per-recipient boost (shown as explicit sum); see also: SCORE 1 above tag level for any recipient? 2 above tag2 level for any recipient? k any recipient declared the message be killed ? T list of triggered SA tests (only in $log_templ and $log_recip_templ) wrap with arguments: width, prefix, indent, string will wrap a string to a multiline string of the specified width; for details see comments before the 'sub wrap_string' in file amavisd; lc lowercases arguments and concatenates them to a single string uc uppercases arguments and concatenates them to a single string substr same as Perl function substr: returns a substring of the first argument, starting at character position specified by argument 2, limited to arg3 characters unless arg3 is not specified; character positions start at 0 index same as Perl function index: locates substring arg2 within arg1, returns -1 if not found, or a character position of the first match len returns string length of its first argument incr returns arg1 incremented by 1 (non-numeric argument interpreted as 0); in the presence of arguments beyond arg1, their sum is added to arg1 decr returns arg1 decremented by 1 (non-numeric argument interpreted as 0) in the presence of arguments beyond arg1, their sum is subtracted from ar1 min returns the smallest number from its arguments, all-whitespace arguments are ignored max returns the largest number from its arguments, all-whitespace arguments are ignored sprintf (just like a Perl function sprintf): formats its arguments, after the first, under control of the format, which is the first argument; the format is a character string which contains three types of objects: plain characters, which are simply copied to resulting string, character escape sequences which are converted and copied to the resulting string, and format specifications, each of which takes the next successive argument and inserts its formatted representation to the resulting string; Note that to get a percent character it needs to be doubled, to avoid its special meaning as a macro call introducer, e.g. [:sprintf|%%s %%.1f|text|[:SCORE]] join (just like a Perl function join): the first argument is a separator string, remaining arguments are strings to be concatenated, with a separator string inserted at every concatenation point; limit takes two arguments: a string size limit and some string, returning the string from the second argument unchanged if its size is below the limit, or cropped to the size limit, with '[...]' appended; dquote encloses its argument in double quotes and doubles existing double quotes within a string (suitable to sanitize Subject header field, e.g. ab"oh"cd -> "ab""oh""cd"); provisional - exact interpretation may change (and has changed, prior to 2.4.5 double quotes were replaced by \", which made parsing tricky as backquotes themselves were not escaped); uquote replaces one or more consecutive space or tab characters by '_', but does not protect existing underlines, which makes it a lossy transformation (suitable for logging of From or To header fields); provisional - exact interpretation may change; mime2utf8 takes a string as its first argument, and an optional truncation length as the second. The string is decoded as a MIME-Header string (understands Q or B character set encodings like =?iso-8859-2?Q?...?=, =?koi8-r?B?...?=) and converted to UTF-8, optionally truncated to the specified size at clean UTF-8 boundaries, and returned as a result. The macro can be useful to decode Subject or From header fields, e.g.: [? [:header_field|Subject]||,\ Subject: [:dquote|[:mime2utf8|[:header_field|Subject]|100]]]# supplementary_info gives access to some additional information provided by content scanners, such as a provided by SpamAssassin API routine get_tag. The macro takes two arguments, the first is a tag name (a name of some attribute which is expected to provide an associated value), the second argument is a sprintf format string and is optional, if missing a %s is assumed. Currently the only available attributes are AUTOLEARN, AUTOLEARNSCORE, SC, SCRULE, SCTYPE, RELAYCOUNTRY, and LANGUAGES. These are nonempty only when an associated SpamAssassin plugin or function is enabled. report_format gives one of the: dsn, arf, attach, plain, resend, according to a report or notification format being generated; feedback_type is expected to give one of the ARF (draft) strings: abuse, fraud, miscategorized, not-spam, opt-out, opt-out-list, virus, other, as supplied in a call to delivery_status_notification(); dkim takes one argument and returns the requested information about valid DKIM or DomainKeys signatures found in a mail header: any (or an empty argument) returns true if there was at least one valid signature present; author is nonempty if a valid author signature is present (matching a From header field), actual value happens to be a signing domain; sender is nonempty if a valid sender signature is present (signature matching a Sender header field, or matching a From if Sender is not present), the actual value happens to be a signing domain; thirdparty is nonempty if there is no valid author signature but at least one valid signature is present, the actual value happens to be a signing domain; envsender is nonempty if a valid signature is present which is matching the envelope sender address, the actual value happens to be a signing domain; identity returns a comma-separated list of signing identies ('i' tag) found in all valid signatures; selector returns a comma-separated list of signing selectors ('s' tag) found in all valid signatures; domain returns a comma-separated list of signing domains ('d' tag) found in all valid signatures; sig_sd returns a comma-separated list of selector:domain pairs found in all valid signatures; multiple occurences of the same selector:domain pair are listed only once; newsig_sd returns a comma-separated list of selector:domain pairs of newly created signatures, added to a mail header section; multiple occurences of the same selector:domain pair are listed only once; any other value (which may contain an '@', but local part is ignored) is interpreted as a 'verifier acceptable signing domain', and the result is true if there exists a valid signature whose signing domain ('d' tag) matches the supplied verifier acceptable signing domain, otherwise the result is false; the domain name in the argument may be prefixed by a '.', in which case subdomains are allowed too; A couple of SpamAssassin look-alike macros with the same names and arguments as in SA, see 'man Mail::SpamAssassin::Conf' for details: AUTOLEARN autolearn status (deprecated, use supplementary_info instead) DATE same as date_rfc2822_local SCORE similar to macro 'c', but returns a single number (sum of SA score and boost), and allows padding as per SA documentation. In a per-message log ($log_templ) when a message has multiple recipients, a minimum value across all recipients is given; STARS score as in macro SCORE, but represented as a bar of characters REPORT a SA terse report of tests hit (for header reports) SUMMARY similar to macro A, but provides a single multiline string, a SA summary of tests hit for standard body reports REMOTEHOSTADDR IP address of your connecting MTA (often 127.0.0.1) REMOTEHOSTNAME your connecting MTA (often [127.0.0.1] or localhost) TESTS similar to %T in logging, but without scores TESTSSCORES similar to %T in logging, but allows to specify a separator YESNO similar to macro '2', but provides Yes/No instead of 1/0 YESNOCAPS similar to macro '2', but provides YES/NO instead of 1/0 REQD minimal tag2 level of all recipients HEADER same as a header_field macro and two additional SA-lookalikes, but with no counterparts in SpamAssassin: LOGID log id (a.k.a. am_id) e.g. 58725-05-2, synonym for macro n MAILID mail_id as in quarantine e.g. jaUETfyBMJHG, synonym for macro i - when $log_recip_templ is expanded (by-recipient log entry), certain macros keep their general semantics but reflect a value for that recipient: %. value is a recipient counter (starting by 1) when $log_recip_templ is expanded, and is undef when other templates are expanded; %R current recipient email address %D recipient email address if mail will be delivered, otherwise empty %O recipient email address if mail will NOT be delivered, otherwise empty %N short DSN if mail will not be delivered for this recip, otherwise empty %c spam level/hits including a per-recipient boost %0 recipient email address belong to local_domains_maps: L or 0 %1 above tag level for this recipient: Y or 0 %2 above tag2 level for this recipient: Y or 0 %k above kill level for this recipient: Y or 0 REQD recipient's tag2_level ccat_maj major category number, takes into account per-recip bypass_* ccat_min minor category number, takes into account per-recip bypass_* ccat_name display name of the c.cat, takes into account per-recip bypass_* remote_mta MTA to which a message was forwarded remote_mta_smtp_response MTA's SMTP response on accepting forwarded message smtp_response either an MTA's SMTP response for forwarded mail, or internally generated SMTP response for mail that was not forwarded score_boost internally generated score points to be added to a SA score The choice of capital letters for lists, and lower case letters for simple strings is purely a convention and is not enforced, neither do all built-in macros adhere to the convention. Further built-in macros are easy to add if special need arises, just append new key/value pairs to the hash which is passed to expand(). Besides a simple string or an array reference, a hash value may also be a subroutine reference which will be called later during macro expansion. This way one can provide a method for obtaining information which is not yet available during initial construction of the hash (such as AV scanner results), or provide a lazy evaluation for more expensive calculations. Subroutine will be evaluated in scalar context. It may return a string or an array reference. The rest of this text explains what expand() does with its arguments. EXPAND This is a simple, yet fully fledged macro processor with proper lexical analysis, call stack, quoting levels, user supplied and builtin macros, three builtin flow-control macros: selector, regexp selector and iterator, plus a macro #, which discards input tokens until newline (like 'dnl' in m4). Also recognized are the usual \c and \nnn forms for specifying special characters, where c can be any of: r, n, f, b, e, a, t. Lexical analysis of the input string is performed only once, macro result values are not in danger of being lexically re-parsed and are treated as plain characters, loosing any special meaning they might have. New macros (with arguments) can be defined by a built-in macro [= name | body ]. Macro calls can be neutral (not re-evaluating the result) or active (pushing result back to input for re-evaluation). Simple caller-provided macros can evaluate to a string (possibly empty or undef), or an array of strings. It can also be a subroutine reference, in which case the subroutine will be called whenever macro is evaluated, supplying it with arguments from the macro call (first argument is a macro name). The subroutine must return a scalar: a string, or an array reference. The result will be treated as if it were specified directly. Two simple forms of macro calls are known: %x and %#x (where x is a single letter macro name, i.e. a key in a user-supplied associative array): %x evaluates to the hash value associated with the name x; if the value is an array ref, the result is a single concatenated string of values separated with comma-space pairs and is not re-evaluated (i.e. it is a form of a neutral macro call) %#x evaluates to a number: if a macro value is a scalar, returns 0 for all-whitespace value, and 1 otherwise. If a value is an array ref, evaluates to the number of elements in the array. Neutral call. A literal percent character can be produced by %% or \%. To take away any special meaning of other characters they can be quoted by a backslash, e.g. \[ or \\ or \"] . A more general form of a macro call provides an ability to call macros with longer names, to provide arguments to a call, and to specify whether the call is a neutral or an active call: [@ name |arg1|arg2|...|argn] is an active macro call, i.e. the result is pushed back to input and evaluated like usual; there may be any number of actual arguments supplied, argument 0 is a macro name; Neither a macro name nor arguments are implicitly quoted, so if it is desired to prevent their evaluation before a call, they should be quoted by enclosing them in a pair of [" and "], e.g.: [@["name"]|["arg1"]|["arg2"]| ... |["argn"]] [: name |arg1|arg2|...|argn] is an neutral macro call, i.e. the result is NOT pushed back to input for evaluation, but goes directly to destination (either straight to output string, or collected as argument if this is a nested macro call); Note that whitespace around macro name is allowed (is optional), and is automatically removed, but space within arguments receives no special treatment, and is passed on to a macro like other characters. There is a further simple form of a neutral macro call which emulates SpamAssassin syntax: _NAME_ or _NAME(...)_ where a macro name must be composed of capital letters only, and an optional string in () is passed to a macro as its first argument (besides argument 0, which is a macro name). Commas within (...) are not special, calls like _TESTS(,)_ and _SPAMMYTOKENS(2,short)_ still provide a single argument: "," or "2,short" respectively, to accommodate SA peculiarity. These all-capitals macros can still be called by a normal general-purpose form of a macro call for greater flexibility, as described above. A macro is evaluated only in non-quoted context. Enclosing strings between tokens [" and "] prevents its evaluation. Quoting may be nested, quote tokens must be balanced. Evaluating a quoted input strips off one level of quotes. As a special feature two built-in macros (selector and iterator) provide implicit quoting of their arguments to keep clutter in these two most common macros calls down to a minimum. Unfortunately this causes some complications in the code, but the feature is kept for backwards compatibility. Built-in macros selector, regexp selector and iterator have the following syntax: [? arg1 | arg2 | ... ] a selector [~ arg1 | arg2 | ... ] a regexp selector [ arg1 | arg2 | ... ] an iterator where [, [?, [~, | and ] are required tokens. Arguments are arbitrary text, possibly multiline, whitespace counts. Nested macro calls are permitted, proper bracket nesting must be observed. SELECTOR lets its first argument be evaluated immediately, and implicitly quotes remaining arguments. The evaluated first argument chooses which of the remaining arguments is selected as a result value. The chosen result is only then evaluated (i.e. this is an active macro call), non-selected arguments are discarded without evaluation. The first argument is usually a number (with optional leading and trailing whitespace). If it is a non-numeric string, it is treated as 0 for all-whitespace and as 1 otherwise. Value 0 selects the very next (second) argument, value 1 selects the one after it, etc. If the value is greater than the number of available arguments, the last one (unless it is the only one) is selected. If there is only one (the first) alternative available but the value is greater than 0, an empty string is returned. Examples: [? 2 | zero | one | two | three ] -> two [? foo | none | any | two | three ] -> any [? 24 | 0 | one | many ] -> many [? 2 |No recipients] -> (empty string) [? %#R |No recipients|One recipient|%#R recipients] [? %q |No quarantine|Quarantined as %q] Note that a selector macro call can be considered a form of if-then-else, except that the 'then' and 'else' parts are swapped! ITERATOR in its full form takes three arguments (and ignores any extra arguments after that): [ %x | body-usually-containing-%x | separator ] All iterator's arguments are implicitly quoted, iterator performs its own substitutions on provided arguments, as described below. The result of an iterator call is a body (the second argument) repeated as many times as there are elements in the array denoted by the first argument. In each instance of a body all occurrences of a token %x in the body are replaced with each consecutive element of the array. Resulting body instances are then glued together with a string given as the third argument. The result is finally pushed back to input (active macro call) for possible further expansion. As a somewhat ugly hack (upwards compatible), it is possible to iterate on built-in macros with names longer than one character: [ long-macro-name | body-usually-containing-%x | separator ] This only works in a full three-argument form of iterator call, and the iterator variable name becomes a hard-wired literal x. There are two simplified forms of iterator call: [ body | separator ] or [ body ] where missing separator is considered a null string, and a missing formal argument name is obtained by looking for the first token of the form %x in the body. If there is no formal argument specified (neither explicitly nor in the body), the result is an empty string, which is potentially useful as a null lexical separator. Examples: [%V| ] a space-separated list of virus names [%V|\n] a newline-separated list of virus names [%V| ] same thing: a newline-separated list of virus names [ %V] a list of virus names, each preceded by NL and spaces [ %R |%s --> <%R>|, ] a comma-space separated list of sender/recipient name pairs where recipient is iterated over the list of recipients. (Only the (first) token %x in the first argument is significant, other characters are ignored.) [%V|[%R|%R + %V|, ]|; ] produce all combinations of %R + %V elements A combined example: [? %#C |#|Cc: [<%C>|, ]] [? %#C ||Cc: [<%C>|, ]\n]# ... same thing evaluates to an empty string if there are no elements in the %C array, otherwise it evaluates to a line: Cc: , , ...\n Whitespace (including newlines) around the first argument %#C of selector call is ignored and can be used for clarity. These all produce the same result: To: [%T|%T|, ] To: [%T|, ] To: %T The '#' removes input characters until and including newline after it. It can be used for clarity to allow newlines be placed in the source text but not resulting in empty lines in the expanded text. In the second example above, a backslash at the end of the line would achieve the same result, although the method is different: \NEWLINE is removed during initial lexical analysis, while # is an internal macro which, when called, actively discards tokens following it, until a NEWLINE (or end of input) is encountered. Further build-in macros are regexp selector and a macro-defining macro. REGEXP SELECTOR macro is somewhat similar to a plain SELECTOR, but it performs a regular expression match on its first argument. The syntax is: [~string|regexp1|then1|regexp2|then2|...|regexpN|thenN|else] It compares string to each regexp in turn, and if a match is found the macro expands to the corresponding 'then' argument, otherwise it expands to the 'else' argument (or empty if the 'else' is missing). For example: [~string|^s.*$|["matches"]|["no match"]] Unlike SELECTOR, arguments are not implicitly quoted, quoting must be explicit if desired (there was no backwards-compatibility need for this newer macro). This is an active macro call, results are pushed back to input for re-evaluation. Tokens %1, %2, ... %9 in arg3 and arg4 are replaced by captured subexpressions (in parenthesis) of a regexp, and %0 is replaced by arg1. DEFINE macro call allows new macros to be dynamically defined: [= name |body] It creates a new macro with a specified name (whitespace around name is trimmed), giving it the argument as a macro body string. No arguments are implicitly quoted, quoting must be explicit (for most uses at least the body string should be quoted in [" ... "] ). Body string may contain tokens %0, %1, %2, ... %9 as formal arguments. These will be substituted with actual arguments (or empty strings for missing arguments) at the time of a call. In most respects these dynamically defined macros behave just like other pre-defined macros. One distinction is that they can only produce a scalar string, they can not produce an array. amavisd-new-2.7.1/README_FILES/README.performance000640 000621 000620 00000031744 10277127437 020545 0ustar00markcmi000000 000000 This file README.performance is part of the amavisd-new distribution, which can be found at http://www.ijs.si/software/amavisd/ Updated: 2002-05-13, 2002-08-01, 2003-01-09, 2005-01-19 Here are some excerpts from my mail(s) on the topic of performance. Mark [...] | What I use now is FreeBSD+Postfix+amavisd+Sophie, Good choice in my opinion. (P.S.: add clamd to the mix) Hopefully hardware matches expectations, fast disks and enough memory are paramount. You may want to put Postfix spool on different disk than /var/amavis, where amavisd does mail unpacking. | is there any suggested configuration for this | environment? Especially if my server is a high loaded | busy mail hub/gateway? Any parameters for performance tuning? | Do I need to increase this number to fit a busy server? | Or any other related parameters should I notice? How many messages per day are we talking about? Both the amavisd child processes, and (to much lesser degree) the Postfix smtpd services consume quite some chunks of memory, so the memory size can determine how many parallel processes you can run. Note that the Perl interpreter in amavisd-new processes occupies the same memory if fork on a Unix system uses copy-on-write for memory pages, as most modern Unixes do. This however does not apply to memory allocated after the child processes have forked. I would start small, e.g. by 2 or 3 child processes per CPU (parameter $max_servers), then see how machine behaves. If you see heavy swapping or load regularly going beyond 2 or 3 (per CPU), decrease the number of parallel streams, otherwise increase it - gradually. This number is probably the most important tuning parameter. Going beyond 10 usually brings no more improvements in overall system throughput, it just wastes memory. If this does not come close to your needs, you may want to place amavisd-new with Sophie on a different host than Postfix. They talk via SMTP so there is no particular advantage in having both MTA and amavisd on the same host. Actually there are now three quite independent modules, which can share the same host, or not: incoming Postfix (MTA-IN) -> amavisd+Sophie -> outgoing Postfix (MTA-OUT) Both MTA-IN and MTA-OUT can be the same single Postfix, but need not be. If you decide to split MTA-IN and MTA-OUT, you can position one of them on the same host as amavisd, although I guess it would be better to either have three boxes, or have MTA-IN and MTA-OUT be a single Postfix, as in the normal setup, while optionally moving amavisd+Sophie to a different host. As amavisd-new is just a regular SMTP server/client to Postfix, one can use the usual load sharing mechanisms as available for normal mail delivery, like having multiple MX records for the content filter (applies to feeding amavisd by the Postfix service smtp, but not to lmtp which does not care for MX). [...] | I would like to know the possibility of email loss? Especially | under unawareness! What if amavisd or Sophie suddenly/abnormally | terminated? Is there any recovery procedures should be take? Mail loss should not be possible (except with disk failure holding MTA spool directories). I am continually testing some awkward situations like disk full, process restarts, child dies, even programming errors :) ... Amavisd never takes the responsibility for mail delivery away from MTA, it just acts as an intermediary between MTA-IN and MTA-OUT. Only when MTA-OUT confirms it has received mail, the MTA-IN does a SMTP session close-down with a success status code. All breakdowns and connection losses are handled by MTA, and Postfix is very good in doing it in a reliable way. The only cause of concern is DoS in some unpackers. This part of code in amavisd-new is still mostly the same as in the amavisd version, and although it does exercise some care, there is still a lot to be desired. Let me tell a heretic secret: if your AV scanner (e.g. Sophie) can handle all archives used by current viruses (except MIME decoding, which is done by amavisd), it is reasonably safe, good and fast to set $bypass_decode_parts to 1 (see amavisd.conf). And more: later Postfix versions can do the MIME syntax checking and enforce 7bit header RFC 2822 requirements (see parameters like: $ postconf | egrep 'mime|[78]bit' ) so you can block invalid MIME even before it hits the MIME::Parser Perl module. Instead of wasting 5 minutes for some particularly nasty archive, Sophie can do it in 5 seconds !!! I have yet to see a virus (in the wild) that Sophos would ONLY detect if first unpacked by amavisd. (P.S. not always true, but most of the time this is so) This does not take care of manual malicious intents, but one can always bring in a virus on a floppy, or download it some other way (e.g. PGP encrypted), if one really wants to. --------- See article by Cor Bosman for a high-end installation: http://www.xs4all.nl/~scorpio/sane2002/paper.ps --------- Limit the number of AV-scanning processes, don't let MTA run arbitrary number of AV-scanning processes (P.S. this is easy to ensure with Postfix, hard to do with pre-queue content filtering like sendmail milter or Postfix smtp proxy). Also limiting based on CPU load (like in sendmail) is not a good idea in my opinion - set the fixed limit based on the number of concurrent AV-checking processes you host (memory,disk,cpu) can handle, not on the current load or mail rate, otherwise when the situation goes bad, it is more likely it will go bad all the way - disk and memory thrashing is the last thing you desire when load goes high. --------- | I have a question about how to distribute amavisd-new directories across | different disks for optimal performance. There are usually 4 directories | in the amavisd-new mail path. | 1) The amavis TEMPBASE directory (Where incoming emails are scanned) | 2) The postfix queue directory | 3) The directory for amavis and mail system logs | 4) The directory where mail is delivered. | What would be the best distribution of these directories over multiple disks? | Obviously, having each one on a different disk would be best. However, if | you only have 3 disks to use, which two services should be combined? If you | have only two disks, which services should be put together? !!! Let amavisd-new log via syslog and make sure your syslogd does not call flush for every log entry !!! (as Linux does by default, but is configurable per log file)! This way the disk with log files becomes non-critical. The disk with Postfix mail queue is likely to be most heavily beaten by file creates/deletes. I would put it on its own disk. The $TEMPBASE (amavis work directory) is probably not as heavily exercised (in the SMTP-in/SMTP-out amavisd-new setup, as with Postfix), unless your mail messages often contain many MIME parts that need to be decoded. If you can afford it, it can even reside on a RAM disk / tempfs or with delayed-syncing without risking any mail loss. --------- Perl running in Unicode mode is reported to be noticably slower than otherwise. It is wise to disable it, e.g. by setting environment variable LC_ALL=C before starting amavisd on systems where this is not a default (Linux RedHat 8.0). See also 'Speed up amavisd-new by using a tmpfs filesystem for $TEMPDIR' at http://www.stahl.bau.tu-bs.de/~hildeb/postfix/amavisd_tmpfs.shtml by Ralf Hildebrandt --------- | define(`confMAX_DAEMON_CHILDREN', 20) | should we limit MaxDaemonChildren in MTA-RX ? ... what would be a magic | formula to define it? I assume it should be based on the number of | amavisd-new child processes (which should match queue runners) | and Max No. of msgs per connection ? Here the charm of dual-sendmail setup (or Postfix setup) is most apparent. The MaxDaemonChildren sendmail option is almost completely independent from the number of amavisd-new child processes. The MaxDaemonChildren in MTA-RX should be sufficiently large so that most of the time all incoming mail connections can each get its own sendmail process which is willing to accept the mail trickle. These smtp server processes are relatively lightweight (hopefully sharing the program code in memory), so they don't cost much. The upper limit is the number of sendmail receiving processes the host can comfortably handle, including disk I/O they produce. One may set this value high and observe the usual number of incoming parallel SMTP sessions during normally busy hours, then set the limit comfortably above that value. This applies to Postfix as well (maxproc for smtpd service on port 25). The number of amavisd-new child processes and the number of queue runners is another matter. Since content filtering (especially with SA enabled) is CPU and memory intensive, the number of content filtering processes is limited by the host power and its memory. Never have this number so high that swapping occurs, or that the time for each individual mail check gets too large, say over a couple of seconds. Long content checking times can also increase the locking contention on the SA Bayes database. P.S.: It is advised to move Bayes database to a SQL server, it need not be on a separate host. A very rough rule of thumb may be that the MaxDaemonChildren can easily be 10 times the number of content filtering processes. --------- > How did you determine this optimal number of child processes? > Is there a nice scientific way to do it or via simple trial and error. Measure and plot a diagram of maximum sustained mail troughput (msgs/h) for a couple of values of max_proc (and a matching value of $max_servers). About 5 or 10 strategically placed data points could already give a useful picture. Measuring throughput for each max_proc value takes a restart of amavisd and a postfix reload (and possibly: postfix flush), the measuring period should last for say 10 minutes or preferably more, assuming that the supply of mail to be processed does not run out and keeps mail processing saturated, i.e. that there are plenty of mail messages in the mail queue waiting to be processed by the content filter. An opportunity for such an experiment on a production machine arises when some backlog of mail accumulates, e.g. after a network outage or some other problem occurred that stopped mail flow. It is certainly possible to create a synthetic mail traffic. For mail sink (if needed) one could use the src/smtpstone/smtp-sink.c from a Postfix distribution, but for generating synthetic mail the smtp-source.c is not realistic regarding the mail contents, so it is best to use a real ham/spam/viruses mail mix, either on a running system, or mail collected and saved for the purpose during normal business hours. The exact mail rate can be deduced from the mail log for each measurement period. I usually choose to do a plot of cumulative message count vs. wall-clock time, which makes it easier to find the slope and ignore startup or other anomalies which stand out more obviously in the plot. A good enough mail rate fugure is given by the amavisd-agent program, which for the first couple of screens shows the counter averages since the start of amavisd (which is probably what we need here), then after about 2.5 minutes its starts reporting exact last 5 minute running averages. The usual counter of interest is the 'InMsgs' (which also happens to be the same as the 'CacheAttempts', which is the first counter on the reported list). One should get a diagram like: msgs/h | | * * * * | * | * | * | | * best |* -----------------------------> max_proc The optimum max_proc is where the function starts to level off, it gives the best throughput at a minimum waste of memory for excessive processes that gain no benefit. > We're running dual 3.0Ghz xeons (64bit fedora core 3) with 2GB or RAM - > what do you think the optimal number should be? > We have local copies of surbl.org and a local dcc server. Not using razor. I wouldn't dare to guess, you just have to measure it. It depends on too many factors. The more network-based high latency SA tests you have enabled, the higher would be the optimum max_proc value. The exact spot varies over daily network conditions and usage patterns, so don't bother to narrow it down too exactly. --------- LDAP lookups note from Michael Hall in reply to Matt Juszczak on Aug 12, 2005: > >You might try to make better indexes on the LDAP server before > >upgrading the hardware. > > Yep, tried this yesterday :) that was the problem. Added another index > for mailRoutingAddress (I already had mailLocalAddress created but I > guess the mailRoutingAddress needed one too) and now we're experiencing > instant mail delivery with no queues :) You definitely want to make sure you have indexes on any attribute used in searches, it can make a huge difference as you found out. If you're using OpenLDAP and 'bdb' databases you also want to make sure and configure the Berkely DB environment with a DB_CONFIG file, and use db_stat to check things. Below is an excerpt from one of our mail servers at work: $ db_stat-4.2 -h /var/db/openldap-data -m ... Correctly sizing the cache can make a big difference as answers can be pulled from it vs accessing the disks. amavisd-new-2.7.1/README_FILES/README.courier-old000640 000621 000620 00000015224 10423370304 020445 0ustar00markcmi000000 000000 How to use amavisd-new with Courier *********************************** by Martin Orr WARNING: This README applies to the older version of the Courier patch which can be used with versions of Net::Server older than 0.90. If you have Net::Server 0.90 or later, please see README.courier and use the newer amavisd-new-courier.patch. There are two different ways of using amavisd-new with Courier. In one of these, the startup and shutdown of amavisd is controlled by courierfilter and in the other it is done independently (probably through its own init script): I shall refer to these as the courierfilter and the manual startup/shutdown modes. It is important to note however that even in manual startup/shutdown mode, amavisd still talks to Courier using the courierfilter protocol. COURIERFILTER STARTUP/SHUTDOWN MODE This is probably the preferable mode of operation, but it alters the original amavisd code more deeply. In this mode amavisd (assuming it is enabled) will be automatically started and stopped whenever Courier starts and stops, and you can enable/disable it with filterctl {start|stop} amavisd If this mode is enabled in amavisd.conf, you must not start, restart or stop amavisd other than through filterctl. Also it may take up to 10 seconds after stopping courierfilter or using "filterctl stop amavisd" before amavisd actually stops. During this time you cannot start amavisd again: this particularly annoying because it means that "courierfilter restart" will not work, since it stops courierfilter and then starts it again immediately (in fact it will appear to work but all messages will be rejected with 432 Mail filters unavailable). MANUAL STARTUP/SHUTDOWN MODE In this mode you need to ensure that amavisd starts as soon as possible after Courier, as mails which arrive after Courier starts but before amavisd will not be filtered. (This includes when Courier is restarted.) However, starting amavisd before Courier will unfortunately not work. If you wish to disable virus checking you must not only stop amavisd but also remove the socket - Courier will refuse to accept mail while the socket exists but amavisd is not running. CONFIGURATION In amavisd.conf: 1. If you are using courierfilter startup/shutdown mode: Comment out the lines setting $daemon_user, $daemon_group: amavisd will be started as whatever user Courier runs as (not root) so is unable to change to another user. If you are using manual startup/shutdown mode: Set $daemon_user to whatever value you like. Set $daemon_group to the same group Courier runs as (probably courier or daemon) so that Courier will be able to write to amavisd's socket. 2. Set $forward_method to undef and $notify_method to 'pipe:flags=q argv=/usr/sbin/sendmail -f ${sender} -- ${recipient}' If you wish to include "local" in your enablefiltering file, or you are using a version of Courier older than 0.49.0, then you must instead use 'pipe:flags=q argv=perl -e $pid=fork();if($pid==-1){exit(75)}elsif($pid==0){exec(@ARGV)}else{exit(0)} /usr/sbin/sendmail -f ${sender} -- ${recipient}' 3. Set $courierfilter_shutdown to 1 if you are using courierfilter startup/shutdown mode, 0 if you are using manual startup/shutdown mode. 4. Set $unix_socketname to DIR/amavisd where DIR is the appropriate courierfilter directory. You need to decide whether you want a mandatory filter, in which case all mail passing through your server will be filtered, or an optional filter, in which case only mail to local users will be filtered, and your users will have the option of overriding filtering (you will almost certainly want a mandatory filter). You should see the courierfilter manpage for more details, and also for the correct directories on your system (on mine, they are /var/lib/courier/allfilters for mandatory filters and /var/lib/courier/filters for optional filters). 5. Replace any existing setting of $interface_policy{'SOCK'} with $interface_policy{'SOCK'} = 'AM-SOCK'; $policy_bank{'AM-SOCK'} = { protocol => 'COURIER' }; If you are using manual startup/shutdown mode, in order to allow amavisd to create its socket you must either start amavisd as root or make the filter directory group-writable. If you are using courierfilter startup/shutdown mode, install amavisd or a link to it in /usr/lib/courier/filters (or whatever directory is correct on your system - again, see the courierfilter manpage). The name of this link must match the name of the socket in $unix_socketname. Make sure that the enablefiltering file exists in your Courier configuration directory (/etc/courier or equivalent) and contains "esmtp". If you wish to include "local", to filter mail sent through the sendmail command, then you must use the long forking value of $notify_method in step 2 above. The patch also places the MTA queue ID into a %Q macro which you can use in log templates to get the Courier queue ID of the message. This is now in the default log template. RELEASING ITEMS FROM QUARANTINE If you wish to be able to release items from quarantine using the AM.PDP protocol, you must set this to use a TCP port. Include the following in amavisd.conf: $inet_socket_port = 9998; $interface_policy{'9998'} = 'AM.PDP'; $policy_bank{'AM.PDP'} = { protocol => 'AM.PDP', inet_acl => [qw( 127.0.0.1 [::1] )] # restrict access to these IP addresses }; Modify the amavisd-release script to use $socketname = '127.0.0.1:9998'. LIMITATIONS All the courierfilter protocol allows is to simply accept or reject a message. In addition it is possible by bypassing the protocol itself to add or remove recipients (thanks to Gordon Messmer for the original implementation of this in courier-pythonfilter). However it cannot modify the headers, rendering amavisd-new/courierfilter of limited use for spam checking. If you want spam checking, you should run spamd separately and pass mail to it either using maildrop or from .courier files. In this version of the patch, it is now possible to use D_BOUNCE and D_DISCARD as message destinies, as well as D_REJECT or D_PASS. D_REJECT is probably still a good choice however as the message is rejected in the original SMTP session. The client MTA receives the response "550 5.7.1 Message content rejected" and it is the client, not Courier, who is responsible for any DSN. ========= See also the PerlStalker's SysAdmin Notes and Tools: Courier+amavisd-new by Randall B. Smith, http://perlstalker.amigo.net/courier/amavisd-new.phtml It applies to the 20040701 release since which time the Courier patch has been completely rewritten. It still contains much useful information, but must be read in conjunction with this README. amavisd-new-2.7.1/README_FILES/README.sendmail000640 000621 000620 00000027700 10233024322 020012 0ustar00markcmi000000 000000 =============================================================================== NOTE: this file is rather old and not well maintained. A recommended sendmail setup is described in file README.sendmail-dual, which describes a dual-MTA setup. The sendmail milter setup as described in README.milter works as well, but with some functionality limitations. =============================================================================== AMaViS & sendmail ***************** Scanning only incoming mail --------------------------- The amavis script is designed to be used in the sendmail.cf configuration file in a similar way to how tcpd is used in /etc/inetd.conf. Amavis helper program receives sender ($f) and recipients ($u) from the command line, and the other arguments after '--' should be the original local delivery agent with original arguments. Amavis will run the original command after scanning for viruses if mail is clean. As most people generate sendmail.cf from a m4 file (we assume sendmail.mc), you should add the following just before the MAILER definitions: MODIFY_MAILER_FLAGS(`LOCAL',`-r')dnl define(`LOCAL_MAILER_ARGS',`amavis $f $u --' LOCAL_MAILER_PATH `-d $u')dnl define(`LOCAL_MAILER_PATH',`/usr/local/sbin/amavis')dnl The resulting Mlocal mailer entry could look like: Mlocal, P=/usr/local/sbin/amavis, F=lsDFMAw5:/|@qPmn9S, S=EnvFromL/HdrFromL, R=EnvToL/HdrToL, T=DNS/RFC822/X-Unix, U=root:amavis, A=amavis $f $u -- /usr/libexec/mail.local -d $u The user and group may be specified with the U option to the mailer. The group name in 'U=root:amavis' should match the chosen group name of the daemon amavisd(-new). This setup is probably the trickiest of them all to get right because of the conflicting daemon UID and file permission requirements of the different components in play. The amavisd daemon should not be running as root for security reasons, whereas the mail.local LDA needs privileges to access user mailboxes. Running amavis helper program as root:amavis retains root privileges for the helper program, while still alowing amavisd daemon process to access the temporary directory in the same group, even if not running as root. Scanning incoming/outgoing and relayed mail ------------------------------------------- The concept for scanning incoming/outgoing and relayed mail is different from the concept described in the AMaViS documentation. If you are running a newer version of sendmail (8.10.0 or better), we recommend to use the milter API. See README.milter for details. We use two different setups (.cf files) for sendmail, one is the original configuration, the second has a different Queue-Directory, another status file and most important a changed Rule Set 0 and the Mailer Definition AMaViS, so that AMaViS is always called first. If no virus is detected, we pass the mail to sendmail again, but advise it to use the original configuration. Note: I assume that sendmail.cf is in /etc - on your system it may be in /etc/mail Setting it up in easy 5 steps (without the m4 way) (please *read* the example configuration section below, too!): Step 1: Copy your /etc/sendmail.cf file to /etc/sendmail.orig.cf Step 2: Change sendmail.cf manually a) open /etc/sendmail.cf in your favorite editor b) change the queue directory, i.e. to O QueueDirectory=/var/spool/mqamavis c) change the status file, i.e. to O StatusFile=/var/log/amavis.st d) change rule set 0 to R$* $: $>Parse0 $1 initial parsing R<@> $#local $: <@> special case error msgs R$* $: $>98 $1 handle local hacks R$* $#amavis $:$1 #R$* $: $>Parse1 $1 final parsing Be careful of tabs, so here's the code again, instead of [tab] press the tab key :-) R$*[tab][tab]$: $>Parse0 $1[tab][tab]initial parsing R<@>[tab][tab]$#local $: <@>[tab][tab]special case error msgs R$*[tab][tab]$: $>98 $1[tab][tab]handle local hacks R$*[tab][tab]$#amavis $:$1 #R$*[tab][tab]$: $>Parse1 $1[tab][tab]final parsing Add the new mailer definition: Mamavis, P=/usr/sbin/amavis, F=nmlsACDFMS5:/|@qhP, S=0, R=0, T=DNS/RFC822/X-Unix, U=amavis:amavis, A=amavis $f $u [Step 3, with older amavis: do a ./configure --enable-relay --enable-sendmail, make and make install (you may add some more flags to configure)] Step 3, with amavisd-new: change the settings of $forward_method and $notify_method in /etc/amavisd.conf: $forward_method= 'pipe:flags=q argv=/usr/sbin/sendmail -i -f ${sender} -- ${recipient}'; $notify_method = 'pipe:flags=q argv=/usr/sbin/sendmail -i -f ${sender} -- ${recipient}'; Step 4: Create /var/spool/mqamavis with the same permissions as /var/spool/mqueue but owner and group should be amavis Step 5: Restart sendmail, i.e. killall -HUP sendmail or with SuSE Linux rcsendmail restart Setting it up in easy 7 steps - doing it the m4 way (please *read* the example configuration section below, too!) Step 1: Copy your /etc/sendmail.cf file to /etc/sendmail.orig.cf Step 2: Copy the provided doc/amavis.m4 file to /usr/share/sendmail/mailer (this is the location for a SuSE Linux system ... please have a look at your .mc file for the "include" macro. It tells you in which path your sendmail m4 stuff is located. Don't forget to put amavis.m4 into the mailer/ directory and not the m4/ dir) Step 3: Copy your .mc file, used for generating sendmail.cf, to amavis.mc Step 4: Change amavis.mc a) in front of the OSTYPE definition, add define(`QUEUE_DIR',`/var/spool/mqamavis')dnl define(`STATUS_FILE',`/var/log/amavis.st')dnl b) add the amavis mailer to the MAILER definitions MAILER(`amavis')dnl [Step 5, with older amavis: do a ./configure --enable-relay --enable-sendmail, make and make install (you may add some more flags to configure) ] Step 5, with amavisd-new: change the settings of $forward_method and $notify_method in /etc/amavisd.conf: $forward_method= 'pipe:flags=q argv=/usr/sbin/sendmail -i -f ${sender} -- ${recipient}'; $notify_method = 'pipe:flags=q argv=/usr/sbin/sendmail -i -f ${sender} -- ${recipient}'; Step 6: Create /var/spool/mqamavis with the same permissions as /var/spool/mqueue but owner and group should be amavis Step 7: Restart sendmail, i.e. killall -HUP sendmail or with SuSE Linux rcsendmail restart Additional information (please read!) ************************************* NOTE: If you decided to copy your original sendmail.cf to another filename than sendmail.orig.cf, you have to specificy the filename with --with-orig-conf= NOTE: This configuration could be made simpler if /etc/sendmail.cf remained untouched, and sendmail could be started simply with sendmail -bd -C/etc/amavis.cf. But for security reasons, sendmail refuses the -C flag if started as root. Therefore, we have to patch sendmail.cf and rename the original file. IMPORTANT NOTE: please have closer look at the mailer definition, especially the F equate (the mailer flags). You may copy the F= stuff out from your original sendmail.cf file, but be careful! You must not use the f flag. You may also add the A flag, otherwise "newaliases" will yell "cannot alias non-local names". NOTE: This concept should be considered *experimental*. NOTE: If mail is deferred, it may get stuck in the queue (this may happen if a delivery attemp fails). Calling /usr/sbin/sendmail -C /etc/sendmail.orig.cf -q via cron is a good idea. Another solution is to call /usr/sbin/sendmail -q5m -C /etc/mail/sendmail.orig.cf In this example, the mail queue is flushed every 5 minutes. EXAMPLE CONFIGURATION (sendmail 8.9.3) -------------------------------------- Here's the configuration I use on my SuSE Linux system with sendmail 8.9.3 (for sendmail 8.11 see below). AMaViS is run as user amavis, group amavis and therfore /var/spool/mqamavis is owned by amavis:amavis /etc/sendmail.cf: * I use the following mailer defintion Mamavis, P=/usr/sbin/amavis, F=nmlsACDFMS5:/|@qhP, S=0, R=0, T=DNS/RFC822/X-Unix, U=amavis:amavis, A=amavis $f $u /etc/sendmail.orig.cf: * to get rid off the X-Authentification-Warning "Processed by amavis with -C /etc/sendmail.orig" and "Processed from queue /var/spool/mqueue" I removed authwarnings from PrivacyOptions, so O PrivacyOptions=novrfy,noexpn NOTE: The "goaway" option is another PrivacyOption. The "goaway" option implies the "authwarnings" option, so with "goaway" you'll get the X-Authentification-Warning. /var/spool/mqueue and /var/spool/mqamavis is owned by amavis. NOTE: as amavis is run as user amavis, /var/virusmails must be owned by amavis and you have to specify a location for the AMaViS logfile that is writable by user amavis, if writing to a log file directly (not via syslog). NOTE: As sendmail will perform most tasks as user amavis now, it may not be able to read the users .forward file anymore! You may consider changing the permissions for the home directories, i.e. access rights for others. EXAMPLE CONFIGURATION (sendmail 8.11) -------------------------------------- Here's the configuration I use on my SuSE Linux system with sendmail 8.11. AMaViS is run as user amavis, group amavis. /etc/mail/sendmail.cf: * I use the following mailer defintion Mamavis, P=/usr/sbin/amavis, F=nmlsACDFMS5:/|@qhP, S=0, R=0, T=DNS/RFC822/X-Unix, U=amavis:amavis, A=amavis $f $u Note: The following entry does *NOT* work Mamavis, P=/usr/sbin/amavis, F=sDFMAw5:/|@qPfhn9, S=0, R=0, T=DNS/RFC822/X-Unix, A=amavis $f $u Hint: F=C (specifies that @domain has to be added to recipient) is needed otherwise you'll get an "user unknown" error. /etc/sendmail.orig.cf: * to get rid off the X-Authentification-Warning "Processed by amavis with -C /etc/sendmail.orig" and "Processed from queue /var/spool/mqueue" I removed authwarnings from PrivacyOptions, so O PrivacyOptions=novrfy,noexpn NOTE: The "goaway" option is another PrivacyOption. The "goaway" option implies the "authwarnings" option, so with "goaway" you'll get the X-Authentification-Warning. The Mlocal entry looks like this Mlocal, P=/usr/bin/procmail, F=lsDFMAw5:/|@qPfhn9, S=EnvFromL/HdrFromL, R=EnvToL/HdrToL, T=DNS/RFC822/X-Unix, A=procmail -Y -a $h -d $u (it seems that in the F= flags neiter the "o" nor "S" must be set ...) The permission of /var/spool/mqueue and /var/spool/mqamavis are the following: drwxrwxr-x 2 amavis root 1024 Sep 2 16:41 mqamavis drwxrwxr-x 2 amavis root 1024 Sep 2 16:41 mqueue As I use procmail als Local Delivery Agent, the setuid-bit for procmail has to be set! (d'oh ...) Note: For some reasons I'm not aware of, the notification messages generated by amavis are not sent immediately. Two solutions do exist for that (the latter one is the one I would recommend) * calling sendmail -C /etc/mail/sendmail.orig.cf -q via a cron job * or (prefered) change the delivery mode in /etc/mail/sendmail.orig.cf to # default delivery mode O DeliveryMode=i # i Deliver interactively (synchronously) NOTE: as amavis is run as amavis /var/virusmails must be owned by amavis and you have to specify a another location for the AMaViS logfile (normally /var/amavis/amavis.log) to which amavis has write access to. NOTE: As sendmail will perform most tasks as user amavis now, it may not be able to read the users .forward file anymore! You may consider changing the permissions for the home directories, i.e. access rights for others. TODO/BUGS --------- * huh? nothing?! that's unbelieveable :-) The author ---------- This stuff was written and tested by Rainer Link Rainer Link , http://rainer.w3.to/ Credits ------- This stuff is based on a patch from gody@master.slon.net and is itself based on the concept from Inflex. Thanks to Paul L. Daniels and (indirectly) to Steve Kehelet via the P.L.Daniels's Inflex scanner. Thanks to Yan Seiner for the m4 stuff, which our amavis.m4 is based upon. Section 'Scanning only incoming mail' updated by Mark Martinec. Thanks ------ Thanks to everyone who reported bugs or problems directly to me or the AMaViS user mailing list, and provided us/me with patches or additional information. amavisd-new-2.7.1/README_FILES/README.sendmail-dual.old000640 000621 000620 00000044725 10314360430 021523 0ustar00markcmi000000 000000 Dual-instance sendmail with amavisd-new --------------------------------------- Mark Martinec, 2003-05-06 (based on initial research by Ricardo Stella) last updated on: 2005-09-22 (added a reference to 'milter-ahead') The most recent version of this document can be found at: http://www.ijs.si/software/amavisd/README.sendmail-dual ========================================================================== The setup is very similar to the one described in README.sendmail (by Rainer Link) in section 'Scanning incoming/outgoing and relayed mail', except that it uses SMTP protocol over inet socket (instead of pipes to commands) to transfer files between MTA and amavisd-new and back, and that it uses a permanently running second sendmail instance in 'queue only' delivery mode, instead of bringing it up every time a new checked mail comes from amavisd. ========================================================================== Comparing the setup described in this document with the sendmail milter setup, as described in README.milter: milter - reasons in favour: - can REJECT on the original SMTP session, instead of generating a bounce (sending a non-delivery notification _after_ the mail has been enqueued); - only one sendmail daemon need be running, only one config file needed, no additional queue area needed (although starting with sendmail 8.12 more than one queue area is already a norm: clientmqueue, queue groups; and MSP already uses a different .cf file); dual-MTA - reasons in favour: - Full amavisd-new functionality is available, including adding spam and virus information header fields, adding address extensions and removing certain recipients from delivery while delivering the same message to the rest (*_lovers). Also a message can be split if different recipients need different header edits. All this is not available when using amavis-milter helper program. - Content scanning need not be performed at the time of mail reception. This allows better control on CPU-intensive content filtering: mail checking can be streamlined and performed at optimum throughput setting (number of content checker processes) so as not to overwhelm host resources, instead of leaving it at the mercy of the current number of incoming SMTP sessions where available crude controls are mostly based on system load. Typically the number of incoming SMTP sessions (tiny processes) is desired to be many times above the number of content filtering processes (heavy resource consumers). - No helper programs needed, MTA communicates with amavisd-new directly via SMTP, saves on creating one directory and one file for each message, and deleting it (at the cost of one additional transfer); - Receiving sendmail daemon (MTA-RX) need not run as root (using option RunAsUser) since it does not need to run any local delivery agents (LDA) or to access user .forward files. This avoids external SMTP clients talking directly to a process running as root. ========================================================================== The following setup is described in this document: ............................ ............................ : sendmail instance MTA-RX : : sendmail instance MTA-TX : : : : : 25 -----> \ (mqueue-rx) : : (mqueue) / -------> forward 587 -----> > -queue- : : -queue- ->-----+ : ^ : / | MAIL_HUB, : : | \ -------> local | : v SMART_HOST : : ^ : delivery msp ...........|................ ....|....................... | ^ loopback interface v | port 10025 loopback interf.| port 10024 | .....|.......................|............. : $inet_socket_port=10024 | : : | : : $forward_method='smtp:127.0.0.1:10025' : : $notify_method ='smtp:127.0.0.1:10025' : : : : amavisd-new : ........................................... The setup is based on the recent sendmail (8.12.9 or later) with its set of m4 configuration macros. Because of several security problems with earlier versions of sendmail it is advised to stick to the most recent version, although the functionality needed for this setup has long been available. If a particular macro or feature is not available with some older version, it is usually possible to achieve the same or similar by manually writing a new 'mailer' specification and/or tweaking the .cf file. We'll prepare two sendmail daemon instances (processes), let's call them MTA-RX (receiving, accepting) and MTA-TX (transmitting, delivering). For convenience we keep the name of the configuration file and the queue (spool) area at default names for one mailer instance, and choose non-default names for the other. Let's choose the MTA-TX to keep default names, and supply non-default names to MTA-RX explicitly. This will make admin utilities like mailq, newaliases, hoststat and purgestat operate on the outgoing mailer instance unless explicitly told otherwise. It can just as well be the other way around. MTA-RX (receiving mailer) will be responsible for accepting mail from the Internet or from internal hosts on port 25, optionally accepting local message submissions on tcp port 587 (rfc2476), and for message submissions via sendmail program. It will forward all mail (both for local and for nonlocal recipients) via SMTP protocol (or LMTP) to 127.0.0.1 (a loopback interface) on tcp port 10024, where amavisd daemon will be listening. - its queue: /var/spool/mqueue-rx - its config file: /etc/mail/sendmail-rx.cf, /etc/mail/submit.cf - the source (.mc) of the configuration file: thishost-rx.mc (where 'thishost' is often by convention the name of the system (uname)) MTA-TX (transmitting mailer) will be responsible for accepting checked mail and notifications from amavisd-new via SMTP on the loopback interface (127.0.0.1) at tcp port 10025, and will forward all mail to its final destinations, either for local delivery, or delivering outgoing mail to the Internet or to other internal mailers. - its queue: /var/spool/mqueue - its config file: /etc/mail/sendmail.cf - the source (.mc) of the configuration file: thishost-tx.mc In-between the two MTAs an amavisd daemon will accept mail via SMTP (or LMTP) protocol on tcp port 10024, check it, and forward checked mail and notifications via SMTP to MTA-TX. If you already have an existing sendmail installation, you already have a queue directory /var/spool/mqueue and the configuration file(s) (.mc source and the compiled .cf file). Most of the existing settings in your .mc file can be reused, and are to be moved to the new files thishost-rx.mc or thishost-tx.mc, or (some of them) to both. The settings pertaining to receiving mail, including recource limits, should go to thishost-rx.mc; settings pertaining to delivering mail (locally or to other mailers) should go to thishost-tx.mc, and general settings should go to both. The MTA-TX should have none or hardly any resource limits, or at least have them larger than MTA-RX. Large messages, common errors in mail, and mail rush-ins should be stopped or limited at their entry to the system. Accepting them first, but choking later can lead to trouble or at least to wasted resources. The file names thishost-rx.mc and thishost-tx.mc are arbitrary, they only serve as source (to the m4 macro processor) for producing .cf files, which control sendmail's behaviour. Sendmail never uses .mc files directly. MTA-TX already got its queue directory during sendmail installation. For MTA-RX a new queue directory needs to be created where incoming mail can be collected. Use the same ownership and protection as used for /var/spool/mqueue, e.g: # mkdir /var/spool/mqueue-rx # chown root:wheel /var/spool/mqueue-rx # chmod 700 /var/spool/mqueue-rx SECURITY NOTE: starting with sendmail 8.12 it is possible to start sendmail daemon as root and let it drop privileges (become user specified by RunAsUser) after binding to port 25. This is normally used by MSP, and it can just as well be used by MTA-RX, since it has no need to access user mailboxes and .forward files. To use this feature, specify user and group in the macro confRUN_AS_USER (file thishost-rx.mc), and set the ownership of mqueue-rx to this user and group: # chown smmsp:smmsp /var/spool/mqueue-rx # chmod 770 /var/spool/mqueue-rx More complex queue setup is possible if needed, like separating sendmail work area and core dump area from actual queues. For details about queue groups see sendmail documentation. Create file thishost-rx.mc: ---cut-here------------------------------ dnl To be used for MTA-RX, the first MTA instance (receiving mail) dnl Insert here the usual .mc preamble, including OSTYPE and DOMAIN calls. dnl Specify here also access controls, relayable domains, anti-spam measures dnl including milter settings if needed, mail submission settings, client dnl authentication, resource controls, maximum mail size and header size, dnl confMIN_FREE_BLOCKS, and other settings needed for receiving mail. dnl dnl NOTE: dnl confMIN_FREE_BLOCKS at MTA-RX should be kept higher than the same dnl setting at MTA-TX, to quench down clients when disk space is low, dnl and not to stop processing the already received mail. dnl dnl In particular, here are some settings to be considered: dnl ( see also http://www.sendmail.org/m4/anti_spam.html ) dnl dnl FEATURE(`access_db',`hash -T /etc/mail/access.db') dnl VIRTUSER_DOMAIN(`sub1.example.com')dnl list valid users here dnl VIRTUSER_DOMAIN(`sub2.example.com')dnl list valid users here dnl FEATURE(`virtusertable', `hash /etc/mail/virtusertable') dnl define(`confUSERDB_SPEC', `/etc/mail/userdb.db') dnl FEATURE(`blacklist_recipients') dnl INPUT_MAIL_FILTER(...) dnl define(`confPRIVACY_FLAGS', `noexpn,novrfy,authwarnings') nobodyreturn ? dnl define(`confDONT_PROBE_INTERFACES') dnl undefine(`USE_CW_FILE')dnl cancel use_cw_file feature, no class {w} extras dnl MASQUERADE_AS(...) FEATURE(`allmasquerade') FEATURE(`masquerade_envelope') dnl define(`confTO_IDENT', `0')dnl Disable IDENT dnl define(`confMAX_MESSAGE_SIZE',`10485760') dnl define(`confMAX_MIME_HEADER_LENGTH', `256/128') dnl define(`confNO_RCPT_ACTION', `add-to-undisclosed') dnl FEATURE(`nocanonify', ...) dnl define(`confBIND_OPTS', ...) dnl define(`confTO_RESOLVER_*... ) dnl define(`confDELAY_LA, 8) dnl define(`confREFUSE_LA', 12) dnl define(`confMAX_DAEMON_CHILDREN',20) dnl define(`confMIN_FREE_BLOCKS', `10000') dnl define(`confDEF_USER_ID', ...) define(`confRUN_AS_USER',`smmsp:smmsp')dnl Drop privileges (see SECURITY NOTE) define(`confPID_FILE', `/var/run/sendmail-rx.pid')dnl Non-default pid file define(`STATUS_FILE', `/etc/mail/stat-rx')dnl Non-default stat file define(`QUEUE_DIR', `/var/spool/mqueue-rx')dnl Non-default queue area define(`confQUEUE_SORT_ORDER',`Modification')dnl Modif or Random are reasonable dnl Match the number of queue runners (R=) to the number of amavisd-new child dnl processes ($max_servers). 2 to 7 OK, 10 is plenty, 20 is too many QUEUE_GROUP(`mqueue', `P=/var/spool/mqueue-rx, R=2, F=f')dnl dnl Direct all mail to be forwarded to amavisd-new at 127.0.0.1:10024 FEATURE(stickyhost)dnl Keep envelope addr "u@local.host" when fwd to MAIL_HUB define(`MAIL_HUB', `esmtp:[127.0.0.1]')dnl Forward all local mail to amavisd define(`SMART_HOST',`esmtp:[127.0.0.1]')dnl Forward all other mail to amavisd define(`confDELIVERY_MODE',`q')dnl Delivery mode: queue only (a must, dnl ... otherwise the advantage of this setup of being able to specify dnl ... the number of queue runners is lost) define(`ESMTP_MAILER_ARGS',`TCP $h 10024')dnl To tcp port 10024 instead of 25 MODIFY_MAILER_FLAGS(`ESMTP', `+z')dnl Speak LMTP (this is optional) define(`SMTP_MAILER_MAXMSGS',`10')dnl Max no. of msgs in a single connection define(`confTO_DATAFINAL',`20m')dnl 20 minute timeout for content checking DAEMON_OPTIONS(`Name=MTA-RX')dnl Daemon name used in logged messages dnl Disable local delivery, as all local mail will go to MAIL_HUB undefine(`ALIAS_FILE')dnl No aliases file, all local mail goes to MAIL_HUB define(`confFORWARD_PATH')dnl Empty search path for .forward files undefine(`UUCP_RELAY')dnl undefine(`BITNET_RELAY')dnl undefine(`DECNET_RELAY')dnl MAILER(smtp) ---end----------------------------------- Create file thishost-tx.mc: ---cut-here------------------------------ dnl To be used for MTA-TX, the second MTA instance dnl (delivering outgoing and local mail) dnl Insert here the usual .mc preamble, including OSTYPE and DOMAIN calls. dnl Specify here also the required outgoing mail processing and dnl local delivery settings such as mailertables, needed mailers, aliases, dnl local delivery mailer settings, smrsh, delivery mode, queue groups, ... dnl Don't use milters here - for all common purposes they belong to MTA-RX. define(`confREFUSE_LA',999)dnl Disable the feature, limiting belongs to MTA-RX define(`confMAX_DAEMON_CHILDREN',0)dnl Disable, limiting belongs to MTA-RX FEATURE(`no_default_msa')dnl No need for another MSA, MTA-RX already has one DAEMON_OPTIONS(`Addr=127.0.0.1, Port=10025, Name=MTA-TX')dnl Listen on lo:10025 define(`confSMTP_LOGIN_MSG', `$w.tx.$m Sendmail $v/$Z; $b')dnl define(`confTO_IDENT', `0')dnl Disable IDENT MAILER(smtp) MAILER(local) ---end----------------------------------- Now macro-expand .mc files into .cf files: (adjust the path if needed to where your cf/m4/cf.m4 file resides) # m4 /usr/share/sendmail/cf/m4/cf.m4 thishost-rx.mc >/etc/mail/sendmail-rx.cf # m4 /usr/share/sendmail/cf/m4/cf.m4 thishost-tx.mc >/etc/mail/sendmail.cf Start MTA-RX and MTA-TX daemons: # /usr/sbin/sendmail -C/etc/mail/sendmail-rx.cf -L sm-mta-rx -bd -qp # /usr/sbin/sendmail -L sm-mta-tx -bd -q15m Start queue runner for the MSP client queue as usual, if using it: # /usr/sbin/sendmail -Ac -L sm-msp-queue -q10m Start amavisd-new: # amavisd Test if MTA-RX is listening: # telnet localhost 25 QUIT Test if MTA-RX is listening on MSA port 587 (a newer sendmail invention) # telnet localhost 587 QUIT Test if MTA-TX is listening: # telnet localhost 10025 QUIT Test if amavisd is listening: # telnet localhost 10024 QUIT For convenience some shell aliases may be defined: alias mailq-rx='mailq -C/etc/mail/sendmail-rx.cf' alias mailq-tx='mailq' alias sendmail-rx='/usr/sbin/sendmail -C/etc/mail/sendmail-rx.cf' alias sendmail-tx='/usr/sbin/sendmail' All done! NOTES - In amavisd.conf file follow the 'POSTFIX or EXIM V4 or dual MTA setup', which is also the default. - The $final_*_destiny should not specify D_REJECT. The D_BOUNCE (or D_PASS or D_DISCARD) is preferred. - To make MTA-RX content-check only some mail but not all, one may use mailertables instead of MAIL_HUB and SMART_HOST. For example setting some recipient domains to be passed to MTA-TX at 127.0.0.1:10025 directly (e.g. via mailer 'esmtp'), while sending all the rest to amavisd at 127.0.0.1:10024. To be able to specify the port number, a new 'mailer' needs to be defined, let's call it 'amavis', with similar settings as the already defined 'esmtp', except with port number 10024. - depending on how local addresses are translated by MTA-RX, the %local_domains (or @local_domains_maps) in amavisd.conf needs to be adjusted accordingly to be able to recognize local domains. Check the amavisd-new log what recipient addresses it sees for local recipients. The '[127.0.0.1]' may need to be added to the @local_domains. - To make MTA-RX reject mail for nonexistent local users by itself (instead of generating a bounce later on), one may use the 'virtusertable' in thishost-rx.mc, listing all known recipients, and rejecting the rest, e.g.: VIRTUSER_DOMAIN(`example.com')dnl FEATURE(`virtusertable', `hash /etc/mail/virtusertable')dnl jim@example.com %1%3 joe@example.com %1%3 postmaster@example.com %1 @example.com error:5.7.0:550 No such user here You may use the righthand side of the map to specify local user (e.g. %1%3, or just jim, without domain name) in which case MAIL_HUB will be used for forwarding, or specify an explicit domain name that is not in the {w} class, in which case the SMART_HOST will get consulted. Perhaps what Stephane Lentz writes is even better: Dictionary attacks and messages to retired accounts can be bounced with sendmail: just replicate your aliases or write some SLocal_check_rcpt rule-set that checks addresses of your domain against a map of valid users (valid_addresses.db). I hope some standard FEATURE will be provided with sendmail - something like FEATURE(checkdomainaddresses) and CHECKDOMAINADDRESSES(mydomain.com). An alternative solution is to use a milter to do address verification against the second MTA in chain. See the milter-ahead project: http://www.milter.info/sendmail/milter-ahead/ PERFORMANCE NOTES - Mail handling is I/O-intensive. For better performance one may place the two mail queue areas (/var/spool/mqueue and /var/spool/mqueue-rx), and the /var/amavis work directory ($TEMPBASE) on three separate disks. The /var/amavis/tmp may be a tmpfs or a RAMdisk or delayed-sync fs. - One of the important arguments for choosing the dual-MTA setup is to be able to keep the number of content filtering processes under control, and not at the mercy of current mail inflow. Don't blow this advantage by setting the number of amavisd-new processes and MTA-RX queue runners too high! Throughput optimum is somewhere between 3 and 10 with fast daemonized av-scanner (or no av scanner) (with or without SpamAssassin), and between 2 and 3 with many command line scanners (regardless of SA). If the host is low on memory and when spam checking (SpamAssassin) is used, even 2 may be a lot for an elderly host. Start conservatively, e.g. at 2 or 3, and if everything works normally and higher throughput is needed, try a bit more. Anything above the point where throughput function levels off is just a waste of memory and gains nothing! The optimum may be higher if high-latency external SpamAssassin tests are enabled (e.g. Razor, RBL), Still, never go beyond available memory. For example with SpamAssassin enabled, the 20..25 processes on a 256 MB machine is where throughput begins to drop rapidly on a way to a swapping tar pit. amavisd-new-2.7.1/README_FILES/README.protocol000640 000621 000620 00000042343 11545464674 020110 0ustar00markcmi000000 000000 AMAVIS POLICY DELEGATION PROTOCOL (AM.PDP) ========================================== Author: Mark Martinec Revisions: 2003-11-10 created 2005-03-18 2005-06-22 2006-08-18 (added attributes: version_server, insheader and quarantine), 2007-05-17 (allow attribute value: request=requeue) 2008-06-21 (new attribute partition_tag=xx on release requests) 2009-11-16 2010-01-22 (new attribute log_id on response) NOTE: at the end of this document there is a description (by Stephane Lentz) of the old protocol, now called AM.CL. Amavis policy delegation protocol (AM.PDP) is intended to replace the old amavis client protocol (AM.CL) as spoken between amavisd-new helper programs (amavis.c or amavis-milter.c) and the amavisd daemon. The server side is implemented by amavisd-new daemon. A sample AM.PDP client in Perl is helper-progs/amavis.pl, and a rewrite by Petr Rehor of the helper program amavis-milter.c to use the new AM.PDP protocol is available as a separate project, see: http://sourceforge.net/projects/amavisd-milter/ The new amavisd client/server protocol is based on the 'Postfix policy delegation protocol', described in the Postfix document file README_FILES/SMTPD_POLICY_README, which can be used to delegate policy decisions to an external server that runs outside Postfix. It is conceptually similar to sendmail/milter mechanism, but based on a documented text protocol which is easier to test and debug, instead of using an undocumented binary protocol such as sendmail/milter. There are a few departures from the original Postfix policy delegation protocol: - Postfix policy delegation protocol terminates lines with LF, AM.PDP protocol terminates lines with CR LF; ( This was changed to facilitate debugging via telnet, and to make it more similar to related protocols, e.g. SMTP, HTTP, FTP, ... ) - Postfix policy delegation protocol restricts the reply to one field only, AM.PDP allows arbitrary number of fields passed from server back to client; ( This allows additional functionality like modifying or removing recipient addresses, adding or editing mail header fields, etc. ) - Postfix policy delegation protocol expects the same attribute name to be used only once; AM.PDP allows repeated appearances of the attribute, which makes passing of lists easier (such as multiple recipient addresses); - attribute value may consist of more than one field. Fields are delimited by exactly one non-encoded space. Spaces within a field must be encoded like any other restricted character (see below). The protocol may be spoken over a Unix STREAM socket, or over an inet tcp socket. The client request is a sequence of name=value attributes, each terminated by CR LF. The sequence is terminated by an empty line (only CR LF). The server reply has the same structure. After client receives server response sequence terminated by an empty line, it may close the session, or issue another request on the same session. During normal operation the server should not close the socket until it has been closed by the client. Only under special circumstances is the server allowed to close the session, e.g. in response to a timeout or fatal error condition. The order of attributes does not matter, except for the 'request' attribute which must appear first in the client request. The policy client as well as the policy server should ignore any attributes that it does not care about. An attribute name must not contain non-encoded characters: "=", "%", space, null, CR or LF. An attribute value must not contain non-encoded characters: "%", space, null, CR, or LF. All restricted characters must be hex-encoded to a sequence of three characters: a percent sign, followed by two hex digits, e.g. %2A. Hexadecimal digits 0..9,a..f or 0..9,A..F are allowed. Any other character MAY be encoded as well. Although not mandated, it is prudent to encode all non-printable characters. The line length is not limited by this protocol. Both the server and the client must be prepared to handle arbitrary line lengths without breaking the protocol or rising security issues. Both are allowed to internally truncate unreasonably long lines to a sensible length, and issue a warning. Neither the client nor the server must make any assumptions that certain characters will not be used in the attribute name or values. E.g. a presence of encoded null or newline or other special character in the attribute name or value must be safely and appropriately handled. If such a character does not comply with the expected syntax, the case should be handled to the best of client or server understanding and capability, e.g. character ignored, attribute ignored, and/or a warning logged. The following example Perl expression may be used for encoding/decoding: to decode attribute name and attribute values: s/%([0-9a-fA-F]{2})/pack("C",hex($1))/eg to encode attribute name: s/([^0-9a-zA-Z_-])/sprintf("%%%02x",ord($1))/eg; to encode attribute values (each space-separated field individually): s/([^\041-\044\046-\176])/sprintf("%%%02x",ord($1))/eg; Attributes in the client request are: ------------------------------------- request=AM.PDP is a required first attribute, its value must be AM.PDP sender= specifies the envelope sender address (reverse-path). The attribute should appear exactly once. The attribute value syntax is specified in rfc5321 as 'Reverse-path' (i.e. smtp-quoted form, enclosed in <>); a null reverse path is specified as <>. recipient= specifies the envelope recipient address. The attribute appears once for each recipient address, the order of addresses must be preserved and might be significant for some setups or functions. The attribute value syntax is specified in rfc5321 as 'Forward-path'. tempdir=/var/amavis/amavis-milter-MWZmu9Di Specifies a temporary work directory to be used for mail unpacking, typically also containing the original mail file - see attribute 'mail_file' below. This attribute should be present exactly once. The server is allowed to use the specified directory to create additional temporary files if it chooses so. As a security precaution, currently amavisd restricts the temporary directory path, which must be a subdirectory under $TEMPBASE or $MYHOME. tempdir_removed_by=client Specifies the client will be responsible for removing the temporary directory. The server must not remove the file email.txt nor the directory, but it may remove temporary files and subdirectories it has created. tempdir_removed_by=server Specifies the server is responsible to remove the temporary directory if/when it deems appropriate. This is a default in the absence of this attribute (for compatibility with traditional amavis clients). mail_file=/var/amavis/amavis-milter-MWZmu9Di/email.txt Specifies a file name (full file path) of a file containing the original mail with header and body. This attribute should be present at most once. In its absence the file name defaults to /email.txt. delivery_care_of=client Specifies that server should NOT actively forward the mail to recipients, but should only report its opinion in its reply, and let the client act on it. This is a default in the absence of this attribute. delivery_care_of=server Specifies that server is responsible to actively forward the mail to recipients if it deems the message appropriate for forwarding. This attribute value indicates that the client has no capability or intention to forward mail by itself. queue_id=qid optional informational attribute: MTA queue id; protocol_name=ESMTP optional informational attribute: the name of the protocol used by the original SMTP client to deliver the mail to the client (or to its associated MTA). Common values are ESMTP, SMTP, LMTP. helo_name=b.example.com optional informational attribute: the value of the HELO or EHLO or LHLO command specified to our MTA by the original SMTP client; client_address=10.2.3.4 optional informational attribute: the IP address of the original SMTP client; client_name=mail.example.com optional informational attribute: the DNS name of the original SMTP client as obtained by DNS reverse mapping of the original SMTP client; policy_bank=TLS,ORIGINATING,MYNETS value is a comma-separated list of policy bank names. Names of nonexistent banks are silently ignored, so are leading and trailing spaces and TABs around each name. The order of policy bank loading generally follows the order in which information about a message were obtained: - interface or socket -based policy banks (when MTA connects to amavisd); - MYNETS (when client's IP address becomes known); - the list as specified in the policy_bank attribute of AM.PDP; - MYUSERS (when sender e-mail address becomes known); Attributes in the server response are: -------------------------------------- The current set of attributes maps almost exactly to the capabilities of libmilter, which should facilitate initial implementation. See sendmail libmilter documentation as well. version_server=2 Since amavisd-new-2.4.3 the 'version_server=2' is inserted, older versions did not provide this attribute (assumed version was 1). With version 2 the following changes to the protocol were made: - version_server=2 is provided by the server as the first attribute; - delheader and chgheader now stand in a response before insheader and addheader, assuming that milter MTA will execute these in the same order; - new attribute insheader, see below; - new attribute quarantine, see below. log_id=xxx Provided since amavisd-new-2.7.0, tells a log_id under which the amavisd daemon logged the request and related debugging events; delrcpt= The specified recipient should be removed from the list of recipients of this mail. The specified address must exactly match the recipient addresses as specified in the client request, i.e. a smtp-quoted form enclosed in <> should be specified; addrcpt= The specified recipient should be added to the list of recipients of this mail. Paired with 'delrcpt' the pair indicates an existing recipient address to be replaced by a modified address, e.g. to add an address extension. delheader=index hdr_head Specifies an existing mail header field should be removed. Index is a decimal integer indicating which of the header fields with the same head should be affected, count starts with 1. 'hdr_head' does not include a colon! chgheader=index hdr_head hdr_body Specifies an existing mail header field should have its body replaced by a new hdr_body. Index is a decimal integer indicating which of the header fields with the same head should be affected, count starts with 1. 'hdr_head' does not include a colon! insheader=index hdr_head hdr_body Similar to addheader, but specifies a mail header field to be inserted to the mail header at a given position, index 0 implies top of the header. Amavisd-new passes a value of $prepend_header_fields_hdridx as the index argument, which is configurable and defaults to 1 for compatibility with dkim-milter and dk-milter signing milters. Alternative useful value is 0, which may be used in absence of other milters inserting their header fields at index 1. Header fields to be prepended are listed in reverse order, the last one listed in AM.PDP is to appear at the top of a resulting mail header. Inserting a header field at an arbitrary position is a later addition to sendmail milter protocol (introduced with sendmail 8.13.0 2004-06-20) addheader=hdr_head hdr_body Specifies a mail header field to be appended to the mail header. Note the use of exactly two value fields, separated by exactly one space. As described above, spaces in each field must be hex-encoded. 'hdr_head' does not include the colon! replacebody=new_body Not implemented - for future consideration. quarantine=reason place message on hold or to a quarantine maintained by MTA, and supply a reason text (e.g. client may call smfi_quarantine milter routine; btw, smfi_quarantine was introduced with sendmail 8.13); For future use - it is currently (2.4.3 or earlier) never used. return_value=val where val can be any of: continue, accept, reject, discard, tempfail This attribute should be present exactly once, and indicates to the client what should be done with the mail and how the original SMTP session should be treated. setreply=rcode xcode message SMTP response code and text suggested by server to the client to be used in response to the original SMTP client; There are exactly three value fields, separated by a single space. exit_code=n Similar in semantics to return_value attribute, but uses sysexits.h -based exit values for the purpose. Useful with old amavis clients, but should be ignored by new clients, which should base its decision on return_value instead. An example ========== Indentation and text in parenthesis in the following example is not part of the actual session. $ telnet 127.0.0.1 9998 Trying 127.0.0.1... Connected to localhost.example.com. Escape character is '^]'. (client->server request) request=AM.PDP sender=me@example.com recipient=user1@example.net recipient=user2@example.net protocol_name=ESMTP client_address=10.2.3.4 tempdir=/var/amavis/amavis-milter-MWZmu9Di (server->client response) setreply=250 2.5.0 Ok,%20id=MWZmu9Di,%20continue%20delivery return_value=continue exit_code=0 (or:) setreply=250 2.7.1 Ok,%20discarded,%20UBE,%20id=mYOljdn2 return_value=discard exit_code=99 (or:) setreply=550 5.7.1 Message%20content%20rejected,%20UBE,%20id=S7uS4qvA return_value=reject exit_code=69 (or:) setreply=451 4.5.0 Error%20in%20processing,%20id=... return_value=tempfail exit_code=75 =============================================================================== Releasing a message from a quarantine: request=release (or request=requeue, available since 2.5.1) mail_id=xxxxxxxxxxxx secret_id=xxxxxxxxxxxx (authorizes a release) partition_tag=xx (optional, but recommended if info is available) quar_type=x F/Z/B/Q/M (file/zipfile/bsmtp/sql/mailbox) mail_file=... (optional: overrides automatics; $QUARANTINEDIR prepended) requested_by= (optional: lands in Resent-From:) sender= (optional: replaces envelope sender) recipient= (optional: replaces envelope recips) recipient= recipient=... In reply, for each recipient a SMTP status response is returned (similar to LMTP) quar_type defaults to Q if spam_quarantine_method is sql:, otherwise to F. =============================================================================== AMAVIS SIMPLE CLIENT/SERVER PROTOCOL (traditional) description by Stephane Lentz 2003-11-06 amavisd is the daemon part of AMAVIS in charge of scanning SMTP messages. It receives messages from other applications (clients) using either SMTP or a simple protocol which is detailed in this document. The protocol being used depends on the MTA and architecture chosen. The "simple protocol" is most often used with sendmail in a MILTER set-up (the client program in such a case is amavis-client). AMAVISD receives messages from clients through a UNIX socket : The UNIX socket used is by default /var/amavis/amavisd.sock . It is defined : - at the amavisd server level in amavisd.conf as $unix_socketname - at the client level when using MILTER (amavis-milter available in helper-progs) as a configure option : --with-sockname swith The protocol used between the client and server is simple (basic & limited). There is no possibility for the server to ask the client to remove/add/change headers. The server can only say if the message was detected as CLEAN, as UNSAFE and to be rejected/discarded) or not analysed successfully due to some errors. PROTOCOL IN DETAILS : The client connects to the AMAVISD server's socket. IF successful then for each incoming message : - the client computes and create a new temporary directory ($tempdir) to store the new incoming message. - the incoming message is stored as $tempdir/email.txt - the client sends the directory name to the SERVER - the server sends \1 to the client if the directory is ok - the client sends the envelope sender recipient address - the server sends \1 to the client if ok - the client sends the envelope recipient addresses one by one to the server: the client trims the address if it is longer than the maximum length possible the client sends this address to the server the server sends \1 to the client if ok - the client sends some request to analyze the message to the SERVER. The character used is EOT (end of transmission) :\3 - the server processes the mail stored in the directory ($tempdir/email.txt) - the server sends a STATUS number to the CLIENT. This number returned is either : EX_OK (2) : message CLEAN EX_UNAVAILABLE (69) : message UNSAFE to be rejected at the SMTP LEVEL (550 reject) 99 : message UNSAFE to be silently (250 code) discarded at the SMTP LEVEL EX_TEMPFAIL (75) : message not processed successfully (error in communication, or server error, ...) amavisd-new-2.7.1/README_FILES/README.milter000640 000621 000620 00000027046 11263154765 017540 0ustar00markcmi000000 000000 =============================================================================== NOTE: A recommended sendmail setup is described in file README.sendmail-dual, which describes a dual-MTA setup. The sendmail milter setup as described in this file works as well, but with some functionality limitations. Please see the introduction section in README.sendmail-dual when deciding between milter setup and a dual-MTA setup. =============================================================================== NOTE1: these instructions describe the amavisd-0.1 installation, but apply in large degree to amavisd-new as well. Don't forget to set the $forward_method to undef in amavisd.conf, and adjust the $notify_method. Also, the $final_*_destiny may use D_REJECT if desired. NOTE2: a rewrite by Petr Rehor of the helper program amavis-milter.c to use the new AM.PDP protocol (README.protocol) is available as a separate project, see: http://sourceforge.net/projects/amavisd-milter/ in FreeBSD ports collection: security/amavisd-milter How To use AMaViS With sendmail/libmilter ***************************************** General Notes ============= By Rob MacGregor SECURITY MILTER is designed such that milter applications do not need to run as root. By not running amavis as root you improve security. Simply put, nothing that can run as an account other than root should be run as root. However, it's important to ensure that you run your virus scanners and both parts of amavis (amavisd and amavis-milter) as the same group. It's worth giving daemonised virus scanners a different account, just to reduce the chance that the scanner modifies the message. If you don't do this then you'll run into permission problems. The account that you run amavis as *MUST* own the /var/amavis directory and the quarantine directory (usually /var/virusmails). Now, create the following account for amavisd and amavis milter: amavis (group amavis) If you use daemonised virus scanners then it is worth creating a separate account for them: vscan (group amavis) GENERAL If you run into problems first check the FAQ at: http://www.amavis.org/amavis-faq.php3 and the list archive at: http://marc.theaimsgroup.com/?l=amavis-user&r=1&w=2 before asking questions on the list. It's highly likely somebody has already come across the same problem and it's been solved. Oh, and don't forget to RTFM :-) SOCKETS Amavis uses 2 sockets for communications. One is for communication between sendmail and amavis-milter process (amavis-milter.sock). The protocol spoken over this socket is MILTER. The other is for communication between the 2 parts of amavis (amavisd.sock), i.e. between amavis-milter process and amavisd daemon. A protocol over this socket is _not_ MILTER, but a private amavis protocol. The first socket is fixed and cannot be changed (short of editing the source). The second can be changed by the configure command. However if you set it to the same as the sendmail-amavis socket strange things will happen. You should receive a message in the log of amavisd-new that indicates what the problem is, e.g.: RX_tempdir FAILED, retry: Invalid temporary directory '\000\000\000\rO'. Other versions of amavis may not produce any such warnings. The short version is: 1) Don't change the socket details unless you know what you're doing. 2) If you do change the socket name, don't use the name of the other socket. CENTRALISING SCANNING (From Dibo ) If you want to place milter-amavis along with amavis daemon on another host, or just prefer inet sockets to Unix sockets, pick a free port number above 1024, and change: - in file sendmail.mc in the call to the macro INPUT_MAIL_FILTER replace: S=local:/var/amavis/amavis-milter.sock with: S=inet:port@hostname (substituting 'port' with your chosen port number, and substituting host name or IP address in place of 'hostname' to specify the host on which milter-amavis daemon is running) - when starting milter-amavis process, change the value of it's option -p: replace -p local:/var/amavis/amavis-milter.sock with: -p inet:port@0.0.0.0 (substituting 'port' with your chosen port number, and optionally limiting the bind address (0.0.0.0) with the desired interface, e.g. 127.0.0.1 to limit bind to the loopback address) Sendmail 8.12.x =============== By Rob MacGregor NOTE: Ensure you're running Sendmail 8.12.10 or later. Earlier versions all have a remotely exploitable vulnerability (see CERT Advisory CA-2003-25 and other advisories). Add the following to /devtools/Site/site.config.m4: APPENDDEF(`confENVDEF', `-DMILTER') Then build sendmail. If you've already built sendmail, clean the old tree by doing "rm -fr obj.*" in the sendmail source directory, or run "./Build -c". Once sendmail has finished building go into the following directories under the sendmail source directory and do a "make" and "make install": libmilter libsm libsmutil Copy the .a files from under obj.*/libsm and obj.*/libsmutil to somewhere the linker can find them (/usr/lib, /usr/local/lib or similar). Building AMAVIS(d) ================== IMPORTANT: Versions of AMAVISd before 0.1 use a different configure command line. If you're running an older version then UPGRADE. Assuming you've copied the libraries to /usr/lib configure amavis(d) with "--enable-milter --with-milter-includes=/usr/include --with-milter-libs=/usr/lib" DO NOT USE the "--enable-relay", "--enable-smtp" or "--with-origconf" options. IMPORTANT: Unless you REALLY understand EXACTLY what you're doing, leave the --with-sockname option alone. The default is correct and changing it may cause you problems. When you run configure check that you see the following lines: checking for sm_errstring in -lsm... yes checking for sm_strlcpy in -lsm... yes checking for libmilter/mfapi.h... yes checking for smfi_main in -lmilter... yes We will use libmilter as the MTA The summary should include: Configured for use with: libmilter Configuration type: sendmail/milter If you don't see this, check that you've put the libmilter.a, libsm.a and libsmutil.a files in a location the linker can find (see above). Check that you've specified the correct paths to the libraries and header files. For building amavisd-new, see files README, INSTALL, and helper-progs/README in its distribution. Finishing Sendmail 8.12.x ========================= In the sendmail.mc file add the following two entries (the first one is mandatory): INPUT_MAIL_FILTER(`milter-amavis', `S=local:/var/amavis/amavis-milter.sock, F=T, T=S:10m;R:10m;E:10m') define(`confMILTER_MACROS_ENVFROM', confMILTER_MACROS_ENVFROM`, r, b') # supply macros b,r to helper Now rebuild your sendmail.cf file and install it (usually /etc/mail/sendmail.cf). Start amavisd and then sendmail. Below is a suitable amavis startup script for a Linux type system. Check syslog for messages (probably /var/log/mail or /var/log/mail/info). You should see something like: Oct 18 16:45:19 host amavis[24606]: starting. amavisd 0.1 Sat Jul 28 10:03:56 UTC 2001 Oct 18 16:45:20 host sm-msp-queue[24618]: starting daemon (8.12.9): queueing@01:00:00 Oct 18 16:45:26 host sm-mta[24631]: starting daemon (8.12.9): SMTP+queueing@01:00:00 The following options can be passed to amavis-milter (0.1.1 or later) to change the default behaviour: -x From 0.1.1 this DISABLES the X_HEADER insertion. -d Disable automatic daemonising of the client. All logging is now performed to stderr instead of stdout. -v Increases the verbosity of the logging, can be repeated for greater verbosity -g Sets the group the client runs as (the amavis account, or your chosen account, MUST be a member of this group). Normally this group will be "smmsp". This is designed for when you run sendmail as non-root and isn't normally required. See the amavis-milter(1) man page for full details. PERFORMANCE NOTES ================= Ok, strictly speaking generic, but always useful. You can (possibly) boost performance in a number of simple ways: * Use a daemonised scanner. This way a new copy of the scanner doesn't have to be launched for every message. Examples include Sophos+Sophie, Trend+Trophie and ClamAV. * Use separate disks/controllers for the amavis spool (/var/amavis) and the sendmail spool (/var/spool/...). * Put amavis on another system (assuming you've got a fast network). This is particularly useful if your mail server is already I/O or processor bound. * Use memory based file systems (TMPFS in Linux and Solaris, MFS in (Free)BSD) for the amavis spool (/var/amavis). Don't do this for the quarantine directory and don't do it for the sendmail spool without reading the tuning section in the Sendmail (Bat) book (3rd edition) by O'Reilly. * It's worth doing some simple checks to see if you're running out of memory or maxing out the processor or disk I/O. The "top" command gives you a start on gathering this information. >>>START /etc/rc.d/init.d/amavisd (or wherever it lives on your system)>>> #!/bin/sh # # chkconfig: 2345 70 30 # description: AMAVISd is an anti-virus scanning interface for \ # common mail servers. # Source amavis configureation. if [ -f /etc/sysconfig/amavis ] ; then . /etc/sysconfig/amavis else AMAVIS_ACCOUNT=amavis MILTER_SOCKET=/var/amavis/amavis-milter.sock MILTER_FLAGS="" # Set the options you want passed to amavis-milter fi # See how we were called. case "$1" in start) # Start daemons. echo -n "Starting amavis-milter: " rm -fr /var/amavis/amavis*.sock su - ${AMAVIS_ACCOUNT} -c /usr/sbin/amavisd sleep 5 su - ${AMAVIS_ACCOUNT} -c "/usr/sbin/amavis-milter ${MILTER_FLAGS} -p local:${MILTER_SOCKET}" RETVAL=$? if [ $RETVAL -eq 0 ]; then echo " [ OK ] " touch /var/lock/subsys/amavis else echo " [ FAIL ] " fi ;; stop) # Stop daemons. echo -n "Shutting down amavis-milter: " if [ -f /var/amavis/amavisd.pid ]; then # *** PAY ATTENTION *** # pkill only seems to exist in LINUX, whereas the -m option to killall only exists in BSD. # You may have to modify the following commands depending on your system. if [ -e /usr/bin/pkill ]; then /usr/bin/pkill amavis-milter else killall -m amavis-milter fi su - ${AMAVIS_ACCOUNT} -c /usr/sbin/amavisd stop echo " [ OK ] " else echo " [ FAIL ] " fi rm -f /var/lock/subsys/amavis ;; restart|reload) $0 stop $0 start RETVAL=$? ;; *) echo "Usage: amavis {start|stop|restart}" exit 1 esac << Last updated 8 March 2004 by Mark Martinec (added milter macro {b}) amavisd-new-2.7.1/README_FILES/README.sendmail-dual000640 000621 000620 00000050537 10502607337 020755 0ustar00markcmi000000 000000 Dual-instance sendmail with amavisd-new --------------------------------------- Mark Martinec, 2003-05-06 (based on initial research by Ricardo Stella) updated on: 2005-09-22 (added a reference to 'milter-ahead'); updated on: 2005-09-29 (added custom rules to reject unknown users outright, provided by Matej Vela, thanks to Simone Marx) updated on: 2006-09-15 (placement of DKIM/DK milters, mention the feature FEATURE(`nocanonify',`canonify_hosts'), the absence of which can make header procesing by sm *very* long) The most recent version of this document can be found at: http://www.ijs.si/software/amavisd/README.sendmail-dual ========================================================================== The setup is very similar to the one described in README.sendmail (by Rainer Link) in section 'Scanning incoming/outgoing and relayed mail', except that it uses SMTP protocol over inet socket (instead of pipes to commands) to transfer files between MTA and amavisd-new and back, and that it uses a permanently running second sendmail instance in 'queue only' delivery mode, instead of bringing it up every time a new checked mail comes from amavisd. ========================================================================== Comparing the setup described in this document with the sendmail milter setup, as described in README.milter: milter - reasons in favour: - can REJECT on the original SMTP session, instead of generating a bounce (sending a non-delivery notification _after_ the mail has been enqueued); - only one sendmail daemon need be running, only one config file needed, no additional queue area needed (although starting with sendmail 8.12 more than one queue area is already a norm: clientmqueue, queue groups; and MSP already uses a different .cf file); dual-MTA - reasons in favour: - Full amavisd-new functionality is available, including adding spam and virus information header fields, adding address extensions and removing certain recipients from delivery while delivering the same message to the rest (*_lovers). Also a message can be split if different recipients need different header edits. All this is not available when using amavis-milter helper program. - Content scanning need not be performed at the time of mail reception. This allows better control on CPU-intensive content filtering: mail checking can be streamlined and performed at optimum throughput setting (number of content checker processes) so as not to overwhelm host resources, instead of leaving it at the mercy of the current number of incoming SMTP sessions where available crude controls are mostly based on system load. Typically the number of incoming SMTP sessions (tiny processes) is desired to be many times above the number of content filtering processes (heavy resource consumers). - No helper programs needed, MTA communicates with amavisd-new directly via SMTP, saves on creating one directory and one file for each message, and deleting it (at the cost of one additional transfer); - Receiving sendmail daemon (MTA-RX) need not run as root (using option RunAsUser) since it does not need to run any local delivery agents (LDA) or to access user .forward files. This avoids external SMTP clients talking directly to a process running as root. ========================================================================== The following setup is described in this document: ............................ ............................ : sendmail instance MTA-RX : : sendmail instance MTA-TX : : : : : 25 -----> \ (mqueue-rx) : : (mqueue) / -------> forward 587 -----> > -queue- : : -queue- ->-----+ : ^ : / | MAIL_HUB, : : | \ -------> local | : v SMART_HOST : : ^ : delivery msp ...........|................ ....|....................... | ^ loopback interface v | port 10025 loopback interf.| port 10024 | .....|.......................|............. : $inet_socket_port=10024 | : : | : : $forward_method='smtp:127.0.0.1:10025' : : $notify_method ='smtp:127.0.0.1:10025' : : : : amavisd-new : ........................................... The setup is based on the recent sendmail (8.12.9 or later) with its set of m4 configuration macros. Because of several security problems with earlier versions of sendmail it is advised to stick to the most recent version, although the functionality needed for this setup has long been available. If a particular macro or feature is not available with some older version, it is usually possible to achieve the same or similar by manually writing a new 'mailer' specification and/or tweaking the .cf file. We'll prepare two sendmail daemon instances (processes), let's call them MTA-RX (receiving, accepting) and MTA-TX (transmitting, delivering). For convenience we keep the name of the configuration file and the queue (spool) area at default names for one mailer instance, and choose non-default names for the other. Let's choose the MTA-TX to keep default names, and supply non-default names to MTA-RX explicitly. This will make admin utilities like mailq, newaliases, hoststat and purgestat operate on the outgoing mailer instance unless explicitly told otherwise. It could just as well be the other way around. MTA-RX (receiving mailer) will be responsible for accepting mail from the Internet or from internal hosts on port 25, optionally accepting local message submissions on tcp port 587 (rfc2476), and for message submissions via sendmail program. It will forward all mail (both for local and for nonlocal recipients) via SMTP protocol (or LMTP) to 127.0.0.1 (a loopback interface) on tcp port 10024, where amavisd daemon will be listening. - its queue: /var/spool/mqueue-rx - its config file: /etc/mail/sendmail-rx.cf, /etc/mail/submit.cf - the source (.mc) of the configuration file: thishost-rx.mc (where 'thishost' is often by convention the name of the system (uname)) MTA-TX (transmitting mailer) will be responsible for accepting checked mail and notifications from amavisd-new via SMTP on the loopback interface (127.0.0.1) at tcp port 10025, and will forward all mail to its final destinations, either for local delivery, or delivering outgoing mail to the Internet or to other internal mailers. - its queue: /var/spool/mqueue - its config file: /etc/mail/sendmail.cf - the source (.mc) of the configuration file: thishost-tx.mc In-between the two MTAs an amavisd daemon will accept mail via SMTP (or LMTP) protocol on tcp port 10024, check it, and forward checked mail and notifications via SMTP to MTA-TX. If you already have an existing sendmail installation, you already have a queue directory /var/spool/mqueue and the configuration file(s) (.mc source and the compiled .cf file). Most of the existing settings in your .mc file can be reused, and are to be moved to the new files thishost-rx.mc or thishost-tx.mc, or (some of them) to both. The settings pertaining to receiving mail, including recource limits, should go to thishost-rx.mc; settings pertaining to delivering mail (locally or to other mailers) should go to thishost-tx.mc, and general settings should go to both. The MTA-TX should have none or hardly any resource limits, or at least have them larger than MTA-RX. Large messages, common errors in mail, and mail rush-ins should be stopped or limited at their entry to the system. Accepting them first, but choking later can lead to trouble or at least to wasted resources. The file names thishost-rx.mc and thishost-tx.mc are arbitrary, they only serve as source (to the m4 macro processor) for producing .cf files, which control sendmail's behaviour. Sendmail never uses .mc files directly. MTA-TX already got its queue directory during sendmail installation. For MTA-RX a new queue directory needs to be created where incoming mail can be collected. Use the same ownership and protection as used for /var/spool/mqueue, e.g: # mkdir /var/spool/mqueue-rx # chown root:wheel /var/spool/mqueue-rx # chmod 700 /var/spool/mqueue-rx SECURITY NOTE: starting with sendmail 8.12 it is possible to start sendmail daemon as root and let it drop privileges (become user specified by RunAsUser) after binding to port 25. This is normally used by MSP, and it can just as well be used by MTA-RX, since it has no need to access user mailboxes and .forward files. To use this feature, specify user and group in the macro confRUN_AS_USER (file thishost-rx.mc), and set the ownership of mqueue-rx to this user and group: # chown smmsp:smmsp /var/spool/mqueue-rx # chmod 770 /var/spool/mqueue-rx More complex queue setup is possible if needed, like separating sendmail work area and core dump area from actual queues. For details about queue groups see sendmail documentation. Create file thishost-rx.mc: ---cut-here------------------------------ dnl To be used for MTA-RX, the first MTA instance (receiving mail) dnl Insert here the usual .mc preamble, including OSTYPE and DOMAIN calls. dnl Specify here also access controls, relayable domains, anti-spam measures dnl including milter settings if needed, mail submission settings, client dnl authentication, resource controls, maximum mail size and header size, dnl confMIN_FREE_BLOCKS, and other settings needed for receiving mail. dnl dnl NOTE: dnl confMIN_FREE_BLOCKS at MTA-RX should be kept higher than the same dnl setting at MTA-TX to quench down clients when disk space is low, dnl and not to stop processing the already received mail. dnl dnl In particular, here are some settings to be considered: dnl ( see also http://www.sendmail.org/m4/anti_spam.html ) dnl dnl FEATURE(`access_db') dnl VIRTUSER_DOMAIN(`sub1.example.com')dnl list valid users here dnl VIRTUSER_DOMAIN(`sub2.example.com')dnl list valid users here dnl FEATURE(`virtusertable') dnl define(`confUSERDB_SPEC', `/etc/mail/userdb.db') dnl FEATURE(`blacklist_recipients') dnl FEATURE(`use_cw_file') dnl FEATURE(`use_ct_file') dnl FEATURE(`nocanonify', `canonify_hosts')dnl dnl INPUT_MAIL_FILTER(...) dnl define(`confPRIVACY_FLAGS', `noexpn,novrfy,authwarnings') nobodyreturn ? dnl define(`confDONT_PROBE_INTERFACES') dnl MASQUERADE_AS(...) FEATURE(`allmasquerade') FEATURE(`masquerade_envelope') dnl define(`confTO_IDENT', `0')dnl Disable IDENT dnl define(`confMAX_MESSAGE_SIZE',`10485760') dnl define(`confMAX_MIME_HEADER_LENGTH', `256/128') dnl define(`confNO_RCPT_ACTION', `add-to-undisclosed') dnl define(`confBIND_OPTS', ...) dnl define(`confTO_RESOLVER_*... ) dnl define(`confDELAY_LA, 8) dnl define(`confREFUSE_LA', 12) dnl define(`confMAX_DAEMON_CHILDREN',20) dnl define(`confMIN_FREE_BLOCKS', `10000') dnl define(`confDEF_USER_ID', ...) define(`confRUN_AS_USER',`smmsp:smmsp')dnl Drop privileges (see SECURITY NOTE) define(`confPID_FILE', `/var/run/sendmail-rx.pid')dnl Non-default pid file define(`STATUS_FILE', `/etc/mail/stat-rx')dnl Non-default stat file define(`QUEUE_DIR', `/var/spool/mqueue-rx')dnl Non-default queue area define(`confQUEUE_SORT_ORDER',`Modification')dnl Modif or Random are reasonable dnl Match the number of queue runners (R=) to the number of amavisd-new child dnl processes ($max_servers). 2 to 7 OK, 10 is plenty, 20 is too many QUEUE_GROUP(`mqueue', `P=/var/spool/mqueue-rx, R=2, F=f')dnl dnl Direct all mail to be forwarded to amavisd-new at 127.0.0.1:10024 FEATURE(stickyhost)dnl Keep envelope addr "u@local.host" when fwd to MAIL_HUB define(`MAIL_HUB', `esmtp:[127.0.0.1]')dnl Forward all local mail to amavisd define(`SMART_HOST', `esmtp:[127.0.0.1]')dnl Forward all other mail to amavisd define(`LOCAL_RELAY',`esmtp:[127.0.0.1]')dnl define(`confDELIVERY_MODE',`q')dnl Delivery mode: queue only (a must, dnl ... otherwise the advantage of this setup of being able to specify dnl ... the number of queue runners is lost) define(`ESMTP_MAILER_ARGS',`TCP $h 10024')dnl To tcp port 10024 instead of 25 MODIFY_MAILER_FLAGS(`ESMTP', `+z')dnl Speak LMTP (this is optional) define(`SMTP_MAILER_MAXMSGS',`10')dnl Max no. of msgs in a single connection define(`confTO_DATAFINAL',`20m')dnl 20 minute timeout for content checking DAEMON_OPTIONS(`Name=MTA-RX')dnl Daemon name used in logged messages dnl Disable local delivery, as all local mail will go to MAIL_HUB undefine(`ALIAS_FILE')dnl No aliases file, all local mail goes to MAIL_HUB define(`confFORWARD_PATH')dnl Empty search path for .forward files undefine(`UUCP_RELAY')dnl undefine(`BITNET_RELAY')dnl undefine(`DECNET_RELAY')dnl MAILER(smtp) dnl The following solution to reject unknown recipients outright dnl is provided by Matej Vela , see: dnl http://groups.google.com/group/comp.mail.sendmail/ dnl browse_thread/thread/88cc72d7c4d3a6e/ee2a9474b3a4558d dnl The FEATURE(stickyhost) short-circuits FEATURE(luser_relay) so that a: dnl define(`LUSER_RELAY',`error:5.1.1:"550 User unknown"') can't be used. dnl A simple solution is to disable FEATURE(stickyhost). If this is not dnl possible, the alternative is to replace FEATURE(luser_relay) with custom dnl rules below. The latter has the advantage of properly handling special dnl aliases like ("|program", "/mailbox", and ":include:/list"). If choosing dnl this route, one should NOT use `undefine(`ALIAS_FILE')dnl', and use the dnl following custom rules: dnl LOCAL_CONFIG Kaliasp hash -m /etc/mail/aliases Kuserp user -m LOCAL_RULESETS SLocal_check_rcpt R$* $: $&{rcpt_addr} R $+ @ $=w $: <@> $1 mark local address R $* @ $* $@ OK ignore remote address R $+ $: <@> $1 mark unqualified user R<@> $+ + $* $: < $(aliasp $1+$2 $: @ $) > $1 + * plussed alias? R<@> $+ + $* $: < $(aliasp $1+$2 $: @ $) > $1 +* alias? R<@> $+ $: < $(aliasp $1 $: @ $) > $1 normal alias? R<@> $+ $: < $(userp $1 $: @ $) > $1 system user? R<@> $+ $#error $@ 5.1.1 $: "550 User unknown" nope, go away ---end----------------------------------- Create file thishost-tx.mc: ---cut-here------------------------------ dnl To be used for MTA-TX, the second MTA instance dnl (delivering outgoing and local mail) dnl Insert here the usual .mc preamble, including OSTYPE and DOMAIN calls. dnl Specify here also the required outgoing mail processing and dnl local delivery settings such as mailertables, needed mailers, aliases, dnl local delivery mailer settings, smrsh, delivery mode, queue groups, ... dnl Don't use milters here - for all common filtering purposes they belong dnl to MTA-RX; an exception to this rule would be DKIM or DomainKeys mail dnl signing milters (signature _verification_ milters still belong to MTA-RX). define(`confREFUSE_LA',999)dnl Disable the feature, limiting belongs to MTA-RX define(`confMAX_DAEMON_CHILDREN',0)dnl Disable, limiting belongs to MTA-RX FEATURE(`no_default_msa')dnl No need for another MSA, MTA-RX already has one FEATURE(`nocanonify')dnl Host/domain names are considered canonical DAEMON_OPTIONS(`Addr=127.0.0.1, Port=10025, Name=MTA-TX')dnl Listen on lo:10025 define(`confSMTP_LOGIN_MSG', `$w.tx.$m Sendmail $v/$Z; $b')dnl define(`confTO_IDENT', `0')dnl Disable IDENT MAILER(smtp) MAILER(local) ---end----------------------------------- Now macro-expand .mc files into .cf files: (adjust the path if needed to where your cf/m4/cf.m4 file resides) # m4 /usr/share/sendmail/cf/m4/cf.m4 thishost-rx.mc >/etc/mail/sendmail-rx.cf # m4 /usr/share/sendmail/cf/m4/cf.m4 thishost-tx.mc >/etc/mail/sendmail.cf Start MTA-RX and MTA-TX daemons: # /usr/sbin/sendmail -C/etc/mail/sendmail-rx.cf -L sm-mta-rx -bd -qp # /usr/sbin/sendmail -L sm-mta-tx -bd -q15m Start queue runner for the MSP client queue as usual, if using it: # /usr/sbin/sendmail -Ac -L sm-msp-queue -q10m Start amavisd-new: # amavisd Test if MTA-RX is listening: # telnet localhost 25 QUIT Test if MTA-RX is listening on MSA port 587 (a newer sendmail invention) # telnet localhost 587 QUIT Test if MTA-TX is listening: # telnet localhost 10025 QUIT Test if amavisd is listening: # telnet localhost 10024 QUIT For convenience some shell aliases may be defined: alias mailq-rx='mailq -C/etc/mail/sendmail-rx.cf' alias mailq-tx='mailq' alias sendmail-rx='/usr/sbin/sendmail -C/etc/mail/sendmail-rx.cf' alias sendmail-tx='/usr/sbin/sendmail' All done! NOTES - In amavisd.conf file follow the 'POSTFIX or EXIM V4 or dual MTA setup', which is also the default. - The $final_*_destiny should not specify D_REJECT. The D_BOUNCE (or D_PASS or D_DISCARD) is preferred. - To make MTA-RX content-check only some mail but not all, one may use mailertables instead of MAIL_HUB and SMART_HOST. For example setting some recipient domains to be passed to MTA-TX at 127.0.0.1:10025 directly (e.g. via mailer 'esmtp'), while sending all the rest to amavisd at 127.0.0.1:10024. To be able to specify the port number, a new 'mailer' needs to be defined, let's call it 'amavis', with similar settings as the already defined 'esmtp', except with port number 10024. - depending on how local addresses are translated by MTA-RX, the %local_domains (or @local_domains_maps) in amavisd.conf needs to be adjusted accordingly to be able to recognize local domains. Check the amavisd-new log what recipient addresses it sees for local recipients. The '[127.0.0.1]' may need to be added to the @local_domains. - To make MTA-RX reject mail for nonexistent local users by itself (instead of generating a bounce later on), one may use the 'virtusertable' in thishost-rx.mc, listing all known recipients, and rejecting the rest, e.g.: VIRTUSER_DOMAIN(`example.com')dnl FEATURE(`virtusertable', `hash /etc/mail/virtusertable')dnl jim@example.com %1%3 joe@example.com %1%3 postmaster@example.com %1 @example.com error:5.7.0:550 No such user here You may use the righthand side of the map to specify local user (e.g. %1%3, or just jim, without domain name) in which case MAIL_HUB will be used for forwarding, or specify an explicit domain name that is not in the {w} class, in which case the SMART_HOST will get consulted. Perhaps what Stephane Lentz writes is even better: Dictionary attacks and messages to retired accounts can be bounced with sendmail: just replicate your aliases or write some SLocal_check_rcpt rule-set that checks addresses of your domain against a map of valid users (valid_addresses.db). I hope some standard FEATURE will be provided with sendmail - something like FEATURE(checkdomainaddresses) and CHECKDOMAINADDRESSES(mydomain.com). An alternative solution is to use a milter to do address verification against the second MTA in chain. See the milter-ahead project: http://www.milter.info/sendmail/milter-ahead/ PERFORMANCE NOTES - Mail handling is I/O-intensive. For better performance one may place the two mail queue areas (/var/spool/mqueue and /var/spool/mqueue-rx), and the /var/amavis work directory ($TEMPBASE) on three separate disks. The /var/amavis/tmp may be a tmpfs or a RAMdisk or delayed-sync fs. - One of the important arguments for choosing the dual-MTA setup is to be able to keep the number of content filtering processes under control, and not at the mercy of current mail inflow. Don't blow this advantage by setting the number of amavisd-new processes and MTA-RX queue runners too high! Throughput optimum is somewhere between 3 and 10 with fast daemonized av-scanner (or no av scanner) (with or without SpamAssassin), and between 2 and 3 with many command line scanners (regardless of SA). If the host is low on memory and when spam checking (SpamAssassin) is used, even 2 may be a lot for an elderly host. Start conservatively, e.g. at 2 or 3, and if everything works normally and higher throughput is needed, try a bit more. Anything above the point where throughput function levels off is just a waste of memory and gains nothing! The optimum may be higher if high-latency external SpamAssassin tests are enabled (e.g. Razor, RBL), Still, never go beyond available memory. For example with SpamAssassin enabled, the 20..25 processes on a 256 MB machine is where throughput begins to drop rapidly on a way to a swapping tar pit. amavisd-new-2.7.1/README_FILES/README.policy-on-notifications000640 000621 000620 00000030350 10233024312 022770 0ustar00markcmi000000 000000 Date: Sat, 23 Aug 2003 00:39:21 -0700 From: Robert LeBlanc Subject: Re: [AMaViS-user] All admins and other amavis users, listen up! To: amavis-user@lists.sourceforge.net Message-id: <5.1.1.6.2.20030822235747.02e41dc8@127.0.0.1> At 11:45 2003/08/22, Peter Surda wrote: >On Thu, Aug 21, 2003 at 02:00:05PM -0500, cdupree@csr.utexas.edu wrote: > > On Thu, Aug 21, 2003 at 04:29:15PM +0200, Olivier Tharan wrote: > > > > People, I urge everyone here to switch off all sender notifications > > > > unconditionally. This is getting out of hand. > > > I completely agree. While it becomes depressing for the admins to see > > > such harm done with a single virus, the (l)users are panicked because > > > they think they have caught a virus on their computer. > > I completely disagree. In this case it's probably useful. But what if > > you were sending me a legitimate mail and it had an infected attachment. >Please show me ONE example from the real world where this actually happened. When I first started using AMaViS (back in the days of amavis-perl-11), I used to think the idea of sending virus notifications to the sender was a really good one--a nice courtesy, and one that the sender would appreciate. In theory, at least, it was possible for a virus to slap a copy of itself on to all outgoing e-mail as an attachment, such that the sender wasn't even aware this was happening. The recipient would receive the legitimate e-mail, but with a mysterious attachment the sender was unaware of, and the fact that the mail itself was legitimate often encouraged the recipient to trust the attachment as well. This moved from the realm of theory to the realm of practice back in 2000 or so, when I recall a particular virus that did exactly this, targeting Outlook and Outlook Express users. While I can't recall the name of the virus anymore, I can attest to the fact that I *have* encountered such a beast, and had a number of users posting to mailing lists and newsgroups, unwittingly spreading their viruses until they were ultimately told to cease and desist and get their systems fixed. If these people were never told that they were spreading a virus, they'd have had no idea they were doing so. That said, I haven't seen a virus like that in several years, possibly because those particular exploits in Outlook and Outlook Express have been patched and there are fewer vulnerable installations out there as a result. These days we're facing a different breed of mass-mailer virus, and none of the current crop of threats is particularly well served by virus notifications. By their very definition, these "mass-mailers" (the viruses that end in "@MM") get their sender and recipient lists from the victims' address books, so notifying the actual sender is all but impossible. When these virus notification e-mails arrive, it's almost always in the mailbox of an innocent party, who then becomes needlessly confused or alarmed (or even indignant!). After they're calmed down and told to disregard the notice, of course, they then program themselves to ignore every *other* virus notification mail they receive, effectively defeating the purpose of such things. Worse, sending out automated virus notifications to all of these supposed senders effectively contributes to the problem by generating an exponential increase in the wasted bandwidth (since in many cases those virus notification e-mails bounce). In the case of worms whose objective is to generate a denial-of-service effect, automated virus notifications only amplify their effectiveness. Even more alarming is the fact that some of these virus notification systems try to be helpful by sending back the original (infected) mail to the supposed sender--complete with the virus attached! When this ends up in the mailboxes of dozens or hundreds of innocent people instead, it puts them unnecessarily at risk of infection. During the recent Sobig.f campaign, my wife (who is a journalist), received more than 400 copies of the virus--and *300* virus notification mails, no less than 100 of which contained more copies of the virus. Interestingly, because of the way some of these mailers incorporated the virus into the body in their notification mail, a dozen or so copies slipped past amavisd-new and the battery of virus scanners running here. The problem is significant enough that some RBLs are adding sites that send out automated virus notifications (e.g. blackholes.five-ten-sg.com, result 127.0.0.10), so that others can block mail from these "offenders". As people get fed up with notifications that they consider "useless" or "alarming", they come to view them as just another form of spam, and want it blocked like the rest. All the best intentions in the world won't change that. In the end, then, I find myself on the other side of things--I no longer send out automated notifications for viruses or spam, as I don't consider it appropriate these days. Modern viruses and spam broadcasters use techniques that render such notification mail less than useless, and cause many more problems than they potentially solve. While there's always a chance that a legitimate e-mail with an infected attachment could be winging its way to one of my users right now, that chance is vanishingly small compared to the harm I'd be causing by sending out automated virus notifications for every instance of every other virus we receive. Do I think that AMaViS should ship with automated notification turned off by default? Not necessarily. What I *would* prefer to see, however, is more information provided to administrators about the consequences of using automated notifications (or not using them). There are certainly trade-offs to be made, and providing inexperienced administrators with some advice on making this decision would be quite helpful, in my opinion. A few paragraphs like the ones above, explaining the rationale for using or not using automated notifications, is all it would take, really. Then let the administrator configure things in accordance to the needs and policies of his organization. Robert LeBlanc Renaissoft, Inc. Date: Mon, 01 Sep 2003 17:18:15 -0700 From: Robert LeBlanc Subject: Re: [AMaViS-user] Using D_Discard to discard trapped emails To: amavis-user@lists.sourceforge.net Message-id: <5.1.1.6.2.20030901165615.01712880@127.0.0.1> ... You'll get a few different answers from the various people on this list, as this is a somewhat controversial topic. Basically there are three main points of view: (1) The "lose no mail" camp believes that a mailer should never discard mail without notifying the sender that the mail was not delivered (and of course if you do notify the sender, the mail was actually "rejected", not "discarded"). If you discard mail, you're effectively creating a "mail sink"--a black hole into which mail vanishes, to be lost forever. The integrity of the Internet mail system would be questionable if this sort of practice were widespread and mail was being lost on a regular basis. When you send mail to someone and that user's mailbox is full, or his mail server is down (hopefully temporarily), you expect to receive a notification to tell you your mail wasn't delivered; without this notice, you'd (wrongly) believe your mail got to its destination. (2) The "acceptable losses" camp generally started out in the "lose no mail" camp, but eventually got frustrated with all the bounces (and bounces from bounces) flooding their mailboxes as a result of automated mechanisms like virus/spam/banned/header alerts. These folks have come to the conclusion that while it's noble and virtuous to never discard mail, it's not a practical solution these days. The volume of "noise" polluting mailboxes and wasting bandwidth across the Internet makes a strong argument for discarding mail, rather than contributing to the problem by sending out more noise. Sure, a few legitimate mail items are likely to get lost this way, but these are considered "acceptable losses" when weighed against the volume of noise being filtered. (3) The "discard safely" camp (to which I now subscribe) believes that discarding mail is acceptable, but only as long as the mail itself is not lost. The sender doesn't need to be notified that the mail was not delivered, *if* the mail is quarantined in a manner that allows the recipient to review it. In a sense, then, the mail *was* delivered, just to an alternate mailbox or quarantine area. The key addition, here, is a quarantine management facility that lets users review the quarantined items and rescue any legitimate items that may have been trapped there. While amavisd-new provides the quarantining mechanisms, it lacks management facilities. If you need something like this, you can try an add-on package like Maia Mailguard 0.9.5a (http://www.renaissoft.com/projects/maia), which provides per-user control for amavisd-new, per-user quarantine management, user administration functions, and stats-gathering/graphing. Robert LeBlanc Renaissoft, Inc. Date: Wed, 21 Jan 2004 01:36:52 -0800 From: Robert LeBlanc Subject: Re: [AMaViS-user] W32/Bagle-A To: amavis-user@lists.sourceforge.net Message-id: <6.0.1.1.2.20040121011109.04e23678@127.0.0.1> [...] >>... Since amavis is smart enough not to include the virus in the DSN, >>the notice the sender receives is at least "clean". > >But the wrong person gets it. > >At least when one of my clients tries sending email through my SMTP >server, they'll get the rejection notice immediately. The mail won't go >anywhere, they won't get any bounce, they just get a message from their >MUA saying that the message was rejected. [Robert LeBlanc talking about Postfix and dual-sendmail-setup, not about the Sendmail milter setup (Mark)] This assumes that your MTA "handles the rejection [from amavis] properly", of course. MTAs know nothing about viruses (or spam for that matter), so they can't do the rejection themselves on this basis. When clients send mail through your MTA, your MTA relays that mail to amavis, which does the content-checking. If amavis finds a virus and uses D_REJECT, the responsibility falls back to your MTA to decide what to do. When MTAs receive a permanent failure SMTP error (i.e. 5xx), they try to be helpful by generating DSNs that include the full original mail, to send back to the sender, explaining the rejection. Your client would then get the virus sent back to him in the DSN. This is harmless (quite helpful, actually) when the sender happens to have an old-style virus, of course, but with the "viruses that fake sender addresses" this is a distribution method. Consider that your user has one of these viruses on his machine and it starts spewing out copies of itself through your mail server, all with fake sender addresses. When your MTA sends out its DSNs to those fake addresses, those DSNs will contain the virus. If you'd used D_BOUNCE instead, the DSNs would be issued by amavis (which is virus-aware) rather than your MTA (which isn't), so the mail going to those fake senders would be virus-free. My point is that your idea of "reject at the MTA" doesn't work as such, because the MTA does not have a virus scanner embedded in its own logic. The rejection takes place at the content-filtering stage (i.e. amavis), so the rejection is handed to the MTA by amavis, rather than from your MTA to the client. Your MTA will still produce a DSN and helpfully send back the original mail--that's what SMTP dictates that it *must* do. You're better off using D_BOUNCE and letting amavis issue its own DSN, so that at least you won't be spreading the virus itself any further. *Both* methods generate spam-by-proxy, but one of them also spreads viruses. The closest thing to an "ideal" solution, in my (admittedly biased) opinion, is what I refer to as the "discard safely" policy--D_DISCARD items like these after quarantining them--provided that you have a quarantine management system (e.g. Maia Mailguard) that lets users access these quarantined items. No DSNs get sent out, as the mail is effectively "accepted" (into quarantine), and the local recipients have access to this special mailbox to retrieve anything they want to keep, so there's no *need* for a DSN--the mail was, for all intents and purposes, successfully "delivered" to the recipients. Robert LeBlanc Renaissoft, Inc. amavisd-new-2.7.1/README_FILES/images/up.png000640 000621 000620 00000002127 10636222147 017746 0ustar00markcmi000000 000000 PNG  IHDR#Yi&PLTE!!))11BBcckkƌΔ޵~bKGDHfIDATxڕK mA֝I+o@^P?'#H];߁Rh~j?g0qF@!eH,0܆x0&p ^J5y=J % P<*ğ{j #7^L~!=&Ṳ; &rgߊFmͩP; ot6BJqC Xdui]}OL4+|) -CtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignatureee9d877396ce267aeb0179d35f81b2ac3'tEXtPage25x24+0+0 IENDB`amavisd-new-2.7.1/README_FILES/images/home.png000640 000621 000620 00000002204 10636222147 020246 0ustar00markcmi000000 000000 PNG  IHDR#Yi&PLTE!!))11BBJJccssƌΔΥ޵GbKGDHIDATxڭQ l)xY,]Ҿ8uu,q¼řN!`RN`M}%,iím )Ϊ + Cs PvBɨ@d6"o;g`Ps+s%ǯ%@S{4ܾU-s"1ч?j<`g eZ* +æ2_Yv*lv~g Z[sY[Ða; =/oJĘ!fclRmRCtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignaturec3ecc1fc5135d1e959695e569d213122riIENDB`amavisd-new-2.7.1/README_FILES/images/toc-minus.png000640 000621 000620 00000000403 10636222147 021233 0ustar00markcmi000000 000000 PNG  IHDR )bKGD#2IDATxch`g  I׏j?}-pZECtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignatureecf413ef47524404f90c44d8c7d12a2e݈ tEXtPage15x9+0+07vIENDB`amavisd-new-2.7.1/README_FILES/images/prev.png000640 000621 000620 00000002154 10636222147 020276 0ustar00markcmi000000 000000 PNG  IHDR#Yi&PLTE!!))11BBJJRRkkssƌΔΥ֭ĪbKGDH{IDATxڵ0 CV%2o%eĶ 8Xz\ #KS!QaĪ (9Wv|YY d=[髂Ng$ C=% CtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignature0f87aaf0b04e24ba165f2a4bfa6bca57kИtEXtPage35x15+0+0~FIENDB`amavisd-new-2.7.1/README_FILES/images/blank.png000640 000621 000620 00000000566 10636222147 020416 0ustar00markcmi000000 000000 PNG  IHDRL\gAMA a-IDATxA 0MF4 z;3'kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` 5kX` `fHfIENDB`amavisd-new-2.7.1/README_FILES/images/caution.png000640 000621 000620 00000002342 10636222147 020763 0ustar00markcmi000000 000000 PNG  IHDRשPLTE!!11BBZZcckkss{{sss{s{cZcc19B191_\bKGDHIDATxuێ@ b̮IcAP`bxsBEU_Oh<`'1wp Wȝt:%su&w,Hȯf 59v:\^ F;`HJ'871ZJ+b2B8Q$wRK v)nRpt}VbTzDv@=rPduvz{Nl;a*CtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignaturec70387830aa4ecd5a4a32a852283b3d6PtEXtPage24x24+0+0r[ 1IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/000751 000621 000620 00000000000 11003707000 020421 5ustar00markcmi000000 000000 amavisd-new-2.7.1/README_FILES/images/warning.png000640 000621 000620 00000002331 10636222147 020764 0ustar00markcmi000000 000000 PNG  IHDRשPLTE!)1BJRZks{{{RRZZ{{99JJ11))))))!!p;bKGDHIDATxm}o0M5%%m szP^UDKnk[;һ?0n!qCtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignaturec42b7d2d564aab588891979703f02b45OߓtEXtPage24x24+0+0r[ 1IENDB`amavisd-new-2.7.1/README_FILES/images/draft.png000640 000621 000620 00000042056 10636222147 020427 0ustar00markcmi000000 000000 PNG  IHDRKZJgAMA a IDATxv۸asV8 BLhp7v8<[ w`9*i$4`-B ! xb^8x5oTGa0 s;&!b;07kBC}b$xkeY?|os$x cx? /󇄎 {w:Rqpt5Ih'xH1' 桕R4EAB;[ ?Hhŏ1}ZBNBHQvC4)rB29턶n}|CCkׯzA= 4ew i}dx4! ZHNWUŸC4b/_a~9B[4l/C4l,+4l)rϟ? 4`3&+m-c 8z%*i@55 WO#xyy9NB7?6ϳ3cNhLkظwNH1&i֮.yB e=9i4M8mrv/ .@b,5ẻVwͬ [ xm=oD% k v R"e2ʽV}Bv=%~n4M0\.OǽKkm{_ބ4Dd桿 8Z9ɧGHhT4{C% μ$ʤ k4!OT,;NB rzAHCyiʲ,NӂZikòvV41} 7}JyeѳIބ4<蛄zӾgiBcB rzi:InL녙~~~.K܅JrWB+iO}%BH$ 3?UNpEz2?,fNnXk]%kg~J~ZK=q ibߛyq|ArBkyݗۉ#1' _Ьvmۮy*[OSIH]Vބ4|ޑZMrnxޣ$eQI w*~o*i{Oθ Rb44Њ.uYTiI_;[IptЩJph:szs {SI8.j#u=M% H44#b=촞p8>OQcN>4! (Hh4! XE4UU" igMӐ{T.r8$u7! eFI??MHHQӄ44;}NLl AvNr.}CӮ$t⬧ iIS 0 iS +Ŗӄ4DND~1)`nEGF% `Hh8ɸ7! `HhxCNvWOFM]bO1U4 i;&9MH ӄ4`O1l.XN?uB>/7" 'ZQI ]p2iB+B@Hhʜ*! B$49sVJ @\H5qaqvuY'"D^Cm[2!אr<6j޻@H^(;4M֕! ;O }N1%oEQ$! /5t4^oy^UUUUί kxI] kPCCi>jh l,zzve8l,iNXNpdrCI>Ӟv,QtkTvF,4-)7Ri546AHث09::! `O@BIرe~iHh(`iqoWHh܎JY? n&qx5]"i2<&ڜ&J4 \*Ih`pnmOXuҴiQIHֆ4 'i)dP8Ih8Ĝ4􅜟󜄆+T!غi8BQy_^^i8AH804\ڭm[Ƙ_~eY%<*iø7pD4vn5rqoĉJqc.˒zq"ڒ{#B wވ40H9(@\q}BN#6Iy}},K9P)4|f~Zko=B$ygYVU1>2D.UU繜 uӈ! lVJYkEQy^UQ ! lBf}A=mnIO\ 9 fNh6ƔeYEe06V&5km4YIaMZ@Nc4 0Ȫ-9+1pFx4^H]ueEQ( kiFHAu]d5cn вjš2WiCHLOITSX;AN# Ba^ XReiY~Qʲmim\nFX8'Cx ! ?niaij^зl ~`~ ooo}?|.Zoa-cw2r"}ߟ1MTUKw] eY>???<8>|^sI=:"y◗5t:ep5,!MF<2i|BCW*RN| D6p/̩_.%w}Ж ._{,\54+*i`38,w&M4ϳXKsߺ,ˤ2:~42HY /diIS׵?U|Ǹ7 / wÕׇ0 r:( 9b+{9CVm+qoWGлC% lm.J)kP0RU/M?Id2i|_Q&$^u~LY]}\duuGa.ޖ-#kss:Haؒ fY2eqs>)y9i/B/7v\}Y[Zske75~QIbuu<[zE% kו;eZQL/b,˺i^-g2ĆeuefY_zW\Z*Q9:YU|c,x¶i j0qW>YKڐ^Xk*7&v)a|Vi vooo]׭Hi}$tQ߷6m4@6,M=;i[%!?69Lؒ F\݃WBN0"9[vG|e ޘ׿_k}:qC#J+Y$@1MӬÚwބPR:+䈎(Xup|ݼWOhHw,G꺮JEvEeMӐP4{uXZ맧`FRCOi.ZNHhB؇y7^u4Mߎ%>KpuCX>Hm:Z(W jmBvc?\IXIӄ&CRX')5{"FT D J,dۉ 4'8_u:8W7&ߑC$e w>NBSUrYsIy F Ry˪ye m$4GH{bq}E0ӫߑC$1KgmHhv/UNo/wxRXB(FԲglGDvZ+GnqZ7_j5mB/Aļ=$J|/ҟ_qW'KetߓEUUOOOө,[އe_n?YF:Y!ӏE#YT-U80 ZIiMI-O>FBe,VJɦfHh܋Jثmjyzz7I[,v,M0PId-BT{wx7ZήUUI}aOi`e)˲;Km+ʷf^6տ#_ WZN1t})VE 6?`ߜ:uݏӮMӐ^W@ve~L~Hh`I#"Z;ϳA$[k-7Z~SwA~X%ҁ.~DHcKKO$t(%9 u5gyJ]EHS}󼪪wNBb4!Nx,zZJjJ8///+w϶u]4A6yXZbpyaMdeZk<ш/&QI#NBk[2x{zǗ?ڲ,1$4B!H+} IDAT)gWUroleOOOr5! $۶dLZ7M|___y}! neOz\&On{'Xk-ےDBk\X`CZ9XW grʯʲl[AbI/^ dOx4ܓ}߯_RWU^N֣cڶm{Gz˧iJ/?1UU$4??pF^__o;/&:^1_x~Ϝ|>;y~}anZeYYeYfY˘Ll;Udw ]j^z_ 1{RC;OOtrʲӠ4r0r䆓K:~DH?xSt:UUUsG7H1d~&W#GH|Z_~eiwG^I74دb4ׯ_n_ZuݮZ+/%_i(a[w>Na 菤vR:}c_J>Ӏo4܏rp]ۮYʲiL68yOd8n|>:5tMlA%v]LN.u\8u!}hkh <#YD++!士m -IM8-/j.(<0b!9Oh+#eY:ajSJMӔD+)4gB L'ڊ8[N])EH?sJeYd`VN\mYk˲trҀ'>UIb_qX;HfsuoD++:HW\'믣 SxO 'J)cL{Vǜm @ABbᔢ*'kpn7OX<^8g 4v ';y1 dkmBv56}2 ]>gq#u]hTeۦQ8Ԟb7r2-f&dY䳋r0 8ק-磆|OWiN)pyu)9ŧ1feNf&)ZّɥdKPtzHZms5<0 Cu@҉!o&bYm[.r+a:Ȁ|A۶m:` -e3R}1( WEc-FʏЏY?~òI۠}O7𰔟ȩ:$+^q-Y00 I$ckmoѢ,KWcfsGy褁QoZWUu( WiX3 <&GsJH5iZ3!mͲ,h0Ƹ:rAo1?@B1MW# #gapr)P@'^celM%W}azGΙ3z^IkuʾnccwZm!Ůicqu?;"3? @1 (O=b{LEa(i|>; &4M0,u km4+{:z_v aGB]Gk&_cpzdQTχFuyElB;Y##Nni첫Oi^Ek4,vrc@9٘b|JEʮ->N:KFe$܈'іڶu:T4־RCZ+.u\ \~q9r#e)> t]v=5kr䟉<_c㑔<ҒZ+gr?Y/j1Ŷ,`wOx ! yyoc$,˲,`VJ}/,9돟$3[)@~1j[ίw;tx؃#WXZ;r߫|tUU5M#ߗeQE~$R巌'ǜ?5nq:RFHo)LNDnZ)U8-ijDKGr|!=G>X-㸮kלNoyB+ny]cO R1Y ޞmz=x qo e`|c?*/Slӷ;TBtwPz<5cRd a7Iΰ10 {r3YFŗs/g2(cEHG9MBozT\)u]dE"ٲH!c8wI,EtJOX˒ǻ]0\ BzӌrGNe܅2އ4 fFAx72Xk0 LBGNvA!`$7IhY<ϳ ^G-[Rt/+jh` $9[!Q2jD냗%654!?^nIbUJxdm9Rb&In q }u{iߣܲ+u]\JmّIJ):i!c{?/_B*޲,<@E$s< )܃s}oNK,˪zm0 {A !yw]z`M.RHw$2-[@lYON\f^^>@o>\`&jۂ&H 崵r(О,TȾ^ߵ"E%izf.NB1#SmNБ%]| bFH'n>2:N׭ga$4-B:}iz"܋;4/pYQ󙄎$?*VOkHB8&*VO;GB8,B@[$4##e_9MB88Bp$4GNÊ9Ihqř$4,C-IhFHݒrwBB;f$뺖7ABGl  oHhIh ! ??MB7rPNo2,}1&sB%Zi71Z,c1J)1F#'ظ4M:ܓqgcLu4 _U˯G]-\Eey/ն/m,*GX%8ھiZf)oRUei[aeiek$8JNoEQcZFɽnk)m CYyWUaxuYmO@i {{{KcH@VZ8 C4A7if8}솱8e GRaƽӉqo㐖sXӆ֧)zim>=0wv7Miǰn+Ƙii~D!=MS۶;3u4z{Qi^^^PsUUeYn}/J\?-eL[k1,3?@"}œHx++qIږI7<,G0 oooZbXV-kS-_ȟ9V7joez=.˲TeYJdYfonQe<8>hHtuu 0 Ȝ=H`|\<BZݵFf9LZe5)7**ߓm}_9xʓЛXa|&r>?&@=͸7} *RV{xezЛ.Z|1rq<^!Б,|l>M|'*JY<n6B1? Iu/Ga}lA8g[Et:i}10 yA&z:׷6MmAw D4Fg{QEYiʲdVxxcHhW꺞贜aby&l{yNO4M1&Ao:eAZ-'R8=b8SsK" ž4HN8!CMGd~QnI䯗Jk-oyv<4bi&,3s454iӓeYQny.bZݐb!w]iGx44M>˲TQr{`]?}dőLodYFB .E91( Wnr5/@NSUo^bkؓ>,Noǎz`_=7 vZ,7;?8;a #!놝\JL;._plJ|d)לz:Czz2}%X|7/7 繓B,yDkm[ n=$iZyd-={8SO-scꖐVө+ɑ4Mn}4G0Io<K Q?өיboN1&˲EvgipHr:=ڢ(VNK*,Ů&#er:1ZkcNn  iźeY npNeDN^_ЪiHƽAJphCNJ-LiX%|!T,e iEN1|g![29g뻻i"wZ+ \%XVG[+Ka-/!؏lֿ$48+ ipcH+]Ӡ׼Nf !#k3B\Ҋ W^7\ *TN'8w BZ#\.qDVEL޽ʸw\}Y(9͸iy$ޑYJ.9f{ǣZ-g{ߛy@;][|{ߛƘ_ťސ$7Gk]%!q}C!-hq|JNhTQv"Ry WH+r:q۶]cLUU(V8^.'b1'ƐVO|IR*ZxZkmuN6뚖1'ސVZ,SRF??aiʳ4EQuROҊvm[W eY]l ^ 90 08Y-icҊ^g]utka6|s~պsuihҊ~ۄ.˒0vҊr9Ϯzz#IDAT. "o6RUUS7ːV weY^CZN$4l"VaBkhB6NHN1yQ/[I*!s({\4 J-rzm&VpuM iuz&5t:4CZ =r+b?.epV"i,^CZn6XNK說؝~H+9--rwB rVI䴿h$8>29qi}dkh"tJZ촞&j9MBa.ծr#;֜秗'ޘ `XIiOO*!"iF݋ƽCW"z,ii<Üj~Z"|&RCw4004_YYk|לw^k$4$לv8}dnBgiCH$& IiREH!œ& a}iFH-& y#6i~І9MBAҏ$Ih8Bz9MBk,Ih8Bځ9]eUU$4 `9,FTOpXKs##st9i:+s(BڟsBڣr,iipƜ&!$4#B:orZ:!Χ9$:-bFH.Ih7Ж&m?`uIENDB`amavisd-new-2.7.1/README_FILES/images/toc-blank.png000640 000621 000620 00000000476 10636222147 021201 0ustar00markcmi000000 000000 PNG  IHDR kd0PLTEO&IbKGDHIDATxc?|` h BhCtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignaturef7e388dabd4ef0097714b5643fdd3cfbb tEXtPage15x9+0+07vIENDB`amavisd-new-2.7.1/README_FILES/images/callouts/3.png000640 000621 000620 00000000536 10636222160 021307 0ustar00markcmi000000 000000 PNG  IHDR s;bKGD#2xIDATx%N@ 4^0+ F``a+&U qXҠq K ]pq˟3&=ۿ-#S:bmR&jQ5cLCtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignature80bbda2726ddace8ab8a01f59de2ebdbutEXtPage12x12+0+0m}IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/4.png000640 000621 000620 00000000531 10636222160 021303 0ustar00markcmi000000 000000 PNG  IHDR s;bKGD#2sIDATx!0C#XdeeP"\o+{%leʰ!b$ci1 q dCwCmJV$6huTj~<_²|㣴 KF6[CtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignature9f82fcac9e039cbdb72380a4591324f5vtEXtPage12x12+0+0m}IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/11.png000640 000621 000620 00000001065 10636222160 021364 0ustar00markcmi000000 000000 PNG  IHDR ˰ pHYsttfxtIME-'kM8okǖejYVǗ˅F C3IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/14.png000640 000621 000620 00000000633 10636222160 021367 0ustar00markcmi000000 000000 PNG  IHDR ˰bKGD pHYsss"tIME x8(IDATx}=@O2\ق۰X"Y;@)lT!H!}=CDZ;9V DDDqf3qӉ~qXkTQp8|.)mUUz~9EQh 0hQE|jǣ,b)ntnwx<.|q~IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/8.png000640 000621 000620 00000000545 10636222160 021314 0ustar00markcmi000000 000000 PNG  IHDR s;bKGD#2IDATx0  v¬a` 544T ?ݻ/TܗW[Б!Dغ[`T3(fpgc31ؿ.0>_ +U99FbCtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignature57be19505c03f92f3847f535e9b114e94kCtEXtPage12x12+0+0m}IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/1.png000640 000621 000620 00000000511 10636222160 021276 0ustar00markcmi000000 000000 PNG  IHDR s;bKGD#2cIDATxU 0.)Bft6#dH('XW 9cAM-!d>0(*?/c}֮5uƌ:x,TCtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignature58a072e070da22f6135cbd3e414546f9hj!tEXtPage12x12+0+0m}IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/6.png000640 000621 000620 00000000543 10636222160 021310 0ustar00markcmi000000 000000 PNG  IHDR s;bKGD#2}IDATx!0    FaPXXj' nn󩺵 oPHl\BuNح!i`d'נ,˖eԸgNLL< V?s8 YCtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignatured25d7176d67a038afc1c56558e3dfb1atEXtPage12x12+0+0m}IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/15.png000640 000621 000620 00000001200 10636222160 021357 0ustar00markcmi000000 000000 PNG  IHDR ˰ pHYsttfxtIME0 JtEXtAuthorH tEXtDescription !# tEXtCopyright:tEXtCreation time5 tEXtSoftware]p: tEXtDisclaimertEXtWarningtEXtSourcetEXtComment̖tEXtTitle'IDATxu1˂`E`59-AZ[֜šhr /h1A-"6B||g":16BTDDD5dN\8纮.v,K墪eYO`8FQ\.kZNjb2H.1"l6{a0FQ\~^{<u6A|>OVefT eX,vrq?j x8mȔL?IDATZ9皦lN|9Nn+bbW*z)r]z=- !HIENDB`amavisd-new-2.7.1/README_FILES/images/callouts/12.png000640 000621 000620 00000001151 10636222160 021361 0ustar00markcmi000000 000000 PNG  IHDR ˰ pHYsttfxtIME.UF/tEXtAuthorH tEXtDescription !# tEXtCopyright:tEXtCreation time5 tEXtSoftware]p: tEXtDisclaimertEXtWarningtEXtSourcetEXtComment̖tEXtTitle'IDATx!`a`R,/+V5 3,+KV--eŏmpSyyBeYQnw<{j$IFQFѤ npX,v=|gk˲N28@$!n @9iX.$5)esLu}ky@ ^7"qI4($WU~a.K窪dY'^E(IDAT~_ŷRf3LLӴm; C!P@kuO4IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/7.png000640 000621 000620 00000000530 10636222160 021305 0ustar00markcmi000000 000000 PNG  IHDR s;bKGD#2rIDATx%0 OV"Y!LO Hd+H퇓e _pDlC0T+ʫ+ VAjݓ{O9lsLGIz>61GVSCtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignature298368142ac43cebd2586d8d1137c8df&9tEXtPage12x12+0+0m}IENDB`amavisd-new-2.7.1/README_FILES/images/callouts/9.png000640 000621 000620 00000000545 10636222160 021315 0ustar00markcmi000000 000000 PNG  IHDR s;bKGD#2IDATx!0 GFVbJ,WX ^YkTb++#{?/Yٗy/j!Rj+~ E#y@!s.gEOr /P8bCtEXtSoftware@(#)ImageMagick 4.2.8 99/08/01 cristy@mystic.es.dupont.com!*tEXtSignature34623e5e4d48310e409b280afe24760214$tEXtPage12x12+0+0m}IENDB`amavisd-new-2.7.1/test-messages/sample.tar.gz.compl000640 000621 000620 00000034426 11027033526 022000 0ustar00markcmi000000 000000 tȣыBh-'@b1l%']i%w╕K^1fx#nn&* )BL8;eE)†e~Xl h ]>n( |>0پ6VЀ?j9vŃVb@ɶ Q06v-N;uOsXA;y;iyi`;0 , saxbc^U;^vϾSfm>&=}?pEFg:Nms mB T.[b;8La~sl2 Z{k+f ?K<,dcG6ݑ󫎣O#3 2ǃ?}(mbbܴ 9d4O%r]/(=qs# sSS*,*|BfbyQgԆu?ya ;-i8ng-B=%RؠstJZVRD3W+2lj)Nś\vmeN16eW=tNjl8eS<MR36VH-*c쩒sJbk3wKC.##+v(81TX/Ŭe;Ms,_ƴ薹@bbj1hIaFa%mvYqIy~1?DVfђs^=beL餔nOhYq\(k!\2Eh0Ht'ss(aRQ3"E1;9=TB{Ā5L:|Ϗ=+/clڱ%TNHŭQc(Q h[P7V_TZ$. O”\!OlBLo J:T`SȦrBD(1gv2ڰ5=CXؘ6}d2, =u2b:e pɕCW/.ocsmPZ0Xd{6DZJKr{{|ae Vb 7h $i!oDV(xN_(rGI$,58 itb]Ty>͑tI=̞)3KSF8oB)Hw5$$%$99&u~ϦD쵔Δtw5}4+QǤX޲<.M9c%ri W"u7StꒉҪXԑlRd$#1kܢ 9dR4w7yiE1[-sJ4NЊ(X-l =dȼ3} H8'G2 $rWXfbC |mN+5]ͯ@0!A1'1;HfVwÒ~WrMIʚCb"$B,^d86ef ZL%oqӖc]';lpRRس,c0 ֌pJI")YH-edt5AāPG[@b8&D`Eζk )(,kZdZ_fV2X&0-^8+БۭH ܙX`zPJlFߨ',2v({ %)YN D v+>͈fqzh<1A/ 'W]Z5mȫ9 %w;/ݧPf'9X:@PUZE?cg}ǔ%#p=SL 2I7S?-+3eECb'yމ\L>9Hf0+e-X\&eܵђ&6xJ33fΨz=4 jRjYQf C`#gBfQ`0 S%UIurgt\!$˘KܢAM'dX)WEFǓE|刧G*<7&8Z%)%yI O) KDCH/DLsEb&+lgH-+Z͗ejS+TA[(+zyғ9l|PJ4,apx>N8"ec(SbAGk/ \&{s)M JW-NM :f$^.x&CUAl~ʝx9f#!r?X"Y8V+jNrL  @2ꔡEU6|w<ȺCe]mWپ-etaA+1prdGh_)S u߷c[a9̪8ԏg:NmsPoZsHsW!p'*in^bZf{ةcyA)v?>>!IPe$`#?/#Tئԅ90 @+ˬˤ0ya+b'|'`48 l<_Fܱ XT*Bbܲ' Mef\C6v-ԅ$m]}*sD`]drOO'c1yqS׉T9 ,8/Ԇul7-C Si#NlߝqA A.Tt)L RvAg?s?WIyvzNqP_dN;69Hp>/]4 $N&?4̺xB (䜷ߓ$Ox ?8!$_VB0:2F7 KHT |~BQU'^ Wx]qv8*Epɖ/-=&2Q CQ|ZVۄdF9FSV.I& o 79!9G z"Q~C`Oe`O%%USjN1}W8Tjj& ۏd)%I.F[!g1kS,xtܽ}_벒/@eKlzrĬZC2ŗP{r 3śS^PcyDd&^\D]#VXK{HFPRLȁX褒 󃱌FfEX~$ID/F{p /nDĎj1],aT AæK?JyЙnT+/u=pԕl<79V:Be?w$JPKSCf45OW>ffRΓ4&9Ҩ~t. O{C̷_Cx)ADlxԊ%)TxQ3Zb>B\<-NVgvΉ 2qX'(wx =ŏՉlJa$h*V&9]P'疉P`*F:J@A8CKOvZq +њu"p!X%$ҪI{]D4u&өȹեJ' ά9Gdr[OeFbShҠi`#7g67{$^NVc !/j,0<%iRfI15k2peؓ<2mNT$F{%oLIkx+)6 ??F&-Y۾οhࡧv.Sp I<"ũpJtKPe0 |B;/LзOo+8P|c?߿ O x5K< G'#,)R8›sGF S ,.S'/ͅEAQ{c\zG>%4h.]!~:dO']ؒu|#a)Ä~2.Oc6x6Ȃdg_y6}crځDj?Ɠt>sEhڛpY /;d}?V|V{#{̈?WM_ S|0Ϳk,NDſbκ0 o&_}|4c/O4("A1B61Yd"1&,|/4:zPb; !IƒT7xY_[Qb5JD#ㇵ!v !e?X\D/!DpN%sxȒ:8Nk\ .4133`']; ;=Aog﫠癈,: (aip #xIWT?2r@:7DzB%2!l~)hY=猘2' ZG({1 %\n5?0+Cg@8+bU< eY4 @e& ؅M-=eQrL(Ʀn>>pOv#rH-Rd$&[UU@t0#Uځoq8cb Hģc5Lg"Ym]"1' /n; faf$裀|'48,8}NHl$Eyf}ȗ]Ϸď7̲lv=$Htxa 9m 4olZfTߎfX?hgZ, )9fhƐ2l:MZˤc 7ey(ڶ0M>CR&?@y{Ad @Q`3Um( #PG 5g.vrRlsˁq`(řF!QL(cq~_{"c7v.sd臨q]wbc2\_;J/*#'Jv>zJŎi X$["]؉8]45:ݖuGGy8}xC^(G}.FTGEEf= {C+@tz;͉/(:JFWX {8?ǯY[! J,V Mΰ_5 13:hRhuDY97%y*Fp(1>r`0mVpwJ'="ngke9.Ge x7RW"{eRyrguXK1O Jthnp/7)ӤN 27n)~#Β IG=H8kתT*w5т[G^~у4}+XQXڈ]OK_A(Bb>ɽ${?<Mebz'ץ#쏧tO>wG|K,P2tlVHmϊ2C-K@=#9rg>ͣ;ҤP~Vx"8RZ,׹yHnMzAKۺ=Ҽ&=c}$pݸx忩n6`}?Vpf[,uJvla]ene/)pڔQrRݟK§VϒH_CSL;:YZfGeN9 34~U -Q4G#.+t} *|O4nZĮMػۨՏ?2Gvs JA_%)NSFgn|w^rIo-K2m:|iG-H] 2pob'>b ;Ȥd)֭0dD?gAPᒈOL@2?83Qȷ2Rs˛tf>b|vs`8@ qkˊ?- *ϩ5z묗`/P( =MxJ] \N"2u/՜(upfLo'c4?Aw`%8:iW9ppP H+xHϊ"X P%&\r zb,LnGa}y(L'^ z| $UYLk @ZF2lo.ߡO|0K (^{ ,(jb+ A$u$#OR z2ȀYUTB)x[[P 8 ٲ"F@li-ÀvQÄDԴfQYyJ3Tl3U J#@݋- o+rD6,)vT T!JUBJgն*ŊbX*ĪJ*fQR d>,RI_rX`DdYNT@YvգSi1Lće]e^g1$e6PO(%o(*5-bt,߻J!ep;PX!.}=˗ZgjW q/+\NUo!OaVeO)#}&`4Sk&‡%1~5;FaiyPF8Uz8a@ e [̎a'Z6r`qMq]3vk+~4-ret>:1D SǤ \n)}4eRG;z \)QX:6/~ypؿ$8*6N[ 4)}deQ_\C|Ƅ~F=:ns;73G =Ow=FSZgs-"AJM:ؽV ڱ.89O --#_gz>?{ ʒeLnn H  2%N0 :&f|h =)wh͜G&6qU[~Zyrf)6bůcgo:8,K?+ahM1+KexVïgc~_~14W~(ؽ:{o E],$.ҼuȎFq&r]ay\~X#[ܮ%ݩ΁Tub\H| 3nYH~y.}[߬%?86bpj o\AC:1\vQ!l,8$hU2 ?81_kkczQ@#*xq_N?tkj:@îK&W7,߃ N+}9/؋mtk&̛ru gD 5H{_-?Io~zo-k_|m+xI$ΎؾusaY;;_t ¾kzּ' ܹϿH3Ev ~7^ }!Ʊ.:6 ?8.*9H}͕ KQ <~*x.$'$_g{-VmŃ<o(\+:3,I0{o>=%(iG_@]sR7Duyòvſ1\5(Xy tC6plm`Γz~/g#~][(M%g>gn|D/EdufXl]{ q.nO 7=J,o7zYPin RXgps#u&FSY^i}һDh .bg>{{` /{c=KPK?"=ST0t߀R\۾/2%>@Ea(dp;:29,&-[H(tA{?ҵaI3t Â\wX{oU36?PRֵV}'JUp8(LU sd W"x`V-$xVT*J h40#ruuH?5#:Ҫ#i4i`@AGZHcA#mHG :vtHci:Ҁ4#mKGZבGHHA#m(u1ґ%i;u42D:XiD:V#,i5H:LH:҄Ց&uiM_x1M,2hĽz&Q;R23+Nc: ^oL N,o ئg`&#iJ0'*0JzΫ4J 1j5f'Z 1{(G$T g+Ĉ?^Ux`d> QQJQOtxH\#>\WVi"Dž t*z YGW[CqG1hfIDgx^v]A]c\Yf=Lno1XԞu򬾃1c~74h,,5ųT wuʮп"uYx Xk@ H:lPRX)dv -x!:5:O3qi0UsFȋbe3Qv}d|JRH27,dcFJޮ8_4lA5=Pt]WF(W  tzDn 1dU]'xMdDzd#\ {t 2Ai|/BwH:*8" ԥ;PC#nb>o*`(Y7  y<\ "֔DyhQXBS>KO'& 0 odyV"ڻD =k-%Kvyj%kiEow2V"Ņ@ZYU?!%8Awz\kxcxYw"h5%`O/NJՁ$I`i=BJg<΀/6/A`X -@?w=,/$`t;C] t2P yx(Dsd2EV3J@|dp ,9 (&@ctD#wF˝\PNZ8͢K :TZiTk 8t,\zC.L.W1~naJ_@sR3r%'xhm䉔->{O3~9N$ܐRI $$芒X.R3`R$Gc*5sձii/9v8F3Vz/F6 LĊ F"v)CH9DU(DV58L(rE[M୾bg̒ "%jllqR {e,oYSz˙e5 \Iئm%)JcO ;q|'zBQ&s:وClNe(Ԅ9NƆo&Zχ