--- amavisd.ori 2011-05-13 19:22:29.277598284 +0200 +++ amavisd 2011-05-13 19:22:38.762672629 +0200 @@ -98,5 +98,5 @@ # Amavis::In::AMCL # Amavis::In::SMTP -#( Amavis::In::Courier ) +# Amavis::In::Courier # Amavis::Out::SMTP::Protocol # Amavis::Out::SMTP::Session @@ -219,5 +219,5 @@ fetch_modules('REQUIRED BASIC MODULES', 1, qw( Exporter POSIX Fcntl Socket Errno Carp Time::HiRes - IO::Handle IO::File IO::Socket IO::Socket::UNIX IO::Socket::INET + IO::Handle IO::File IO::Select IO::Socket IO::Socket::UNIX IO::Socket::INET IO::Stringy Digest::MD5 Unix::Syslog File::Basename Compress::Zlib MIME::Base64 MIME::QuotedPrint MIME::Words @@ -9281,5 +9281,5 @@ ### but before binding to sockets sub post_configure_hook { -# umask(0007); # affect protection of Unix sockets created by Net::Server + umask(0007); # affect protection of Unix sockets created by Net::Server } @@ -9292,4 +9292,34 @@ ### Net::Server hook +### This hook takes place immediately after the "->run()" method is called. +### This hook allows for setting up the object before any built in configuration +### takes place. This allows for custom configurability. +sub configure_hook { + my($self) = @_; + if ($courierfilter_shutdown) { + # Duplicate the courierfilter pipe to another fd since STDIN is closed if we + # daemonize + $self->{courierfilter_pipe} = IO::File->new('<&STDIN') + or die "Can't duplicate courierfilter shutdown pipe: $!"; + } +} + +### Net::Server hook +### This hook occurs just after the bind process and just before any +### chrooting, change of user, or change of group occurs. At this point +### the process will still be running as the user who started the server. +sub post_bind_hook { + my ($self) = @_; + if (c('protocol') eq 'COURIER') { + # Allow courier to write to the socket + chmod(0660, $unix_socketname); + } + if ($self->{courierfilter_pipe}) { + # Watch for courierfilter telling us to shut down + $self->{server}->{select}->add($self->{courierfilter_pipe}); + } +} + +### Net::Server hook ### This hook occurs in the parent (master) process after chroot, ### after change of user, and change of group has occured. @@ -9344,4 +9374,15 @@ } $spamcontrol_obj->init_pre_fork if $spamcontrol_obj; + if ($courierfilter_shutdown) { + # Tell courierfilter we have finished initialisation by closing fd 3 + # But make sure it's a pipe (and not the courierfilter shutdown pipe) + # first: if we have been started using filterctl (i.e. not when + # courierfilter itself starts) then there is no initial pipe on fd 3 so + # it could be assigned to another file + open(my $fh3, '<&3'); + if (-p $fh3 && $self->{courierfilter_pipe}->fileno() != 3) { + POSIX::close(3); + } + } my(@modules_extra) = grep {!exists $modules_basic{$_}} keys %INC; if (@modules_extra) { @@ -9706,5 +9747,7 @@ $amcl_in_obj->process_policy_request($sock, $conn, \&check_mail, 0); } elsif ($suggested_protocol eq 'COURIER') { - die "unavailable support for protocol: $suggested_protocol"; + # courierfilter client + $courier_in_obj = Amavis::In::Courier->new if !$courier_in_obj; + $courier_in_obj->process_courier_request($sock, $conn, \&check_mail); } elsif ($suggested_protocol eq 'QMQPqq') { die "unavailable support for protocol: $suggested_protocol"; @@ -9784,4 +9827,24 @@ } +### Net::Server hook +### Called in child process when any filehandle Net::Server is selecting on +### becomes readable +### Used to detect EOF on the courierfilter pipe +sub can_read_hook { + my($self, $fh) = @_; + if ($self->{courierfilter_pipe} + && fileno($fh) == $self->{courierfilter_pipe}->fileno()) + { + do_log(0, "Instructed by courierfilter to shutdown"); + kill('TERM', $self->{server}->{ppid}); + # Wait for the parent to kill us + sleep(1); + # Still here? Close down ourselves + $self->child_finish_hook; + exit; + } + return undef; +} + ### Child is about to be terminated ### user customizable Net::Server hook @@ -13679,4 +13742,9 @@ Amavis::Conf::supply_after_defaults(); +# courierfilter shutdown needs can_read_hook, added in Net::Server 0.90 +if ($courierfilter_shutdown && Net::Server->VERSION < 0.90) { + die "courierfilter shutdown needs Net::Server 0.90 or better"; +} + if (defined $desired_user && $daemon_user ne '') { local($1); @@ -14161,4 +14229,6 @@ : $inet_socket_bind), listen => defined $listen_queue_size ? $listen_queue_size : undef, + # need to set multi_port for can_read_hook + multi_port => $courierfilter_shutdown ? 1 : undef, max_servers => $max_servers, # number of pre-forked children !defined($min_servers) ? () @@ -17132,5 +17202,424 @@ no warnings 'uninitialized'; -BEGIN { die "Code not available for module Amavis::In::Courier" } +BEGIN { + require Exporter; + use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); + $VERSION = '2.102'; + @ISA = qw(Exporter); + import Amavis::Conf qw(:platform :confvars ca c); + import Amavis::Util qw(do_log am_id untaint debug_oneshot snmp_counters_init + switch_to_my_time switch_to_client_time + read_text orcpt_encode xtext_decode); + import Amavis::Lookup qw(lookup); + import Amavis::Lookup::IP qw(lookup_ip_acl); + import Amavis::rfc2821_2822_Tools qw(quote_rfc2821_local qquote_rfc2821_local + unquote_rfc2821_local); + import Amavis::Timing qw(section_time); + import Amavis::TempDir; + import Amavis::In::Message; +} + +use IO::File; + +# Amavis::In::Courier->new() +# Creates a new Amavis::In::Courier object +sub new() { + my($class) = @_; + my $tempdir = Amavis::TempDir->new; + bless { tempdir => $tempdir }, $class; +} + +# courier_in_obj->process_courier_request(socket, conn, check_mail) +# Processes a request from Courier to check a single message +# socket: the socket to communicate with courierfilter +# conn: Amavis::In::Connection object +# check_mail: reference to the MTA-independent function called to check the +# message +sub process_courier_request($$$) { + my($self, $socket, $conn, $check_mail) = @_; + + # Save the policy bank so that it can be restored at the end + my %baseline_policy_bank = %current_policy_bank; + + eval { + $self->init_request($conn); + $self->read_courierfilter_socket($socket); + $self->open_mail_text(); + $self->change_policy_bank(); + $self->call_check_mail($conn, $check_mail); + $self->process_result(); + 1; + } or do { # An exception occurred + my($eval_stat) = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat; + my $msg = "Error in processing: $eval_stat"; + do_log(-2, "TROUBLE in process_courier_request: 451 4.5.0 %s", $msg); + # Close the mail text file + $self->{msginfo}->mail_text->close() if ($self->{msginfo}->mail_text); + $self->{msginfo}->mail_text(undef); + # Send a temporary failure to Courier + $self->{smtp_resp} = "451 4.5.0 $msg"; + }; + + # Send the SMTP reponse back to Courier (done outside the eval to ensure that + # it always happens exactly once, whether or not there is an exception) + do_log(3, "Mail checking ended: %s", $self->{smtp_resp}); + send($socket, "$self->{smtp_resp}\n", 0); + + # Record time + section_time('send response'); + do_log(2, Amavis::Timing::report()); + + # Restore the policy bank + %current_policy_bank = %baseline_policy_bank; + + # Clean up object + $self->{per_recip_data} = undef; + $self->{control_files} = undef; +} + +# courier_in_obj->init_request( ) +# Begins processing for a single request: initialises global variables and +# creates msginfo object +# conn: Amavis::In::Connection object +sub init_request($) { + my($self, $conn) = @_; + + # Set up globals + am_id("$$-$Amavis::child_invocation_count"); + Amavis::Timing::init(); + snmp_counters_init(); + + # Create msginfo object + $self->{msginfo} = Amavis::In::Message->new; + $self->{msginfo}->rx_time(time); + $self->{msginfo}->log_id(am_id()); + $self->{msginfo}->conn_obj($conn); + + $conn->appl_proto('courierfilter'); +} + +# courier_in_obj->read_courierfilter_socket(socket) +# Reads the courierfilter socket, which specifies the path to the mail text and +# the control files, storing the path to the mail text in the msginfo object +# Also reads the control files and stores their data in msginfo +# socket: The courierfilter socket +sub read_courierfilter_socket($) { + my($self, $socket) = @_; + + # Just make sure + local $/ = "\n"; + + # Read the path to the mail text + switch_to_client_time("start receiving message text path"); + my $text_path = $socket->getline; + switch_to_my_time("received message text path"); + $text_path || die "Can't read message text path: $!"; + chomp($text_path); + $text_path = untaint($text_path) if ($text_path =~ m{^[A-Za-z0-9/._=+-]+\z}); + $self->{msginfo}->mail_text_fn($text_path); + + # Read control files + $self->{control_files} = []; + my $path; + switch_to_client_time("start receving control file paths"); + for ($! = 0; defined($path = $socket->getline); $! = 0) { + chomp($path); + # courierfilter indicates end of control files by sending a blank line + $path || last; + + switch_to_my_time("received control file path"); + $path = untaint($path) if ($path =~ m{^[A-Za-z0-9/._=+-]+\z}); + push(@{ $self->{control_files} }, $path); + $self->read_control_file($path); + switch_to_client_time("receiving control file paths"); + } + switch_to_my_time("finished receiving control file paths"); + + # Check we did actually get a control file + @{ $self->{control_files} } || die "No control files specified"; + # Record the recipients in msginfo + $self->{msginfo}->per_recip_data($self->{per_recip_data}); + + # Record time + section_time('read control'); +} + +# courier_in_obj->read_control_file(path) +# Reads a single Courier control file, adding its recipients to +# $self->{per_recip_data} and storing other information in msginfo. +# $self->{per_recip_data} is an array of Amavis::In::Message::PerRecip objects. +# (Note that this method will overwrite any previous settings for sender, etc, +# but if there are multiple control files they should contain the same +# information.) +# path: the path to the control file +sub read_control_file($) { + my($self, $path) = @_; + + do_log(3, "Reading Courier control file %s", $path); + + # Read the file + my $control_data = read_text($path); + my ($rcpt_idx, $recip) = (0, undef); + foreach (split(/\n/, $control_data)) { + # Parse a line of the control file + # Sender + do_log(4, "Courier control file line: %s", $_); + if (/^s ( .*? (?: \[ (?: \\. | [^\]\\] )* \] + | [^@"<>\[\]\\\s] )* + ) \z/xs) { + my $sender_quoted = $1; + my $sender_unquoted = unquote_rfc2821_local($sender_quoted); + $self->{msginfo}->sender_smtp('<'.$sender_quoted.'>'); + $self->{msginfo}->sender($sender_unquoted); + } + + # Recipient + if (/^r ( .*? (?: \[ (?: \\. | [^\]\\] )* \] + | [^@"<>\[\]\\\s] )* + ) \z/xs) { + $recip = Amavis::In::Message::PerRecip->new; + my $addr_quoted = $1; + my $addr_unquoted = unquote_rfc2821_local($addr_quoted); + $recip->recip_addr_smtp('<'.$addr_quoted.'>'); + $recip->recip_addr($addr_unquoted); + $recip->courier_control_file($path); + $recip->courier_recip_index($rcpt_idx); + $recip->recip_destiny(D_PASS); # Default destiny + push(@{ $self->{per_recip_data} }, $recip); + $rcpt_idx++; + } + + # Original Recipient (RFC 3461) + if (/^R ( [!-~]+ ) \z/xs) { $recip->dsn_orcpt($1) } + # RFC 3461 NOTIFY value + if (/^N ( [FSDN]+ ) \z/xs) { + my %notify_values = ( F => 'FAILURE', S => 'SUCCESS', D => 'DELAY', N => 'NEVER' ); + $recip->dsn_notify([ map { $notify_values{$_} } split(m//, $1) ]); + } + + # DSN RET parameter (RFC 3461) + if (/^t F \z/xs) { $self->{msginfo}->dsn_ret('FULL') } + if (/^t H \z/xs) { $self->{msginfo}->dsn_ret('HDRS') } + # Envid (RFC 3461) + if (/^e ( [!-~]+ ) \z/xs) { $self->{msginfo}->dsn_envid($1) } + + # Authenticated submitter (RFC 2554) + if (/^i ( [!-~]+ ) \z/xs) { $self->{msginfo}->auth_submitter(xtext_decode($1)) } + + # Received-From-MTA + if (/^f .*? ;\s* ( [A-Za-z0-9\.-]+ | \[ [0-9A-Fa-f\.:]+ \] ) \s* + \( ( [A-Za-z0-9\.-]* ) \s* \[ ( [0-9A-Fa-f\.:]+ ) \] \) + \z/xs) { + $self->{msginfo}->client_helo($1); + $self->{msginfo}->client_name($2); + $self->{msginfo}->client_addr($3); + } + + # Courier queue ID + if (/^M ( [0-9A-Fa-f]+ \. [0-9A-Fa-f]+ \. [0-9A-Fa-f]+ ) + \z/xs) { + $self->{msginfo}->queue_id($1); + } + } +} + +# courier_in_obj->open_mail_text( ) +# Opens the mail text file, whose path has been read into msginfo->mail_text_fn +# The file handle is stored in msginfo->mail_text +sub open_mail_text() { + my($self) = @_; + + # Open the file + my $fh = IO::File->new($self->{msginfo}->mail_text_fn, 'r'); + $fh || die "Can't open ", $self->{msginfo}->mail_text_fn, ": $!"; + + # Disable UTF-8 decoding of input data + if ($unicode_aware) { + binmode($fh, ':bytes') || die "Can't cancel :utf8 mode: $!"; + } + + # Store file handle + $self->{msginfo}->mail_text($fh); + + # Record time + section_time('open text'); +} + +# courier_in_obj->change_policy_bank( ) +# Loads a new policy bank if necessary +# Also enables debug_oneshot if necessary, and sets msginfo->client_addr_mynets +sub change_policy_bank() { + my($self) = @_; + my $cl_ip = $self->{msginfo}->client_addr; + my $sender = $self->{msginfo}->sender; + + # Enable debug_oneshot if set for this sender + debug_oneshot(1) if lookup(0, $sender, @{ ca('debug_sender_maps') }); + + # Load MYNETS policy bank if client IP is local + my $cl_ip_mynets = ($cl_ip eq '' ? undef + : lookup_ip_acl($cl_ip, @{ ca('mynetworks_maps') })); + $self->{msginfo}->client_addr_mynets($cl_ip_mynets); + if (($cl_ip_mynets?1:0) > (c('originating')?1:0)) { + $current_policy_bank{'originating'} = $cl_ip_mynets; + } + if ($cl_ip_mynets && defined($policy_bank{'MYNETS'})) { + Amavis::load_policy_bank('MYNETS'); + } + + # Load MYUSERS policy bank if sender is local + if ($sender ne '' && defined($policy_bank{'MYUSERS'}) + && lookup(0, $sender, @{ ca('local_domains_maps') })) + { + Amavis::load_policy_bank('MYUSERS'); + } +} + +# courier_in_obj->call_check_mail(conn, check_mail) +# Calls the check_mail function to check a message - the properties of msginfo +# must already be set +# Also handles the tempdir and closes the mail_text file afterwards +# Saves the STMP response returned by check_mail in $self->{smtp_resp} +# conn: Amavis::In::Connection object +# check_mail: reference to the function to call +sub call_check_mail($$) { + my($self, $conn, $check_mail) = @_; + + # Initialise variables + Amavis::check_mail_begin_task(); + + # Prepare temporary directory + $self->{tempdir}->prepare(); + $self->{msginfo}->mail_tempdir($self->{tempdir}->path); + + # Courier is responsible for relaying the message, and so for success DSNs + $self->{msginfo}->dsn_passed_on(c('forward_method') eq '' ? 1 : 0); + + # Log the message + do_log(1, 'Courier %s %s: %s -> %s%s', + $self->{msginfo}->queue_id, $self->{tempdir}->path, + $self->{msginfo}->sender_smtp, + join(',', map { $_->recip_addr_smtp } + @{ $self->{msginfo}->per_recip_data }), + join('', + !$self->{msginfo}->auth_submitter || + $self->{msginfo}->auth_submitter eq '<>' ? (): + ' AUTH='.$self->{msginfo}->auth_submitter, + !$self->{msginfo}->dsn_ret ? () : + ' RET='.$self->{msginfo}->dsn_ret, + !$self->{msginfo}->dsn_envid ? () : + ' ENVID='.xtext_decode($self->{msginfo}->dsn_envid), + )); + + # The temporary directory is about to become non-empty + $self->{tempdir}->empty(0); + # Do the work + my ($smtp_resp, $exit_code, $preserve_evidence) + = $check_mail->($conn, $self->{msginfo}, 0); + # Preserve evidence if necessary + $preserve_evidence && $self->{tempdir}->preserve(1); + + # Clean the temporary directory + $self->{tempdir}->clean(); + + # Close the mail text file + $self->{msginfo}->mail_text->close() || die "Can't close temp file: $!"; + $self->{msginfo}->mail_text(undef); + + # Save the SMTP response + $self->{smtp_resp} = $smtp_resp; +} + +# courier_in_obj->process_result( ) +# Processes the result of mail scanning - recipient addition/deletion +# Before calling this, the SMTP response must be stored in $self->{smtp_resp} +# and may be altered +# This does not send the SMTP response back to Courier +sub process_result() { + my($self) = @_; + + if ($self->{smtp_resp} =~ /^25/) { + foreach my $r (@{ $self->{msginfo}->per_recip_data }) { + my ($addr, $newaddr) = ($r->recip_addr, $r->recip_final_addr); + + if ($r->recip_done) { + # Deleted recipient + $self->delete_recipient($r) if defined $addr; + + } elsif (!defined($r->courier_control_file)) { + # Newly added recipient + $self->add_recipient($newaddr, '', $r->dsn_notify); + + } elsif ($newaddr ne $addr) { + # Recipient with address changed + $r->recip_smtp_response("251 2.1.5 Amavisd replaced recip with <$newaddr>"); + $self->delete_recipient($r) if defined $addr; + + my $orcpt = $r->dsn_orcpt || orcpt_encode($r->recip_addr_smtp); + $self->add_recipient($newaddr, $orcpt, $r->dsn_notify); + } + } + } + + # Record time + section_time('process result'); +} + +# delete_recipient(recip) +# Deletes a recipient by marking them as successfully delivered in the control +# file. If the same recipient appears more than once in the control files, +# every instance will be marked as done. +# recip: Amavis::In::Message::PerRecip object +sub delete_recipient($) { + my($self, $recip) = @_; + + do_log(1, "Amavis::In::Courier: Deleting recipient <%s>: %s", + $recip->recip_addr, $recip->recip_smtp_response); + + my $filename = $recip->courier_control_file; + my $control_file = IO::File->new($filename, 'a'); + # Not sure why we do the seek when the file is already opened for append, + # but courier-pythonfilter does it so it's probably a good idea + seek($control_file, 0, 2); + # Courier may still append to the control file after calling the filter, + # so write a long blank line first to ensure that its additional records + # only overwrite the blank line + $control_file->print(" " x 254, "\n"); + + # Tell Courier the message is delivered + $control_file->printf("I%d R %s\n", $recip->courier_recip_index, + $recip->recip_smtp_response); + $control_file->printf("S%d %d\n", $recip->courier_recip_index, time); + + $control_file->close or die "Error closing control file $filename: $!"; +} + +# add_recipient(recip) +# Adds a recipient to the last control file for the message. +# address: recipient address to add +# orig_recip: RFC 3461 original recipient (if any) +# notify: reference to array containing values of the DSN NOTIFY value +sub add_recipient($$$) { + my ($self, $address, $orig_recip, $notify) = @_; + + do_log(1, "Amavis::In::Courier: Adding recipient <%s>", $address); + + # Convert $notify array into character string + my $notify_str = join('', map { substr($_, 0, 1) } @$notify); + + # Open the last control file + my $filename = $self->{control_files}->[-1]; + my $control_file = IO::File->new($filename, 'a'); + # Take care with the control file: see comments in delete_recipient + seek($control_file, 0, 2); + $control_file->print(" " x 254, "\n"); + + # Add recipient to control file + $control_file->print("r$address\n"); + $control_file->print("R$orig_recip\n"); + $control_file->print("N$notify_str\n"); + + $control_file->close or die "Error closing control file $filename: $!"; +} 1; --- amavisd.conf-sample.ori 2011-05-13 19:22:25.123703387 +0200 +++ amavisd.conf-sample 2011-05-13 19:22:38.762672629 +0200 @@ -148,4 +148,11 @@ #$notify_method = $forward_method; +# COURIER using courierfilter +#$forward_method = undef; # no explicit forwarding, Courier does it itself +#$notify_method = 'pipe:flags=q argv=perl -e $pid=fork();if($pid==-1){exit(75)}elsif($pid==0){exec(@ARGV)}else{exit(0)} /usr/sbin/sendmail -f ${sender} -- ${recipient}'; +# Only set $courierfilter_shutdown to 1 if you are using courierfilter to +# control the startup and shutdown of amavis +#$courierfilter_shutdown = 1; # (default 0) + # prefer to collect mail for forwarding as BSMTP files? #$forward_method = "bsmtp:$MYHOME/out-%i-%n.bsmtp"; @@ -221,5 +228,5 @@ # (default is true) -# AMAVIS-CLIENT PROTOCOL INPUT SETTINGS (e.g. with amavisd-release, or +# AMAVIS-CLIENT AND COURIER PROTOCOL INPUT SETTINGS (e.g. amavisd-release, or # sendmail milter through helper clients like amavis-milter.c and amavis.c) # option(s) -p overrides $inet_socket_port and $unix_socketname @@ -227,4 +234,5 @@ #$unix_socketname = undef; # disable listening on a unix socket # (default is undef, i.e. disabled) +#$unix_socketname = "/var/lib/courier/allfilters/amavisd"; # Courier socket # (usual setting is $MYHOME/amavisd.sock) @@ -2508,4 +2516,7 @@ # }; +# Needed for Courier: speak courier protocol on the socket +#$interface_policy{'SOCK'} = 'AM-SOCK'; +#$policy_bank{'AM-SOCK'} = {protocol => 'COURIER'}; ## Now we can assign policy banks to amavisd tcp port numbers listed in