Add 'dhcp-probe' support

dhcp-probe script is used to scan the network for unauthorized DHCP
servers.

README.md

@@ -8,6 +8,9 @@ failover configuration. Alternatively, you can configure an DHCP relay on
 a host connected to multiple networks which will relay DHCP/BOOTP messages
 to your DHCP server.
 
+`dhcp-probe` script will be used to scan the network for unauthorized DHCP
+servers and notify administrators if they are found.
+
 ### Installation
 
 This role requires at least Ansible `v1.7.0`. To install it, run:

defaults/main.yml

@@ -17,6 +17,8 @@
 #
 # - ``relay``: host is an ISC DHCP relay, see dhcrelay(8)
 #
+# - ``probe``: configure only ``dhcp-probe`` when enabled
+#
 dhcpd_mode: 'server'
 
 
@@ -32,6 +34,7 @@ dhcpd_ipversion: '4'
 dhcpd_base_packages_map:
   'server': [ 'isc-dhcp-server' ]
   'relay':  [ 'isc-dhcp-relay' ]
+  'probe':  []
 
 
 # --------------------------------
@@ -215,3 +218,52 @@ dhcpd_includes: []
 # DHCP failover configuration. See :ref:`dhcpd_failovers` for more details.
 dhcpd_failovers: []
 
+
+# -----------------------------
+#   dhcp-probe configuration
+# -----------------------------
+
+# .. envvar:: dhcpd_probe
+#
+# Enable or disable ``dhcp-probe`` script
+dhcpd_probe: True
+
+
+# .. envvar:: dhcpd_probe_mail_to
+#
+# List of mail recipients which will receive messages about unauthorized DHCP
+# servers. Set to ``[]`` to disable.
+dhcpd_probe_mail_to: [ 'root@{{ ansible_domain }}' ]
+
+
+# .. envvar:: dhcpd_probe_page_to
+#
+# Alternative list of mail recipients which will receive mail messages. Meant
+# to be used as a "pager service", you can use ``debops.smstools`` role to
+# setup a mail-SMS gateway and send the SMS messages that way.
+dhcpd_probe_page_to: []
+
+
+# .. envvar:: dhcpd_probe_mail_timeout
+#
+# Number of seconds between to wait between sending new mail messages
+dhcpd_probe_mail_timeout: '{{ (20 * 60) }}'
+
+
+# .. envvar:: dhcpd_probe_page_timeout
+#
+# Number of seconds between to wait between sending new pager messages
+dhcpd_probe_page_timeout: '{{ (20 * 60) }}'
+
+
+# .. envvar:: dhcpd_probe_legal_servers
+#
+# List of IP addresses of the host which are authorized DHCP servers.
+dhcpd_probe_legal_servers: []
+
+
+# .. envvar:: dhcpd_probe_options
+#
+# Additional ``dhcp-probe`` options specified as a YAML text block.
+dhcpd_probe_options: ''
+

docs/introduction.rst

@@ -6,6 +6,9 @@ standalone or in a 2-host failover configuration. Alternatively, you can
 configure an DHCP relay on a host connected to multiple networks which will
 relay DHCP/BOOTP messages to your DHCP server.
 
+``dhcp-probe`` script will be used to scan the network for unauthorized DHCP
+servers and notify administrators if they are found.
+
 .. _ISC DHCP Server: https://www.isc.org/downloads/dhcp/
 
 ..

handlers/main.yml

@@ -10,3 +10,14 @@
     name: 'isc-dhcp-relay'
     state: 'restarted'
 
+- name: Restart dhcp-probe
+  service:
+    name: 'dhcp-probe'
+    state: 'stopped'
+  notify: [ 'Start dhcp-probe' ]
+
+- name: Start dhcp-probe
+  service:
+    name: 'dhcp-probe'
+    state: 'started'
+

meta/ansigenome.yml

@@ -22,4 +22,6 @@ ansigenome_info:
     failover configuration. Alternatively, you can configure an DHCP relay on
     a host connected to multiple networks which will relay DHCP/BOOTP messages
     to your DHCP server.
-
+    
+    `dhcp-probe` script will be used to scan the network for unauthorized DHCP
+    servers and notify administrators if they are found.

tasks/dhcp-probe.yml

@@ -0,0 +1,28 @@
+---
+
+- name: Create dhcp-probe lib directory
+  file:
+    path: '{{ ansible_local.root.lib + "/dhcp-probe" }}'
+    state: 'directory'
+    owner: 'root'
+    group: 'root'
+    mode: '0755'
+
+- name: Manage dhcp-probe notification scripts
+  template:
+    src: 'usr/local/lib/dhcp-probe/{{ item }}.j2'
+    dest: '{{ ansible_local.root.lib + "/dhcp-probe/" + item }}'
+    owner: 'root'
+    group: 'root'
+    mode: '0755'
+  with_items: [ 'dhcp_probe_notify2', 'mail-throttled' ]
+
+- name: Configure dhcp-probe
+  template:
+    src: 'etc/dhcp_probe.cf.j2'
+    dest: '/etc/dhcp_probe.cf'
+    owner: 'root'
+    group: 'root'
+    mode: '0644'
+  notify: [ 'Restart dhcp-probe' ]
+

tasks/main.yml

@@ -18,7 +18,9 @@
     name: '{{ item }}'
     state: 'present'
     install_recommends: False
-  with_items: dhcpd_base_packages_map[dhcpd_mode]
+  with_flattened:
+    - dhcpd_base_packages_map[dhcpd_mode]
+    - [ '{{ "dhcp-probe" if (dhcpd_probe|d() and dhcpd_probe) else [] }}' ]
 
 - name: Reconfigure ISC DHCP relay
   command: dpkg-reconfigure --frontend=noninteractive isc-dhcp-relay
@@ -51,3 +53,6 @@
   command: touch /var/lib/dhcp/dhcpd6.leases creates=/var/lib/dhcp/dhcpd6.leases
   when: dhcpd_ipversion == '6'
 
+- include: dhcp-probe.yml
+  when: dhcpd_probe|d() and dhcpd_probe
+

templates/etc/dhcp_probe.cf.j2

@@ -0,0 +1,17 @@
+# {{ ansible_managed }}
+
+# Send mail messages about unauthorized DHCP servers
+alert_program_name2 {{ ansible_local.root.lib + "/dhcp-probe/dhcp_probe_notify2" }}
+
+{% if dhcpd_probe_legal_servers|d() and dhcpd_probe_legal_servers %}
+# Legal DHCP servers
+{% for address in dhcpd_probe_legal_servers %}
+legal_server {{ address }}
+{% endfor %}
+
+{% endif %}
+{% if dhcpd_probe_options|d() and dhcpd_probe_options %}
+# Other options
+{{ dhcpd_probe_options }}
+
+{% endif %}

templates/usr/local/lib/dhcp-probe/dhcp_probe_notify2.j2

@@ -0,0 +1,247 @@
+#!/usr/bin/perl -w
+
+# {{ ansible_managed }}
+
+# dhcp_probe_notify2 -p calling_prog_name -I interface_name -i IPaddress -m MACaddress [-y yiaddr]
+#
+# An external program called by dhcp_probe upon response of a response from an unexpected BootP/DHCP server.
+#
+# Called via specification of 'alert_program_name2' in /etc/dhcp_probe.cf file.
+# This version obeys the syntax provided by the 'alert_program_name2' statement,
+# not the syntax provided by the older 'alert_program_name' statement.
+#
+# Required options:
+#   -p calling_prog_name     the name of the calling program (e.g. 'dhcp_probe')
+#   -I interface_name        the name of the interface on which the unexpected response packet arrived (e.g. 'qfe0')
+#   -i IPaddress             the IP source address of the unexpected response packet (e.g. '192.168.0.1')
+#   -m MACaddress            the Ethernet source address of the unexpected response packet (e.g. '0:1:2:3:4:5')
+#
+# Optional options:
+#   -y yiaddr                the response packet's non-zero yiaddr value, when it falls within a "Lease Network of Interest" (e.g. '172.16.1.2')
+#
+# May send email subject to throttling.
+# May send page subject to throttling.
+#
+# You will need to edit the definitions below.
+#
+# Irwin Tillman
+
+
+use Sys::Hostname;
+use Sys::Syslog qw(:DEFAULT setlogsock);
+use Time::HiRes qw(gettimeofday); # from CPAN
+use Getopt::Std;
+use strict;
+
+#############################################################################
+#
+# Definitions you may need to edit
+
+my $SYSLOG_FACILITY="daemon";                   # name of facility to use if syslogging (e.g. 'daemon')
+my $SYSLOG_OPT = 'pid,cons';                    # comma-separated syslog options to use if syslogging (e.g. 'pid,cons') , ignored if not syslogging
+
+use vars qw($VERBOSE);
+$VERBOSE = 1;	# set to true to produce more verbose messages
+
+# We may send one piece of mail this way, subject to frequency throttling.
+# Use this set of definitions for a piece of email that is delivered as regular email (not a page).
+use vars qw($THROTTLE_MAIL_CMD $THROTTLE_MAIL_TIMEOUT $THROTTLE_MAIL_FROM $THROTTLE_MAIL_RECIPIENT $THROTTLE_MAIL_RECIPIENT_TEST $THROTTLE_MAIL_SUBJECT);
+{% if dhcpd_probe_mail_to|d() and dhcpd_probe_mail_to %}
+$THROTTLE_MAIL_CMD = "{{ ansible_local.root.lib + '/dhcp-probe/mail-throttled' }}"; # set to "" to disable
+{% else %}
+$THROTTLE_MAIL_CMD = ""; # set to "" to disable
+{% endif %}
+$THROTTLE_MAIL_TIMEOUT = {{ dhcpd_probe_mail_timeout | default('600') }}; # seconds
+$THROTTLE_MAIL_FROM = "root"; # e.g. "root"
+$THROTTLE_MAIL_RECIPIENT = "{{ dhcpd_probe_mail_to | join(' ') | replace('@','\@') }}"; # space-separated email addresses, remember to escape '@' characters
+$THROTTLE_MAIL_SUBJECT = "Unexpected BOOTP/DHCP server";
+
+
+# We may also send another piece of email this way, subject to frequency throttling.
+# Use this set of definitions for a piece of email that is delivered to a pager (not as regular email).
+use vars qw($THROTTLE_PAGE_CMD $THROTTLE_PAGE_TIMEOUT $THROTTLE_PAGE_RECIPIENT);
+{% if dhcpd_probe_page_to|d() and dhcpd_probe_page_to %}
+$THROTTLE_PAGE_CMD = "{{ ansible_local.root.lib + '/dhcp-probe/mail-throttled' }}"; # set to "" to disable
+{% else %}
+$THROTTLE_PAGE_CMD = ""; # set to "" to disable
+{% endif %}
+$THROTTLE_PAGE_TIMEOUT = {{ dhcpd_probe_page_timeout | default('600') }}; # seconds
+$THROTTLE_PAGE_RECIPIENT = "{{ dhcpd_probe_page_to | join(' ') | replace('@','\@') }}"; # space-separated email addresses, remember to escape '@' characters
+
+
+# End of definitions you may need to edit
+#
+#############################################################################
+
+(my $prog = $0) =~ s/.*\///;
+
+# init our use of syslog
+# setlogsock('unix'); # talk to syslog with UNIX domain socket, not INET domain. XXX causes failure in Solaris 7
+openlog($prog, $SYSLOG_OPT, $SYSLOG_FACILITY);
+
+#############################################################################
+#
+# Parse options and arguments
+
+# We must use getopt() instead of getopts() to avoid throwing an error
+# if we are passed an unrecognized option.
+# We must silently ignore unrecognized options to be forward compatible with enhancements to dhcp_probe.
+use vars qw($opt_i $opt_I $opt_m $opt_p $opt_y);
+# &getopt('p:i:I:m:y:');
+&getopt('piImy');
+
+# Required options
+my $calling_program = $opt_p;
+my $ifname = $opt_I;
+my $ip_src = $opt_i;
+my $ether_src = $opt_m;
+#
+# Optional options
+my $yiaddr = $opt_y || "";
+
+
+# Enforce presence of required options
+unless ($calling_program) {
+	my_message('LOG_ERR', "${prog}: missing -p calling_program option");
+	exit 100;
+}
+unless ($ifname)  {
+	my_message('LOG_ERR', "${prog}: missing -I interface_name option");
+	exit 101;
+}
+unless ($ip_src)  {
+	my_message('LOG_ERR', "${prog}: missing -i ip_src_address option");
+	exit 102;
+}
+unless ($ether_src)  {
+	my_message('LOG_ERR', "${prog}: missing -m ether_src_address option");
+	exit 103;
+}
+
+# Done parsing options and arguments
+#
+#############################################################################
+
+# Miscellaneous Initialization
+
+my $hostname = hostname();
+
+my ($seconds, $microseconds) = gettimeofday;    # from Time::HiRes on CPAN
+my $timestamp = scalar(localtime($seconds));
+$timestamp =~ s/(\d\d:\d\d:\d\d)/$1.$microseconds/; # glue microsends to end of seconds
+
+
+
+# When we pass a key to THROTTLE_MAIL_CMD, we want the key to also include an indication
+# of whether the yiaddr option was specified.  (If we didn't include such an indication,
+# the following could happen: The first response we detect from a rogue server doesn't
+# distribute an "address of concern", so the yiaddr option wasn't specified.  We alert
+# based on that.  The next response we detect from a rogue server does distribute an
+# "address of concern", so the yiaddr is specified.  Our alert gets throttled since
+# the first alert we sent was "recent".  But this means the administrator doesn't get
+# notified that the rogue distributed an "address of interest".  So we need to
+# include the state of yiaddr option in the key we pass to THROTTLE_MAIL_CMD,
+# to ensure the second response isn't throttled using the same key as the first response.)
+# 
+# We can't just use the value of yiaddr itself as the string to incorporate into the key,
+# as that would result in a unqiue key for each distributed IP address.  Instead, we
+# will use the state of yiaddr (set or unset).
+#
+# Note this means that when a rogue DHCP server is distributing IP addresses that fall into "Networks of Concern",
+# we will very likely send more than one notification for it within each throttle period.
+# That's because while some of the responses from the rogue will have a yiaddr within a "Networks of Concern",
+# others (for example, DHCPNAK responses) will not.  This is unfortunate, but is better than the
+# alternative approach (of not taking into account the yiaddr state in the key), since that will
+# sometimes cause you to not be alerted at all to the "yiaddr falls into a Network of Concern" situation.
+#
+# Create a string based on the state of yiaddr, for later incorporation into the key.
+my $yiaddr_option_state = $yiaddr ? "yiaddr=set" : "yiaddr=unset";
+
+#############################################################################
+
+if ($THROTTLE_MAIL_CMD) {
+	# This command suppresses the message if it's sent a message to 'key' within 'throttle_seconds'
+	# I use the calling program's name and the offender's hardware address as the key.
+
+	my $subject_yiaddr_addendum = "";
+	$subject_yiaddr_addendum = ", YIADDR=$yiaddr" if $yiaddr;
+
+	unless (open(THROTTLE_MAIL, "| $THROTTLE_MAIL_CMD -l -k ${calling_program}_mail_${ether_src}_$yiaddr_option_state -t $THROTTLE_MAIL_TIMEOUT -f \"$THROTTLE_MAIL_FROM\" -r \"$THROTTLE_MAIL_RECIPIENT\" -s\"$THROTTLE_MAIL_SUBJECT (MAC=${ether_src}, IP=${ip_src}${subject_yiaddr_addendum})\"")) {
+
+		my_message('LOG_ERR', "${prog}: failure trying to send throttled email: can't execute '${THROTTLE_MAIL_CMD}': open(): $!");
+		exit 20;
+	}
+
+	print THROTTLE_MAIL
+			$timestamp, "\n",
+			"\n",
+			"$calling_program detected an unexpected BOOTP/DHCP server.\n",
+			"Host=$hostname, interface=${ifname}, IP source=${ip_src}, Ethernet source=${ether_src}\n";
+	print THROTTLE_MAIL "\nThis means that *there is* a rogue BOOTP/DHCP server operating.\n" if $VERBOSE;
+	if ($yiaddr) {
+		print THROTTLE_MAIL
+			"The server distributed IP address $yiaddr, which falls into a network of special concern.\n";
+	}
+
+	unless (close(THROTTLE_MAIL)) {
+		my_message('LOG_ERR',
+					"${prog}: failure trying to send throttled email: error executing '${THROTTLE_MAIL_CMD}': close(): " .
+						($! ?
+							"syserr closing pipe: $!"
+							:
+							"wait status $? from pipe"
+						) .
+					"\n"
+				);
+		exit 21;
+
+	}
+}
+
+
+
+
+if ($THROTTLE_PAGE_CMD) {
+	# This command suppresses the message if it's sent a message to 'key' within 'throttle_seconds'
+	# I use the calling program's name and the offender's hardware address as the key.
+
+	unless (open(THROTTLE_PAGE, "| $THROTTLE_PAGE_CMD -l -k ${calling_program}_page_${ether_src}_$yiaddr_option_state -t $THROTTLE_PAGE_TIMEOUT -r \"$THROTTLE_PAGE_RECIPIENT\"")) {
+		my_message('LOG_ERR', "${prog}: failure trying to send throttled page: can't execute '${THROTTLE_PAGE_CMD}': open(): $!\n");
+		exit 30;
+	}
+
+	print THROTTLE_PAGE "Rogue DHCP server IP=$ip_src MAC=$ether_src seen via host $hostname interface $ifname\n";
+	print THROTTLE_PAGE "This means *there is* a rogue BOOTP/DHCP server operating.\n" if $VERBOSE;
+	if ($yiaddr) {
+		print THROTTLE_PAGE
+			"Rogue server distributed yiaddr=$yiaddr, a special concern.\n";
+	}
+
+	unless (close(THROTTLE_PAGE)) {
+		my_message('LOG_ERR',
+					"${prog}: failure trying to send throttled page: error executing '${THROTTLE_PAGE_CMD}': close(): " .
+						($! ?
+							"syserr closing pipe: $!"
+							:
+							"wait status $? from pipe"
+						) .
+					"\n"
+				);
+		exit 31;
+	}
+
+}
+
+exit 0;
+
+#############################################################################
+
+sub my_message {
+	# Call with a syslog priority constant and a message string.
+	# We write the message to syslog, using the specified priority.
+	# Your message should not contain a newline.
+
+	my($priority, $msg) = @_;
+	syslog($priority, $msg);
+	return;
+}

templates/usr/local/lib/dhcp-probe/mail-throttled.j2

@@ -0,0 +1,173 @@
+#!/usr/bin/perl -w
+
+# {{ ansible_managed }}
+
+# $Header: /usr/local/etc/RCS/mail-throttled,v 1.11 2008/12/06 01:50:28 root Exp $
+
+# mail-throttled [-l] [-D dbm_file] -k key -t throttle_seconds [-f from] -r recipient [-d] [-T tie_attempts_max] [-S tie_retry_sleep] [-s subject]
+#
+# Sends mail body (read from STDIN) to 'recipient', but avoids doing so "too frequently."
+#
+# You provide a 'key', which is an arbitrary string used to identify this notification.
+# You also provide 'throttle_seconds', an integer.  If we've sent anything that
+# specified this 'key' within the last 'throttle_seconds', we do not send the message.
+# Otherwise, we send the message, and the remember that we've sent a message for this 'key'
+# at the current time.
+#
+# This key/timesent tuples are stored on-disk, in a dbm.  As a result, the 'key'
+# you supply must satisfy the syntactic requirements for dbm keys.  
+# The caller needs to have permission to read and write this DBM (and create it if
+# it does not already exist).  If you fail to specify a dbm_file, we'll use a default
+# value, which may not be what you want (since the caller might not be able to r/w that
+# particular DBM).
+# We never clean this dbm.  You can safely erase it entirely, if you don't mind losing
+# the state, and you know the caller has permission to create a new instance of the DBM.
+#
+# The 'recipient' should be a valid email address.  Naturally, it should not
+# be one that will cause any ack or bounce mail to return to us!
+# If there are several addresses (delimited by spaces), be sure to quote them as a single arg.
+#
+# If a subject is specified, be sure to quote it if it contains any spaces or other shell
+# metachars.
+#
+# If -l is specified, then any errors, warnings, or debugging output is written to syslog
+# in addition to its usual destination (STDERR).  This is helpful if you call this from an
+# environment where STDERR may get lost.
+#
+# Irwin Tillman
+
+use Getopt::Std;
+use GDBM_File;
+use Errno qw(EAGAIN);
+use Sys::Syslog qw(:DEFAULT setlogsock);
+
+use strict;
+use warnings;
+
+use vars qw($DBM_FILE_DEFAULT $MAILCMD $MAILCMD_OPTS $FROM_DEFAULT);
+$DBM_FILE_DEFAULT = '{{ ansible_local.root.lib + "/dhcp-probe/mail-throttled.gdbm" }}';
+$MAILCMD = "/usr/lib/sendmail";
+# $MAILCMD_OPTS = "-t -ODeliveryMode=queueonly";
+$MAILCMD_OPTS = "-t";
+$FROM_DEFAULT = "root";
+
+my $SYSLOG_FACILITY="daemon";                   # name of facility to use if syslogging
+my $SYSLOG_OPT = 'pid,cons';                    # syslog options to use if syslogging, ignored otherwise
+my $SYSLOG_PRIORITY = 'LOG_ERR';
+
+# The tie() call sometimes fails with EAGAIN.
+# Perhaps that's due to some other process having the DBM open;
+# in fact, that may be more likely if the process that calls us may calls us multiple times in quick succession.
+# So when tie() fails with EAGAIN, we can sleep and retry some number of times before giving up entirely.
+my $TIE_ATTEMPTS_MAX = 3;	#  number of times to try tie() before giving up
+my $TIE_RETRY_SLEEP = 1;	#  seconds to sleep before retrying tie()
+
+(my $prog = $0) =~ s/.*\///;
+
+use vars qw($opt_f $opt_D $opt_d $opt_k $opt_l $opt_r $opt_s $opt_S $opt_t $opt_T);
+&getopts('dD:f:k:lr:s:S:t:T:');
+
+my $debug = $opt_d || "";
+my $dbm_file = $opt_D || $DBM_FILE_DEFAULT;
+my $key = $opt_k || "";
+my $from = $opt_f || $FROM_DEFAULT;
+my $recipient = $opt_r || "";
+my $throttle_secs = $opt_t || 1;
+my $subject = $opt_s || "";
+my $also_syslog = $opt_l || "";
+my $tie_attempts_max = $opt_T || $TIE_ATTEMPTS_MAX; # we deliberately override if CLI option specifies 0, as that makes no sense
+my $tie_retry_sleep = $opt_S || $TIE_RETRY_SLEEP;
+
+if ($also_syslog) {
+	# init our use of syslog
+	# setlogsock('unix'); # talk to syslog with UNIX domain socket, not INET domain. XXX causes failure in Solaris 7
+	openlog($prog, $SYSLOG_OPT, $SYSLOG_FACILITY);
+}
+
+my_warn("${prog}:\nkey=$key\nthrottle_secs=$throttle_secs\nfrom=$from\nrecipient=$recipient\ntie_attempts_max=$tie_attempts_max\ntie_retry_sleep=$tie_retry_sleep\nsubject=$subject") if $debug;
+
+# certain options and args are required
+&Usage() unless ($key && $throttle_secs && $recipient);
+
+my %last_sent = ();
+my $tie_succeeded = 0;
+my $tie_attempts_left = $TIE_ATTEMPTS_MAX;
+
+while ($tie_attempts_left--) {
+	if (tie(%last_sent, 'GDBM_File', $dbm_file, &GDBM_WRCREAT, 0644)) {
+		$tie_succeeded = 1;
+		last;
+	}
+	# the tie() failed
+
+	if ($! == EAGAIN) {
+		# The failure may be due to a transient problem.
+		# Retrying may help.
+		sleep $TIE_RETRY_SLEEP;
+		next;
+
+	} else {
+		# Some other (presumably more serious) error.
+		last;
+	}
+}
+unless ($tie_succeeded) {
+	my_warn("${prog}: can't tie ${dbm_file}: $!");
+	exit 10;
+}
+
+my @mailbody = "";
+@mailbody = <STDIN>; # read it even if we decide not to send it
+
+my $now = time;
+
+my_warn("now = $now") if $debug;
+
+$last_sent{$key} = 0 unless defined($last_sent{$key}); # so it's defined before we use it in subtraction (placate use strict)
+
+if ($now - $last_sent{$key} >= $throttle_secs) {
+	my_warn("last_sent = $last_sent{$key}, will send") if $debug;
+	unless (open(MAIL, "| $MAILCMD $MAILCMD_OPTS -f\"$from\"")) {
+		my_warn("${prog}: error executing '${MAILCMD}': open(): $!");
+		exit 20;
+	}
+	print MAIL	"From: $from\n",
+				"To: $recipient\n",
+				($subject ? "Subject: $subject\n" : "") ,
+				"\n",
+				@mailbody;
+	unless (close(MAIL)) {
+		my_warn("${prog}: error executing '${MAILCMD}': close(): " .
+			($! ?
+				"syserror closing pipe: $!"
+				:
+				"wait status $? from pipe"
+			)
+		);
+		exit 21;
+	}
+
+	$last_sent{$key} = $now;
+} else {
+	my_warn("last_sent = $last_sent{$key}, suppressing") if $debug;
+}
+
+untie %last_sent;
+
+exit 0;
+
+
+
+sub Usage {
+	my_warn("Usage: $prog [-l] [-D dbm_file] -k key -t throttle_seconds [-f from] -r recipient [-T tie_attempts_max] [-S tie_retry_sleep] [-s subject]");
+	exit 1;
+}
+
+
+sub my_warn {
+	# Just a wrapper for warn, but with a possible copy to syslog too.
+	my $msg = shift;
+	warn $msg, "\n";
+	syslog($SYSLOG_PRIORITY, $msg) if $also_syslog;
+	return;
+}