From: doi@jrd.dec.com (Hitoshi Doi)
Subject: Re: Wanted: script for mailstat
Organization: DEC Japan Research and Development Center
Date: Tue, 12 Jan 1993 01:08:17 GMT

In article <vhc.726780735@abacus.hgs.se>, vhc@abacus.hgs.se (Mikael Larsson) writes:
# I am looking for a good perl-script that makes a nice-looking toplist of 
# the users who gets and sends mots mails on this machine.

This is what I use.  It isn't be 100% accurate..
Running mstat on my machine says:

usagi.jrd.dec.com> mstat -all
mail summary for usagi.jrd.dec.com

from              count      bytes
---------------  ------  ---------
JRD                  23      68473
DEC                   5      18784
non-DEC              22      62870
---------------  ------  ---------
total                50     150127

from                   count      bytes
--------------------  ------  ---------
            \@.*.edu       5      20202
            \@.*.org       3       9997
            \@.*.com       1        781
             \@.*.jp      13      31890
--------------------  ------  ---------

                   sent              received
local user         mails      bytes  mails      bytes
-----------------  -----  ---------  -----  ---------
              doi      0          0     49     148520
-----------------  -----  ---------  -----  ---------
                       0          0     49     148520

Here's the program.

#! /usr/local/bin/perl
#
#	$Id: mstat,v 1.24 92/12/03 11:15:31 doi Exp $
#
#	mstat - get local mail statistics
#
#	this program reads the system mail log to calculate some
#	mail statistics.
#	- it doesn't work 100% percent correct for (cont) lines.
#	- there may be some DEC specific stuff in here.
#	- you should edit the first few lines to match your system.
#	- you should edit the list of domains to match your requirements.
#
#	original hack by Hitoshi Doi (doi@jrd.dec.com), 1991/03/03
#	this program is public domain.  do whatever you wish with it.
#	this program is offered "as is".  there is no warrantee.
#
#	---------------------------------------------------------------
#	modifications
#	1991/04/01	Hitoshi Doi
#		added options to print out specific stats
#		-totalonly, -useronly, -hostname
#
#	1991/05/17	Hitoshi Doi
#		some minor bugs fixes to make is more correct
#
#	1991/08/27	Hitoshi Doi
#		more minor fixes to make output correct
#
#	1991/10/09	Hitoshi Doi
#		added option to calculate stats for certain domains
#
#	1992/01/08	Hitoshi Doi
#		sort users
#		added -site option
#
#	1992/10/22	Hitoshi Doi
#		SunOS support
#
#	1992/12/03	Hitoshi Doi
#		modified calculation of stats for listed domains
#	---------------------------------------------------------------

	#
	# local system specific stuff
	#
	$company_name = 'DEC';			# company name (for summary)
	$company_addr = '\@.*.dec.com';		# company address (for match)
	$localsite = 'JRD';			# local sub-domain

	chop ($localhost = `hostname`);
	chop ($uname = `uname`);
	$me = $0;
	$me =~ s/.*\///;

	if ($uname =~ m/osf1/i) {
		$syslog = "/var/adm/syslog/mail.log";	# OSF/1
	}
	elsif ($uname =~ m/ultrix/i) {
		$syslog = "/var/spool/mqueue/syslog";	# ULTRIX
	}
	elsif ($uname =~ m/sunos/i) {
		$syslog = "/var/log/syslog";		# SunOS
	}

	$verbose = 0;
	$calc_total = 1;
	$calc_user = 0;
	$calc_from = 0;
	while ($aa = shift(@ARGV)) {
		if ($aa =~ m/-stdin/i) {
			$fdin = STDIN;
			$syslog = "";
		}
		elsif ($aa =~ m/-verbose/i) {
			$verbose = 1;
		}
		elsif ($aa =~ m/-hostname/i) {
			$aa = shift(@ARGV);
			$localhost = $aa;
		}
		elsif ($aa =~ m/-site/i) {
			$aa = shift(@ARGV);
			$localsite = $aa;
		}
		elsif ($aa =~ m/-user/i) {
			$calc_user = 1;
		}
		elsif ($aa =~ m/-from/i) {
			$calc_from = 1;
		}
		elsif ($aa =~ m/-nototal/i) {
			$calc_total = 0;
		}
		elsif ($aa =~ m/-all/i) {
			$calc_from = 1;
			$calc_user = 1;
		}
		elsif ($aa =~ m/-help/i) {
			print "
$me - get local mail statistics

usage: $me [options] [syslog]
options: -verbose            give lots of details
         -hostname host      set the hostname
         -site site          set the local site [default = $localsite]
         -stdin              take the syslog information from stdin
         -user               print out the local user information
         -from               calculate from stats per various domains
         -all                -from|-user
the default syslog is $syslog
";
			exit(0);
		}
		else {
			$syslog = $aa;
		}
	}
	if ($syslog) {
		if (!open(S, "<$syslog")) {
			print "$me: can't open $syslog\n";
			exit(1);
		}
		$fdin = S;
	}

	#
	# if we are counting the stats from various domains,
	# initialize the from stats.
	# the $site_code[$i] should be the expression for the domain.
	# if $site_noprint[$i] exists, this site will be counted,
	#  but not displayed in the report.
	#
	if ($calc_from) {
		$site_total = 0;
		$site_code[$site_total++] = '\@.*.edu';
		$site_code[$site_total++] = '\@.*.org';
		$site_noprint[$site_total] = 1;
		$site_code[$site_total++] = '\@.*.dec.com';
		$site_code[$site_total++] = '\@.*.com';
		$site_code[$site_total++] = '\@.*.jp';
		for ($i = 0; $i < $site_total; $i++) {
			$site_ct[$i] = 0;
			$site_bytes[$i] = 0;
		}
	}

	$localname = $localhost;
	$localname =~ s/\..*//;
	$localdomain = $localhost;
	$localdomain =~ s/[a-z0-9_]+\.//i;
	#
	# go through the syslog file and fill in the data
	#  @mails, $mailct, @froms, @sizes, @tos
	#
	$mailct = 0;
	while ($l = <$fdin>) {
		next if (!($l =~ m/sendmail/i));

		chop($l);
		if ($l =~ m/message-id=\</i) {
			$mails[$mailct++] = do get_id($l);
		}
		elsif ($l =~ m/from=/i) {
			if (($mi = do find_mail(do get_id($l))) < 0) {
				next;
			}
			$sizes[$mi] = do get_size($l);
			$froms[$mi] = do get_from($l);
		}
		elsif (($l =~ m/to=/i) && !($l =~ m/deffered:/i)) {
			if (($mi = do find_mail(do get_id($l))) < 0) {
				next;
			}
			$to = do get_to($l);
			if ($tos[$mi]) {
				$tos[$mi] .= "\t$to";
			}
			else {
				$tos[$mi] = "$to";
			}
		}
	}
	if ($syslog) {
		close(S);
	}
	#
	# if verbose, print info on all mail
	#
	if ($verbose) {
		for ($i = 0; $i < $mailct; $i++) {
			print "$mails[$i]: $froms[$i] ($sizes[$i]) -> ";
			print "$tos[$i]\n";
		}
	}

	#
	# find mail sent by local users
	#  @lusers, $luserct, @lumct, @lumbytes, $locct, $locbytes
	# find mail sent to local users
	#  @tumct, @tumbytes, $tocct, $tocbytes
	# get stats about mail from local sites, DEC, non-DEC
	#
	$totbytes = 0;
	$luserct = 0;
	$locct = 0;
	$locbytes = 0;
	$tocct = 0;
	$tocbytes = 0;
	$sitct = 0;
	$sitbytes = 0;
	$decct = 0;
	$decbytes = 0;
	#
	# process each mail
	#
	for ($i = 0; $i < $mailct; $i++) {
		$totbytes += int($sizes[$i]);
		#
		# from local user
		#
		if ($froms[$i] && !($froms[$i] =~ m/\@/)) {
			if (($uid = do find_luser($froms[$i])) < 0) {
				$uid = $luserct;
				$lusers[$luserct++] = $froms[$i];
			}
			$lumct[$uid]++;
			$lumbytes[$uid] += int($sizes[$i]);
			$locct++;
			$locbytes += int($sizes[$i]);
		}
		#
		# to local user
		#
		@tt = split(/\t/, $tos[$i]);
		to_user: foreach $tl (@tt) {
			$tu = $tl;
			if ($tl =~ m/\@/) {
				$th = $tl;
				$th =~ s/.*\@//;
				#
				# to match local user..
				# user@localdomain or user@localhost.domain
				#
				if ($th ne $localdomain) {
					$th =~ s/\..*//;
					next to_user if ($th ne $localname);
				}
				$tu = $tl;
				$tu =~ s/\@.*//;
			}
			#
			# local users shouldn't have period (.) or (!)
			#
			next to_user if ($tu =~ m/\./);
			next to_user if ($tu =~ m/\!/);
			$tu =~ s/^[ ]+//;
			$tu =~ s/[ ]+$//;
			next to_user if (!$tu);

			if (($uid = do find_luser($tu)) < 0) {
				$uid = $luserct;
				$lusers[$luserct++] = $tu;
			}
			$tumct[$uid]++;
			$tumbytes[$uid] += int($sizes[$i]);
			$tocct++;
			$tocbytes += int($sizes[$i]);
		}
		#
		# from local hosts or same company
		#
		if (!($froms[$i] =~ m/\@/) || ($froms[$i] =~ m/$localsite/i)) {
			$sitct++;
			$sitbytes += int($sizes[$i]);
		}
		elsif ($froms[$i] =~ m/$company_addr/i) {
			$decct++;
			$decbytes += int($sizes[$i]);
		}
		#
		# from certain domains
		#
		if ($calc_from) {
			for ($j = 0; $j < $site_total; $j++) {
				if ($froms[$i] =~ m/$site_code[$j]/i) {
					$site_ct[$j]++;
					$site_bytes[$j] += int($sizes[$i]);
					last;
				}
			}
		}
	}
	#
	# print out summaries
	#
	print "mail summary for $localhost\n";
	if ($calc_total) {
	    print "\n";
	    print "from              count      bytes\n";
	    print "---------------  ------  ---------\n";
	    printf "%-17s%6d%11d\n", $localsite, $sitct, $sitbytes;
	    printf "%-17s%6d%11d\n", $company_name, $decct, $decbytes;
	    printf "non-%-13s%6d%11d\n", $company_name,
		$mailct - ($decct + $sitct),
		$totbytes - ($decbytes + $sitbytes);
	    print "---------------  ------  ---------\n";
	    printf "total            %6d%11d\n", $mailct, $totbytes;
	}
	if ($calc_from) {
	    print "\n";
	    print "from                   count      bytes\n";
	    print "--------------------  ------  ---------\n";
	    for ($j = 0; $j < $site_total; $j++) {
		next if ($site_noprint[$j]);
		printf "%20s%8d%11d\n", $site_code[$j],
			$site_ct[$j], $site_bytes[$j];
	    }
	    print "--------------------  ------  ---------\n";
	}
	if ($calc_user) {
	    #
	    # sort user data
	    #
	    for ($i = 0; $i < $luserct; $i++) {
		$lstat[$i] = "$lusers[$i]\t$i";
	    }
	    @lsnew = sort cmp_lstat @lstat;
	    print "\n";
	    print "                   sent              received        \n";
	    print "local user         mails      bytes  mails      bytes\n";
	    print "-----------------  -----  ---------  -----  ---------\n";
	    for ($j = 0; $j < $luserct; $j++) {
		($lu, $i) = split(/\t/, $lsnew[$j]);
		printf "%17s%7d%11d%7d%11d\n", $lusers[$i],
			$lumct[$i], $lumbytes[$i],
			$tumct[$i], $tumbytes[$i];
	    }
	    print "-----------------  -----  ---------  -----  ---------\n";
	    printf "%24d%11d%7d%11d\n",
		$locct, $locbytes, $tocct, $tocbytes;
	}
	print "\n";

	exit(0);

#
#	subroutines
#

#
# find the index into the local user data arrays
#
sub find_luser
{
	local($f) = $_[0];
	local($i);

	for ($i = 0; $i < $luserct; $i++) {
		return ($i) if ($lusers[$i] eq $f);
	}
	return(-1);
}

#
# find the index into the data arrays for the specified mail id
#
# search from the back because the probability is greater that the id
# will be found closer toward the back
#
sub find_mail
{
	local($id) = $_[0];
	local($i);

	for ($i = $mailct - 1; $i >= 0; $i--) {
		return($i) if ($mails[$i] eq $id);
	}
	return(-1);
}

#
# get the mail id
#
sub get_id
{
	local($id) = $_[0];

	$id =~ s/.*sendmail: //i;
	$id =~ s/:.*//;
	return($id);
}

#
# get the size
#
sub get_size
{
	local($s) = $_[0];

	$s =~ s/.*size=//i;
	$s =~ s/,.*//;
	return($s);
}

#
# get the to addresses
#
sub get_to
{
	local($l) = $_[0];
	local(@a, $t, $tl, $tu, $th);

	$l =~ s/.*to=//i;
	$l =~ s/, delay=.*//i;
	$l =~ s/\(.*\)//g;	# "real name" in parenthesis
	$l =~ s/\<//g;
	$l =~ s/\>//g;
	$l =~ y/A-Z/a-z/;
	$tl ="";
	@a = split(/,/, $l);
	foreach $t (@a) {
		if ($t =~ m/::/) {
			$t = do decnet2ip($t);
		}
		#
		# the following are not real to addresses
		#
		elsif (($t =~ m/\|/) ||
		       ($t =~ m/\(cont\)/) ||
		       ($t =~ m/\[cont\]/) ||
		       ($t =~ m/\[\&c\]/)) {
			next;
		}
		$t = do address_canon($t);
		if ($tl) {
			$tl .= "\t$t";
		}
		else {
			$tl = "$t";
		}
	}
	return($tl);
}

#
# make the address canonical
#
sub address_canon
{
	local($a) = $_[0];
	local($tu, $th);

	if ($a =~ m/\%.*\@/) {
		$tu = $a;
		$tu =~ s/\%.*//;
		$th = $a;
		$th =~ s/.*\%(.*)\@.*/$1/;
		$a = "$tu@$th";
	}
	if ($a =~ m/\@.*\@/) {
		$tu = $a;
		$tu =~ s/\@.*//;
		$th = $a;
		$th =~ s/.*\@(.*)\@.*/$1/;
		$a = "$tu@th";
	}
	if ($a =~ m/:/) {
		$tu = $a;
		$tu =~ s/.*://;
		$th = $a;
		$th =~ s/:.*//;
		$a = "$tu@$th";
	}
	if ($a =~ m/\.dnet/) {
		$a =~ s/dnet/enet.dec.com/;
	}
	if ($a =~ m/^\\/) {
		$a =~ s/^\\//;
	}
	return($a);
}

#
# get the from address
#
sub get_from
{
	local($f) = $_[0];
	local(@x);

	$f =~ s/.*from=//;
	$f =~ s/, size=.*//;
	if ($f =~ m/\<.*\>/) {
		$f =~ s/.*\<(.*)\>/$1/;
	}
	$f =~ s/\<//;
	$f =~ s/\>//;
	$f =~ y/A-Z/a-z/;
	if ($f =~ m/::/) {
		$f = do decnet2ip($f);
	}
	#
	# get rid of the "real name" in parenthesis
	#
	if ($f =~ m/\(.*\)/) {
		@ff = split(/ /, $f);
		$f = $ff[0];
	}
	$f = do address_canon($f);
	return($f);
}

#
# recursively turn the DECnet style mail address into a normal address
#
sub decnet2ip
{
	local($f) = $_[0];
	local($u, $h);

	$u = $f;
	$u =~ s/[a-z0-9_\.]+:://;
	$h = $f;
	$h =~ s/([a-z0-9_\.]+)::.*/$1/;
	if ($u =~ m/::/) {
		$f = do decnet2ip($u);
	}
	elsif ($u =~ m/\@/) {
		$f = $u;
		$f =~ s/\"//g;
	}
	else {
		$f = "$u@$h.enet.dec.com";
	}
	return($f);
}

#
# compare the two local users
# return -1 if $a should come before $b [used by sort]
#
sub cmp_lstat
{
	local(@da, @db);

	@da = split(/\t/, $a);
	@db = split(/\t/, $b);
	return -1 if ($lumct[$da[1]] > $lumct[$db[1]]);		# sent
	return  1 if ($lumct[$da[1]] < $lumct[$db[1]]);
	return -1 if ($tumct[$da[1]] > $tumct[$db[1]]);		# received
	return  1 if ($tumct[$da[1]] < $tumct[$db[1]]);
	return -1 if ($lumbytes[$da[1]] > $lumbytes[$db[1]]);
	return  1 if ($lumbytes[$da[1]] < $lumbytes[$db[1]]);
	return -1 if ($tumbytes[$da[1]] > $tumbytes[$db[1]]);
	return  1 if ($tumbytes[$da[1]] < $tumbytes[$db[1]]);
	return -1 if ($da[0] gt $db[0]);
	return 1;
}

--
Hitoshi Doi, International Open Systems Engineering        doi@jrd.dec.com
Japan Research and Development Center               decwrl!jrd.dec.com!doi
Digital Equipment Corporation Japan      [from JUNET: doi@jrd.dec-j.co.jp]