From: lindner@mudhoney (Paul Lindner)
Subject: Re: Clarinet Information
Original-To: TDTRUE%PUCC.BITNET@mail.unet.umn.edu (Tom True)
Date: Tue, 6 Oct 92 9:27:02 CDT
Original-Cc: gopher@boombox.micro.umn.edu, OBERST%PUCC.BITNET@mail.unet.umn.edu,
        D8897%PUCC.BITNET@mail.unet.umn.edu,
        gopher-news@boombox.micro.umn.edu (Gopher News Mailing List)
Reply-To: lindner@boombox.micro.umn.edu
X-Mailer: ELM [version 2.3 PL11]
Newsgroups: comp.infosystems.gopher
Distribution: comp

Tom True sez:
. 
. Could you please send us the current information on your scheme for
. displaying and searching the clarinet news.
.  

Okay, here's the new scheme, it's only three perl scripts now.  Much,
much nicer, and more automatic.

The script perlnews basically does the scanning.  It finds the new
articles and puts them into a data directory according to the date in
the header.

It then builds the index for the daily articles (defined as yesterday
and today) and the monthly index.

Then it makes links for "ThisMonth" and "LastMonth" in the Index and
data directories.

Note that these scripts could also be used to index and archive other
portions of USENET rather easily....

-------------snip--------------snip---------------

#! /bin/sh
# This is a shell archive, meaning:
# 1. Remove everything above the #! /bin/sh line.
# 2. Save the resulting text in a file.
# 3. Execute the file with /bin/sh (not csh) to create:
#	perlnews
#	index-todays-upi
#	index-upi-month
# This archive created: Tue Oct  6 09:22:42 1992
export PATH; PATH=/bin:/usr/bin:$PATH
if test -f 'perlnews'
then
	echo shar: "will not over-write existing file 'perlnews'"
else
cat << \SHAR_EOF > 'perlnews'
#!/usr/local/bin/perl
#  A better way of automating the gopher-izing of Clarinet
#     - Mark McCahill
#
# An ever better perl way
#     - Paul Lindner
#
#  check for existance of the lock file...
#  we don't want two of the scripts running simultaneously.
#
$Base      = "/home/mrfrosty/gopherd/massage";
$LockFile  = "$Base/LOCK.paperboy";
$LogFile   = "$Base/logfile";
$IndexHome = "/home/mrfrosty/gopherd/UPI-indexes";
$DataHome  = "/home/mrfrosty/gopherd/UPI-data";
$NewsDir   = "/home/paperboy/clari";
$TimeFile  = "$Base/lastscan";
$TodayIndex= "$Base/index-todays-upi";
$MonthIndex= "$Base/index-upi-month";


#######################################

if (-e $LockFile) {
    `ls -l $LockFile | /usr/ucb/mail -s "problem updating today's UPI news" gopher@boombox.micro.umn.edu`;
    exit;
} else {
    system("touch $LockFile");
}

rename("$LogFile", "$LogFile.1");
open(Log, ">$LogFile");
$| = 1;  #Buffer output nicely...

#
# Do all the news if we can't find the timefile
#

if (! -f $TimeFile) {
    `touch $TimeFile`;
    utime 0, 0, $TimeFile;
    print Log "Initial scan, get some coffee man!\n";
}

$startscan = time - 360;  # give or take three minutes, just in case.


open(lsproc, "ls $NewsDir|");
while (<lsproc>) {
    chop;
    push(@control, $_);
}
close(lsproc);

#
#  set up the directories
#
foreach $i (@control) {
    print Log "Searching $i...\n";
    open(findproc, "find $NewsDir/$i -newer $TimeFile -type f -print|");
    while (<findproc>) {
	chop;
	&cleanfile($DataHome, $_, $i);
    }
}

#
# Now build indexes...
#
open(today, "$TodayIndex|");
while (<today>) {
	print Log;
}

($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);

@DaytoMon  = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec");

open(month, "$MonthIndex $DaytoMon[$mon] 19$year $IndexHome/$DaytoMon[$mon]19$year|");
while (<month>) {
	print Log;
}

#
# Make links for this month, today etc.
#

unlink("$DataHome/LastMonth");
unlink("$DataHome/ThisMonth");
unlink("$DataHome/Today");
unlink("$IndexHome/ThisMonth");
unlink("$IndexHome/LastMonth");
system("ln -s $DataHome/19$year/$DaytoMon[$mon] $DataHome/ThisMonth");
system("ln -s \"$DataHome/19$year/$DaytoMon[$mon]/$DaytoMon[$mon] $mday\" $DataHome/Today");
system("ln -s $IndexHome/$DaytoMon[$mon]19$year $IndexHome/ThisMonth");

if ($mon != 0) {
	$Lastmo = "19$year/$DaytoMon[$mon-1]";
	$Indexmo = "$DaytoMon[$mon-1]19$year";
} else {
	$myear = $year - 1;
	$Lastmo = "19$myear/Dec";
	$Indexmo = "Dec19$year";
}

system("ln -s $DataHome/$Lastmo $DataHome/LastMonth");
system("ln -s $IndexHome/$Lastmo $IndexHome/LastMonth");

unlink($LockFile);


#
#  Set the time of the timefile for the next scan
#
utime $startscan, $startscan, $TimeFile;


#
# Remove headers and make the file pretty...
#

sub cleanfile {
    local ($base, $file, $area) = @_;

    print Log "Cleaning: $file $area\n";

    open( INFILE, $file );
    open(tmphead, "+>/tmp/headers$$");
    $HeaderFlag = 1;
    while (<INFILE>) {
	#
	# Get rid of Headers we don't want
	#
	if ((/^Received:/)||
	    (/^Message-Id:/)||
	    (/^Cc:/)) {
	    next;
	}
	
	if (/^Subject:/) {
	    $SubjectLine = $_;
	    $SubjectLine =~ s/^Subject: //;
	}
	
	if (/^Date: /) {
	    $Date = $_;
	    $Date =~ s/^Date: //;
	    $Date =~ s/^.*, //;  #get rid of weekday
	    $Date =~ m/^([0-9]+) ([a-zA-Z]+) ([0-9]+)/;
	    
	    $Day = $1;
	    $Mon = $2;
	    $Year = $3;
	    $Year = "19$3" if (length($3) < 4);
	    &Makedirs($base, $Mon, $Day, $Year);
	    $SaveDir = "$Year/$Mon/$Mon $Day";
	    mkdir("$base/$SaveDir/$area", 0755);
	    $SaveDir = "$SaveDir/$area";
	    print Log "Saving:   $SaveDir\n";
	}

	if ((/^$/) && ($HeaderFlag == 1)) {
	    # End of headers

	    # clean up the SubjectLine varaible so that it
	    # only has nice characters, and is a nice length
	    
	    #($cleansubject = $SubjectLine) =~ tr/a-zA-Z0-9@#$%^&()\-+=`'":;?,.<> //cs;
            ($cleansubject = $SubjectLine) =~ s,[^\040-\176],,g;
            $cleansubject =~ s,/,-,g;

	    if ( length ( $cleansubject ) > 60 ) {
		$cleansubject = substr($cleansubject, 0, 60 );
	    }

	    open(Outfile, ">$base/$SaveDir/$cleansubject");
	    $HeaderFlag = 0;

	    print Log "Subject:  $cleansubject\n";

	    seek(tmphead, 0,0);
	    while (<tmphead>) {
		if ((/^Dat:/)     || (/^From:/) ||
		    (/^Subject:/)  || (/^Keywords:/)||
		    (/^Location:/) || (/^ACategory/)) {
		    
		    print Outfile;
		}
	    }
	    close(tmphead);
	    unlink "/tmp/headers$$";
	    
	    print Outfile "\n";
	}
	
	# Filter out the header lines
	
	if ($HeaderFlag == 1) {
	    print tmphead;
	}
	else {
	    print Outfile;
	}
    }
    close Outfile;
    next;
}

 

sub Makedirs {
    local($base, $mon, $day, $year) = @_;
    
    if (! -d $base) {
	die "No such directory $base";
    }

    mkdir("$base/$year",0755) if (! -d "$base/$year");
    mkdir("$base/$year/$mon",0755) if (! -d "$base/$year/$mon");
    mkdir("$base/$year/$mon/$mon $day",0755) if (! -d "$base/$year/$mon $day");


}
SHAR_EOF
chmod +x 'perlnews'
fi
if test -f 'index-todays-upi'
then
	echo shar: "will not over-write existing file 'index-todays-upi'"
else
cat << \SHAR_EOF > 'index-todays-upi'
#!/usr/local/bin/perl
#
# This script makes indexes for the days daily
#

$DataHome = "/home/mrfrosty/gopherd/UPI-data";
$TmpDir = "/home/mrfrosty/gopherd/massage/tmp-today";
$IndexHome = "/home/mrfrosty/gopherd/UPI-indexes/Today";
$TmpHome  = "/tmp/index$$";


($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
@DaytoMon  = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec");

#
# Copy in information added in the past day (when all the back issues are
# in we will probably change directorys to the 
#
$Today = "$DataHome/19$year/$DaytoMon[$mon]/$DaytoMon[$mon] $mday";
$ymday = $mday -1;
if ($ymday != 0) {
	$Yesterday = "\"$DataHome/19$year/$DaytoMon[$mon]/$DaytoMon[$mon] $ymday\"";
} else {
	$Yesterday = "";
}

chdir($DataHome) || die ("Daily directory has dissappeared!!");

$i = 0;

while (!( -d $Today)) {
	print "Couldn't find $Today\n";
	sleep(60);
	print ".\n";
	$i ++;
        # wait 30 minutes
	if ($i > 30) {
		print "No news this hour!\n";
		exit;
	}
}
system("rm -rf $DataHome/.index");
mkdir("$TmpHome",0755) if (! -d "$TmpHome");
mkdir("$TmpHome/.index",0755) if (! -d "$TmpHome/.index");
system("ln -s $TmpHome/.index $DataHome/.index");
print "Building Index... $IndexHome\n";
$Result = system("/usr/bin/ixBuild -V -f ascii -i $DataHome/.index/index.ixif \"$Today\" $Yesterday");

if ($Result != 0) {
	print "Problems indexing Today's daily... Help\n";
	print $!;
	exit;
}

system("mv \"$TmpHome/.index/index.ixif\" \"$IndexHome/.index/index.ixif\"");
system("rm -rf $TmpHome");
SHAR_EOF
chmod +x 'index-todays-upi'
fi
if test -f 'index-upi-month'
then
	echo shar: "will not over-write existing file 'index-upi-month'"
else
cat << \SHAR_EOF > 'index-upi-month'
#!/usr/local/bin/perl
#
# This script makes indexes for specific month
# 
# It leaves the index in $DataHome/.index
#

$TmpHome = "/tmp/upisection$$";
$DataHome = "/home/mrfrosty/gopherd/UPI-data";
$TmpDir = "/home/mrfrosty/gopherd/massage/tmp-today";

if ($#ARGV != 2) {
        print "Usage: $0 Month Year Indexdir\n";
        exit;
}

$Year =  @ARGV[1];
$Mon  =  @ARGV[0];
$IndexHome = @ARGV[2];
#
$Data = "$Year/$Mon";

chdir($DataHome) || die ("Daily directory has dissappeared!!");

$i = 0;

die "No such directory" if (! -d $Data);

system("rm -rf $DataHome/.index");
mkdir("$TmpHome",0755);
mkdir("$TmpHome/.index",0755);
system("ln -s $TmpHome/.index $DataHome");
system("cp $IndexHome/.index/index.ixif $TmpHome/.index/index.ixif") if
	( -f "$IndexHome/.index/index.ixif");
print "Building Index... $DataHome/$Data\n";
$Result = system("/usr/bin/ixBuild -V -f ascii -i $DataHome/.index/index.ixif \"$DataHome/$Data\"");

if ($Result != 0) {
	print "Problems indexing Today's daily... Help\n";
	print $!;
	exit;
}

mkdir("$IndexHome", 0755) if (! -d "$IndexHome");
mkdir("$IndexHome/.index",0755) if (! -d "$IndexHome/.index");

system("mv \"$TmpHome/.index/index.ixif\" \"$IndexHome/.index/index.ixif\"");
SHAR_EOF
chmod +x 'index-upi-month'
fi
exit 0
#	End of shell archive
Newsgroups: comp.infosystems.gopher