#!/usr/bin/perl
#
# A script to rip thru mail messages to identify the header and grab
# pertinant info
# Then spit out only the usefull part of the header in a nice order Sub, From, Date  :)
#
# Notes:
# 1) Strip out ',[,] from subject titles - doesn't grep!
#
# WORKS Well!!!!
#
# Mime headers are "incomplete"
# Content-Type: text/plain;
#         charset="iso-8859-1"
# Content-Transfer-Encoding: quoted-printable
#
# NOTE: Quoted articles means that there is more "^Subject" lines than articles!
#
# [root@localhost Search]# cat hcos01/* | perl find-header.pl 
# Good: 2145, Bad: 188, Short: 418
# [root@localhost Search]# grep "^Subject" hcos01/* | wc
#    2189   15315  135614
#

# Fixes:
#  April 27, 2001 - increased the number of "___" in a row for a line to be deleted
#

$VERBOSE = 0;

# if invert is set; show the lines I'm throwing out and the ones I'd print
$INVERT = 0;

$Line_Number = 0;
$num_headers_found = 0;
$num_bad_headers_found = 0;
$num_short_headers_found = 0;
$header_lines = 0;
$have_date = 0;
$have_from = 0;
$have_subject = 0;
$have_extended = 0;		# do we have an extended line?
$have_quoted_header = 0;

$lines_in_email = 0;		# how many lines in a piece of email?
$email_index = 0;		# how many emails printed?  [unique number for the file]

$last_Clean_From = '';
$Clean_From = '';

while (<>){
$Line_Number++;

# Strip out the garbage!

if (/Yahoo! Photos - Share your holiday/){
                print $_ if $INVERT;
                next; }
if (/photos\.yahoo\.com/){
                print $_ if $INVERT;
                next; }
if (/To leave this mailing list, send mail/){
                print $_ if $INVERT;
                next; }
if (/the message UNSUBSCRIBE H-COSTUME/){
                print $_ if $INVERT;
                next; }
if (/_________________________________/){
                print $_ if $INVERT;
                next; }
if (/Get Your Private, Free E-mail/){
                print $_ if $INVERT;
                next; }
if (/Do You Yahoo/){
                print $_ if $INVERT;
                next; }
if (/MSN Messenger/i){
                print $_ if $INVERT;
                next; }
if (/largest e-mail service with MSN Hotmail/i){
                print $_ if $INVERT;
                next; }
# > http://www.hotmail.com
if (/MSN Photos is the easiest /i){
                print $_ if $INVERT;
                next; }
if (/photos.msn.com.support.worldwide/){
                print $_ if $INVERT;
                next; }

# Strip indented h-cosutme tailer
#> h-costume mailing list
#> h-costume@mail.indra.com
#> http://mail.indra.com/mailman/listinfo/h-costume
if (/> *h\-costume mailing list/){
                print $_ if $INVERT;
                next; }

if (/> *h\-costume\@mail.indra.com/){
                print $_ if $INVERT;
                next; }
if (/> *http:..mail.indra.com\/mailman\/listinfo\/h\-costume/){
                print $_ if $INVERT;
                next; }

if (/free/i) {
	if (/ MSN /){
		print $_ if $INVERT;
		next; }
	if (/ yahoo /i){
		print $_ if $INVERT;
		next; }
	}

print $_ if $VERBOSE > 2;

# First line of email header:
# From h-costume-admin... (2002)
# From ???@??? Wed  (1997)
#

	if ($have_extended == 1){
# if we have a tab or spaces at start then junk the line
		next if (/^[	 ]+/);
#		next if (/^[	 ]+[A-Za-z]+/);   This was too restrictive
		}

# Handle case of embedded quotes that are prefexed
# Note we must treat this as text to be added into message body though!
# Jump out of quoted headers if we have a non-header line

if ($have_quoted_header == 1){
	print "Ignore Quoted Header $_\n" if $VERBOSE > 2;
	$have_quoted_header = 0 if (!/^[a-zA-Z][A-Za-z\-]+: /);
	}

if (/---Original Message---/) {
	$have_quoted_header = 1;
	print "Found Quoted Header\n" if $VERBOSE > 2;
	}

# Possibly the first line of the header
# This IS the first line of an email - ALWAYS!
	if (/^From / && $have_quoted_header == 0){
		$have_extended = 0;
		$header_lines++;
		next; }
	if (/^Date: / && $have_quoted_header == 0){
		$Date = $_;
		$have_date = 1;
		$have_extended = 0;
		$header_lines++;
		next; }
	if (/^From: / && $have_quoted_header == 0){
		$From = $_;
		$have_from = 1;
		$have_extended = 0;
		$header_lines++;

        	tr/A-Z/a-z/;            # downcase the text
        	tr/'/ /;                # replace quotes with a space
        	s/"//g;                # strip quotes
        	s/\n//;                # strip \n \r
        	s/\r//;                # strip \n \r
        	tr/[/(/;                # replace square braces
        	tr/]/)/;                # replace square braces
        	s/from: *//gi;          # replace "subject: "
        	s/  */ /g;              # strip multiple spaces anywhere
        	s/  *$//g;              # strip spaces at end of line
        	s/^  *//g;              # strip spaces at start of line
		$last_Clean_From = $Clean_From;
		$last_Clean_From = $_ if $Clean_From eq '';
		$Clean_From = $_;
		next; }
	if (/^Subject: / && $have_quoted_header == 0){
		$Subject = $_;
		tr/A-Z/a-z/;		# downcase the text
		tr/'/ /;		# replace quotes with a space
		tr/[/(/;		# replace square braces
		tr/]/)/;		# replace square braces
		s/subject: */CSuX:/gi;		# replace "subject: "
		s/re: *//gi;		# strip "re: "
		s/fw: *//gi;		# strip "fw: "
		s/fwd: *//gi;		# strip "fwd: "
		s/h\-cost: *//gi;		# strip "h-cost: "
# Subject: Re: [h-cost] Re: Cheap linen at JoAnn's
		s/\(h\-cost\)//gi;		# strip "[h-cost]"
		s/  */ /g;		# strip multiple spaces anywhere
		s/  *$//g;		# strip spaces at end of line
		s/^  *//g;		# strip spaces at start of line
		$Clean_Subject = $_;
		$have_subject = 1;
		$have_extended = 0;
		$header_lines++;
		next; }

# Now identify continued Recieved: (and other unimportant) lines etc
# Received: from server.indra.com (server.indra.com [204.144.142.2])
#         by sca.uwaterloo.ca (8.9.3/8.9.3) with ESMTP id IAA05286
#         for <fashion@sca.uwaterloo.ca>; Wed, 1 Sep 1999 08:29:37 -0400
# Note can get lower case starting headers ie bcc, cc
	if (/^[a-zA-Z][A-Za-z\-]+: / && $have_quoted_header == 0){
		$header_lines++;
		$have_extended = 1;
		next;
		}
# Here I either have a header ending or text body (between headers)
$have_extended = 0;
if ($header_lines > 0){
	if ($header_lines < 3){
		print "SHORT header Line: $Line_Number ($header_lines)\n" if $VERBOSE;
		$num_short_headers_found++; }
	else{
	if ($have_date + $have_from + $have_subject < 3){
		print "INCOMPLETE header ($header_lines), Line: $Line_Number At: ($_)\n" if $VERBOSE;
		print stderr "INCOMPLETE header ($header_lines), Line: $Line_Number At: ($_)\n";		# Eric ADDED ERIC
		$num_bad_headers_found++; }
	else {
		if ($lines_in_email > 1){
#			print "$last_Clean_From[$lines_in_email,$email_index]$Clean_Subject_last$Subject_last$From_last$Date_last\n" if ($INVERT == 0);
			print "$last_Clean_From\[$lines_in_email,$email_index\]$Clean_Subject_last$Subject_last$From_last$Date_last" if ($INVERT == 0);
			for ($i=0; $i<$lines_in_email; $i++){
#			do{		# print out any email body we've found so far
				print $Buffer[$i] if ($INVERT == 0);
				}
#				$lines_in_email--;
#				} while ($lines_in_email >= 0);
			$lines_in_email = 0;
			$email_index++;
			}
		}
#		print "$last_Clean_Subject$Subject$From$Date\n" if ($INVERT == 0);
		$Clean_Subject_last = $Clean_Subject;
		$Subject_last = $Subject;
		$From_last = $From;
		$Date_last = $Date;
		print "HEADER !! $Subject$From$Date\n" if $VERBOSE > 1;
		$num_headers_found++;
		}
	$header_lines = 0;
	$have_date = 0;
	$have_from = 0;
	$have_subject = 0;
	$have_extended = 0;
#	next;
	}
else{		# we have a normal line - print it
#	print $_ if ($INVERT == 0);
	$Buffer[$lines_in_email] = $_;
	$lines_in_email++;
	}

}

# print the last article if there was one [should be!]
#
if ($lines_in_email > 1){
# Eric	print "poster[$lines_in_email,$email_index]$Clean_Subject$Subject$From$Date\n" if ($INVERT == 0);
	print "$Clean_From\[$lines_in_email,$email_index\]$Clean_Subject$Subject$From$Date" if ($INVERT == 0);
	for ($i=0; $i<$lines_in_email; $i++){
		print $Buffer[$i] if ($INVERT == 0);
		}
	}

print "Good: $num_headers_found, Bad: $num_bad_headers_found, Short: $num_short_headers_found\n" if $VERBOSE;
1;
