Index: /other-projects/book-stumper/trunk/DOWNLOAD.sh
===================================================================
--- /other-projects/book-stumper/trunk/DOWNLOAD.sh	(revision 33353)
+++ /other-projects/book-stumper/trunk/DOWNLOAD.sh	(revision 33353)
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+wget --mirror -np  --convert-links --random-wait  --adjust-extension --page-requisites http://www.logan.com/loganberry/solved-a.html
Index: /other-projects/book-stumper/trunk/EXTRACT-POSTS.sh
===================================================================
--- /other-projects/book-stumper/trunk/EXTRACT-POSTS.sh	(revision 33353)
+++ /other-projects/book-stumper/trunk/EXTRACT-POSTS.sh	(revision 33353)
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+./extract-posts.pl www.logan.com/loganberry/
+
Index: /other-projects/book-stumper/trunk/extract-posts.pl
===================================================================
--- /other-projects/book-stumper/trunk/extract-posts.pl	(revision 33353)
+++ /other-projects/book-stumper/trunk/extract-posts.pl	(revision 33353)
@@ -0,0 +1,745 @@
+#!/usr/bin/perl -w
+
+use v5.10;
+use strict;
+
+use Mojo::DOM;
+
+my $FOUT = undef;
+my $HOUT = undef;
+my $REFNUM = 1;
+
+sub tidy_header_field
+{
+    my ($prefix,$untidy_text) = @_;
+    my $tidy_text = $untidy_text;
+    $tidy_text =~ s/^$prefix:\s*//;
+    
+    $tidy_text =~ s/<\s*>//gs;
+
+    return $tidy_text;
+}
+
+sub store_header_field
+{
+    my ($untidy_text,$header_record) = @_;
+
+    my ($header_label,$header_value) = ($untidy_text =~ m/^(.*?)\s*:\s*(.*)$/s);
+
+    $header_value =~ s/<\s*>//gs;
+
+    $header_record->{$header_label} = $header_value;
+}
+
+sub print_csv_escaped
+{
+    my ($text) = @_;
+
+    my $escaped_text = $text;
+
+    $escaped_text =~ s/\"/\"\"/g;
+    $escaped_text =~ s/\n/ /gs;
+    
+    print $FOUT ",\"$escaped_text\"";
+}
+
+
+# plain text
+#   <Row ss:AutoFitHeight="0">
+#    <Cell ss:Index="2" ss:StyleID="s62"><Data ss:Type="String">On Sat, 1 Jan 2000, Richard Griscom wrote: &gt; &gt; So. It still works.</Data></Cell>
+#   </Row>
+
+
+# Hyperlinked cell
+#  <Row>
+#   <Cell ss:StyleID="s63" ss:HRef="foo.html">
+#      <Data ss:Type="String">Test Boldfoo.html</Data>
+#    </Cell>
+#  </Row>
+
+sub print_spreadsheet_text_cell_xml
+{
+    my ($text) = @_;
+    
+    $text =~ s/&/&amp;/g;
+    
+    $text =~ s/</&lt;/g;
+    $text =~ s/>/&gt;/g;
+    $text =~ s/"/&quot;/g;
+
+    print $FOUT "    <Cellss:StyleID=\"sPlainText\"><ss:Data ss:Type=\"String\">$text</ss:Data></Cell>\n";
+}
+
+# html
+#   <Row ss:Index="7" ss:AutoFitHeight="0" ss:Height="13.5">
+#    <Cell ss:Index="2" ss:StyleID="s66"><ss:Data ss:Type="String"
+#      xmlns="http://www.w3.org/TR/REC-html40"><I><Font html:Color="#000000">YYYY Happy </Font></I><B><I><Font
+#         html:Color="#000000">New</Font></I></B><I><Font html:Color="#000000"> Year Dick&#10; et al !!</Font></I></ss:Data></Cell>
+#   </Row>
+
+sub print_spreadsheet_html_cell_xml_orig
+{
+    my ($html) = @_;
+
+#    $html =~ s/<br\s*\/?>/&#10;/ig;
+    $html =~ s/<br\s*>/&#10;/ig;
+    $html =~ s/<\/br>//ig;
+
+    $html =~ s/<p[^>]*>(.*?)<\/p>/$1&#10;/ig;
+    $html =~ s/<div[^>]*>(.*?)<\/div>/$1&#10;/ig;
+    
+    $html =~ s/<[^>]*>//g;
+#    $html =~ s/<\/?(font|big|small)[^>]*>//ig;
+
+    
+    
+    print $FOUT "    <Cell ss:StyleID=\"sHtmlWrap\"><ss:Data ss:Type=\"String\" xmlns=\"http://www.w3.org/TR/REC-html40\">$html</ss:Data></Cell>\n";
+}
+
+sub ucattrs
+{
+    my ($inner) = @_;
+
+    ##$inner =~ s/(\w+)=("[^"]+")/"".uc($1)."=".$2/ge;
+    $inner =~ s/(\w+)=("[^"]+")/\u$1=$2/g;
+
+    return $inner;
+}
+
+sub ucattrsNS
+{
+    my ($tag,$inner) = @_;
+
+    if ($tag =~ m/^Font$/i) {
+	$inner =~ s/(\w+)=("[^"]+")/html:\u$1=$2/g;
+    }
+    else {
+	$inner =~ s/(\w+)=("[^"]+")/\u$1=$2/g;
+    }
+    
+    return $inner;
+}
+
+
+sub print_spreadsheet_html_cell_xml_b_i_font
+{
+    my ($html) = @_;
+
+    $html =~ s/<br([^>]*)>/<br$1\/>/ig;
+    $html =~ s/<hr([^>]*)>/<hr$1\/>/ig;
+    $html =~ s/<img([^>]*)>/<img$1\/>/ig;
+    
+    $html =~ s/<br\s*\/?>/&#10;/ig;
+#    $html =~ s/<\/br>//ig;
+
+    $html =~ s/<p[^>]*>(.*?)<\/p>/$1&#10;/isg;
+#    $html =~ s/<p[^>]*>/&#10;/ig;
+    $html =~ s/<div[^>]*>(.*?)<\/div>/$1&#10;/isg;
+    
+#    $html =~ s/<[^>]*>//g;
+
+    $html =~ s/<table/<div/ig;
+    $html =~ s/<\/table/<\/div/ig;
+
+    $html =~ s/<tr/<div/ig;
+    $html =~ s/<\/tr/<\/div/ig;
+
+    $html =~ s/<td/<div/ig;
+    $html =~ s/<\/td/<\/div/ig;
+
+    # Make HTML all uppercase, as this seems to be needed for spreadsheet format
+
+#    $html =~ s/<(\w+)([^>]*)>/"<".uc($1).$2.">"/ge;
+#    $html =~ s/<\/(\w+)([^>]*)>/"<\/".uc($1).$2.">"/ge;
+
+    $html =~ s/<(\w+)([^>]*)>/"<".ucfirst($1).$2.">"/ge;
+    $html =~ s/<\/(\w+)([^>]*)>/"<\/".ucfirst($1).$2.">"/ge;
+    
+    ##$html =~ s/<([^>]*)(\w+="[^"]+")([^>]*)>/"<".$1.uc($2).$3.">"/ge;
+    $html =~ s/<(\w+)([^>]*)>/"<".$1.ucattrs($2).">"/ge;
+
+    $html =~ s/<Font(.*?)Size=\"[^"]*\"/<Font$1/gs;
+    $html =~ s/<Font(.*?)Face=\"[^"]*\"/<Font$1/gs;
+    $html =~ s/<Big[^>]*>(.*?)<\/Big>/$1/sg;
+    $html =~ s/<Span[^>]*>(.*?)<\/Span>/$1/sg;
+
+    $html =~ s/<[^>]*>//g;
+    
+    $html =~ s/^(&#10;)+//s;
+    
+    
+    print $FOUT "    <Cell ss:StyleID=\"sHtmlWrap\"><ss:Data ss:Type=\"String\" xmlns=\"http://www.w3.org/TR/REC-html40\">$html</ss:Data></Cell>\n";
+}
+
+    
+sub print_spreadsheet_html_cell_xml
+{
+    my ($html) = @_;
+    
+    $html =~ s/<br([^>]*)>/<br$1\/>/ig;
+    $html =~ s/<hr([^>]*)>/<hr$1\/>/ig;
+    $html =~ s/<img([^>]*)>/<img$1\/>/ig;
+
+    $html =~ s/<br[^\/]*\/>/&#10;/ig;
+    
+#    $html =~ s/<\/br>//ig;
+
+    $html =~ s/<p[^>]*>(.*?)<\/p>/&#10;$1/isg;
+    $html =~ s/<blockquote[^>]*>(.*?)<\/blockquote>/&#10;--$1/isg;
+    
+#    $html =~ s/<p[^>]*>/&#10;/ig;
+##    $html =~ s/<div[^>]*>(.*?)<\/div>/$1&#10;/isg;
+
+    $html =~ s/<(Font)([^>]*)>/AABBCCDD$1$2DDCCBBAA/ig;
+    $html =~ s/<\/(Font)>/AABBCCDD\/$1DDCCBBAA/ig;
+    $html =~ s/<(I|B)\s+([^>]+)>/AABBCCDD$1 $2DDCCBBAA/ig;
+    $html =~ s/<(I|B)>/AABBCCDD$1DDCCBBAA/ig;
+    $html =~ s/<\/(I|B)>/AABBCCDD\/$1DDCCBBAA/ig;
+    
+    $html =~ s/<[^>]*>//g;
+
+    $html =~ s/AABBCCDD/</g;
+    $html =~ s/DDCCBBAA/>/g;
+    
+    # Make HTML all uppercase, as this seems to be needed for spreadsheet format
+
+##    $html =~ s/<(\w+)([^>]*)>/"<".uc($1).$2.">"/ge;
+##    $html =~ s/<\/(\w+)([^>]*)>/"<\/".uc($1).$2.">"/ge;
+
+    $html =~ s/<(\w+)([^>]*)>/"<".ucfirst($1).$2.">"/ge;
+    $html =~ s/<\/(\w+)([^>]*)>/"<\/".ucfirst($1).$2.">"/ge;
+    
+#    ##$html =~ s/<([^>]*)(\w+="[^"]+")([^>]*)>/"<".$1.uc($2).$3.">"/ge;
+    $html =~ s/<(\w+)([^>]*)>/"<".$1.ucattrs($2).">"/ge;
+
+    $html =~ s/<Font(.*?)Size=\"((?:\+|-)\d+)[^"]*\"/"<Font".$1." Size=\"".(12+$2)."\""/egs;
+#    $html =~ s/<Font(.*?)Face=\"[^"]*\"/<Font$1/gs;
+# #   $html =~ s/<Big[^>]*>(.*?)<\/Big>/$1/sg;
+# #   $html =~ s/<Span[^>]*>(.*?)<\/Span>/$1/sg;
+
+#    $html =~ s/<[^>]*>//g;
+    $html =~ s/Style="color: rgb\(64, 0, 128\)(;?)\"/Color="#400080"/sig;
+    $html =~ s/Style="color: rgb\(0, 0, 0\)(;?)\"/Color="#000000"/sig;
+    
+    # consolidate sequence of <Font> tags
+    while ($html =~ m/^(.*?)<Font ([^>]+)><Font ([^>]+)>(.*?)<\/Font><\/Font>(.*$)/is) {
+	my $front=$1;
+	my $attr1=$2;
+	my $attr2=$3;
+	my $inner=$4;
+	my $back=$5;
+
+	#$attr1 =~ s/Style="color: rgb\(64, 0, 128\)(;?)/Color="#400080"/ig;
+	#$attr2 =~ s/Style="color: rgb\(64, 0, 128\)(;?)/Color="#400080"/ig;
+
+	my ($attr_name1) = ($attr1 =~ m/^(.*?)=/);
+	my ($attr_name2) = ($attr2 =~ m/^(.*?)=/);
+
+	if ($attr_name1 ne $attr_name2) {
+#	if (index($attr1,$attr_name2)<0) {
+	    $html = "$front<Font $attr1 $attr2>$inner<\/Font>$back";
+	}
+	else {
+	    $html = "$front<Font $attr2>$inner<\/Font>$back";
+	}
+    }
+    
+    $html =~ s/\s+/ /sg;
+    
+    $html =~ s/^(&#10;)+//s;
+    
+    # parse
+    my $tidy_dom = Mojo::DOM->new($html);
+    my $child_font_collection = $tidy_dom->find('font>font');
+    
+    for my $child_font_elem (@$child_font_collection) {
+	my $children = $child_font_elem->child_nodes();
+	$child_font_elem->replace($children->join()->say());
+    }
+	
+    my $tidy_html = $tidy_dom->content;
+    $tidy_html =~ s/<(\w+)([^>]*)>/"<".ucfirst($1).$2.">"/ge;
+    $tidy_html =~ s/<\/(\w+)([^>]*)>/"<\/".ucfirst($1).$2.">"/ge;
+    $tidy_html =~ s/<(\w+)([^>]*)>/"<".$1.ucattrsNS($1,$2).">"/ge;
+	    
+    $tidy_html =~ s/$/&#10;/mg;
+    
+    print $FOUT "    <Cell ss:StyleID=\"sHtmlWrap\"><ss:Data ss:Type=\"String\" xmlns=\"http://www.w3.org/TR/REC-html40\">$tidy_html</ss:Data></Cell>\n";
+    
+
+}
+
+
+
+sub str_cat_eval
+{
+    my ($str) = @_;
+    $str =~ s/\+/./g;
+    
+    return eval $str;
+}
+
+sub process_html_message
+{
+    my ($html_filename,$message_id) = @_;
+    print "    Processing: $html_filename\n";
+
+    open (my $FH, '<:encoding(UTF-8)', $html_filename)
+	|| die "Could not open file '$html_filename' $!";
+
+    my $content = "";
+    while (my $line = <$FH>) {
+	$content .= $line;
+    }
+
+    close($FH);
+
+    my $dom = Mojo::DOM->new($content);
+    
+    my $li_collection = $dom->find('div[class="block"]>ul>li');
+    
+#    my $from    = tidy_header_field("From",   $li_collection->[0]->all_text);
+#    my $to      = tidy_header_field("To",     $li_collection->[1]->all_text);
+#    my $subject = tidy_header_field("Subject",$li_collection->[2]->all_text);
+#    my $date    = tidy_header_field("Date",   $li_collection->[3]->all_text);
+#    print "    From    : $from\n";
+#    print "    To      : $to\n";
+#    print "    Subject : $subject\n";
+#    print "    Date    : $date\n";
+    
+    my $header_record = {};
+
+    for my $li_elem (@$li_collection) {
+	store_header_field($li_elem->all_text,$header_record);
+    }
+    
+#    for my $header_label (sort keys %$header_record) {
+#	print "    $header_label : $header_record->{$header_label}\n";
+#    }
+
+
+    print $FOUT "  <Row>\n";
+
+    #print $FOUT "\"$message_id\"";
+    print_spreadsheet_text_cell_xml($message_id);
+
+    if (defined $header_record->{'From'}) {
+	my $from = $header_record->{'From'};
+	#print_csv_escaped($from);
+	print_spreadsheet_text_cell_xml($from)
+    }
+    else {
+	# leave empty cell in CSV file
+	#print_csv_escaped("");
+	print_spreadsheet_text_cell_xml("")
+    }
+
+#    if (defined $header_record->{'To'}) {
+#	my $to = $header_record->{'To'};
+#	#print_csv_escaped($to);
+#	print_spreadsheet_text_cell_xml($to)
+#    }
+#    else {
+#	# leave empty cell in CSV file
+#	#print_csv_escaped("");
+#	print_spreadsheet_text_cell_xml("")
+#    }
+
+    if (defined $header_record->{'Date'}) {
+	my $date = $header_record->{'Date'};
+	#print_csv_escaped($date);
+	print_spreadsheet_text_cell_xml($date)
+    }
+    else {
+	# leave empty cell in CSV file
+	#print_csv_escaped("");
+	print_spreadsheet_text_cell_xml("")
+    }
+
+    if (defined $header_record->{'Subject'}) {
+	my $subject = $header_record->{'Subject'};
+	#print_csv_escaped($subject);
+	print_spreadsheet_text_cell_xml($subject)
+    }
+    else {
+	# leave empty cell in CSV file
+	#print_csv_escaped("");
+	print_spreadsheet_text_cell_xml("")
+    }
+    
+    # Extract article
+    my $posting_div = $dom->at('#ActionHeader + div');
+    if (!defined $posting_div) {
+	print_spreadsheet_html_cell_xml("Empty article posted");
+	
+	print "Empty article!!!\n";
+	print "=============\n";
+	
+	# print $FOUT "\n";
+	print $FOUT "  </Row>\n";
+	
+	$REFNUM++;
+
+	return;
+    }
+    
+    my $posting = $posting_div->child_nodes;
+    my $posting_array = $posting->to_array();
+
+#    if (scalar(@$posting_array)>1) {
+#	print "*****### dealing with multi value posting array: ", scalar(@$posting_array), "\n";
+#    }
+    
+    foreach my $elem (@$posting_array) {
+	if ($elem =~ m/^<hr\s+[^>]*>/si) {
+
+	    my ($article) = ($elem =~ m/<!--X-Body-of-Message-->(.*)<!--X-Body-of-Message-End-->/s);
+	    $article =~ s/<script[^>]*>.*?document\.write\((.*?)\).*?<\/script>/str_cat_eval($1)/sge;
+
+	    my $cutpos = rindex($article,"To    leave    MLA-L,    send    the    command    SIGNOFF    MLA-L    to");
+	    my $tidy_article = substr($article,0,$cutpos);
+	    $article = $tidy_article;
+	    $article =~ s/\*{70,}<br>$//s;
+	    
+	    $article =~ s/^\s*//s;
+	    $article =~ s/\s*$//s;
+#	    if ($article =~ m/Get Your Private/) {
+#		print "***** $article\n\n";
+#	    }
+	    
+	    print_spreadsheet_html_cell_xml($article);
+
+	    # clean up tail, lock onto last occurrance of 'To   leave ...'
+
+##	    my $cutpos = rindex($elem,"To    leave    MLA-L,    send    the    command    SIGNOFF    MLA-L    to");
+##	    my $tidy_elem = substr($elem,0,$cutpos);
+##	    $elem = $tidy_elem;
+	    
+#	    $elem =~ s/To    leave    MLA-L,    send    the    command    SIGNOFF    MLA-L    to<br>.*?$//s;
+#	    $elem =~ s/\*{70,}<br>$//s;
+#	    $elem =~ s/\s*$//s;
+	    
+#	    # clean up head
+#	    $elem =~ s/^.*<!--X-Body-of-Message-->//s;
+#	    $elem =~ s/^\s*//s;
+		    
+#	    $elem =~ s/<script[^>]*>.*?document\.write\((.*?)\).*?<\/script>/str_cat_eval($1)/sge;
+
+#	    if ($elem =~ m/Get Your Private/) {
+#		print "***** $elem\n\n";
+#	    }
+	    
+    
+	    ##print "    <article>\n$elem\n    </article>\n";
+	    #print_csv_escaped("<html>$elem</html>");
+#	    print_spreadsheet_html_cell_xml($elem)
+	    
+	}
+    }
+    # <!--X-Body-of-Message-->
+    # ....
+    # *************************************************************************<br>
+    # To    leave    MLA-L,    send    the    command    SIGNOFF    MLA-L    to<br>	
+
+    
+    print "=============\n";
+
+    # print $FOUT "\n";
+    print $FOUT "  </Row>\n";
+    
+    $REFNUM++;
+}
+
+sub spreadsheet_header
+{
+    my ($group) = @_;
+    
+    open (my $FH, '<:encoding(UTF-8)', "spreadsheet-template-header.xml")
+	|| die "Could not open file 'spreadsheet-template-header' $!";
+
+    my $content = "";
+    while (my $line = <$FH>) {
+	$line =~ s/####Worksheet####/$group/g;	
+	$content .= $line;
+    }
+
+    close($FH);
+
+    print $FOUT $content;
+}
+
+sub spreadsheet_footer
+{
+    open (my $FH, '<:encoding(UTF-8)', "spreadsheet-template-footer.xml")
+	|| die "Could not open file 'spreadsheet-template-footer' $!";
+
+    my $content = "";
+    while (my $line = <$FH>) {
+	$content .= $line;
+    }
+
+    close($FH);
+
+    print $FOUT $content;
+}
+
+
+sub process_html_file
+{
+    my ($group, $html_filename) = @_;
+    print "    Processing: $html_filename\n";
+
+    open (my $FH, '<:encoding(windows-1252)', $html_filename)
+	|| die "Could not open file '$html_filename' $!";
+
+    my $content = "";
+    while (my $line = <$FH>) {
+	$content .= $line;
+    }
+
+    close($FH);
+
+    my $dom = Mojo::DOM->new($content);
+
+    # my $article_dom = $dom->find('center')->first()->following();
+
+    my $last_post = 0;
+
+    my $posts = [];
+    my $missing_titles = [];
+    my $missing_body = [];
+    
+    
+    for my $hr_elem ($dom->find('hr[width="100%"]')->each) {
+
+	my $opt_icons = [];
+	my $title_line = undef;
+	my $body_html = "";
+	
+	for my $e ($hr_elem->following()->each) {
+
+	    if ($e->tag eq "center") {
+		$last_post = 1;
+		last;
+	    }
+	    
+	    if ($e->tag eq "hr") {
+		last;
+	    }
+
+#	    if ($e->tag eq "br") {
+#		next;
+#	    }
+
+	    my $opt_head = $e->find('font[size="+3"]');
+	    my $opt_font_color = $e->find('font[color="#400080"]');
+	    my $opt_style = $e->find('[style*="color: rgb(64, 0, 128)"]');
+				     
+	    my $is_title_line = ((scalar(@$opt_head) > 0) || (scalar(@$opt_font_color)>0) || (scalar(@$opt_style)>0));
+
+	    # print "*** testing for title: $e\n";
+	    if (!$is_title_line) {
+		if ($e->tag eq "font"){
+		    my $opt_size = $e->{'size'};
+		    if (defined $opt_size && $opt_size eq "+3") {
+			$is_title_line = 1;
+		    }
+		    my $opt_color = $e->{'color'};
+		    if (defined $opt_color && $opt_color eq "#400080") {
+			$is_title_line = 1;
+		    }
+		}
+		elsif ($e->tag eq "big") {
+		    my $opt_font_color = $e->find('font[color="#400080"]');
+		    if (scalar(@$opt_font_color)>0) {
+			$is_title_line = 1;
+		    }
+		    else {
+			my $opt_style = $e->{'style'};
+			if (defined $opt_style && $opt_style =~ m/color:\s*rgb\(64,\s*0,\s*128\)/) {
+			    $is_title_line = 1;
+			}
+		    }
+
+		    
+		}
+	    }
+	    
+	    if ($e->tag eq "a") {
+		push(@$opt_icons,$e);
+	    }
+	    elsif ((!defined $title_line) && ($is_title_line)) {
+		$title_line = $e;
+	    }
+	    else {
+		$body_html .= $e;		
+	    }
+##	    print "  ==\n";
+	}
+	
+	if (defined $title_line) {
+	    my $visible_title = $title_line;
+	    $visible_title =~ s/<[^>]*>//g;
+	    $visible_title =~ s/\s+/ /gs;
+
+
+	    if ($body_html ne "") {
+
+		push(@$posts,{ 'title' => $title_line, 'body' => $body_html, 'opt_icons' => scalar(@$opt_icons)});
+		
+		print "Title: $visible_title\n";
+		
+#		print "Title Line: $title_line\n";
+#		print "Body: $body_html\n";
+	    }
+	    else {
+		print "Warning: No BODY HTML to go with TITLE LINE: '$visible_title'\n";
+		push(@$missing_body, $title_line);
+	    }
+	}
+	else {
+	    if ($body_html eq "") {
+		print "Skipping entry with no title or body text\n";
+	    }
+	    else {
+		my $visible_text = $body_html;
+		$visible_text =~ s/<[^>]*>//g;
+		if ($visible_text =~ m/^\s*$/s) {
+		    print "Skipping entry with no title or body text\n";
+		}
+		else {
+		    if ($body_html !~ m/^<table/i) {
+			print "Warning: article has BODY TEXT but no TITLE: '$body_html'\n";
+		    }
+		    push(@$missing_titles, $body_html);
+		}
+	    }
+	}
+	print "+++====++++\n";
+
+	if ($last_post) {
+	    last;
+	}
+    }
+
+    my ($tail_filename) = ($html_filename =~ m/^.*\/(.*)$/);
+    my $anchored_html_filename = "anchored-$tail_filename";
+
+    my $i = 1;    
+    foreach my $rec (@$posts) {
+	print $FOUT "  <Row>\n";
+	my $ref_id = "Ref-$group-".sprintf("%03d",$i);
+
+	print_spreadsheet_html_cell_xml($ref_id);
+	print_spreadsheet_html_cell_xml($rec->{'title'});
+	print_spreadsheet_html_cell_xml($rec->{'body'});
+	print_spreadsheet_html_cell_xml($rec->{'opt_icons'});
+	print $FOUT "    <Cell ss:StyleID=\"sPlainText\" ss:HRef=\"$anchored_html_filename#$ref_id\"><ss:Data ss:Type=\"String\">View external HTML $ref_id</ss:Data></Cell>\n";
+
+	print $FOUT "  </Row>\n";
+
+	$rec->{'title'}->prepend("<div><i><a name=\"$ref_id\" >Waikato added $ref_id</a></i></div>");
+	$i++;
+    }
+
+    print "*\n";
+    print "* Generated anchored version $anchored_html_filename\n";
+    print "*\n";
+
+    #  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+    $dom->at('meta[http-equiv]')->replace('<meta http-equiv="Content-Type" content="text/html; charset=utf-8">');
+
+        ## https://web.archive.org/web/20070218055720/http://www.logan.com/loganberry/solved-carbonel.jpg
+##    print "*** looking for html tags\n";
+    my $a_collection = $dom->find('a');
+    for my $a_elem (@$a_collection) {
+	my $href = $a_elem->attr('href');
+	if (defined $href) {
+	    if ($href =~ m/^http:\/\/www\.logan\.com\/loganberry\/.*\.(jpg|gif|png)$/i) {
+		$a_elem->attr({ 'href' => "https://web.archive.org/web/20030218055720/$href"});
+	    }
+	}
+    }
+
+    my $img_collection = $dom->find('img');
+    for my $img_elem (@$img_collection) {
+	# <img alt="click for image of book" src="http://www.logan.com/loganberry/camera1.gif" width="64" height="64" border="0" align="LEFT">
+	    
+	my $src = $img_elem->attr('src');
+#	print "*** src = $src\n";
+	if ($src =~ m/^http:\/\/www\.logan\.com\/loganberry\/.*\.(jpg|gif|png)$/i) {
+##	    print "**** relplacing src with https://web.archive.org/web/20170218055720/$src\n";
+#	    if ($src =~ m/camera1\.gif$/i) {
+#		$img_elem->attr({ 'src' => "https://web.archive.org/web/20120213150518/$src"});		
+#	    }
+#	    else {
+		$img_elem->attr({ 'src' => "https://web.archive.org/web/20020218055720/$src"});
+##		$img_elem->attr({ 'src' => "https://web.archive.org/web/20170218055720/$src"});
+#	    }
+	}	    
+    }
+
+
+    
+    open(my $AOUT, '>:encoding(UTF-8)', $anchored_html_filename)
+	|| die "Failed to open $anchored_html_filename: $!\n";
+    print $AOUT $dom->content;
+    close($AOUT);
+
+    
+    print "*****\n";
+
+    print "* num posts      = ", scalar(@$posts), "\n";
+    print "* missing titles = ", scalar(@$missing_titles), "\n";
+    print "* missing body   = ", scalar(@$missing_body), "\n";
+    print "*****\n";
+}
+    
+sub main
+{
+    my $ARGC = scalar(@ARGV);
+
+    if ($ARGC != 1) {
+	print STDERR "Usage: $0 directory\n";
+	exit(1);
+    }
+
+    my $input_dir = $ARGV[0];
+    $input_dir .= "/" unless ($input_dir =~ /\/$/);
+    
+    print "Reading sub-directories that match: solved-[a-z]+.html\n";
+    
+    opendir(my $DIRIN, $input_dir) || die "can't opendir $input_dir: $!";
+    my @input_files = sort grep { $_ =~ /^solved-[a-z]+.html$/ } readdir($DIRIN);    
+    closedir $DIRIN;
+
+    print "Number of matching files: ", scalar(@input_files), "\n";
+
+##    my $prev_year = "0000";
+    
+    foreach my $file (@input_files) {
+
+	my $full_filename = "$input_dir$file";
+	print "  Processing file: $full_filename\n";
+
+	my ($group) = ($file =~ m/^solved-([a-z]+).html$/);
+
+	my $csv_filename = "solved-$group.xml";
+	    
+	open $FOUT, '>:encoding(UTF-8)', $csv_filename
+	    || die "Failed to open $csv_filename: $!\n";
+	
+	spreadsheet_header($group);
+	process_html_file($group,$full_filename);
+	spreadsheet_footer();
+
+	close($FOUT);
+    }
+
+##    spreadsheet_footer();
+##    close($FOUT);
+}
+
+main();
+
+
Index: /other-projects/book-stumper/trunk/perl-cpan/SETUP.bash
===================================================================
--- /other-projects/book-stumper/trunk/perl-cpan/SETUP.bash	(revision 33353)
+++ /other-projects/book-stumper/trunk/perl-cpan/SETUP.bash	(revision 33353)
@@ -0,0 +1,6 @@
+
+cd Mojolicious-7.24/lib
+export PERL5LIB=`pwd`
+cd ../..
+
+
