Index: /gs3-extensions/hathitrust-downloadfrom/trunk/htc-get-pd-docs.pl
===================================================================
--- /gs3-extensions/hathitrust-downloadfrom/trunk/htc-get-pd-docs.pl	(revision 26441)
+++ /gs3-extensions/hathitrust-downloadfrom/trunk/htc-get-pd-docs.pl	(revision 26442)
@@ -41,5 +41,9 @@
 	print STDERR "**** Status:  ", $response->status_line, "\n";
 	print "------\n";
-	print STDERR "**** Content: ", $response->content, "\n";
+	my $text_only_content = $response->content();
+	$text_only_content =~ s/<[^>]*>//g;
+	$text_only_content =~ s/^\s*$//mg;
+
+	print STDERR "**** Content: $text_only_content\n";
 	print "------\n";
 	
@@ -58,16 +62,37 @@
 	print STDERR "Downloading PageImage $htid/$seq_num\n";
 
+	my $retryCount = 0;
+      PageImageRetry:
 	my $response = _data_api("pageimage",$htid, $seq_num );
-	my $content = $response->content();
-
-	if (open(IMGOUT,">$ofilename")) {       
-	    binmode(IMGOUT);	   
-	    print IMGOUT $content;
-	    close(IMGOUT);
-	}
-	else {
-	    print STDERR "Error: Failed to open $ofilename for binary output\n";
-	    print STDERR "       $!\n";
-	}    
+	if (defined $response) {
+	    $retryCount = 0; # reset it
+	    my $content = $response->content();
+
+	    if (open(IMGOUT,">$ofilename")) {       
+		binmode(IMGOUT);	   
+		print IMGOUT $content;
+		close(IMGOUT);
+	    }
+	    else {
+		print STDERR "Error: Failed to open $ofilename for binary output\n";
+		print STDERR "       $!\n";
+	    }    
+	}
+	else {
+	    $retryCount++;
+	    print STDERR "Failed to download PageImage\n";
+
+	    if ($retryCount<2) {
+		print STDERR "Sleeping to 60 seconds\n";
+		sleep(60);
+		print STDERR "Retry attempt $retryCount\n";
+		goto PageImageRetry;
+	    }
+	    else {
+		print STDERR "Maximum number of attempts reached.  Stopping.\n";
+		exit -1;
+	    }
+	}
+	
     }
     else {
@@ -88,16 +113,40 @@
 	|| (!defined $ofilename)) {
 	print STDERR "Downloading PageOCR (text) $htid/$seq_num\n";
+
+	my $retryCount = 0;
+      PageOcrRetry:
+
+	my $response = _data_api("pageocr",$htid, $seq_num );
+
+	if (defined $response) {
+	    $retryCount = 0; # reset it
+
+	    $content = $response->content();
 	    
-	my $response = _data_api("pageocr",$htid, $seq_num );
-	$content = $response->content();
-	
-	if (open(TXTOUT,">$ofilename")) {       
-	    print TXTOUT $content;
-	    close(TXTOUT);
-	}
-	else {
-	    print STDERR "Error: Failed to open $ofilename for binary output\n";
-	    print STDERR "       $!\n";
-	}    
+	    if (open(TXTOUT,">$ofilename")) {       
+		print TXTOUT $content;
+		close(TXTOUT);
+	    }
+	    else {
+		print STDERR "Error: Failed to open $ofilename for binary output\n";
+		print STDERR "       $!\n";
+	    }    
+	}
+	else {
+	    $retryCount++;
+	    print STDERR "Failed to download PageOCR\n";
+
+	    if ($retryCount<2) {
+		print STDERR "Sleeping to 60 seconds\n";
+		sleep(60);
+		print STDERR "Retry attempt $retryCount\n";
+		goto PageOcrRetry;
+	    }
+	    else {
+		print STDERR "Maximum number of attempts reached.  Stopping.\n";
+		exit -1;
+	    }
+	}
+	
     }
     else {
@@ -198,11 +247,15 @@
 sub rec_paged_image_structure
 {
-    my ($this_div,$pagenum,$elem_name,$depth,$htid,$file_id_map,$resource_output_dir) = @_;
+    my ($this_div,$pagenum,$depth,$htid,$file_id_map,$resource_output_dir) = @_;
 
     my ($local_output_dir) = ($resource_output_dir =~ m/^.*\/(.*?)$/);
 
-    print PIOUT "  " x $depth, "<$elem_name>\n";
 
     my $fptr_entry = $this_div->{'METS:fptr'};
+
+    if (defined $this_div->{'METS:div'}) {
+	# Only want Greenstones <PageGroup> tag if not a METS leaf div
+	print PIOUT "  " x $depth, "<PageGroup>\n";
+    }
     
     if (defined $fptr_entry) {
@@ -220,4 +273,5 @@
 	my $imgfile = undef;
 	my $txtfile = undef;
+
 
 	foreach my $fptr_hash (@$fptr_array) {
@@ -229,4 +283,5 @@
 	    my $seq  = $file->{'SEQ'};
 	    my $href = $file->{'METS:FLocat'}->{'xlink:href'};
+
 
 	    if ($file->{'USE'} =~ m/\bimage\b/i) {		
@@ -240,4 +295,5 @@
 		pageocr_data_api($htid,$seq,$full_txtfile);
 	    }
+
 	}
 	# Generate line along the following lines
@@ -268,14 +324,19 @@
 	}
 
+	print STDERR "+ Processing ", scalar(@$div_array), " sections\n";
+
 	foreach my $div_hash (@$div_array) {
 
 	    my $pagenum = $div_hash->{'ORDER'};
 	    
-	    rec_paged_image_structure($div_hash,$pagenum,"PageGroup",$depth+1,$htid,$file_id_map,$resource_output_dir);
-	}
-    }
-
-
-    print PIOUT "  " x $depth, "</$elem_name>\n";
+	    rec_paged_image_structure($div_hash,$pagenum,$depth+1,$htid,$file_id_map,$resource_output_dir);
+	}
+    }
+
+    if (defined $this_div->{'METS:div'}) {
+	# Only want Greenstones <PageGroup> tag if not a METS leaf div
+	print PIOUT "  " x $depth, "</PageGroup>\n";
+    }
+
 
 }
@@ -294,6 +355,12 @@
     if (open(PIOUT,">$ofilename")) {
 	binmode(PIOUT,":utf8");
-
-	rec_paged_image_structure($toplevel_div,1,"PageDocument",0,$htid,$file_id_map,$resource_output_dir);
+	
+	print PIOUT "<PagedDocument>\n";
+#	print PIOUT "  <PageGroup>\n";
+
+	rec_paged_image_structure($toplevel_div,1,1,$htid,$file_id_map,$resource_output_dir);
+
+#	print PIOUT "  </PageGroup>\n";
+	print PIOUT "</PagedDocument>\n";
 
 	close(PIOUT);
@@ -308,4 +375,6 @@
 }
 
+
+my $pdCount = 0;
 
 sub download_ht_doc
@@ -367,5 +436,9 @@
 ##    print "**** json_content = $json_content_utf8\n\n";
 
-    exit 0;
+    $pdCount++;
+
+#    if ($pdCount>5) {
+#	exit 0;
+#    }
 
 }
@@ -395,5 +468,17 @@
     my $primary_cat_key = shift @record_keys;
 
-    my $items_array = $json_data->{'items'};
+    my $items_entry = $json_data->{'items'};
+    my $items_array;
+
+    print STDERR "*** ref: ", ref $items_entry, "\n\n";
+
+
+    if (ref $items_entry eq "HASH") {
+	$items_array = [ $items_entry ];
+    }
+    else {
+	$items_array = $items_entry;
+    }
+
     my $num_items = scalar(@$items_array);
 
@@ -441,5 +526,5 @@
     
     if (opendir(DIN, $full_dir)) {
-	my @dir_content = grep { $_ !~ m/^\./ }  readdir(DIN);
+	my @dir_content = grep { $_ !~ m/^\./ }  sort readdir(DIN);
 	closedir DIN;
 
