Changeset 34124

Timestamp:

2020-05-27T18:03:58+12:00 (4 years ago)

Author:

ak19

Message:

Decoding the title and text using the encoding seemed to have turned into a problem, both for windows 1252 but also <td valign=top>

File:

• main/trunk/greenstone2/perllib/plugins/NutchTextDumpPlugin.pm (modified) (4 diffs)

main/trunk/greenstone2/perllib/plugins/NutchTextDumpPlugin.pm

@@ -90 +90 @@
 # Example record in dump.txt to process:
     # https://www.whanau-tahi.school.nz/    key:    nz.school.whanau-tahi.www:https/
+    # OR: http://yutaka.it-n.jp/apa/750010010.html        key:    jp.it-n.yutaka:http/apa/750010010.html
     # baseUrl:  null
     # status:   2 (status_fetched)
@@ -609 +610 @@
     # (and contains characters above codepoint 255). You can't decode it again."
     if($title_meta && $title_meta ne "" && $title_meta ne "null") {
-    $title_meta = $self->to_utf8($encoding, $title_meta) if ($encoding);
+    #$title_meta = $self->to_utf8($encoding, $title_meta) if ($encoding);
     } else { # if we have "null" as title metadata, set it to the record URL?
     #my $srcURLs = $doc_obj->get_metadata($cursection, "ex.srcURL");
@@ -615 +616 @@
     #$title_meta = $srcURLs->[0] if (scalar @$srcURLs > 0);
     my $srcURL = $doc_obj->get_metadata_element($cursection, "srcURL", 1); # TODO: why does ex.srcURL not work, nor srcURL without 3rd param
+    my ($basicURL) = $srcURL =~ m@^https?://(?:www\.)?(.*)$@; # use basicURL for title instead of srcURL, else many docs get classified under "Htt" bucket for https
     if(defined $srcURL) {
-        print STDERR "@@@@ null/empty title to be replaced with ".$srcURL."\n"
+        print STDERR "@@@@ null/empty title to be replaced with ".$basicURL."\n"
         if $self->{'verbosity'} > 3;
-        $title_meta = $srcURL;
+        $title_meta = $basicURL;
     }
     }
@@ -659 +661 @@
         $$textref = $1;
         if($$textref !~ m/^\s*$/) {
-        $$textref = $self->to_utf8($encoding, $$textref) if ($encoding);
+        #$$textref = $self->to_utf8($encoding, $$textref) if ($encoding);
         $$textref = "<pre>\n".$$textref."\n</pre>";
         $no_text = 0;