Changeset 34123

Timestamp:

2020-05-26T02:18:44+12:00 (4 years ago)

Author:

ak19

Message:

Some more minor changes

File:

• main/trunk/greenstone2/perllib/plugins/NutchTextDumpPlugin.pm (modified) (4 diffs)

main/trunk/greenstone2/perllib/plugins/NutchTextDumpPlugin.pm

@@ -31 +31 @@
 #
 # For a commoncrawl collection of siteID-labelled folders containing dump.txt files each,
-# set <importOption name="OIDtype" value="dirname"/>
-# And create 2 List browsing classifiers (with bookshelf_type set to always) on ex.siteID and ex.srcDomain
+# - set <importOption name="OIDtype" value="dirname"/>
+# - Create 2 List browsing classifiers (with bookshelf_type set to always) on ex.siteID and ex.srcDomain
 # both sorted by ex.srcURL, and an ex.Title classifier.
 # For the ex.srcDomain classifier, set removeprefix to: https?\:\/\/(www\.)?
 # An alternative is to build that List classifier on ex.basicDomain instead of ex.srcDomain.
+# Set this List classifier's "partition_type_within_level" option to "per_letter".
+# - Add search indexes on text (default), Title, basicDomain, siteID, Identifier, srcURL (not working)
+#
 # Finally, in the "display" format statement, add the following before the "wrappedSectionText" to
 # display the most relevant metadata of each record:
@@ -50 +53 @@
   #       <dd>
   #         <gsf:metadata name="ex.Title"/>
+  #       </dd>
+  #   <dt>Identifier:</dt>
+  #       <dd>
+  #         <gsf:metadata name="Identifier"/>
   #       </dd>
   #       <dt>SiteID:</dt>
@@ -475 +482 @@
     my $cursection = $doc_obj->get_top_section();
     
+    # https://stackoverflow.com/questions/27801561/turn-off-binmodestdout-utf8-locally
+    #binmode STDERR, ':utf8'; ## FOR DEBUGGING! To avoid "wide character in print" messages
     
     #print STDERR "---------------\nDUMP.TXT\n---------\n", $$textref, "\n------------------------\n";
@@ -489 +498 @@
     
     foreach my $line (@lines) {
+    #$line =~ s@\{@\\{@g; # escape open curly braces for newer perl
+    
     # first line is special and contains the URL (no metaname)
     # and the inverted URL labelled with metaname "key"