Changeset 32109

Timestamp:

2018-01-18T23:28:54+13:00 (5 years ago)

Author:

davidb

Message:

Changes made after testing through YARN

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Files:

• JSONLIST-YARN-INGEST.sh (modified) (1 diff)
• scripts/JSONLIST-RUN-YARN-SPARK.sh (added)
• scripts/_RUN.sh (modified) (2 diffs)
• src/main/java/org/hathitrust/extractedfeatures/PerVolumeUtil.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/JSONLIST-YARN-INGEST.sh

@@ -1 +1 @@
 #!/bin/bash
 
-nohup ./scripts/JSONLIST-RUN-YARN-SPARK.sh TMP-faceted-htrc-fictsample-ef20 &
+nohup ./scripts/JSONLIST-RUN-YARN-SPARK.sh \
+      pair-tree-annika-1k-fiction-vol-ids.txt TMP-faceted-htrc-fictsample-ef20 &

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/scripts/_RUN.sh

@@ -27 +27 @@
 
     if [ "x$input_dir" != "x" ] ; then
-    if [ $show_usage = "json" ] ; then
+    if [ $class_mode = "json" ] ; then
         show_usage=0
     fi
@@ -130 +130 @@
     #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $seq_file $*"
 else
-    cmd="$cmd --properties ef-solr.properties $json_filelist $input_dir $*"
-    #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $json_filelist $input_dir $*"
+    cmd="$cmd --properties ef-solr.properties $input_dir $json_filelist $*"
+    #cmd="$cmd --properties /homea/dbbridge/extracted-features-solr/solr-ingest/ef-solr.properties $input_dir $json_filelist $*"
 
 fi

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeUtil.java

@@ -2 +2 @@
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
-
+    
 import org.apache.hadoop.io.Text;
 import org.apache.spark.api.java.function.FlatMapFunction;
@@ -14 +15 @@
 import org.json.JSONObject;
 
-/*
-class PagedJSON implements Function<String, Boolean> {
-
+public class PerVolumeUtil implements Serializable
+{
     private static final long serialVersionUID = 1L;
-
-    public Boolean call(String s) { return s.contains("a"); }
-}
- */
-
-
-//public class PerVolumeJSON implements VoidFunction<String> 
-public class PerVolumeUtil
-{
-    //private static final long serialVersionUID = 1L;
     protected String _input_dir;
     protected String _whitelist_filename;