Changeset 30918

Timestamp:

2016-10-25T14:49:36+13:00 (6 years ago)

Author:

davidb

Message:

More flexible command-line args

Location:

other-projects/hathitrust/solr-extracted-features/trunk

Files:

• RUN.bash (modified) (1 diff)
• src/main/java/org/hathitrust/PagedJSON.java (modified) (1 diff)
• src/main/java/org/hathitrust/PrepareForIngest.java (modified) (4 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/RUN.bash

@@ -1 +1 @@
 #!/bin/bash
 
-spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target/htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step10000.txt pd-ef-json-files pd-solr-files $* 
+input_dir=pd-ef-json-files
+output_dir=pd-solr-json-files
+
+master_opt="--master local[4]"
+self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar
+base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar"
+
+if [ $# -ge 1 ] ; then
+    file_listing=shift $*
+    $base_cmd --json-filelist="$file_listing" $input_dir $output_dir $*
+else
+    echo "****"
+    echo "* Processing all files in: $input_dir"
+    echo "****"
+    $base_cmd $input_dir $output_dir $*
+fi
+
+#    spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target/htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step10000.txt pd-ef-json-files pd-solr-json-files $* 
 
 # spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target\htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step1000.txt json-files solr-files $*

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

@@ -32 +32 @@
 }
  */
+
+
+/* 
+URI uri = URI.create (“hdfs://host:port/file path”);
+Configuration conf = new Configuration();
+FileSystem file = FileSystem.get(uri, conf);
+FSDataInputStream in = file.open(new Path(uri));
+
+*/
 
 class PagedJSON implements FlatMapFunction<String, String> 

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java

@@ -35 +35 @@
 */
 
-    //protected int _num_cores;
-    protected String _input_dir;
-    protected String _json_list_filename;
+    //protected int _num_cores;
+    protected String _input_dir;
+    protected String _json_list_filename;
     protected String _output_dir;
     
@@ -44 +44 @@
         //_num_cores = num_cores;
         _input_dir = input_dir;
-        _json_list_filename = json_list_filename;
+        _json_list_filename = (json_list_filename != null) ? json_list_filename : input_dir;
         _output_dir = output_dir;
     }
@@ -115 +115 @@
 
         //cmd.hasOption("json-filelist")
-        String json_list_filename = cmd.getOptionValue("json-filelist","pd-file-listing.txt");
+        String json_list_filename = cmd.getOptionValue("json-filelist");
         //int num_cores = Integer.parseInt(num_cores_str);
 
@@ -126 +126 @@
         
         if (filtered_args.length != 2) {
-            System.err.println("Usage: RUN.bat [options] input-dir output-dir");
+                System.err.println("Usage: RUN.bat [options] input-dir output-dir");
+            formatter.printHelp("utility-name", options);
+
+            //System.err.println("Usage: RUN.bat [options] input-dir output-dir");
             //System.err.println("  Where 'filename.txt' contains a list of JSON files, one per line,");
             //System.err.println("  which use the HathiTrust Extracted Feature JSON format");