Changeset 30934 for other-projects/hathitrust

Timestamp:

2016-10-26T11:05:28+13:00 (7 years ago)

Author:

davidb

Message:

Providing json-filelist now a compulsory argument, rather than an option

Location:

other-projects/hathitrust/solr-extracted-features/trunk

Files:

• _RUN.bash (modified) (1 diff)
• src/main/java/org/hathitrust/PrepareForIngest.java (modified) (7 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/_RUN.bash

@@ -24 +24 @@
 fi
 
-echo
-echo "****"
-echo "* Checking for Spark and Hadoop daemons"
-echo "****"
-jps | sed 's/^/* /g'
-echo "****"
-echo "* Done"
-echo "****"
-echo
+run_jps=0
+run_jps_daemons=""
+run_jps_daemons_suffix="daemon"
+
+if [ "x${input_dir##hdfs://*}" = "x" ] || [ "x${output_dir##hdfs://*}" = "x" ] ; then
+    # Evidence of running command over HDFS
+    run_jps=1
+    run_jps_daemons="Spark"
+fi
+
+if [ "x${master_op##--master spark://*}" = "x" ] ; then
+    # Evidence of running command submitted to Spark cluster
+    run_jps=1
+    if [ "x$run_jps_daemons" != "x" ] ; then
+        run_jps_daemons="$run_jps_daemons and Hadoop"
+    run_jps_daemons_suffix="daemons"
+    else
+        run_jps_daemons="Hadoop"
+    fi
+fi
+
+if [ "$run_jps" = "1" ] ; then
+  echo
+  echo "****"
+  echo "* Checking for $run_jps_daemons $run_jps_daemons_suffix"
+  echo "****"
+  jps | sed 's/^/* /g'
+  echo "****"
+  echo "* Done"
+  echo "****"
+  echo
+
+fi
 
 self_contained_jar=target/htrc-ef-ingest-0.9-jar-with-dependencies.jar
 base_cmd="spark-submit --class org.hathitrust.PrepareForIngest $master_opt $self_contained_jar"
 
-cmd="$base_cmd --json-filelist=\"$json_filelist\" $input_dir $output_dir $*"
+cmd="$base_cmd --verbosity 1 $json_filelist $input_dir $output_dir $*"
 
 echo "****"

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java

@@ -19 +19 @@
     private static final long serialVersionUID = 1L;
 
-/*
-    class ContainsA implements Function<String, Boolean> {
-        
-        private static final long serialVersionUID = 1L;
-
-        public Boolean call(String s) { return s.contains("a"); }
-    }
-
-    class ConvertJSON implements Function<String, Boolean> {
-
-        private static final long serialVersionUID = 1L;
-
-        public Boolean call(String s) { return s.contains("a"); }
-    }
-*/
-
-    //protected int _num_cores;
     protected String _input_dir;
     protected String _json_list_filename;
     protected String _output_dir;
+    protected int    _verbosity;
     
-    public PrepareForIngest(String input_dir, String json_list_filename, String output_dir)
+    public PrepareForIngest(String input_dir, String json_list_filename, String output_dir, int verbosity)
     {
-        //_num_cores = num_cores;
         _input_dir = input_dir;
         _json_list_filename = (json_list_filename != null) ? json_list_filename : input_dir;
         _output_dir = output_dir;
+        _verbosity = verbosity;
     }
 
@@ -66 +50 @@
         }).count();
 
-        System.out.println("####**** Lines with a: " + numAs + ", lines with b: " + numBs);
+        System.out.println("#### Lines with a: " + numAs + ", lines with b: " + numBs);
     */
         long num_ids = json_ids.count();
-        System.out.println("####**** number of IDS: " + num_ids);
+        System.out.println("");
+        System.out.println("############");
+        System.out.println("# number of IDS: " + num_ids);
+        System.out.println("############");
+        System.out.println("");
         
         sc.close();
@@ -81 +69 @@
         //.withType(Integer.class)
 
-        options.addOption(OptionBuilder.withLongOpt("json-filelist")
-                .withDescription("Explicit list of JSON files to read in")
+        options.addOption(OptionBuilder.withLongOpt("verbosity")
+                .withDescription("Set to control the level of debugging output [0=none, 1=some, 2=lots]")
                 .hasArg()
-                .withArgName("f")
+                .withArgName("v")
                 .isRequired(false)
                 .create());
@@ -102 +90 @@
         catch (ParseException e) {
             System.err.println(e.getMessage());
-            System.err.println("Usage: RUN.bat [options] input-dir output-dir");
-            formatter.printHelp("utility-name", options);
+            //System.err.println("Usage: RUN.bat [options] json-file-list.txt input-dir output-dir");
+            formatter.printHelp("RUN.bash/RUN.bat [options] json-file-list.txt input-dir output-dir", options);
             //System.err.println("  Where 'filename.txt' contains a list of JSON files, one per line,");
             //System.err.println("  which use the HathiTrust Extracted Feature JSON format");
@@ -115 +103 @@
 
         //cmd.hasOption("json-filelist")
-        String json_list_filename = cmd.getOptionValue("json-filelist");
-        //int num_cores = Integer.parseInt(num_cores_str);
+        String verbosity_str = cmd.getOptionValue("verbosity","0");
+        int verbosity = Integer.parseInt(verbosity_str);
 
         //System.out.println(inputFilePath);
@@ -125 +113 @@
         String[] filtered_args = cmd.getArgs();
         
-        if (filtered_args.length != 2) {
-                System.err.println("Usage: RUN.bat [options] input-dir output-dir");
-            formatter.printHelp("utility-name", options);
+        if (filtered_args.length != 3) {
+                //System.err.println("Usage: RUN.bat [options] json-filelist.txt input-dir output-dir");
+                formatter.printHelp("RUN.bash/RUN.bat  [options] json-filelist.txt input-dir output-dir", options);
 
             //System.err.println("Usage: RUN.bat [options] input-dir output-dir");
             //System.err.println("  Where 'filename.txt' contains a list of JSON files, one per line,");
             //System.err.println("  which use the HathiTrust Extracted Feature JSON format");
-            System.exit(1);
+                System.exit(1);
         }
-        String input_dir  = filtered_args[0];
-        String output_dir = filtered_args[1];
+        String json_list_filename = filtered_args[0];
+        String input_dir  = filtered_args[1];
+        String output_dir = filtered_args[2];
         
 
@@ -142 +131 @@
         //int num_cores = 2;
 
-        PrepareForIngest prep_for_ingest = new PrepareForIngest(input_dir,json_list_filename,output_dir);
+        PrepareForIngest prep_for_ingest = new PrepareForIngest(input_dir,json_list_filename,output_dir,verbosity);
         prep_for_ingest.exec();