Changeset 32106 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

Timestamp:

2018-01-16T22:39:16+13:00 (6 years ago)

Author:

davidb

Message:

Rekindle ability to process a json-filelist.txt using Spark

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -142 +142 @@
     
     /*
-    public void execPerVolume()
+    public void execPerVolumeJSONFileList()
     {   
         String spark_app_name = generateSparkAppName("Per Volume");     
@@ -168 +168 @@
         boolean strict_file_io = Boolean.getBoolean("wcsa-ef-ingest.strict-file-io");
         
-        PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, 
-                                                       _solr_url,_output_dir,_verbosity, progress_accum,per_vol,
-                                                       icu_tokenize,strict_file_io);
+        ArrayList<String> solr_endpoints = extrapolateSolrEndpoints(_solr_collection);
+        
+        //PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, 
+        //                                             _solr_url,_output_dir,_verbosity, progress_accum,per_vol,
+        //                                             icu_tokenize,strict_file_io);
+        PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, _langmap_directory,
+                                                        solr_endpoints,_output_dir,_verbosity,
+                                                        icu_tokenize,strict_file_io);
 
         //json_list_data_rp.foreach(per_vol_json);