Changeset 31028 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

Timestamp:

2016-11-02T14:17:45+13:00 (7 years ago)

Author:

davidb

Message:

Support for randonly choosing Solr endpoints added in

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (5 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -6 +6 @@
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.ArrayList;
+
 import org.apache.commons.cli.*;
 
 import org.apache.spark.api.java.*;
 import org.apache.spark.util.DoubleAccumulator;
+import org.apache.spark.util.LongAccumulator;
 import org.hathitrust.extractedfeatures.PerPageJSONFlatmap;
 import org.json.JSONObject;
@@ -103 +106 @@
         jsc.close();
     }
-
+    public ArrayList<String> extrapolateSolrEndpoints()
+    {
+        ArrayList<String> solr_endpoints = new ArrayList<String>();
+        
+        String solr_endpoint_ips = System.getProperty("wcsa-ef-ingest.solr-endpoint-ips",null);
+        if (solr_endpoint_ips != null) {
+            String [] ips = solr_endpoint_ips.split(",");
+            for (String ip : ips) {
+                String solr_endpoint = _solr_url.replaceFirst("//.*?:", "//"+ip+":");
+                solr_endpoints.add(solr_endpoint);
+            }
+        }
+        else {
+            if (_solr_url != null) {
+                solr_endpoints.add(_solr_url);
+            }
+        }
+
+        return solr_endpoints;
+    }
+    
     public void execPerPage()
     {   
@@ -129 +152 @@
         //long num_page_ids = per_page_jsonobjects.count(); // trigger lazy eval of: flatmap:per-vol
 
-        DoubleAccumulator per_page_progress_accum = jsc.sc().doubleAccumulator("Pages Processed");
+        LongAccumulator per_page_progress_accum = jsc.sc().longAccumulator("Pages Processed");
+        ArrayList<String> solr_endpoints = extrapolateSolrEndpoints();
         
         PerPageJSONMap paged_json_id_map 
-            = new PerPageJSONMap(_input_dir,_solr_url,_output_dir,_verbosity, per_page_progress_accum,1.0);
+            = new PerPageJSONMap(_input_dir,solr_endpoints,_output_dir,_verbosity, per_page_progress_accum,1);
         JavaRDD<String> per_page_ids = per_page_jsonobjects.map(paged_json_id_map);
 
@@ -219 +243 @@
 
         
-        String verbosity_str = cmd.getOptionValue("verbosity","0");
+        String verbosity_str = cmd.getOptionValue("verbosity","1");
         int verbosity = Integer.parseInt(verbosity_str);
 
@@ -269 +293 @@
             = new ProcessForSolrIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity);
         
-        String process_json_mode = System.getProperty("ef-solr.process-json-mode","per-page");
-        if (process_json_mode.equals("per-volume")) { 
+        String process_ef_json_mode = System.getProperty("wcsa-ef-ingest.process-ef-json-mode","per-page");
+        if (process_ef_json_mode.equals("per-volume")) { 
             prep_for_ingest.execPerVolume();
         }