Changeset 31028 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONMap.java

Timestamp:

2016-11-02T14:17:45+13:00 (7 years ago)

Author:

davidb

Message:

Support for randonly choosing Solr endpoints added in

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONMap.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONMap.java

@@ -1 +1 @@
 package org.hathitrust.extractedfeatures;
+
+import java.util.ArrayList;
 
 import org.apache.spark.api.java.function.Function;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.util.DoubleAccumulator;
+import org.apache.spark.util.LongAccumulator;
 import org.json.JSONObject;
 
@@ -12 +15 @@
     
     protected String _input_dir;
-    protected String _solr_url;
     protected String _output_dir;
     protected int    _verbosity;
     
-    protected DoubleAccumulator _progress_accum;
-    protected double            _progress_step;
+    protected final ArrayList<String> _solr_endpoints;
+    protected final int _solr_endpoints_len;
+    
+    protected LongAccumulator _progress_accum;
+    protected long            _progress_step;
     
     
-    public PerPageJSONMap(String input_dir, String solr_url, String output_dir, int verbosity, 
-                          DoubleAccumulator progress_accum, double progress_step)
+    public PerPageJSONMap(String input_dir, ArrayList<String> solr_endpoints, String output_dir, int verbosity, 
+                          LongAccumulator progress_accum, long progress_step)
     {
         _input_dir  = input_dir;
-        _solr_url   = solr_url;
         _output_dir = output_dir;
         _verbosity  = verbosity;
+        
+        _solr_endpoints   = solr_endpoints;
+        _solr_endpoints_len = _solr_endpoints.size();
         
         _progress_accum = progress_accum;
@@ -45 +52 @@
         }
         
-                    
-        if (_solr_url != null) {
+        String solr_url = null;
+        if (_solr_endpoints_len > 0) {
+            int random_choice = (int)(_solr_endpoints_len * Math.random());
+            solr_url = _solr_endpoints.get(random_choice);
+        }
+                
+        if (solr_url != null) {
             if ((_verbosity >=2) && (random_test)) {
                 System.out.println("==================");
-                System.out.println("Posting to: " + _solr_url);
+                System.out.println("Posting to: " + solr_url);
                 System.out.println("==================");
             }
-            SolrDocJSON.postSolrDoc(_solr_url, solr_add_doc_json);
+            SolrDocJSON.postSolrDoc(solr_url, solr_add_doc_json);
         }