Changeset 31451 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Timestamp:

2017-03-02T23:28:38+13:00 (7 years ago)

Author:

davidb

Message:

shift to using solr-base-url and a specified solr-collection

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerVolumeJSON.java (modified) (5 diffs)
• ProcessForSolrIngest.java (modified) (10 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -32 +32 @@
     protected String _langmap_directory;
     
-    protected String _solr_url;
+    protected final ArrayList<String> _solr_endpoints;
+    protected final int _solr_endpoints_len;
+    
+    //protected String _solr_url;
     protected String _output_dir;
     
@@ -44 +47 @@
 
     public PerVolumeJSON(String input_dir, String whitelist_filename, String langmap_directory,
-                         String solr_url, String output_dir, int verbosity, 
+                         ArrayList<String> solr_endpoints, String output_dir, int verbosity, 
                          boolean icu_tokenize, boolean strict_file_io)
     {
@@ -53 +56 @@
         _langmap_directory = langmap_directory;
         
-        _solr_url   = solr_url;
+        _solr_endpoints = solr_endpoints;
+        _solr_endpoints_len = solr_endpoints.size();
+        
+        //_solr_url   = solr_url;
         _output_dir = output_dir;
         _verbosity  = verbosity;
@@ -77 +83 @@
         int ef_num_pages = 0;
 
+        String solr_url = null;
+        if (_solr_endpoints_len > 0) {
+            int random_choice = (int)(_solr_endpoints_len * Math.random());
+            solr_url = _solr_endpoints.get(random_choice);
+        }
+        
         try {
 
@@ -125 +137 @@
 
 
-                        if (_solr_url != null) {
+                        if (solr_url != null) {
                             if ((_verbosity >=2) && (i==20)) {
                                 System.out.println("==================");
-                                System.out.println("Posting to: " + _solr_url);
+                                System.out.println("Posting to: " + solr_url);
                                 System.out.println("==================");
                             }
-                            SolrDocJSON.postSolrDoc(_solr_url, solr_add_doc_json);
+                            SolrDocJSON.postSolrDoc(solr_url, solr_add_doc_json);
                         }
 

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -27 +27 @@
     
     protected String _input_dir;
-    //protected String _json_list_filename;
+    protected String _solr_base_url;
+    protected String _solr_collection;
+    
     protected String _whitelist_filename;
     protected String _langmap_directory;
     
-    protected String _solr_url;
+    //protected String _solr_url;
     protected String _output_dir;
     
     protected int    _verbosity;
 
-    public ProcessForSolrIngest(String input_dir, /*String json_list_filename,*/ 
-                                String solr_url, String output_dir, int verbosity)
+    public ProcessForSolrIngest(String input_dir, String solr_collection,
+                                String solr_base_url, String output_dir, int verbosity)
     {
         _input_dir = input_dir;
-        //_json_list_filename = (json_list_filename != null) ? json_list_filename : input_dir;
-
+        _solr_collection = solr_collection;
+        
         boolean use_whitelist = Boolean.getBoolean("wcsa-ef-ingest.use-whitelist");
         _whitelist_filename = (use_whitelist) ?  System.getProperty("wcsa-ef-ingest.whitelist-filename") : null;
@@ -49 +51 @@
         
         
-        _solr_url   = solr_url;
+        _solr_base_url   = solr_base_url;
         _output_dir = output_dir;
         _verbosity  = verbosity;
@@ -57 +59 @@
     {
         String spark_app_name = "[" + exec_mode + "] Extract Features: Process for Solr Ingest";
-        //spark_app_name += " [" + _json_list_filename + "]";
-
-        if (_solr_url != null) { 
-            spark_app_name += " solr_url=" + _solr_url;
+        spark_app_name += " [" + _solr_collection + "]";
+
+        if (_solr_base_url != null) { 
+            spark_app_name += " solr_base_url=" + _solr_base_url;
         }
         
@@ -70 +72 @@
     }
     
-    public ArrayList<String> extrapolateSolrEndpoints()
+    public ArrayList<String> extrapolateSolrEndpoints(String solr_collection)
     {
         ArrayList<String> solr_endpoints = new ArrayList<String>();
         
-        if (_solr_url != null) {
+        if (_solr_base_url != null) {
+            String solr_url = _solr_base_url + "/" + solr_collection + "/update";
+            
             String solr_cloud_nodes = System.getProperty("wcsa-ef-ingest.solr-cloud-nodes",null);
             if (solr_cloud_nodes != null) {
                 String [] cloud_nodes = solr_cloud_nodes.split(",");
                 for (String cn : cloud_nodes) {
-                    String solr_endpoint = _solr_url.replaceFirst("//.*?:\\d+/", "//"+cn+"/");
+                    String solr_endpoint = solr_url.replaceFirst("//.*?:\\d+/", "//"+cn+"/");
                     solr_endpoints.add(solr_endpoint);
                 }
             }
             else {
-                solr_endpoints.add(_solr_url);
+                solr_endpoints.add(solr_url);
             }
         }
@@ -109 +113 @@
         boolean strict_file_io = Boolean.getBoolean("wcsa-ef-ingest.strict-file-io");
         
+        ArrayList<String> solr_endpoints = extrapolateSolrEndpoints(_solr_collection);
+        
         System.out.println("*** away to create PerVolumeJSON class, _langmap_directory = " + _langmap_directory);
         PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, _langmap_directory,
-                                                       _solr_url,_output_dir,_verbosity,
+                                                       solr_endpoints,_output_dir,_verbosity,
                                                        icu_tokenize,strict_file_io);
 
@@ -250 +256 @@
     public static void print_usage(HelpFormatter formatter, Options options)
     {
-        formatter.printHelp("RUN.bash [options] input-dir json-filelist.txt", options);
+        formatter.printHelp("RUN.bash [options] input-dir solr-collection", options);
     }
     
@@ -271 +277 @@
         options.addOption(output_dir_opt);
         
-        Option solr_url_opt = new Option("u", "solr-url", true, 
-                "If specified, the URL to post the Solr JSON data to");
-        solr_url_opt.setRequired(false);
-        options.addOption(solr_url_opt);
+        Option solr_base_url_opt = new Option("u", "solr-base-url", true, 
+                "If specified, the base URL to post the Solr JSON data to");
+        solr_base_url_opt.setRequired(false);
+        options.addOption(solr_base_url_opt);
         
         Option read_only_opt = new Option("r", "read-only", false, 
@@ -304 +310 @@
     
         String output_dir = cmd.getOptionValue("output-dir",null);
-        String solr_url   = cmd.getOptionValue("solr-url",null);
+        String solr_base_url   = cmd.getOptionValue("solr-base-url",null);
         boolean read_only   = cmd.hasOption("read-only");
         
         String[] filtered_args = cmd.getArgs();
 
-        if (filtered_args.length != 1) {
+        if (filtered_args.length != 2) {
             print_usage(formatter,options);
             System.exit(1);
@@ -331 +337 @@
         }
         
-        if (!read_only && ((output_dir == null) && (solr_url==null))) {
-            System.err.println("Need to specify either --solr-url or --output-dir otherwise generated files are not ingested/saved");
+        if (!read_only && ((output_dir == null) && (solr_base_url==null))) {
+            System.err.println("Need to specify either --solr-base-url or --output-dir otherwise generated files are not ingested/saved");
             print_usage(formatter,options);
             System.exit(1);
@@ -339 +345 @@
             // For this case, need to ensure solr-url and output-dir are null
             output_dir = null;
-            solr_url = null;
+            solr_base_url = null;
         }
         
         String input_dir  = filtered_args[0];
-        //String json_list_filename = filtered_args[1];
+        String solr_collection = filtered_args[1];
         
         ProcessForSolrIngest prep_for_ingest 
-            = new ProcessForSolrIngest(input_dir,/*json_list_filename,*/solr_url,output_dir,verbosity);
+            = new ProcessForSolrIngest(input_dir,solr_collection,solr_base_url,output_dir,verbosity);
             
         prep_for_ingest.execPerVolumeSequenceFile();