Changeset 31375 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

Timestamp:

2017-01-31T21:35:50+13:00 (7 years ago)

Author:

davidb

Message:

Initial cut at including POS information to solr index

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (4 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -21 +21 @@
     private static final long serialVersionUID = 1L;
 
-    // Following details on number of partitions to use given in 
-    //  "Parallelized collections" section of:
-    //   https://spark.apache.org/docs/2.0.1/programming-guide.html
-    //
-    // For a more detailed discussion see:
-    //   http://blog.cloudera.com/blog/2015/03/how-to-tune-your-apache-spark-jobs-part-2/
-    
     protected static final int DEFAULT_NUM_CORES = 10;
     protected static final int MINIMUM_NUM_PARTITIONS = 10*DEFAULT_NUM_CORES; 
@@ -36 +29 @@
     //protected String _json_list_filename;
     protected String _whitelist_filename;
+    protected String _langmap_directory;
+    
     protected String _solr_url;
     protected String _output_dir;
@@ -49 +44 @@
         boolean use_whitelist = Boolean.getBoolean("wcsa-ef-ingest.use-whitelist");
         _whitelist_filename = (use_whitelist) ?  System.getProperty("wcsa-ef-ingest.whitelist-filename") : null;
+        
+        boolean use_langmap = Boolean.getBoolean("wcsa-ef-ingest.use-langmap");
+        _langmap_directory = (use_langmap) ?  System.getProperty("wcsa-ef-ingest.langmap-directory") : null;
+        
         
         _solr_url   = solr_url;
@@ -110 +109 @@
         boolean strict_file_io = Boolean.getBoolean("wcsa-ef-ingest.strict-file-io");
         
-        PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, 
+        PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, _langmap_directory,
                                                        _solr_url,_output_dir,_verbosity,
                                                        icu_tokenize,strict_file_io);