Changeset 31260 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Timestamp:

2016-12-21T00:12:10+13:00 (7 years ago)

Author:

davidb

Message:

Language counting

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Files:

• scripts/FULL-RUN-MASTER-SPARK-LANG-COUNT.sh (added)
• src/main/java/org/hathitrust/extractedfeatures/PerVolumeLangStreamFlatmap.java (added)
• src/main/java/org/hathitrust/extractedfeatures/ProcessForLangCount.java (added)
• src/main/java/org/hathitrust/extractedfeatures/ProcessForPOSCount.java (modified) (2 diffs)
• src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForPOSCount.java

@@ -45 +45 @@
     protected String generateSparkAppName(String exec_mode)
     {
-        String spark_app_name = "[" + exec_mode + "] Extracted Features: Process for POS";
+        String spark_app_name = "[" + exec_mode + "] Extracted Features: Process for POS Count";
         spark_app_name += " [" + _json_list_filename + "]";
     
@@ -79 +79 @@
         //boolean icu_tokenize = Boolean.getBoolean("wcsa-ef-ingest.icu-tokenize");
         
-        PerVolumePOSStreamFlatmap paged_solr_wordfreq_flatmap 
+        PerVolumePOSStreamFlatmap paged_solr_posfreq_flatmap 
             = new PerVolumePOSStreamFlatmap(_input_dir,_verbosity, 
                                      per_vol_progress_accum,per_vol,
                                      strict_file_io);
-        JavaRDD<String> pos_list = json_list_data.flatMap(paged_solr_wordfreq_flatmap); 
+        JavaRDD<String> pos_list = json_list_data.flatMap(paged_solr_posfreq_flatmap); 
         pos_list.setName("pos-stream");
         

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -13 +13 @@
 import java.util.Iterator;
 import org.apache.commons.compress.compressors.CompressorException;
+import org.json.JSONArray;
 import org.json.JSONObject;
 import org.apache.lucene.analysis.TokenStream;
@@ -294 +295 @@
     }
     
+    public static ArrayList<String> generateTokenPosCountLangLabels(String volume_id, String page_id, JSONObject ef_page) 
+    {
+        ArrayList<String> lang_list = new ArrayList<String>();;
+
+        if (ef_page != null) {
+            JSONArray ef_languages = ef_page.getJSONArray("languages");
+            if (ef_languages != null) {
+                
+                int lang_len = ef_languages.length();
+                for (int i=0; i<lang_len; i++) {
+                    JSONObject lang_rec = ef_languages.getJSONObject(i);
+
+                    Iterator<String> lang_key_iter = lang_rec.keys();
+                    while (lang_key_iter.hasNext()) {
+                        String lang_label = lang_key_iter.next();
+
+                        lang_list.add(lang_label);
+                    }
+                }
+            }
+            else {
+                System.err.println("Warning: empty languages field for '" + page_id + "'");
+            }
+
+        }
+        else {
+            System.err.println("Warning: null page for '" + page_id + "'");
+        }
+
+        return lang_list;
+    }
+    
     public static void saveSolrDoc(JSONObject solr_add_doc_json, String output_file_json_bz2)
     {