Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/POSString.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/POSString.java	(revision 31375)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/POSString.java	(revision 31375)
@@ -0,0 +1,24 @@
+package org.hathitrust.extractedfeatures;
+
+public class POSString {
+
+	String _str;
+	String[] _pos_tags;
+	
+	public POSString(String str, String[] pos_tags)
+	{
+		_str = str;
+		_pos_tags = pos_tags;
+	}
+	
+	public String getString()
+	{
+		return _str;
+	}
+	
+	public String[] getPOSTags()
+	{
+		return _pos_tags;
+	}
+	
+}
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java	(revision 31374)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java	(revision 31375)
@@ -33,5 +33,5 @@
 	
 	protected WhitelistBloomFilter _whitelist_bloomfilter;
-
+	protected UniversalPOSLangMap _universal_langmap = null;
 	
 	protected DoubleAccumulator _progress_accum;
@@ -136,5 +136,7 @@
 					// Convert to Solr add form
 					JSONObject solr_add_doc_json 
-						= SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, _whitelist_bloomfilter,_icu_tokenize);
+						= SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, 
+													_whitelist_bloomfilter, _universal_langmap,
+													_icu_tokenize);
 					solr_add_doc_json.put("filename_json_bz2", output_json_bz2);
 
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java	(revision 31374)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java	(revision 31375)
@@ -3,4 +3,5 @@
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
 
@@ -29,4 +30,5 @@
 	protected String _input_dir;
 	protected String _whitelist_filename;
+	protected String _langmap_directory;
 	
 	protected String _solr_url;
@@ -36,11 +38,10 @@
 	
 	protected WhitelistBloomFilter _whitelist_bloomfilter;
-	
-
-	
-	 boolean _icu_tokenize;
-	 boolean _strict_file_io;
-	 
-	public PerVolumeJSON(String input_dir, String whitelist_filename,
+	protected UniversalPOSLangMap _universal_langmap;
+
+	boolean _icu_tokenize;
+	boolean _strict_file_io;
+
+	public PerVolumeJSON(String input_dir, String whitelist_filename, String langmap_directcory,
 				         String solr_url, String output_dir, int verbosity, 
 					     boolean icu_tokenize, boolean strict_file_io)
@@ -48,4 +49,5 @@
 		_input_dir  = input_dir;
 		_whitelist_filename = whitelist_filename;
+		_langmap_directory = langmap_directcory;
 		
 		_solr_url   = solr_url;
@@ -57,4 +59,5 @@
 		
 		_whitelist_bloomfilter = null;
+		_universal_langmap = null;
 	}
 	
@@ -66,5 +69,8 @@
 			_whitelist_bloomfilter = new WhitelistBloomFilter(_whitelist_filename,true);
 		}
-
+		
+		if ((_langmap_directory != null) && (_universal_langmap == null)) {
+			_universal_langmap = new UniversalPOSLangMap(_langmap_directory);
+		}
 		int ef_num_pages = 0;
 
@@ -107,5 +113,5 @@
 						// Convert to Solr add form
 						JSONObject solr_add_doc_json 
-						= SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, _whitelist_bloomfilter, _icu_tokenize);
+						= SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, _whitelist_bloomfilter, _universal_langmap, _icu_tokenize);
 
 
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java	(revision 31374)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java	(revision 31375)
@@ -21,11 +21,4 @@
 	private static final long serialVersionUID = 1L;
 
-	// Following details on number of partitions to use given in 
-	//  "Parallelized collections" section of:
-	//   https://spark.apache.org/docs/2.0.1/programming-guide.html
-	//
-	// For a more detailed discussion see:
-	//   http://blog.cloudera.com/blog/2015/03/how-to-tune-your-apache-spark-jobs-part-2/
-	
 	protected static final int DEFAULT_NUM_CORES = 10;
 	protected static final int MINIMUM_NUM_PARTITIONS = 10*DEFAULT_NUM_CORES; 
@@ -36,4 +29,6 @@
 	//protected String _json_list_filename;
 	protected String _whitelist_filename;
+	protected String _langmap_directory;
+	
 	protected String _solr_url;
 	protected String _output_dir;
@@ -49,4 +44,8 @@
 		boolean use_whitelist = Boolean.getBoolean("wcsa-ef-ingest.use-whitelist");
 		_whitelist_filename = (use_whitelist) ?  System.getProperty("wcsa-ef-ingest.whitelist-filename") : null;
+		
+		boolean use_langmap = Boolean.getBoolean("wcsa-ef-ingest.use-langmap");
+		_langmap_directory = (use_langmap) ?  System.getProperty("wcsa-ef-ingest.langmap-directory") : null;
+		
 		
 		_solr_url   = solr_url;
@@ -110,5 +109,5 @@
 		boolean strict_file_io = Boolean.getBoolean("wcsa-ef-ingest.strict-file-io");
 		
-		PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, 
+		PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, _langmap_directory,
 											           _solr_url,_output_dir,_verbosity,
 											           icu_tokenize,strict_file_io);
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java	(revision 31374)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java	(revision 31375)
@@ -11,5 +11,8 @@
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.Set;
+
 import org.apache.commons.compress.compressors.CompressorException;
 import org.json.JSONArray;
@@ -88,8 +91,8 @@
 	}
 	
-	protected static ArrayList<String> getTokenPosCountWordsArrayList(JSONObject ef_token_pos_count, String page_id,
+	protected static ArrayList<POSString> getTokenPosCountWordsArrayList(JSONObject ef_token_pos_count, String page_id,
 			boolean icu_tokenize)
 	{
-		ArrayList<String> words = new ArrayList<String>();
+		ArrayList<POSString> words = new ArrayList<POSString>();
 
 		if (ef_token_pos_count != null) {
@@ -99,4 +102,9 @@
 				String word_token = word_token_iter.next();
 
+				JSONObject pos_json_object = ef_token_pos_count.getJSONObject(word_token);
+				Set<String> pos_keys = pos_json_object.keySet();	
+				int pos_keys_len = pos_keys.size();
+				String[] pos_tags = (pos_keys_len>0) ? pos_keys.toArray(new String[pos_keys_len]) : null;
+				
 				if (icu_tokenize == true) {
 					Reader reader = new StringReader(word_token);
@@ -114,5 +122,8 @@
 						while (token_stream.incrementToken()) {
 							String term = charTermAttribute.toString();
-							words.add(term);
+							
+							POSString pos_string = new POSString(term,pos_tags);
+						
+							words.add(pos_string);
 						}
 
@@ -125,5 +136,7 @@
 				}
 				else {
-					words.add(word_token);
+					POSString pos_word_token = new POSString(word_token,pos_tags);
+					
+					words.add(pos_word_token);
 				}
 			}				
@@ -135,9 +148,11 @@
 		return words;
 	}
-	protected static ArrayList<String> getTokenPosCountWordsMapCaseInsensitive(ArrayList<String> words_in)
-	{
-		ArrayList<String> words_out = new ArrayList<String>();
-
-		for (String word: words_in) {
+	protected static ArrayList<POSString> getTokenPosCountWordsMapCaseInsensitive(ArrayList<POSString> words_in)
+	{
+		ArrayList<POSString> words_out = new ArrayList<POSString>();
+
+		for (POSString pos_word: words_in) {
+			String word = pos_word.getString();
+			String[] pos_tags = pos_word.getPOSTags();
 			
 			Reader reader = new StringReader(word);
@@ -154,5 +169,7 @@
 				while (token_stream.incrementToken()) {
 					String term = charTermAttribute.toString();
-					words_out.add(term);
+					
+					POSString pos_term = new POSString(term,pos_tags);
+					words_out.add(pos_term);
 				}
 
@@ -169,13 +186,13 @@
 	}
 	
-	protected static ArrayList<String> getTokenPosCountWordsMapWhitelist(ArrayList<String> words_in,
+	protected static ArrayList<POSString> getTokenPosCountWordsMapWhitelist(ArrayList<POSString> words_in,
 																	     WhitelistBloomFilter whitelist_bloomfilter)
 	{
-		ArrayList<String> words_out = new ArrayList<String>();
-
-		for (String word: words_in) {
-			
+		ArrayList<POSString> words_out = new ArrayList<POSString>();
+
+		for (POSString pos_word: words_in) {
+			String word = pos_word.getString();
 			if (whitelist_bloomfilter.contains(word)) {
-				words_out.add(word);
+				words_out.add(pos_word);
 			}
 		}
@@ -259,11 +276,13 @@
 	}
 	
-	protected static ArrayList<String> filterSolrTextFields(JSONObject ef_token_pos_count, String page_id,
-											   WhitelistBloomFilter whitelist_bloomfilter, boolean icu_tokenize)
-	{
-		ArrayList<String> cs_tokens = getTokenPosCountWordsArrayList(ef_token_pos_count, page_id,icu_tokenize);
-		ArrayList<String> lc_tokens = getTokenPosCountWordsMapCaseInsensitive(cs_tokens);
-		
-		ArrayList<String> tokens = null;
+	protected static ArrayList<POSString> filterSolrTextFields(JSONObject ef_token_pos_count, String page_id,
+											   WhitelistBloomFilter whitelist_bloomfilter, 
+											   UniversalPOSLangMap universal_langmap,
+											   boolean icu_tokenize)
+	{
+		ArrayList<POSString> cs_tokens = getTokenPosCountWordsArrayList(ef_token_pos_count, page_id,icu_tokenize);
+		ArrayList<POSString> lc_tokens = getTokenPosCountWordsMapCaseInsensitive(cs_tokens);
+		
+		ArrayList<POSString> tokens = null;
 		if (whitelist_bloomfilter != null) {
 			tokens =  getTokenPosCountWordsMapWhitelist(lc_tokens,whitelist_bloomfilter);
@@ -276,5 +295,6 @@
 	}
 
-	protected static void addSolrLanguageTextFields(JSONObject ef_page, ArrayList<String> text_al,
+	protected static void addSolrLanguageTextFields(JSONObject ef_page, ArrayList<POSString> text_al,
+													UniversalPOSLangMap universal_langmap,
 												    JSONObject solr_doc_json)
 	{
@@ -293,6 +313,5 @@
 					String lang_label = lang_key_iter.next();
 
-					String solr_field = lang_label + "_htrctoken";
-					lang_list[i] = solr_field;
+					lang_list[i] = lang_label;
 				}
 			}
@@ -301,12 +320,54 @@
 			
 			for (int li=0; li<lang_len; li++) {
-				String lang_text_field = lang_list[li];
-			
-				JSONArray json_values = new JSONArray();
-				for (int ti=0; ti<text_len; ti++) {
-					String text_value = text_al.get(ti);
-					json_values.put(text_value);
-				}
-				solr_doc_json.put(lang_text_field, json_values); 
+				String lang_key = lang_list[li];
+				
+				if (universal_langmap.containsLanguage(lang_key))
+				{
+					HashMap<String,JSONArray> pos_lang_text_field_map = new HashMap<String,JSONArray>();
+					
+					for (int ti=0; ti<text_len; ti++) {
+						POSString pos_text_value = text_al.get(ti);
+						String text_value = pos_text_value.getString();
+						
+						String[] pos_tags = pos_text_value.getPOSTags();
+						int pos_tags_len = pos_tags.length;
+						
+						for (int pti=0; ti<pos_tags_len; pti++) {
+							String opennlp_pos_key = pos_tags[pti];
+							
+							String upos = universal_langmap.getUniversalLanguagePOS(lang_key, opennlp_pos_key);
+							String pos_lang_text_field = lang_key + "_" + upos + "_htrctoken";
+							
+							if (!pos_lang_text_field_map.containsKey(pos_lang_text_field)) {
+								JSONArray empty_json_values = new JSONArray();
+								pos_lang_text_field_map.put(pos_lang_text_field, empty_json_values);
+							}
+							pos_lang_text_field_map.get(pos_lang_text_field).put(text_value);
+						}
+					}
+
+					// Now add each of the POS language fields into solr_doc_json
+					Set<String> pos_lang_field_keys = pos_lang_text_field_map.keySet();
+					for (String plf_key : pos_lang_field_keys) {
+						String lang_text_field = plf_key;
+						JSONArray json_values = pos_lang_text_field_map.get(plf_key);
+						
+						solr_doc_json.put(lang_text_field, json_values); 
+					}
+				}
+				else {
+					String lang_text_field = lang_key + "_htrctoken";
+					
+					JSONArray json_values = new JSONArray();
+					for (int ti=0; ti<text_len; ti++) {
+						POSString pos_text_value = text_al.get(ti);
+						String text_value = pos_text_value.getString();
+						json_values.put(text_value);
+					}
+					solr_doc_json.put(lang_text_field, json_values); 
+					
+				}
+				
+				
 			}
 			
@@ -314,5 +375,7 @@
 	}
 	protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page,
-											        WhitelistBloomFilter whitelist_bloomfilter, boolean icu_tokenize) 
+											        WhitelistBloomFilter whitelist_bloomfilter, 
+											        UniversalPOSLangMap universal_langmap,
+											        boolean icu_tokenize) 
 	{
 		JSONObject solr_update_json = null;
@@ -326,5 +389,5 @@
 					JSONObject solr_add_json = new JSONObject();
 	
-					ArrayList<String> text_al = filterSolrTextFields(ef_token_pos_count,page_id,whitelist_bloomfilter,icu_tokenize);
+					ArrayList<POSString> text_al = filterSolrTextFields(ef_token_pos_count,page_id,whitelist_bloomfilter,universal_langmap,icu_tokenize);
 					
 					JSONObject solr_doc_json = new JSONObject();
@@ -332,5 +395,5 @@
 					solr_doc_json.put("volumeid_s", volume_id);
 					if (text_al.size()>0) {
-						addSolrLanguageTextFields(ef_page,text_al, solr_doc_json);
+						addSolrLanguageTextFields(ef_page,text_al, universal_langmap, solr_doc_json);
 						//solr_doc_json.put("eftext_txt", text_al.toString()); // ****
 					}
@@ -502,4 +565,6 @@
 
 
+		System.out.println("**** post_url = " + post_url);
+		
 		try {
 			HttpURLConnection httpcon = (HttpURLConnection) ((new URL(post_url).openConnection()));
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/UniversalPOSLangMap.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/UniversalPOSLangMap.java	(revision 31375)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/UniversalPOSLangMap.java	(revision 31375)
@@ -0,0 +1,79 @@
+package org.hathitrust.extractedfeatures;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class UniversalPOSLangMap 
+{
+
+	protected HashMap<String,HashMap<String,String>> _all_langmaps;
+	
+	public UniversalPOSLangMap(String langmap_directory) {
+		System.out.println("Constructing: UniversalPOS Language Map");
+		
+		_all_langmaps = new HashMap<String,HashMap<String,String>>();
+		 
+		List<Path> langmap_paths = null;
+		
+		try (Stream<Path> stream_paths = Files.walk(Paths.get(langmap_directory))) {
+			langmap_paths = stream_paths
+	                .filter(Files::isRegularFile)
+	                .collect(Collectors.toList());
+
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+		
+		// For-each language file
+		langmap_paths.forEach(langmap_path -> {
+			File langmap_file = langmap_path.toFile();
+			String lang_key = langmap_file.getName().substring(0,2);
+			
+			HashMap<String,String> pos_lookup = new HashMap<String,String>();
+			
+			// For-each line within that language file
+			try (Stream<String> lang_lines = Files.lines(langmap_path)) {
+				lang_lines.forEach(line -> {
+					String[] line_parts = line.split("\\t");
+					if (line_parts.length == 2) {
+						String pos_key = line_parts[0];
+						String pos_val = line_parts[1];
+						pos_lookup.put(pos_key, pos_val);
+					}
+				});
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+
+			_all_langmaps.put(lang_key, pos_lookup);
+		});
+		
+	
+	
+
+	}
+	public boolean containsLanguage(String lang_key)
+	{
+		return _all_langmaps.containsKey(lang_key);
+	}
+
+	public String getUniversalLanguagePOS(String lang_key,String opennlp_pos_key)
+	{
+		String universal_pos = null;
+		
+		HashMap<String,String> langmap = _all_langmaps.get(lang_key);
+		if (langmap != null) {
+			universal_pos = langmap.get(opennlp_pos_key);
+		}
+		
+		return universal_pos;
+	} 
+	
+}
