Index: other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java
===================================================================
--- other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java	(revision 31244)
+++ other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java	(revision 31245)
@@ -24,15 +24,16 @@
 public class SolrDocJSON {
 
-	protected static String generateSolrText(JSONObject ef_token_pos_count, WhitelistBloomFilter whitelist_bloomfilter)
-		{
-			boolean solr_icu_tokenize = true;
-		
-		    ArrayList<String> tokens = new ArrayList<String>();
-		    
+	protected static ArrayList<String> getTokenPosCountWords(JSONObject ef_token_pos_count, String page_id)
+	{
+		boolean solr_icu_tokenize = true;
+		
+	    ArrayList<String> words = new ArrayList<String>();
+		
+		if (ef_token_pos_count != null) {
+
 			Iterator<String> token_iter = ef_token_pos_count.keys();
-		    
 			while (token_iter.hasNext()) {
 				String token = token_iter.next();
-
+				
 				if (solr_icu_tokenize == true) {
 					Reader reader = new StringReader(token);
@@ -48,5 +49,5 @@
 						while (tokenizer.incrementToken()) {
 						    String term = charTermAttribute.toString();
-						    tokens.add(term);
+						    words.add(term);
 						}
 						
@@ -59,17 +60,52 @@
 				}
 				else {
-					tokens.add(token);
-				}
-			}
-			
-			StringBuilder sb = new StringBuilder();
-			
-			if (whitelist_bloomfilter == null) {
-
-				boolean first_append = true;
-				
-				for (int i=0; i<tokens.size(); i++) {
-					String token = tokens.get(i);
-									
+					words.add(token);
+				}
+			}				
+		}
+		else {
+			System.err.println("Warning: empty tokenPosCount field for '" + page_id + "'");
+		}
+		
+		/* Alternative way to get at keys
+		Set<String> token_keys = ef_token_pos_count.keySet();
+		for (String token : token_keys) {
+				sb.append(token + " ");
+		}
+*/
+		return words;
+	}
+	
+	
+	protected static String generateSolrText(JSONObject ef_token_pos_count, String page_id,
+											WhitelistBloomFilter whitelist_bloomfilter)
+	{
+		ArrayList<String> tokens = getTokenPosCountWords(ef_token_pos_count, page_id);
+
+		StringBuilder sb = new StringBuilder();
+
+		if (whitelist_bloomfilter == null) {
+
+			boolean first_append = true;
+
+			for (int i=0; i<tokens.size(); i++) {
+				String token = tokens.get(i);
+
+				if (!first_append) {
+					sb.append(" ");
+				}
+				else {
+					first_append = false;
+				}
+				sb.append(token);
+			}
+		}
+		else {
+			boolean first_append = true;
+
+			for (int i=0; i<tokens.size(); i++) {
+				String token = tokens.get(i);
+
+				if (whitelist_bloomfilter.contains(token)) {
 					if (!first_append) {
 						sb.append(" ");
@@ -79,33 +115,12 @@
 					}
 					sb.append(token);
-				}
-			}
-			else {
-				boolean first_append = true;
-				
-				for (int i=0; i<tokens.size(); i++) {
-					String token = tokens.get(i);
-					
-					if (whitelist_bloomfilter.contains(token)) {
-						if (!first_append) {
-							sb.append(" ");
-						}
-						else {
-							first_append = false;
-						}
-						sb.append(token);
-					}					
-				}
-					
-			}
-			/*
-			Set<String> token_keys = ef_token_pos_count.keySet();
-			for (String token : token_keys) {
-					sb.append(token + " ");
-			}
-	*/
-			
-		   return sb.toString();
-		}
+				}					
+			}
+
+		}
+
+
+		return sb.toString();
+	}
 
 	protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page,
@@ -122,5 +137,5 @@
 					JSONObject solr_add_json = new JSONObject();
 	
-					String text = generateSolrText(ef_token_pos_count,whitelist_bloomfilter);
+					String text = generateSolrText(ef_token_pos_count,page_id,whitelist_bloomfilter);
 					
 					JSONObject solr_doc_json = new JSONObject();
@@ -200,5 +215,5 @@
 	protected static ArrayList<String> generateTokenPosCountText(String volume_id, String page_id, JSONObject ef_page) 
 	{
-		ArrayList<String> word_list = new ArrayList<String>();
+		ArrayList<String> word_list = null;
 		
 		if (ef_page != null) {
@@ -206,15 +221,5 @@
 			if (ef_body != null) {
 				JSONObject ef_token_pos_count = ef_body.getJSONObject("tokenPosCount");
-				if (ef_token_pos_count != null) {
-	
-					Iterator<String> token_iter = ef_token_pos_count.keys();
-					while (token_iter.hasNext()) {
-						String token = token_iter.next();
-						word_list.add(token);
-					}				
-				}
-				else {
-					System.err.println("Warning: empty tokenPosCount field for '" + page_id + "'");
-				}
+				word_list = getTokenPosCountWords(ef_token_pos_count,page_id);
 			}
 			else {
