Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java	(revision 31219)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java	(revision 31220)
@@ -25,4 +25,7 @@
 	
 	protected String _input_dir;
+	
+	protected WhitelistBloomFilter _whitelist_bloomfilter;
+
 	protected String _solr_url;
 	protected String _output_dir;
@@ -34,9 +37,15 @@
 	boolean _strict_file_io;
 	
-	public PerPageJSONFlatmap(String input_dir, String solr_url, String output_dir, int verbosity, 
+	public PerPageJSONFlatmap(String input_dir, String whitelist_filename, 
+							  String solr_url, String output_dir, int verbosity, 
 					 		  DoubleAccumulator progress_accum, double progress_step,
 					 		  boolean strict_file_io)
 	{
 		_input_dir  = input_dir;
+		
+		if (whitelist_filename != null) {
+			_whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
+		}
+		
 		_solr_url   = solr_url;
 		_output_dir = output_dir;
@@ -51,11 +60,8 @@
 	
 	public Iterator<JSONObject> call(String json_file_in) throws IOException
-	//public void call(String json_file_in) 
 	{ 
-	        //ClusterFileIO.memory_usage("Before BZIP2 JSON file read");
-		String full_json_file_in = _input_dir + "/" + json_file_in;
+	 	String full_json_file_in = _input_dir + "/" + json_file_in;
 		JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(full_json_file_in);
-		//ClusterFileIO.memory_usage("After BZIP2 JSON file read");
-
+	
 		ArrayList<JSONObject> json_pages = new ArrayList<JSONObject>();
 		
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java	(revision 31219)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java	(revision 31220)
@@ -20,4 +20,7 @@
 	private static final long serialVersionUID = 1L;
 	protected String _input_dir;
+
+	protected WhitelistBloomFilter _whitelist_bloomfilter;
+
 	protected String _solr_url;
 	protected String _output_dir;
@@ -27,8 +30,14 @@
 	protected double            _progress_step;
 	
-	public PerVolumeJSON(String input_dir, String solr_url, String output_dir, int verbosity, 
+	public PerVolumeJSON(String input_dir, String whitelist_filename,
+				         String solr_url, String output_dir, int verbosity, 
 					     DoubleAccumulator progress_accum, double progress_step)
 	{
 		_input_dir  = input_dir;
+		
+		if (whitelist_filename != null) {
+			_whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
+		}
+		
 		_solr_url   = solr_url;
 		_output_dir = output_dir;
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java	(revision 31219)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java	(revision 31220)
@@ -33,4 +33,5 @@
 	protected String _input_dir;
 	protected String _json_list_filename;
+	protected String _whitelist_filename;
 	protected String _solr_url;
 	protected String _output_dir;
@@ -44,4 +45,7 @@
 		_json_list_filename = (json_list_filename != null) ? json_list_filename : input_dir;
 
+		boolean use_whitelist = Boolean.getBoolean("wcsa-ef-ingest.use-whitelist");
+		_whitelist_filename = (use_whitelist) ?  System.getProperty("wcsa-ef-ingest.whitelist-filename") : null;
+		
 		_solr_url   = solr_url;
 		_output_dir = output_dir;
@@ -117,5 +121,6 @@
 		System.err.println();
 		
-		PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+		PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, 
+											           _solr_url,_output_dir,_verbosity, progress_accum,per_vol);
 
 		json_list_data.foreach(per_vol_json);
@@ -160,5 +165,6 @@
 				
 		PerPageJSONFlatmap paged_solr_json_flatmap 
-			= new PerPageJSONFlatmap(_input_dir,_solr_url,_output_dir,_verbosity, 
+			= new PerPageJSONFlatmap(_input_dir,_whitelist_filename,
+								     _solr_url,_output_dir,_verbosity, 
 								     per_vol_progress_accum,per_vol,
 								     strict_file_io);
@@ -309,16 +315,5 @@
 		ProcessForSolrIngest prep_for_ingest 
 			= new ProcessForSolrIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity);
-		
-		
-		boolean use_whitelist = Boolean.getBoolean("wcsa-ef-ingest.use-whitelist");
-				
-		if (use_whitelist) {
-			String whitelist_filename = System.getProperty("wcsa-ef-ingest.whitelist-filename");
-
-			WhitelistBloomFilter whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
-			whitelist_bloomfilter.contains("foo");
-		}
-		
-		
+			
 		String process_ef_json_mode = System.getProperty("wcsa-ef-ingest.process-ef-json-mode","per-page");
 		if (process_ef_json_mode.equals("per-volume")) { 
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java	(revision 31219)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java	(revision 31220)
@@ -16,18 +16,32 @@
 public class SolrDocJSON {
 
-	protected static String generateSolrText(JSONObject ef_token_pos_count)
+	protected static String generateSolrText(JSONObject ef_token_pos_count, WhitelistBloomFilter whitelist_bloomfilter)
 		{
 			StringBuilder sb = new StringBuilder();
 	
 			Iterator<String> token_iter = ef_token_pos_count.keys();
-			while (token_iter.hasNext()) {
-				String token = token_iter.next();
-			    
-				sb.append(token);
-				if (token_iter.hasNext()) {
-					sb.append(" ");
-				}
-			}
-			
+		    
+			if (whitelist_bloomfilter == null) {
+
+				while (token_iter.hasNext()) {
+					String token = token_iter.next();
+					sb.append(token);
+					if (token_iter.hasNext()) {
+						sb.append(" ");
+					}
+				}
+			}
+			else {
+				while (token_iter.hasNext()) {
+					String token = token_iter.next();
+					if (whitelist_bloomfilter.contains(token)) {
+						sb.append(token);
+						if (token_iter.hasNext()) {
+							sb.append(" ");
+						}
+					}
+				}
+				
+			}
 			/*
 			Set<String> token_keys = ef_token_pos_count.keySet();
@@ -40,5 +54,6 @@
 		}
 
-	protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page) 
+	protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page,
+											        WhitelistBloomFilter whitelist_bloomfilter) 
 	{
 		JSONObject solr_update_json = null;
@@ -52,11 +67,15 @@
 					JSONObject solr_add_json = new JSONObject();
 	
-					String text = generateSolrText(ef_token_pos_count);
+					String text = generateSolrText(ef_token_pos_count,whitelist_bloomfilter);
 					
 					JSONObject solr_doc_json = new JSONObject();
 					solr_doc_json.put("id", page_id);
 					solr_doc_json.put("volumeid_s", volume_id);
-					solr_doc_json.put("eftext_txt", text);
-				   
+					if (!text.equals("")) {
+						solr_doc_json.put("eftext_txt", text);
+					}
+					else {
+						solr_doc_json.put("efnotext_b", true);
+					}
 					solr_add_json.put("commitWithin", 5000);
 					solr_add_json.put("doc", solr_doc_json);
