Index: /other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java
===================================================================
--- /other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java	(revision 30969)
+++ /other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java	(revision 30970)
@@ -4,6 +4,10 @@
 import java.io.BufferedWriter;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.Set;
 
 import org.apache.commons.compress.compressors.CompressorException;
@@ -39,6 +43,4 @@
 	protected JSONObject readJSONFile(String filename)
 	{
-		//Path path = Paths.get(filename);
-
 		StringBuilder sb = new StringBuilder();
 
@@ -59,7 +61,144 @@
 		JSONObject json_obj = new JSONObject(sb.toString());
 		
+		
 		return json_obj;
 	}
 
+	protected String generateSolrText(JSONObject ef_token_pos_count)
+	{
+		StringBuilder sb = new StringBuilder();
+
+		Iterator<String> token_iter = ef_token_pos_count.keys();
+		while (token_iter.hasNext()) {
+			String token = token_iter.next();
+		    
+			sb.append(token);
+			if (token_iter.hasNext()) {
+				sb.append(" ");
+			}
+		}
+		
+		/*
+		Set<String> token_keys = ef_token_pos_count.keySet();
+		for (String token : token_keys) {
+				sb.append(token + " ");
+		}
+*/
+		
+	   return sb.toString();
+	}
+	
+	protected JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page) 
+	{
+		JSONObject solr_add_json = null;
+		
+		if (ef_page != null) {
+			JSONObject ef_body = ef_page.getJSONObject("body");
+			if (ef_body != null) {
+				JSONObject ef_token_pos_count = ef_body.getJSONObject("tokenPosCount");
+				if (ef_token_pos_count != null) {
+
+					solr_add_json = new JSONObject();
+
+					String text = generateSolrText(ef_token_pos_count);
+		
+					
+					
+					JSONObject solr_doc_json = new JSONObject();
+					solr_doc_json.put("id", page_id);
+					solr_doc_json.put("volumeid", volume_id);
+					solr_doc_json.put("text_t", text);
+				    solr_doc_json.put("commitWithin", 5000);
+				   
+					solr_add_json.put("add", solr_doc_json);
+				}
+				else {
+					System.err.println("Warning: empty tokenPosCount field for '" + page_id + "'");
+				}
+			}
+			else {
+				System.err.println("Warning: empty body field for '" + page_id + "'");
+			}
+			
+		}
+		else {
+			System.err.println("Warning: null page for '" + page_id + "'");
+		}
+		
+		
+	    /*
+	      
+	     /update/json/docs
+	     */
+	    
+		// For Reference ...
+		// Example documentation on Solr JSON syntax:
+		//   https://cwiki.apache.org/confluence/display/solr/Uploading+Data+with+Index+Handlers
+		//     #UploadingDatawithIndexHandlers-JSONFormattedIndexUpdates
+		
+		/*
+		curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/my_collection/update' --data-binary '
+		{
+		  "add": {
+		    "doc": {
+		      "id": "DOC1",
+		      "my_boosted_field": {         use a map with boost/value for a boosted field 
+		        "boost": 2.3,
+		        "value": "test"
+		      },
+		      "my_multivalued_field": [ "aaa", "bbb" ]    Can use an array for a multi-valued field 
+		    }
+		  },
+		  "add": {
+		    "commitWithin": 5000,           commit this document within 5 seconds 
+		    "overwrite": false,             don't check for existing documents with the same uniqueKey 
+		    "boost": 3.45,                  a document boost 
+		    "doc": {
+		      "f1": "v1",                   Can use repeated keys for a multi-valued field 
+		      "f1": "v2"
+		    }
+		  },
+		 
+		  "commit": {},
+		  "optimize": { "waitSearcher":false },
+		 
+		  "delete": { "id":"ID" },          delete by ID 
+		  "delete": { "query":"QUERY" }     delete by query 
+		}'
+		*/
+		
+		//return solr_doc_json;
+		return solr_add_json;
+	}
+	
+	protected void postSolrDoc(JSONObject solr_add_doc_json)
+	{
+		// "http://10.11.0.53:8983/solr/"
+		String post_url = "http://10.11.0.53:8983/solr/htrc-pd-ef/update";
+			
+		//String curl_popen = "curl -X POST -H 'Content-Type: application/json'";
+		//curl_popen += " 'http://10.11.0.53:8983/solr/htrc-pd-ef/update'";
+		//curl_popen += " --data-binary '";
+		//curl_popen += "'"
+
+
+		try {
+			HttpURLConnection httpcon = (HttpURLConnection) ((new URL(post_url).openConnection()));
+			httpcon.setDoOutput(true);
+			httpcon.setRequestProperty("Content-Type", "application/json");
+			httpcon.setRequestProperty("Accept", "application/json");
+			httpcon.setRequestMethod("POST");
+			httpcon.connect();
+
+			byte[] outputBytes = solr_add_doc_json.toString().getBytes("UTF-8");
+			OutputStream os = httpcon.getOutputStream();
+			os.write(outputBytes);
+			os.close();
+		}
+		catch (Exception e) {
+			e.printStackTrace();
+		}
+		
+	}
 	public Iterator<String> call(String json_file_in) 
 	{ 
@@ -69,7 +208,9 @@
 		
 		
-		String id = extracted_feature_record.getString("id");
+		String volume_id = extracted_feature_record.getString("id");
 		
 		//JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
+		//String title= ef_metadata.getString("title");
+		
 		JSONObject ef_features = extracted_feature_record.getJSONObject("features");
 		
@@ -89,10 +230,9 @@
 		String page_json_dir = json_dir + "/pages";
 		ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
-		//System.out.println("mkdir: " + _output_dir + "/" + page_json_dir);
 		
 		ArrayList<String> ids = new ArrayList<String>(ef_num_pages);
 		for (int i = 0; i < ef_page_count; i++) {
 			String formatted_i = String.format("page-%06d", i);
-			String page_id = id + "." + formatted_i;
+			String page_id = volume_id + "." + formatted_i;
 		
 			if (_verbosity >= 2) {
@@ -107,16 +247,36 @@
 			}
 			
-			// create JSON obj of just the page (for now), and write it out
 			JSONObject ef_page = ef_pages.getJSONObject(i);
-			try {
-				BufferedWriter bw = ClusterFileIO.getBufferedWriterForCompressedFile(_output_dir + "/" + output_json_bz2);
-				bw.write(ef_page.toString());
-				bw.close();
-			} catch (IOException e) {
-				e.printStackTrace();
-			} catch (CompressorException e) {
-				e.printStackTrace();
-			}
-						
+
+			if (ef_page != null) {
+				// Convert to Solr add form
+				JSONObject solr_add_doc_json = generateSolrDocJSON(volume_id, page_id, ef_page);
+
+				if (i==20) {
+					System.out.println("Sample output Solr add JSON [page 20]: " + solr_add_doc_json.toString());
+					System.out.println("==================");
+					//System.out.println("Sample text [page 20]: " + solr_add_doc_json.getString("text_t"));
+				}
+				
+				// create JSON obj of just the page (for now), and write it out
+				// write out the JSONOBject as a bz2 compressed file
+				/*
+				try {
+					BufferedWriter bw = ClusterFileIO.getBufferedWriterForCompressedFile(_output_dir + "/" + output_json_bz2);
+					bw.write(ef_page.toString());
+					bw.close();
+				} catch (IOException e) {
+					e.printStackTrace();
+				} catch (CompressorException e) {
+					e.printStackTrace();
+				}
+				*/
+								
+				//postSolrDoc(solr_add_doc_json);
+			
+			}
+			else {
+				System.err.println("Skipping: " + page_id);
+			}
 			
 		}
@@ -140,5 +300,5 @@
 		
 		
-		ids.add(id);
+		ids.add(volume_id);
 		
 		return ids.iterator();
