Index: /other-projects/hathitrust/solr-extracted-features/trunk/COMPILE.bat
===================================================================
--- /other-projects/hathitrust/solr-extracted-features/trunk/COMPILE.bat	(revision 30898)
+++ /other-projects/hathitrust/solr-extracted-features/trunk/COMPILE.bat	(revision 30898)
@@ -0,0 +1,4 @@
+
+
+::mvn package
+mvn assembly:assembly -DdescriptorId=jar-with-dependencies
Index: /other-projects/hathitrust/solr-extracted-features/trunk/RUN.bat
===================================================================
--- /other-projects/hathitrust/solr-extracted-features/trunk/RUN.bat	(revision 30898)
+++ /other-projects/hathitrust/solr-extracted-features/trunk/RUN.bat	(revision 30898)
@@ -0,0 +1,9 @@
+
+::spark-submit --class SimpleApp --master local[2] target\htrc-ef-ingest-0.9.jar %1 %2 %3 %4
+
+::spark-submit --class PrepareForIngest --master local[2] target\htrc-ef-ingest-0.9.jar %1 %2 %3 %4
+
+
+spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target\htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=quick-test.txt json-files solr-files %1 %2 %3 %4
+
+:: spark-submit --class org.hathitrust.PrepareForIngest --master local[4] target\htrc-ef-ingest-0.9-jar-with-dependencies.jar --json-filelist=pd-file-listing-step1000.txt json-files solr-files %1 %2 %3 %4
Index: /other-projects/hathitrust/solr-extracted-features/trunk/pom.xml
===================================================================
--- /other-projects/hathitrust/solr-extracted-features/trunk/pom.xml	(revision 30898)
+++ /other-projects/hathitrust/solr-extracted-features/trunk/pom.xml	(revision 30898)
@@ -0,0 +1,79 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                      http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <name>HTRC Extracted Features Solr Ingest</name>
+  <url>analytics.hathitrust.org</url>
+  <groupId>org.hathitrust</groupId>
+  <artifactId>htrc-ef-ingest</artifactId>
+  <version>0.9</version>
+  <packaging>jar</packaging>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-core_2.11</artifactId>
+      <version>2.0.1</version>
+      <scope>provided</scope>
+    </dependency>
+
+    <!--
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <version>2.7.1</version>
+    </dependency>
+    -->
+
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+      <version>1.10</version>
+    </dependency>
+    
+    <dependency>
+      <groupId>org.json</groupId>
+      <artifactId>json</artifactId>
+      <version>20160212</version>
+    </dependency>
+    
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+      <version>1.2</version>
+    </dependency>
+
+  </dependencies>
+
+
+   <build>
+     <plugins>
+       <!--
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>2.3.2</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+-->
+	    
+       <plugin>
+	 <groupId>org.apache.maven.plugins</groupId>
+	 <artifactId>maven-assembly-plugin</artifactId>
+	 <configuration>
+	   <archive>
+	     <manifest>
+               <mainClass>org.hathitrust.PrepareForIngest</mainClass>
+	     </manifest>
+	   </archive>
+	 </configuration>
+       </plugin>
+     </plugins>
+   </build>
+  
+</project>
Index: /other-projects/hathitrust/solr-extracted-features/trunk/scripts/PD-DOWNLOAD-EVERY-1000.sh
===================================================================
--- /other-projects/hathitrust/solr-extracted-features/trunk/scripts/PD-DOWNLOAD-EVERY-1000.sh	(revision 30898)
+++ /other-projects/hathitrust/solr-extracted-features/trunk/scripts/PD-DOWNLOAD-EVERY-1000.sh	(revision 30898)
@@ -0,0 +1,1 @@
+rsync -pav --progress --files-from=pd-file-listing-step1000.txt data.analytics.hathitrust.org::pd-features/ json-files/.
Index: /other-projects/hathitrust/solr-extracted-features/trunk/scripts/PD-SELECT-EVERY-1000.sh
===================================================================
--- /other-projects/hathitrust/solr-extracted-features/trunk/scripts/PD-SELECT-EVERY-1000.sh	(revision 30898)
+++ /other-projects/hathitrust/solr-extracted-features/trunk/scripts/PD-SELECT-EVERY-1000.sh	(revision 30898)
@@ -0,0 +1,1 @@
+sed -n '1~1000p' pd-file-listing.txt > pd-file-listing-step1000.txt 
Index: /other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java
===================================================================
--- /other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java	(revision 30898)
+++ /other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java	(revision 30898)
@@ -0,0 +1,140 @@
+package org.hathitrust;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.spark.api.java.function.FlatMapFunction;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+/*
+class PagedJSON implements Function<String, Boolean> {
+
+	private static final long serialVersionUID = 1L;
+
+	public Boolean call(String s) { return s.contains("a"); }
+}
+ */
+
+class PagedJSON implements FlatMapFunction<String, String> 
+{
+	private static final long serialVersionUID = 1L;
+
+	protected String _input_dir;
+	
+	public PagedJSON(String input_dir)
+	{
+		_input_dir = input_dir;
+	}
+	
+	protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn) 
+			throws FileNotFoundException, UnsupportedEncodingException, CompressorException {
+	    FileInputStream fin = new FileInputStream(fileIn);
+	    BufferedInputStream bis = new BufferedInputStream(fin);
+	    CompressorInputStream input = new CompressorStreamFactory().createCompressorInputStream(bis);
+	    BufferedReader br2 = new BufferedReader(new InputStreamReader(input,"UTF8"));
+	    return br2;
+	}
+	
+	protected JSONObject readJSONFile(String filename)
+	{
+		//Path path = Paths.get(filename);
+
+		StringBuilder sb = new StringBuilder();
+
+		try {
+
+			String str;
+			BufferedReader br = getBufferedReaderForCompressedFile(_input_dir + "/" + filename);
+			while ((str = br.readLine()) != null) {
+				sb.append(str);
+			    //System.out.println(str);
+			}
+
+	        br.close();
+	        
+	        //System.err.println("*****" + sb.toString());
+	               
+	        /*
+			List<String> lines = Files.readAllLines(path,StandardCharsets.UTF_8);
+
+
+			for (String line : lines) {
+				sb.append(line);
+
+			}
+			*/
+	        
+		} 
+		catch (Exception e) {
+			e.printStackTrace();
+		}
+
+		JSONObject json_obj = new JSONObject(sb.toString());
+		
+		
+		return json_obj;
+		
+		//return sb.toString();
+	}
+
+	public Iterator<String> call(String s) 
+	{ 
+		JSONObject extracted_feature_record = readJSONFile(s);
+		
+		String id = extracted_feature_record.getString("id");
+		
+		JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
+		JSONObject ef_features = extracted_feature_record.getJSONObject("features");
+		
+		
+		int ef_page_count = ef_features.getInt("pageCount");
+		
+		JSONArray ef_pages = ef_features.getJSONArray("pages");
+		int ef_num_pages = ef_pages.length();
+		
+		ArrayList<String> ids = new ArrayList<String>(ef_num_pages);
+		for (int i = 0; i < ef_page_count; i++) {
+			ids.add(id + "." + i);
+		}
+		
+		/*
+		for (int i = 0; i < ef_num_pages; i++)
+		{
+		    //String post_id = ef_pages.getJSONObject(i).getString("post_id");
+		    //......
+		}
+		*/
+		//String pageName = json_obj.getJSONObject("pageInfo").getString("pageName");
+/*
+		JSONArray arr = obj.getJSONArray("posts");
+		for (int i = 0; i < arr.length(); i++)
+		{
+		    String post_id = arr.getJSONObject(i).getString("post_id");
+		    ......
+		}
+*/
+		
+		
+		ids.add(id);
+		
+		return ids.iterator();
+	}
+}
+
Index: /other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java
===================================================================
--- /other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java	(revision 30898)
+++ /other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java	(revision 30898)
@@ -0,0 +1,145 @@
+package org.hathitrust;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+import org.apache.commons.cli.*;
+
+import org.apache.spark.api.java.*;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.Function;
+
+public class PrepareForIngest implements Serializable
+{
+	private static final long serialVersionUID = 1L;
+
+/*
+	class ContainsA implements Function<String, Boolean> {
+		
+		private static final long serialVersionUID = 1L;
+
+		public Boolean call(String s) { return s.contains("a"); }
+	}
+
+	class ConvertJSON implements Function<String, Boolean> {
+
+		private static final long serialVersionUID = 1L;
+
+		public Boolean call(String s) { return s.contains("a"); }
+	}
+*/
+
+	//protected int _num_cores;
+	protected String _input_dir;
+	protected String _json_list_filename;
+    protected String _output_dir;
+    
+	public PrepareForIngest(String input_dir, String json_list_filename, String output_dir)
+	{
+		//_num_cores = num_cores;
+		_input_dir = input_dir;
+		_json_list_filename = json_list_filename;
+		_output_dir = output_dir;
+	}
+
+	public void exec()
+	{	
+		SparkConf conf = new SparkConf().setAppName("HTRC-EF: Prepare for Solr Ingest");
+		JavaSparkContext sc = new JavaSparkContext(conf);
+
+		JavaRDD<String> json_list_data = sc.textFile(_json_list_filename).cache();
+
+		JavaRDD<String> json_ids = json_list_data.flatMap(new PagedJSON(_input_dir));
+		
+
+		//long numAs = json_list_data.filter(new ContainsA()).count();
+
+	
+/*
+		long numBs = json_list_data.filter(new Function<String, Boolean>() {
+			public Boolean call(String s) { return s.contains("b"); }
+		}).count();
+
+		System.out.println("####**** Lines with a: " + numAs + ", lines with b: " + numBs);
+	*/
+		long num_ids = json_ids.count();
+		System.out.println("####**** number of IDS: " + num_ids);
+		
+		sc.close();
+	}
+
+	public static void main(String[] args) {
+
+
+		Options options = new Options();
+
+		//.withType(Integer.class)
+
+		options.addOption(OptionBuilder.withLongOpt("json-filelist")
+				.withDescription("Explicit list of JSON files to read in")
+				.hasArg()
+				.withArgName("f")
+				.isRequired(false)
+				.create());
+
+		//Option num_cores_opt = new Option("n", "num-cores", true, "Number of cores to use");
+		//num_cores_opt.setRequired(false);
+		//options.addOption(num_cores_opt);
+
+		//CommandLineParser parser = new DefaultParser(); // 1.3 and above
+		CommandLineParser parser = new GnuParser();
+		HelpFormatter formatter = new HelpFormatter();
+		CommandLine cmd;
+
+		try {
+			cmd = parser.parse(options, args);
+		}
+		catch (ParseException e) {
+			System.err.println(e.getMessage());
+			System.err.println("Usage: RUN.bat [options] input-dir output-dir");
+			formatter.printHelp("utility-name", options);
+			//System.err.println("  Where 'filename.txt' contains a list of JSON files, one per line,");
+			//System.err.println("  which use the HathiTrust Extracted Feature JSON format");
+
+			System.exit(1);
+			return;
+		}
+
+		//value = ((Integer)cmdLine.getParsedOptionValue("num-cores")).intValue();
+		//value = ((Integer)cmdLine.getOptionValue("num-cores","2")).intValue();
+
+		//cmd.hasOption("json-filelist")
+		String json_list_filename = cmd.getOptionValue("json-filelist","pd-file-listing.txt");
+		//int num_cores = Integer.parseInt(num_cores_str);
+
+		//System.out.println(inputFilePath);
+		//System.out.println(outputFilePath);
+
+
+
+		String[] filtered_args = cmd.getArgs();
+		
+		if (filtered_args.length != 2) {
+			System.err.println("Usage: RUN.bat [options] input-dir output-dir");
+			//System.err.println("  Where 'filename.txt' contains a list of JSON files, one per line,");
+			//System.err.println("  which use the HathiTrust Extracted Feature JSON format");
+			System.exit(1);
+		}
+		String input_dir  = filtered_args[0];
+		String output_dir = filtered_args[1];
+		
+
+		//String json_list_filename = cmd.getArgs()[0]; // args[0];
+		//String json_list_filename = args[0];
+		//int num_cores = 2;
+
+		PrepareForIngest prep_for_ingest = new PrepareForIngest(input_dir,json_list_filename,output_dir);
+		prep_for_ingest.exec();
+
+	}
+}
