Changeset 31272 for other-projects/hathitrust

Timestamp:

2017-01-04T09:37:56+13:00 (7 years ago)

Author:

davidb

Message:

Use disk and memory to store main language RDD

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• JSONClusterFileIO.java (modified) (2 diffs)
• ProcessForLangCount.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/JSONClusterFileIO.java

@@ -14 +14 @@
             StringBuilder sb = new StringBuilder();
             
-            String str;
             BufferedReader br = ClusterFileIO.getBufferedReaderForCompressedFile(filename);
 
@@ -23 +22 @@
 
             /*
+            String str;
             while ((str = br.readLine()) != null) {
                 sb.append(str);

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForLangCount.java

@@ -11 +11 @@
 import org.apache.spark.api.java.function.Function2;
 import org.apache.spark.api.java.function.PairFunction;
+import org.apache.spark.storage.StorageLevel;
 import org.apache.spark.util.DoubleAccumulator;
 import scala.Tuple2;
@@ -91 +92 @@
                                      strict_file_io);
         JavaRDD<String> lang_list = json_list_data_rp.flatMap(paged_solr_langfreq_flatmap); 
+        lang_list.persist(StorageLevel.MEMORY_AND_DISK());
         lang_list.setName("lang-stream");