Changeset 31362 for other-projects

Timestamp:

2017-01-27T16:38:08+13:00 (7 years ago)

Author:

davidb

Message:

use Spark sample() to make for smaller test with Sequence files

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForCatalogLangCount.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForCatalogLangCount.java

@@ -122 +122 @@
         String packed_sequence_path = "hdfs:///user/capitanu/data/packed-ef";
 
-        JavaPairRDD<String, Text> inputRdd = jsc.sequenceFile(packed_sequence_path, String.class, Text.class);
-        JavaRDD<Text> jsonTextRdd = inputRdd.map(Tuple2::_2);
-
+        JavaPairRDD<Text, Text> input_pair_rdd = jsc.sequenceFile(packed_sequence_path, Text.class, Text.class);
+        //JavaRDD<Text> jsonTextRdd = input_pair_rdd.map(Tuple2::_2);
+        JavaRDD<Text> json_text_rdd = input_pair_rdd.map(item -> item._2);
+        
+        JavaRDD<Text> json_text_sample_rdd = json_text_rdd.sample(false,0.0001);
+        
         /*
         jsonTextRdd.map(
@@ -135 +138 @@
         PerVolumeCatalogLangSequenceFileMap volume_catalog_langfreq_map
             = new PerVolumeCatalogLangSequenceFileMap(_input_dir,_verbosity,strict_file_io);
-        JavaRDD<String> catalog_lang_list = jsonTextRdd.map(volume_catalog_langfreq_map); 
+        JavaRDD<String> catalog_lang_list = json_text_sample_rdd.map(volume_catalog_langfreq_map); 
         //catalog_lang_list.persist(StorageLevel.MEMORY_AND_DISK());
         catalog_lang_list.setName("catalog-lang-stream");