Changeset 31502 for other-projects/hathitrust/wcsa

Timestamp:

2017-03-13T14:16:15+13:00 (7 years ago)

Author:

davidb

Message:

Comment out section, useful for controlling a smaller run

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -107 +107 @@
         
         JavaPairRDD<Text, Text> input_pair_rdd = jsc.sequenceFile(packed_sequence_path, Text.class, Text.class);
-    
+        //JavaPairRDD<String, String> input_pair_rdd = jsc.wholeTextFiles(packed_sequence_path);
+
+        //JavaPairRDD<Text, Text> input_pair_sampled_rdd = input_pair_rdd.sample(false,0.5,42);
+
+        //JavaRDD<Text> json_text_rdd = input_pair_sampled_rdd.map(item -> item._2);
+        //JavaRDD<Text> json_text_rdd = input_pair_rdd.map(item -> new Text(item._2));
         JavaRDD<Text> json_text_rdd = input_pair_rdd.map(item -> item._2);