Changeset 31369

Timestamp:

2017-01-30T10:34:01+13:00 (7 years ago)

Author:

davidb

Message:

Trial new save

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForCatalogLangCount.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForCatalogLangCount.java

@@ -130 +130 @@
     }
     
+     public static class ConvertToWritableTypes implements PairFunction<Tuple2<Text, Text>, String, String> {
+            /**
+         * 
+         */
+        private static final long serialVersionUID = 1L;
+
+            public Tuple2<String, String> call(Tuple2<Text, Text> record) {
+              return new Tuple2(record._1.toString(), record._2.toString());
+            }
+          }
+     
     public void sampleDown100()
     {
@@ -146 +157 @@
         JavaPairRDD<Text, Text> json_text_sample_repart_rdd = json_text_sample_rdd.repartition(120);
         
+        //JavaPairRDD<Text, Text> json_text_sample_repart_rdd = json_text_sample_rdd.repartition(120);
+        
         String output_directory = "packed-full-ef-100";
         //json_text_sample_repart_rdd.saveAsTextFile(output_directory);
         //json_text_sample_repart_rdd.saveAsSequenceFile(output_directory);
-        json_text_sample_repart_rdd.saveAsHadoopFile(output_directory, Text.class, Text.class, SequenceFileOutputFormat.class);
-        
-        
+        //json_text_sample_repart_rdd.saveAsHadoopFile(output_directory, Text.class, Text.class, SequenceFileOutputFormat.class);
+        
+        
+        JavaPairRDD<String,String> result = json_text_sample_repart_rdd.mapToPair(new ConvertToWritableTypes());
+        result.saveAsHadoopFile(output_directory, String.class, String.class, SequenceFileOutputFormat.class);
+        
         jsc.close();
     }