Changeset 31371 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Timestamp:

2017-01-31T00:06:08+13:00 (7 years ago)

Author:

davidb

Message:

Trying to get saveAsSequenceFile working

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForCatalogLangCount.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForCatalogLangCount.java

@@ -8 +8 @@
 import org.apache.commons.cli.*;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+//import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
 import org.apache.spark.api.java.*;
 import org.apache.spark.api.java.function.Function2;
@@ -162 +165 @@
         //json_text_sample_repart_rdd.saveAsTextFile(output_directory);
         //json_text_sample_repart_rdd.saveAsSequenceFile(output_directory);
-        //json_text_sample_repart_rdd.saveAsHadoopFile(output_directory, Text.class, Text.class, SequenceFileOutputFormat.class);
-        
-        
-        JavaPairRDD<String,String> result = json_text_sample_repart_rdd.mapToPair(new ConvertToWritableTypes());
-        result.saveAsHadoopFile(output_directory, String.class, String.class, SequenceFileOutputFormat.class);
+        json_text_sample_repart_rdd.saveAsHadoopFile(output_directory, Text.class, Text.class, SequenceFileOutputFormat.class);
+        
+        //SequenceFileOutputFormat<Text,Text> sfof = new SequenceFileOutputFormat<Text,Text>();
+        // //sfof.setOutputCompressionClass(BZip2Codec.class);
+        // //sfof.setOutputCompressorClass(conf);
+        // json_text_sample_repart_rdd.saveAsNewAPIHadoopFile(output_directory, Text.class, Text.class, sfof);
+        //org.apache.hadoop.mapred.SequenceFileAsBinaryOutputFormat
+        //org.apache.hadoop.mapred.
+        //json_text_sample_repart_rdd.saveAsObjectFile(output_directory);
+        
+        //JavaPairRDD<String,String> result = json_text_sample_repart_rdd.mapToPair(new ConvertToWritableTypes());
+        //result.saveAsHadoopFile(output_directory, String.class, String.class, SequenceFileOutputFormat.class);
         
         jsc.close();