Changeset 31371
- Timestamp:
- 2017-01-31T00:06:08+13:00 (7 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForCatalogLangCount.java
r31369 r31371 8 8 import org.apache.commons.cli.*; 9 9 import org.apache.hadoop.io.Text; 10 import org.apache.hadoop.io.compress.BZip2Codec; 11 import org.apache.hadoop.mapreduce.OutputFormat; 10 12 import org.apache.hadoop.mapred.SequenceFileOutputFormat; 13 //import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 11 14 import org.apache.spark.api.java.*; 12 15 import org.apache.spark.api.java.function.Function2; … … 162 165 //json_text_sample_repart_rdd.saveAsTextFile(output_directory); 163 166 //json_text_sample_repart_rdd.saveAsSequenceFile(output_directory); 164 //json_text_sample_repart_rdd.saveAsHadoopFile(output_directory, Text.class, Text.class, SequenceFileOutputFormat.class); 165 166 167 JavaPairRDD<String,String> result = json_text_sample_repart_rdd.mapToPair(new ConvertToWritableTypes()); 168 result.saveAsHadoopFile(output_directory, String.class, String.class, SequenceFileOutputFormat.class); 167 json_text_sample_repart_rdd.saveAsHadoopFile(output_directory, Text.class, Text.class, SequenceFileOutputFormat.class); 168 169 //SequenceFileOutputFormat<Text,Text> sfof = new SequenceFileOutputFormat<Text,Text>(); 170 // //sfof.setOutputCompressionClass(BZip2Codec.class); 171 // //sfof.setOutputCompressorClass(conf); 172 // json_text_sample_repart_rdd.saveAsNewAPIHadoopFile(output_directory, Text.class, Text.class, sfof); 173 //org.apache.hadoop.mapred.SequenceFileAsBinaryOutputFormat 174 //org.apache.hadoop.mapred. 175 //json_text_sample_repart_rdd.saveAsObjectFile(output_directory); 176 177 //JavaPairRDD<String,String> result = json_text_sample_repart_rdd.mapToPair(new ConvertToWritableTypes()); 178 //result.saveAsHadoopFile(output_directory, String.class, String.class, SequenceFileOutputFormat.class); 169 179 170 180 jsc.close();
Note:
See TracChangeset
for help on using the changeset viewer.