Changeset 31368 for other-projects

Timestamp:

2017-01-30T10:02:27+13:00 (7 years ago)

Author:

davidb

Message:

downsample-100 added

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForCatalogLangCount.java (modified) (5 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForCatalogLangCount.java

@@ -8 +8 @@
 import org.apache.commons.cli.*;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.spark.api.java.*;
 import org.apache.spark.api.java.function.Function2;
@@ -46 +47 @@
     public void execCatalogLangCountSparkDirect()
     {   
-        SparkConf conf = new SparkConf().setAppName("Spark-Direct + Per Volume: Downsample");
+        SparkConf conf = new SparkConf().setAppName("Spark-Direct + Per Volume");
         JavaSparkContext jsc = new JavaSparkContext(conf);
         
@@ -108 +109 @@
     }
 
-    public void sampleDown()
-    {
-        String spark_app_name = generateSparkAppName("Spark Cluster + Per Volume");     
+    public void sampleDown10000()
+    {
+        String spark_app_name = generateSparkAppName("Spark Cluster + Per Volume: Downsample 10000");       
         
         SparkConf conf = new SparkConf().setAppName(spark_app_name);
@@ -126 +127 @@
         
         
-        
+        jsc.close();
+    }
+    
+    public void sampleDown100()
+    {
+        String spark_app_name = generateSparkAppName("Spark Cluster + Per Volume: Downsample 100");     
+        
+        SparkConf conf = new SparkConf().setAppName(spark_app_name);
+        JavaSparkContext jsc = new JavaSparkContext(conf);
+        jsc.hadoopConfiguration().set("io.compression.codec.bzip2.library", "java-builtin");
+        
+        String packed_sequence_path = "hdfs:///user/capitanu/data/packed-ef";
+
+        JavaPairRDD<Text, Text> input_pair_rdd = jsc.sequenceFile(packed_sequence_path, Text.class, Text.class);
+
+        JavaPairRDD<Text, Text> json_text_sample_rdd = input_pair_rdd.sample(false,0.01,42);
+        
+        JavaPairRDD<Text, Text> json_text_sample_repart_rdd = json_text_sample_rdd.repartition(120);
+        
+        String output_directory = "packed-full-ef-100";
+        //json_text_sample_repart_rdd.saveAsTextFile(output_directory);
+        //json_text_sample_repart_rdd.saveAsSequenceFile(output_directory);
+        json_text_sample_repart_rdd.saveAsHadoopFile(output_directory, Text.class, Text.class, SequenceFileOutputFormat.class);
+        
+        
+        jsc.close();
     }
     public void execCatalogLangCount()
@@ -255 +281 @@
         
         //prep_for_lang.execCatalogLangCount();
-        prep_for_lang.sampleDown();
+        prep_for_lang.sampleDown100();
     
     }