Changeset 31255 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java

Timestamp:

2016-12-20T15:37:26+13:00 (7 years ago)

Author:

davidb

Message:

Changed to using lambda functions

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java

@@ -92 +92 @@
         counts.setName("word-frequency");
         
+        /*
         JavaPairRDD<Integer, String> swapped_pair = counts.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() {
                @Override
@@ -102 +103 @@
         
         JavaPairRDD<Integer, String> sorted_swapped_pair = swapped_pair.sortByKey(false,num_partitions);
-        
         sorted_swapped_pair.setName("descending-sorted-frequency-word");
         
@@ -112 +112 @@
             });
         sorted_swaped_back_pair.setName("descending-word-frequency");
-        
-/*
-        JavaPairRDD<Integer, String> counts_swapped_pair 
+        */
+        
+
+        JavaPairRDD<Integer, String> counts_swapped_pair
             = counts.mapToPair(item -> item.swap());
+        counts_swapped_pair.setName("frequency-word-swap");
+        
         JavaPairRDD<Integer, String> counts_swapped_pair_sorted 
-            = counts_swapped_pair.sortByKey(true, 1);
-        JavaPairRDD<String, Integer> count_sorted = counts_swapped_pair_sorted.mapToPair(item -> item.swap());
-        */
-        
+            = counts_swapped_pair.sortByKey(true, num_partitions);
+        counts_swapped_pair_sorted.setName("descending-sorted-frequency-word");
+        
+        JavaPairRDD<String, Integer> count_sorted 
+            = counts_swapped_pair_sorted.mapToPair(item -> item.swap());
+        count_sorted.setName("descending-word-frequency");
         
         String filename_root = _json_list_filename.replaceAll(".*/","").replaceAll("\\..*$","");
         String output_directory = "whitelist-" + filename_root + "-out";
-        sorted_swaped_back_pair.saveAsTextFile(output_directory);
-
+        
+        //sorted_swaped_back_pair.saveAsTextFile(output_directory);
+        count_sorted.saveAsTextFile(output_directory);
+        
         
         //System.out.println("");