Changeset 31246 for other-projects/hathitrust/wcsa

Timestamp:

2016-12-18T18:25:02+13:00 (7 years ago)

Author:

davidb

Message:

Experimenting with sorting

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java

@@ -95 +95 @@
         });
 
-        
-        //JavaRDD<String> per_page_ids = per_page_jsonobjects.map(paged_json_id_map);
-        
-        //long num_page_ids = per_page_ids.count(); // trigger lazy eval of: flatmap:per-vol -> map:per-page
-
-        counts.saveAsTextFile(_json_list_filename + ".out");
-
-
+        //counts.map(lambda (x,y): (y,x));
+        
+        
+        JavaPairRDD<Integer, String> swappedPair = counts.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() {
+               @Override
+               public Tuple2<Integer, String> call(Tuple2<String, Integer> item) throws Exception {
+                   return item.swap();
+               }
+
+            });
+        JavaPairRDD<Integer, String> sorted_swapped_pair = swappedPair.sortByKey(true, 1); // 1st arg configures ascending sort, 2nd arg configures one task
+
+        JavaPairRDD<String, Integer> sorted_swaped_back_pair = sorted_swapped_pair.mapToPair(new PairFunction<Tuple2<Integer, String>, String, Integer>() {
+               @Override
+               public Tuple2<String, Integer> call(Tuple2<Integer, String> item) throws Exception {
+                   return item.swap();
+               }
+
+            });
+
+        /*
+
+        JavaPairRDD<String, Integer> sorted_counts 
+            = counts.map(item -> item.swap()) // interchanges position of entries in each tuple
+                .sortByKey(true, 1)         // 1st arg configures ascending sort, 2nd arg configures one task
+                .map(item -> item.swap());
+        
+        */
+        
+        
+        //sorted_counts.saveAsTextFile(_json_list_filename + ".out");
+
+        sorted_swaped_back_pair.saveAsTextFile(_json_list_filename + ".out");
+
+        
         //System.out.println("");
         //System.out.println("############");