Changeset 31255


Ignore:
Timestamp:
12/20/16 15:37:26 (4 years ago)
Author:
davidb
Message:

Changed to using lambda functions

File:
1 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java

    r31252 r31255  
    9292        counts.setName("word-frequency");
    9393       
     94        /*
    9495        JavaPairRDD<Integer, String> swapped_pair = counts.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() {
    9596               @Override
     
    102103       
    103104        JavaPairRDD<Integer, String> sorted_swapped_pair = swapped_pair.sortByKey(false,num_partitions);
    104        
    105105        sorted_swapped_pair.setName("descending-sorted-frequency-word");
    106106       
     
    112112            });
    113113        sorted_swaped_back_pair.setName("descending-word-frequency");
    114        
    115 /*
    116         JavaPairRDD<Integer, String> counts_swapped_pair
     114        */
     115       
     116
     117        JavaPairRDD<Integer, String> counts_swapped_pair
    117118            = counts.mapToPair(item -> item.swap());
     119        counts_swapped_pair.setName("frequency-word-swap");
     120       
    118121        JavaPairRDD<Integer, String> counts_swapped_pair_sorted
    119             = counts_swapped_pair.sortByKey(true, 1);
    120         JavaPairRDD<String, Integer> count_sorted = counts_swapped_pair_sorted.mapToPair(item -> item.swap());
    121         */
    122        
     122            = counts_swapped_pair.sortByKey(true, num_partitions);
     123        counts_swapped_pair_sorted.setName("descending-sorted-frequency-word");
     124       
     125        JavaPairRDD<String, Integer> count_sorted
     126            = counts_swapped_pair_sorted.mapToPair(item -> item.swap());
     127        count_sorted.setName("descending-word-frequency");
    123128       
    124129        String filename_root = _json_list_filename.replaceAll(".*/","").replaceAll("\\..*$","");
    125130        String output_directory = "whitelist-" + filename_root + "-out";
    126         sorted_swaped_back_pair.saveAsTextFile(output_directory);
    127 
     131       
     132        //sorted_swaped_back_pair.saveAsTextFile(output_directory);
     133        count_sorted.saveAsTextFile(output_directory);
     134       
    128135       
    129136        //System.out.println("");
Note: See TracChangeset for help on using the changeset viewer.