Changeset 31255 for other-projects

Show
Ignore:
Timestamp:
20.12.2016 15:37:26 (3 years ago)
Author:
davidb
Message:

Changed to using lambda functions

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java

    r31252 r31255  
    9292        counts.setName("word-frequency"); 
    9393         
     94        /* 
    9495        JavaPairRDD<Integer, String> swapped_pair = counts.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() { 
    9596               @Override 
     
    102103         
    103104        JavaPairRDD<Integer, String> sorted_swapped_pair = swapped_pair.sortByKey(false,num_partitions); 
    104          
    105105        sorted_swapped_pair.setName("descending-sorted-frequency-word"); 
    106106         
     
    112112            }); 
    113113        sorted_swaped_back_pair.setName("descending-word-frequency"); 
    114          
    115 /* 
    116         JavaPairRDD<Integer, String> counts_swapped_pair  
     114        */ 
     115         
     116 
     117        JavaPairRDD<Integer, String> counts_swapped_pair 
    117118            = counts.mapToPair(item -> item.swap()); 
     119        counts_swapped_pair.setName("frequency-word-swap"); 
     120         
    118121        JavaPairRDD<Integer, String> counts_swapped_pair_sorted  
    119             = counts_swapped_pair.sortByKey(true, 1); 
    120         JavaPairRDD<String, Integer> count_sorted = counts_swapped_pair_sorted.mapToPair(item -> item.swap()); 
    121         */ 
    122          
     122            = counts_swapped_pair.sortByKey(true, num_partitions); 
     123        counts_swapped_pair_sorted.setName("descending-sorted-frequency-word"); 
     124         
     125        JavaPairRDD<String, Integer> count_sorted  
     126            = counts_swapped_pair_sorted.mapToPair(item -> item.swap()); 
     127        count_sorted.setName("descending-word-frequency"); 
    123128         
    124129        String filename_root = _json_list_filename.replaceAll(".*/","").replaceAll("\\..*$",""); 
    125130        String output_directory = "whitelist-" + filename_root + "-out"; 
    126         sorted_swaped_back_pair.saveAsTextFile(output_directory); 
    127  
     131         
     132        //sorted_swaped_back_pair.saveAsTextFile(output_directory); 
     133        count_sorted.saveAsTextFile(output_directory); 
     134         
    128135         
    129136        //System.out.println("");