Changeset 31264 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForLangCount.java

Timestamp:

2016-12-21T13:47:56+13:00 (7 years ago)

Author:

davidb

Message:

Switching to 'long' in counts to allow higher number representation

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForLangCount.java (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForLangCount.java

@@ -86 +86 @@
     
         
-        JavaPairRDD<String, Integer> lang_pairs = lang_list.mapToPair(s -> new Tuple2<String, Integer>(s, 1));
+        JavaPairRDD<String, Long> lang_pairs = lang_list.mapToPair(s -> new Tuple2<String, Long>(s, 1L));
         lang_pairs.setName("single-lang-count");
         
-        JavaPairRDD<String, Integer> lang_counts = lang_pairs.reduceByKey((a, b) -> a + b);
+        JavaPairRDD<String, Long> lang_counts = lang_pairs.reduceByKey((a, b) -> a + b);
         lang_counts.setName("lang-frequency");
         
-        JavaPairRDD<Integer, String> lang_counts_swapped_pair
+        JavaPairRDD<Long, String> lang_counts_swapped_pair
             = lang_counts.mapToPair(item -> item.swap());
         lang_counts_swapped_pair.setName("frequency-lang-swap");
         
-        JavaPairRDD<Integer, String> lang_counts_swapped_pair_sorted 
+        JavaPairRDD<Long, String> lang_counts_swapped_pair_sorted 
             = lang_counts_swapped_pair.sortByKey(false, num_partitions);
         lang_counts_swapped_pair_sorted.setName("descending-sorted-frequency-lang");
         
-        JavaPairRDD<String, Integer> lang_count_sorted 
+        JavaPairRDD<String, Long> lang_count_sorted 
             = lang_counts_swapped_pair_sorted.mapToPair(item -> item.swap());
         lang_count_sorted.setName("descending-lang-frequency");