Changeset 31263 for other-projects/hathitrust/wcsa/extracted-features-solr

Timestamp:

2016-12-21T13:26:31+13:00 (7 years ago)

Author:

davidb

Message:

Change to using long for higher word counts

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForPOSCount.java (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForPOSCount.java

@@ -98 +98 @@
         */
         
-        JavaPairRDD<String, Integer> pos_pairs = pos_list.mapToPair(s -> new Tuple2<String, Integer>(s, 1));
+        JavaPairRDD<String, Long> pos_pairs = pos_list.mapToPair(s -> new Tuple2<String, Long>(s, (long)1));
         pos_pairs.setName("single-pos-count");
         
-        JavaPairRDD<String, Integer> pos_counts = pos_pairs.reduceByKey((a, b) -> a + b);
+        JavaPairRDD<String, Long> pos_counts = pos_pairs.reduceByKey((a, b) -> a + b);
         pos_counts.setName("pos-frequency");
         
-        JavaPairRDD<Integer, String> pos_counts_swapped_pair
+        JavaPairRDD<Long, String> pos_counts_swapped_pair
             = pos_counts.mapToPair(item -> item.swap());
         pos_counts_swapped_pair.setName("frequency-pos-swap");
         
-        JavaPairRDD<Integer, String> pos_counts_swapped_pair_sorted 
+        JavaPairRDD<Long, String> pos_counts_swapped_pair_sorted 
             = pos_counts_swapped_pair.sortByKey(false, num_partitions);
         pos_counts_swapped_pair_sorted.setName("descending-sorted-frequency-pos");
         
-        JavaPairRDD<String, Integer> pos_count_sorted 
+        JavaPairRDD<String, Long> pos_count_sorted 
             = pos_counts_swapped_pair_sorted.mapToPair(item -> item.swap());
         pos_count_sorted.setName("descending-pos-frequency");