Changeset 31264 for other-projects

Timestamp:

2016-12-21T13:47:56+13:00 (7 years ago)

Author:

davidb

Message:

Switching to 'long' in counts to allow higher number representation

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• ProcessForLangCount.java (modified) (1 diff)
• ProcessForPOSCount.java (modified) (1 diff)
• ProcessForWhitelist.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForLangCount.java

@@ -86 +86 @@
     
         
-        JavaPairRDD<String, Integer> lang_pairs = lang_list.mapToPair(s -> new Tuple2<String, Integer>(s, 1));
+        JavaPairRDD<String, Long> lang_pairs = lang_list.mapToPair(s -> new Tuple2<String, Long>(s, 1L));
         lang_pairs.setName("single-lang-count");
         
-        JavaPairRDD<String, Integer> lang_counts = lang_pairs.reduceByKey((a, b) -> a + b);
+        JavaPairRDD<String, Long> lang_counts = lang_pairs.reduceByKey((a, b) -> a + b);
         lang_counts.setName("lang-frequency");
         
-        JavaPairRDD<Integer, String> lang_counts_swapped_pair
+        JavaPairRDD<Long, String> lang_counts_swapped_pair
             = lang_counts.mapToPair(item -> item.swap());
         lang_counts_swapped_pair.setName("frequency-lang-swap");
         
-        JavaPairRDD<Integer, String> lang_counts_swapped_pair_sorted 
+        JavaPairRDD<Long, String> lang_counts_swapped_pair_sorted 
             = lang_counts_swapped_pair.sortByKey(false, num_partitions);
         lang_counts_swapped_pair_sorted.setName("descending-sorted-frequency-lang");
         
-        JavaPairRDD<String, Integer> lang_count_sorted 
+        JavaPairRDD<String, Long> lang_count_sorted 
             = lang_counts_swapped_pair_sorted.mapToPair(item -> item.swap());
         lang_count_sorted.setName("descending-lang-frequency");

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForPOSCount.java

@@ -98 +98 @@
         */
         
-        JavaPairRDD<String, Long> pos_pairs = pos_list.mapToPair(s -> new Tuple2<String, Long>(s, (long)1));
+        JavaPairRDD<String, Long> pos_pairs = pos_list.mapToPair(s -> new Tuple2<String, Long>(s, 1L));
         pos_pairs.setName("single-pos-count");
         

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java

@@ -91 +91 @@
         words.setName("tokenized-words");
         
-        JavaPairRDD<String, Integer> pairs = words.mapToPair(new PairFunction<String, String, Integer>() {
-            public Tuple2<String, Integer> call(String s) { return new Tuple2<String, Integer>(s, 1); }
+        JavaPairRDD<String, Long> pairs = words.mapToPair(new PairFunction<String, String, Long>() {
+            public Tuple2<String, Long> call(String s) { return new Tuple2<String, Long>(s, 1L); }
         });
         pairs.setName("single-word-count");
         
-        JavaPairRDD<String, Integer> counts = pairs.reduceByKey(new Function2<Integer, Integer, Integer>() {
-            public Integer call(Integer a, Integer b) { return a + b; }
+        JavaPairRDD<String, Long> counts = pairs.reduceByKey(new Function2<Long, Long, Long>() {
+            public Long call(Long a, Long b) { return a + b; }
         });
         counts.setName("word-frequency");
         
         /*
-        JavaPairRDD<Integer, String> swapped_pair = counts.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() {
+        JavaPairRDD<Long, String> swapped_pair = counts.mapToPair(new PairFunction<Tuple2<String, Long>, Long, String>() {
                @Override
-               public Tuple2<Integer, String> call(Tuple2<String, Integer> item) throws Exception {
+               public Tuple2<Long, String> call(Tuple2<String, Long> item) throws Exception {
                    return item.swap();
                }
@@ -111 +111 @@
         swapped_pair.setName("frequency-word-swap");
         
-        JavaPairRDD<Integer, String> sorted_swapped_pair = swapped_pair.sortByKey(false,num_partitions);
+        JavaPairRDD<Long, String> sorted_swapped_pair = swapped_pair.sortByKey(false,num_partitions);
         sorted_swapped_pair.setName("descending-sorted-frequency-word");
         
-        JavaPairRDD<String, Integer> sorted_swaped_back_pair = sorted_swapped_pair.mapToPair(new PairFunction<Tuple2<Integer, String>, String, Integer>() {
+        JavaPairRDD<String, Long> sorted_swaped_back_pair = sorted_swapped_pair.mapToPair(new PairFunction<Tuple2<Long, String>, String, Long>() {
                @Override
-               public Tuple2<String, Integer> call(Tuple2<Integer, String> item) throws Exception {
+               public Tuple2<String, Long> call(Tuple2<Long, String> item) throws Exception {
                    return item.swap();
                }
@@ -124 +124 @@
         
 
-        JavaPairRDD<Integer, String> counts_swapped_pair
+        JavaPairRDD<Long, String> counts_swapped_pair
             = counts.mapToPair(item -> item.swap());
         counts_swapped_pair.setName("frequency-word-swap");
         
-        JavaPairRDD<Integer, String> counts_swapped_pair_sorted 
+        JavaPairRDD<Long, String> counts_swapped_pair_sorted 
             = counts_swapped_pair.sortByKey(false, num_partitions);
         counts_swapped_pair_sorted.setName("descending-sorted-frequency-word");
         
-        JavaPairRDD<String, Integer> count_sorted 
+        JavaPairRDD<String, Long> count_sorted 
             = counts_swapped_pair_sorted.mapToPair(item -> item.swap());
         count_sorted.setName("descending-word-frequency");