Changeset 31202 for other-projects/hathitrust

Timestamp:

2016-12-12T13:53:06+13:00 (7 years ago)

Author:

davidb

Message:

Turns out Spark uses Guava 14.0 not 20.0. Additional code to fill in some gaps

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/WhitelistBloomFilter.java (modified) (5 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/WhitelistBloomFilter.java

@@ -9 +9 @@
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Serializable;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -14 +16 @@
 import java.util.stream.Stream;
 
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
 import com.google.common.hash.BloomFilter;
 import com.google.common.hash.Funnel;
 import com.google.common.hash.Funnels;
+import com.google.common.hash.PrimitiveSink;
 
 public class WhitelistBloomFilter {
+    
+    
 
     protected BloomFilter<CharSequence> _bloomFilter;
@@ -68 +76 @@
                 num_lines = countLines(dictionary_filename);
 
+                
+                 
                 Funnel<CharSequence> string_funnel = Funnels.stringFunnel(StandardCharsets.UTF_8);
                 _bloomFilter = BloomFilter.create(string_funnel, num_lines,FALSE_POSITIVE_PERCENTAGE);
@@ -138 +148 @@
             BufferedInputStream bfis = new BufferedInputStream(fis);
 
-            Funnel<CharSequence> string_funnel = Funnels.stringFunnel(StandardCharsets.UTF_8);
+            //Funnel<CharSequence> string_funnel = Funnels.stringFunnel(StandardCharsets.UTF_8);
+            Funnel<CharSequence> string_funnel = stringFunnel(StandardCharsets.UTF_8);
             bloomFilter = BloomFilter.readFrom(bfis,string_funnel);
 
@@ -154 +165 @@
 
 
+    // Spark uses Guava 14.0, the following is future-ported from Guava 20.0
+    // Added in here, rather then Funnel, and StringCharsetFunnel -> MyStringCharsetFunnel
+    
+    public static Funnel<CharSequence> stringFunnel(Charset charset) {
+
+        return new MyStringCharsetFunnel(charset);
+
+      }
+    
+    private static class MyStringCharsetFunnel implements Funnel<CharSequence>, Serializable {
+
+        private static final long serialVersionUID = 1L;
+        
+        private final Charset charset;
+
+        MyStringCharsetFunnel(Charset charset) {
+          this.charset = Preconditions.checkNotNull(charset);
+        }
+
+        public void funnel(CharSequence from, PrimitiveSink into) {
+          into.putString(from, charset);
+        }
+
+        @Override
+        public String toString() {
+          return "Funnels.stringFunnel(" + charset.name() + ")";
+        }
+
+        @Override
+        public boolean equals(@Nullable Object o) {
+
+          if (o instanceof MyStringCharsetFunnel) {
+            MyStringCharsetFunnel funnel = (MyStringCharsetFunnel) o;
+            return this.charset.equals(funnel.charset);
+          }
+          return false;
+        }
+
+        @Override
+        public int hashCode() {
+          return MyStringCharsetFunnel.class.hashCode() ^ charset.hashCode();
+        }
+
+        Object writeReplace() {
+          return new SerializedForm(charset);
+        }
+
+        private static class SerializedForm implements Serializable {
+          private final String charsetCanonicalName;
+
+          SerializedForm(Charset charset) {
+            this.charsetCanonicalName = charset.name();
+          }
+
+          private Object readResolve() {
+            return stringFunnel(Charset.forName(charsetCanonicalName));
+          }
+
+          private static final long serialVersionUID = 0;
+
+        }
+
+      }
+    
+    
 }