Changeset 31204 for other-projects

Timestamp:

2016-12-12T14:28:20+13:00 (7 years ago)

Author:

davidb

Message:

Splicing in Guava verion 20 of BloomFilter into code as own class (now BloomFilterAdvanced). This is because Spark runs with older version of Guava (14.0). Written code makes use of newer features.

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java

Files:

• com (added)
• com/google (added)
• com/google/common (added)
• com/google/common/hash (added)
• com/google/common/hash/BloomFilterAdvanced.java (added)
• org/hathitrust/extractedfeatures/WhitelistBloomFilter.java (modified) (8 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/WhitelistBloomFilter.java

@@ -3 +3 @@
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
@@ -9 +11 @@
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.Serializable;
 import java.nio.charset.Charset;
@@ -18 +21 @@
 import javax.annotation.Nullable;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import com.google.common.base.Preconditions;
-import com.google.common.hash.BloomFilter;
+//import com.google.common.hash.BloomFilter;
+import com.google.common.hash.BloomFilterAdvanced;
+//import com.google.common.hash.BloomFilterStrategies.BitArray;
 import com.google.common.hash.Funnel;
 import com.google.common.hash.Funnels;
 import com.google.common.hash.PrimitiveSink;
+import com.google.common.primitives.SignedBytes;
+import com.google.common.primitives.UnsignedBytes;
 
 public class WhitelistBloomFilter {
@@ -28 +37 @@
     
 
-    protected BloomFilter<CharSequence> _bloomFilter;
+    protected BloomFilterAdvanced<CharSequence> _bloomFilter;
     protected static final String SERIALIZED_SUFFIX = "-serialized";
     protected static final double FALSE_POSITIVE_PERCENTAGE = 0.01;
@@ -78 +87 @@
                 //Funnel<CharSequence> string_funnel = Funnels.stringFunnel(StandardCharsets.UTF_8);
                 Funnel<CharSequence> string_funnel = stringFunnel(StandardCharsets.UTF_8);
-                _bloomFilter = BloomFilter.create(string_funnel, num_lines,FALSE_POSITIVE_PERCENTAGE);
+                _bloomFilter = BloomFilterAdvanced.create(string_funnel, num_lines,FALSE_POSITIVE_PERCENTAGE);
             } 
             catch (IOException e) {
@@ -139 +148 @@
     }
 
-    protected static BloomFilter<CharSequence> serializeIn(File ser_file)
-    {
-        BloomFilter<CharSequence> bloomFilter = null;
+    protected static BloomFilterAdvanced<CharSequence> serializeIn(File ser_file)
+    {
+        BloomFilterAdvanced<CharSequence> bloomFilter = null;
     
         try {
@@ -149 +158 @@
             //Funnel<CharSequence> string_funnel = Funnels.stringFunnel(StandardCharsets.UTF_8);
             Funnel<CharSequence> string_funnel = stringFunnel(StandardCharsets.UTF_8);
-            bloomFilter = BloomFilter.readFrom(bfis,string_funnel);
+            bloomFilter = BloomFilterAdvanced.readFrom(bfis,string_funnel);
 
             bfis.close();
@@ -225 +234 @@
 
         }
-
       }