Changeset 31222 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/WhitelistBloomFilter.java

Timestamp:

2016-12-12T23:22:33+13:00 (7 years ago)

Author:

davidb

Message:

Changed to using ClusterFileIO supporting methods

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/WhitelistBloomFilter.java (modified) (6 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/WhitelistBloomFilter.java

@@ -9 +9 @@
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.Serializable;
-import java.nio.charset.Charset;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -16 +15 @@
 import java.util.stream.Stream;
 
-import javax.annotation.Nullable;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 
-import com.google.common.base.Preconditions;
 import com.google.common.hash.BloomFilter;
 import com.google.common.hash.Funnel;
 import com.google.common.hash.Funnels;
-import com.google.common.hash.PrimitiveSink;
 
 public class WhitelistBloomFilter {
@@ -30 +30 @@
     protected static final double FALSE_POSITIVE_PERCENTAGE = 0.01;
     
-    
     // http://stackoverflow.com/questions/453018/number-of-lines-in-a-file-in-java
-    public static int countLines(String filename) throws IOException 
+    protected static int countLines(String filename) throws IOException 
     {
         InputStream is = new BufferedInputStream(new FileInputStream(filename));
@@ -59 +58 @@
         System.out.println("Constructing: WhitelistBloomFilter");
 
-        File ser_dictionary_file = new File(dictionary_filename + SERIALIZED_SUFFIX);
         
-        if (ser_dictionary_file.exists()) {
+        String ser_dictionary_filename = dictionary_filename + SERIALIZED_SUFFIX;
+        
+        
+        //File ser_dictionary_file = new File(dictionary_filename + SERIALIZED_SUFFIX);
+        
+        //if (ser_dictionary_file.exists()) {
+        if (ClusterFileIO.exists(ser_dictionary_filename)) {
             System.out.println("Loading Serialized Bloom filter ...");
-            _bloomFilter = serializeIn(ser_dictionary_file);
+            _bloomFilter = serializeIn(ser_dictionary_filename);
             System.out.println("... done");
         }
@@ -103 +107 @@
             System.out.println("Serializing Bloom filter ...");
 
-            File ser_dictionary = new File(filename + SERIALIZED_SUFFIX);
-            serializeOut(ser_dictionary);
+            String ser_filename = filename + SERIALIZED_SUFFIX;
+            
+            //File ser_dictionary = new File(ser_filename);
+            serializeOut(ser_filename);
 
             System.out.println("... done");
@@ -116 +122 @@
     }
 
-    protected void serializeOut(File ser_file)
+    //protected void serializeOut(File ser_file)
+    protected void serializeOut(String ser_filename)
     {
+        //FileSystem fs = ClusterFileIO.getFileSystemInstance(ser_filename);
+        
         try {
-            FileOutputStream fos = new FileOutputStream(ser_file);
+            //URI ser_uri = URI.create (ser_filename);
+            //Path ser_path = new Path(ser_uri);
+            
+            BufferedOutputStream bos = ClusterFileIO.getBufferedOutputStream(ser_filename);
+            
+            //FileOutputStream fos = new FileOutputStream(ser_file);
 
-            BufferedOutputStream bfos = new BufferedOutputStream(fos);
+            //BufferedOutputStream bfos = new BufferedOutputStream(fos);
 
-            _bloomFilter.writeTo(bfos);
+            _bloomFilter.writeTo(bos);
 
-            bfos.close();
+            bos.close();
         }
         catch (FileNotFoundException e) {
-            System.err.println("Unable to open Bloom file:" + ser_file.getAbsolutePath());
+            System.err.println("Unable to open Bloom file:" + ser_filename);
+            //System.err.println("Unable to open Bloom file:" + ser_file.getAbsolutePath());
+            
             e.printStackTrace();
         } catch (IOException e) {
-            System.err.println("Error reading in Bloom file:" + ser_file.getAbsolutePath());
+            System.err.println("Error reading in Bloom file:" + ser_filename);
+            //System.err.println("Error reading in Bloom file:" + ser_file.getAbsolutePath());
             e.printStackTrace();
         }
     }
 
-    protected static BloomFilter<CharSequence> serializeIn(File ser_file)
+    //protected static BloomFilter<CharSequence> serializeIn(File ser_file)
+    protected static BloomFilter<CharSequence> serializeIn(String ser_filename)
     {
         BloomFilter<CharSequence> bloomFilter = null;
     
+        //FileSystem fs = ClusterFileIO.getFileSystemInstance(ser_filename);
+        
         try {
-            FileInputStream fis = new FileInputStream(ser_file);
-            BufferedInputStream bfis = new BufferedInputStream(fis);
+            //URI ser_uri = URI.create (ser_filename);
+            //Path ser_path = new Path(ser_uri);
+
+            //FSDataInputStream fsdis = fs.open(ser_path);
+            //BufferedInputStream bis = new BufferedInputStream(fsdis);
+            
+            BufferedInputStream bis = ClusterFileIO.getBufferedInputStream(ser_filename);
+            
+            //FileInputStream fis = new FileInputStream(ser_file);
+            //BufferedInputStream bfis = new BufferedInputStream(fis);
 
             Funnel<CharSequence> string_funnel = Funnels.stringFunnel(StandardCharsets.UTF_8);
-            bloomFilter = BloomFilter.readFrom(bfis,string_funnel);
+            bloomFilter = BloomFilter.readFrom(bis,string_funnel);
 
-            bfis.close();
+            bis.close();
         }
         catch (FileNotFoundException e) {
-            System.err.println("Unable to open Bloom file:" + ser_file.getAbsolutePath());
+            //System.err.println("Unable to open Bloom file:" + ser_file.getAbsolutePath());
+            //System.err.println("Unable to open Bloom file:" + ser_path.getName());
+            System.err.println("Unable to open Bloom file:" + ser_filename);
+            
             e.printStackTrace();
         } catch (IOException e) {
-            System.err.println("Error writing out Bloom file:" + ser_file.getAbsolutePath());
+            //System.err.println("Error writing out Bloom file:" + ser_file.getAbsolutePath());
+            //System.err.println("Error writing out Bloom file:" + ser_path.getName());
+            System.err.println("Error writing out Bloom file:" + ser_filename);
             e.printStackTrace();
         }