Changeset 31227

Timestamp:

2016-12-13T14:00:15+13:00 (7 years ago)

Author:

davidb

Message:

Code tidy up

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• ClusterFileIO.java (modified) (3 diffs)
• WhitelistBloomFilter.java (modified) (5 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ClusterFileIO.java

@@ -45 +45 @@
 
     
-    public static FileSystem getFileSystemInstance(String input_file_or_dir)
+    protected static FileSystem getFileSystemInstance(String input_file_or_dir)
     {
         FileSystem fs = null;
@@ -73 +73 @@
         FileSystem fs = getFileSystemInstance(file);
         
-        //Configuration conf = jsc.hadoopConfiguration();
-        //FileSystem fs = org.apache.hadoop.fs.FileSystem.get(conf);
         boolean exists = false;
         
@@ -81 +79 @@
             exists = fs.exists(path);
         } catch (IllegalArgumentException e) {
-            e.printStackTrace();
             exists = false;
         } catch (IOException e) {
-            e.printStackTrace();
             exists = false;
         }

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/WhitelistBloomFilter.java

@@ -3 +3 @@
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
-import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.stream.Stream;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 
 import com.google.common.hash.BloomFilter;
@@ -57 +49 @@
     public WhitelistBloomFilter(String dictionary_filename, boolean serialize) {
         System.out.println("Constructing: WhitelistBloomFilter");
-
         
         String ser_dictionary_filename = dictionary_filename + SERIALIZED_SUFFIX;
         
-        
-        //File ser_dictionary_file = new File(dictionary_filename + SERIALIZED_SUFFIX);
-        
-        //if (ser_dictionary_file.exists()) {
-        System.err.println("***** checking serialized dictionary:" + ser_dictionary_filename);
         if (ClusterFileIO.exists(ser_dictionary_filename)) {
             System.out.println("Loading Serialized Bloom filter ...");
@@ -109 +95 @@
 
             String ser_filename = filename + SERIALIZED_SUFFIX;
-            
-            //File ser_dictionary = new File(ser_filename);
             serializeOut(ser_filename);
 
@@ -123 +107 @@
     }
 
-    //protected void serializeOut(File ser_file)
     protected void serializeOut(String ser_filename)
     {
-        //FileSystem fs = ClusterFileIO.getFileSystemInstance(ser_filename);
-        
         try {
-            //URI ser_uri = URI.create (ser_filename);
-            //Path ser_path = new Path(ser_uri);
-            
             BufferedOutputStream bos = ClusterFileIO.getBufferedOutputStream(ser_filename);
-            
-            //FileOutputStream fos = new FileOutputStream(ser_file);
-
-            //BufferedOutputStream bfos = new BufferedOutputStream(fos);
-
             _bloomFilter.writeTo(bos);
-
             bos.close();
         }
         catch (FileNotFoundException e) {
             System.err.println("Unable to open Bloom file:" + ser_filename);
-            //System.err.println("Unable to open Bloom file:" + ser_file.getAbsolutePath());
-            
-            e.printStackTrace();
+            //e.printStackTrace();
         } catch (IOException e) {
             System.err.println("Error reading in Bloom file:" + ser_filename);
-            //System.err.println("Error reading in Bloom file:" + ser_file.getAbsolutePath());
-            e.printStackTrace();
+            //e.printStackTrace();
         }
     }
 
-    //protected static BloomFilter<CharSequence> serializeIn(File ser_file)
     protected static BloomFilter<CharSequence> serializeIn(String ser_filename)
     {
         BloomFilter<CharSequence> bloomFilter = null;
     
-        //FileSystem fs = ClusterFileIO.getFileSystemInstance(ser_filename);
-        
         try {
-            //URI ser_uri = URI.create (ser_filename);
-            //Path ser_path = new Path(ser_uri);
-
-            //FSDataInputStream fsdis = fs.open(ser_path);
-            //BufferedInputStream bis = new BufferedInputStream(fsdis);
-            
             BufferedInputStream bis = ClusterFileIO.getBufferedInputStream(ser_filename);
-            
-            //FileInputStream fis = new FileInputStream(ser_file);
-            //BufferedInputStream bfis = new BufferedInputStream(fis);
-
+          
             Funnel<CharSequence> string_funnel = Funnels.stringFunnel(StandardCharsets.UTF_8);
             bloomFilter = BloomFilter.readFrom(bis,string_funnel);
@@ -179 +136 @@
         }
         catch (FileNotFoundException e) {
-            //System.err.println("Unable to open Bloom file:" + ser_file.getAbsolutePath());
-            //System.err.println("Unable to open Bloom file:" + ser_path.getName());
-            System.err.println("Unable to open Bloom file:" + ser_filename);
-            
-            e.printStackTrace();
+            System.err.println("Unable to open Bloom file:" + ser_filename);    
+            //e.printStackTrace();
         } catch (IOException e) {
-            //System.err.println("Error writing out Bloom file:" + ser_file.getAbsolutePath());
-            //System.err.println("Error writing out Bloom file:" + ser_path.getName());
             System.err.println("Error writing out Bloom file:" + ser_filename);
-            e.printStackTrace();
+            //e.printStackTrace();
         }
         return bloomFilter;
     }
-
-
-
-    
-    
 }