Show
Ignore:
Timestamp:
13.12.2016 14:00:15 (3 years ago)
Author:
davidb
Message:

Code tidy up

Location:
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures
Files:
2 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ClusterFileIO.java

    r31223 r31227  
    4545 
    4646     
    47     public static FileSystem getFileSystemInstance(String input_file_or_dir) 
     47    protected static FileSystem getFileSystemInstance(String input_file_or_dir) 
    4848    { 
    4949        FileSystem fs = null; 
     
    7373        FileSystem fs = getFileSystemInstance(file); 
    7474         
    75         //Configuration conf = jsc.hadoopConfiguration(); 
    76         //FileSystem fs = org.apache.hadoop.fs.FileSystem.get(conf); 
    7775        boolean exists = false; 
    7876         
     
    8179            exists = fs.exists(path); 
    8280        } catch (IllegalArgumentException e) { 
    83             e.printStackTrace(); 
    8481            exists = false; 
    8582        } catch (IOException e) { 
    86             e.printStackTrace(); 
    8783            exists = false; 
    8884        } 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/WhitelistBloomFilter.java

    r31224 r31227  
    33import java.io.BufferedInputStream; 
    44import java.io.BufferedOutputStream; 
    5 import java.io.File; 
    65import java.io.FileInputStream; 
    76import java.io.FileNotFoundException; 
    8 import java.io.FileOutputStream; 
    97import java.io.IOException; 
    108import java.io.InputStream; 
    11 import java.net.URI; 
    129import java.nio.charset.StandardCharsets; 
    1310import java.nio.file.Files; 
    1411import java.nio.file.Paths; 
    1512import java.util.stream.Stream; 
    16  
    17 import org.apache.hadoop.fs.FSDataInputStream; 
    18 import org.apache.hadoop.fs.FSDataOutputStream; 
    19 import org.apache.hadoop.fs.FileSystem; 
    20 import org.apache.hadoop.fs.Path; 
    2113 
    2214import com.google.common.hash.BloomFilter; 
     
    5749    public WhitelistBloomFilter(String dictionary_filename, boolean serialize) { 
    5850        System.out.println("Constructing: WhitelistBloomFilter"); 
    59  
    6051         
    6152        String ser_dictionary_filename = dictionary_filename + SERIALIZED_SUFFIX; 
    6253         
    63          
    64         //File ser_dictionary_file = new File(dictionary_filename + SERIALIZED_SUFFIX); 
    65          
    66         //if (ser_dictionary_file.exists()) { 
    67         System.err.println("***** checking serialized dictionary:" + ser_dictionary_filename); 
    6854        if (ClusterFileIO.exists(ser_dictionary_filename)) { 
    6955            System.out.println("Loading Serialized Bloom filter ..."); 
     
    10995 
    11096            String ser_filename = filename + SERIALIZED_SUFFIX; 
    111              
    112             //File ser_dictionary = new File(ser_filename); 
    11397            serializeOut(ser_filename); 
    11498 
     
    123107    } 
    124108 
    125     //protected void serializeOut(File ser_file) 
    126109    protected void serializeOut(String ser_filename) 
    127110    { 
    128         //FileSystem fs = ClusterFileIO.getFileSystemInstance(ser_filename); 
    129          
    130111        try { 
    131             //URI ser_uri = URI.create (ser_filename); 
    132             //Path ser_path = new Path(ser_uri); 
    133              
    134112            BufferedOutputStream bos = ClusterFileIO.getBufferedOutputStream(ser_filename); 
    135              
    136             //FileOutputStream fos = new FileOutputStream(ser_file); 
    137  
    138             //BufferedOutputStream bfos = new BufferedOutputStream(fos); 
    139  
    140113            _bloomFilter.writeTo(bos); 
    141  
    142114            bos.close(); 
    143115        } 
    144116        catch (FileNotFoundException e) { 
    145117            System.err.println("Unable to open Bloom file:" + ser_filename); 
    146             //System.err.println("Unable to open Bloom file:" + ser_file.getAbsolutePath()); 
    147              
    148             e.printStackTrace(); 
     118            //e.printStackTrace(); 
    149119        } catch (IOException e) { 
    150120            System.err.println("Error reading in Bloom file:" + ser_filename); 
    151             //System.err.println("Error reading in Bloom file:" + ser_file.getAbsolutePath()); 
    152             e.printStackTrace(); 
     121            //e.printStackTrace(); 
    153122        } 
    154123    } 
    155124 
    156     //protected static BloomFilter<CharSequence> serializeIn(File ser_file) 
    157125    protected static BloomFilter<CharSequence> serializeIn(String ser_filename) 
    158126    { 
    159127        BloomFilter<CharSequence> bloomFilter = null; 
    160128     
    161         //FileSystem fs = ClusterFileIO.getFileSystemInstance(ser_filename); 
    162          
    163129        try { 
    164             //URI ser_uri = URI.create (ser_filename); 
    165             //Path ser_path = new Path(ser_uri); 
    166  
    167             //FSDataInputStream fsdis = fs.open(ser_path); 
    168             //BufferedInputStream bis = new BufferedInputStream(fsdis); 
    169              
    170130            BufferedInputStream bis = ClusterFileIO.getBufferedInputStream(ser_filename); 
    171              
    172             //FileInputStream fis = new FileInputStream(ser_file); 
    173             //BufferedInputStream bfis = new BufferedInputStream(fis); 
    174  
     131           
    175132            Funnel<CharSequence> string_funnel = Funnels.stringFunnel(StandardCharsets.UTF_8); 
    176133            bloomFilter = BloomFilter.readFrom(bis,string_funnel); 
     
    179136        } 
    180137        catch (FileNotFoundException e) { 
    181             //System.err.println("Unable to open Bloom file:" + ser_file.getAbsolutePath()); 
    182             //System.err.println("Unable to open Bloom file:" + ser_path.getName()); 
    183             System.err.println("Unable to open Bloom file:" + ser_filename); 
    184              
    185             e.printStackTrace(); 
     138            System.err.println("Unable to open Bloom file:" + ser_filename);     
     139            //e.printStackTrace(); 
    186140        } catch (IOException e) { 
    187             //System.err.println("Error writing out Bloom file:" + ser_file.getAbsolutePath()); 
    188             //System.err.println("Error writing out Bloom file:" + ser_path.getName()); 
    189141            System.err.println("Error writing out Bloom file:" + ser_filename); 
    190             e.printStackTrace(); 
     142            //e.printStackTrace(); 
    191143        } 
    192144        return bloomFilter; 
    193145    } 
    194  
    195  
    196  
    197      
    198      
    199146}