Changeset 30932 for other-projects

Show
Ignore:
Timestamp:
26.10.2016 10:16:06 (3 years ago)
Author:
davidb
Message:

Support both  file:// and  hdfs://

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

    r30924 r30932  
    33import java.io.BufferedInputStream; 
    44import java.io.BufferedReader; 
    5 //import java.io.FileInputStream; 
    6 //import java.io.FileNotFoundException; 
     5import java.io.FileInputStream; 
    76import java.io.IOException; 
    87import java.io.InputStreamReader; 
    9 //import java.io.UnsupportedEncodingException; 
    108import java.net.URI; 
    11 //import java.nio.charset.StandardCharsets; 
    12 //import java.nio.file.Files; 
    13 //import java.nio.file.Path; 
    14 //import java.nio.file.Paths; 
    159import java.util.ArrayList; 
    16 //import java.util.Arrays; 
    1710import java.util.Iterator; 
    18 //import java.util.List; 
    1911 
    2012import org.apache.commons.compress.compressors.CompressorException; 
     
    5042    } 
    5143     
    52     protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn)  
    53             throws CompressorException, IOException  
     44    protected static BufferedInputStream getBufferedInputStream(String fileIn)  
     45            throws IOException  
    5446    { 
     47        BufferedInputStream bis = null; 
     48         
     49        if (fileIn.startsWith("hdfs://")) { 
    5550        URI uri = URI.create (fileIn); 
    5651        Configuration conf = new Configuration(); 
    5752        FileSystem file = FileSystem.get(uri, conf); 
    5853        FSDataInputStream fin = file.open(new Path(uri)); 
    59              
    60         //FileInputStream fin = new FileInputStream(fileIn); 
    61         BufferedInputStream bis = new BufferedInputStream(fin); 
    62         CompressorInputStream input = new CompressorStreamFactory().createCompressorInputStream(bis); 
    63         BufferedReader br2 = new BufferedReader(new InputStreamReader(input,"UTF8")); 
    64         return br2; 
     54          
     55        bis = new BufferedInputStream(fin); 
     56        } 
     57        else { 
     58          FileInputStream fin = new FileInputStream(fileIn); 
     59          bis = new BufferedInputStream(fin); 
     60        } 
     61         
     62        return bis; 
     63         
     64    } 
     65    protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn)  
     66            throws IOException, CompressorException  
     67    { 
     68        BufferedInputStream bis = getBufferedInputStream(fileIn); 
     69        CompressorInputStream comp_input = new CompressorStreamFactory().createCompressorInputStream(bis); 
     70        BufferedReader br = new BufferedReader(new InputStreamReader(comp_input,"UTF8")); 
     71        return br; 
    6572    } 
    6673