Ignore:
Timestamp:
10/26/16 10:16:06 (4 years ago)
Author:
davidb
Message:

Support both file:// and hdfs://

File:
1 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

    r30924 r30932  
    33import java.io.BufferedInputStream;
    44import java.io.BufferedReader;
    5 //import java.io.FileInputStream;
    6 //import java.io.FileNotFoundException;
     5import java.io.FileInputStream;
    76import java.io.IOException;
    87import java.io.InputStreamReader;
    9 //import java.io.UnsupportedEncodingException;
    108import java.net.URI;
    11 //import java.nio.charset.StandardCharsets;
    12 //import java.nio.file.Files;
    13 //import java.nio.file.Path;
    14 //import java.nio.file.Paths;
    159import java.util.ArrayList;
    16 //import java.util.Arrays;
    1710import java.util.Iterator;
    18 //import java.util.List;
    1911
    2012import org.apache.commons.compress.compressors.CompressorException;
     
    5042    }
    5143   
    52     protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn)
    53             throws CompressorException, IOException
     44    protected static BufferedInputStream getBufferedInputStream(String fileIn)
     45            throws IOException
    5446    {
     47        BufferedInputStream bis = null;
     48       
     49        if (fileIn.startsWith("hdfs://")) {
    5550        URI uri = URI.create (fileIn);
    5651        Configuration conf = new Configuration();
    5752        FileSystem file = FileSystem.get(uri, conf);
    5853        FSDataInputStream fin = file.open(new Path(uri));
    59            
    60         //FileInputStream fin = new FileInputStream(fileIn);
    61         BufferedInputStream bis = new BufferedInputStream(fin);
    62         CompressorInputStream input = new CompressorStreamFactory().createCompressorInputStream(bis);
    63         BufferedReader br2 = new BufferedReader(new InputStreamReader(input,"UTF8"));
    64         return br2;
     54         
     55        bis = new BufferedInputStream(fin);
     56        }
     57        else {
     58          FileInputStream fin = new FileInputStream(fileIn);
     59          bis = new BufferedInputStream(fin);
     60        }
     61       
     62        return bis;
     63       
     64    }
     65    protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn)
     66            throws IOException, CompressorException
     67    {
     68        BufferedInputStream bis = getBufferedInputStream(fileIn);
     69        CompressorInputStream comp_input = new CompressorStreamFactory().createCompressorInputStream(bis);
     70        BufferedReader br = new BufferedReader(new InputStreamReader(comp_input,"UTF8"));
     71        return br;
    6572    }
    6673   
Note: See TracChangeset for help on using the changeset viewer.