Changeset 30932

Timestamp:

2016-10-26T10:16:06+13:00 (6 years ago)

Author:

davidb

Message:

Support both ​file:// and hdfs://

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java (modified) (2 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

@@ -3 +3 @@
 import java.io.BufferedInputStream;
 import java.io.BufferedReader;
-//import java.io.FileInputStream;
-//import java.io.FileNotFoundException;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-//import java.io.UnsupportedEncodingException;
 import java.net.URI;
-//import java.nio.charset.StandardCharsets;
-//import java.nio.file.Files;
-//import java.nio.file.Path;
-//import java.nio.file.Paths;
 import java.util.ArrayList;
-//import java.util.Arrays;
 import java.util.Iterator;
-//import java.util.List;
 
 import org.apache.commons.compress.compressors.CompressorException;
@@ -50 +42 @@
     }
     
-    protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn) 
-            throws CompressorException, IOException 
+    protected static BufferedInputStream getBufferedInputStream(String fileIn) 
+            throws IOException 
     {
+        BufferedInputStream bis = null;
+        
+        if (fileIn.startsWith("hdfs://")) {
         URI uri = URI.create (fileIn);
         Configuration conf = new Configuration();
         FileSystem file = FileSystem.get(uri, conf);
         FSDataInputStream fin = file.open(new Path(uri));
-            
-        //FileInputStream fin = new FileInputStream(fileIn);
-        BufferedInputStream bis = new BufferedInputStream(fin);
-        CompressorInputStream input = new CompressorStreamFactory().createCompressorInputStream(bis);
-        BufferedReader br2 = new BufferedReader(new InputStreamReader(input,"UTF8"));
-        return br2;
+         
+        bis = new BufferedInputStream(fin);
+        }
+        else {
+          FileInputStream fin = new FileInputStream(fileIn);
+          bis = new BufferedInputStream(fin);
+        }
+        
+        return bis;
+        
+    }
+    protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn) 
+            throws IOException, CompressorException 
+    {
+        BufferedInputStream bis = getBufferedInputStream(fileIn);
+        CompressorInputStream comp_input = new CompressorStreamFactory().createCompressorInputStream(bis);
+        BufferedReader br = new BufferedReader(new InputStreamReader(comp_input,"UTF8"));
+        return br;
     }