Changeset 30921

Timestamp:

2016-10-25T23:23:08+13:00 (7 years ago)

Author:

davidb

Message:

Code change to read in JSON file over HDFS

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java (modified) (4 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

@@ -8 +8 @@
 import java.io.InputStreamReader;
 import java.io.UnsupportedEncodingException;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+//import java.nio.file.Files;
+//import java.nio.file.Path;
+//import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -20 +21 @@
 import org.apache.commons.compress.compressors.CompressorInputStream;
 import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.spark.api.java.function.FlatMapFunction;
 import org.json.JSONArray;
@@ -35 +40 @@
 
 /* 
-URI uri = URI.create (“hdfs://host:port/file path”);
+URI uri = URI.create ("hdfs://host:port/file path");
 Configuration conf = new Configuration();
 FileSystem file = FileSystem.get(uri, conf);
@@ -54 +59 @@
     
     protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn) 
-            throws FileNotFoundException, UnsupportedEncodingException, CompressorException {
-        FileInputStream fin = new FileInputStream(fileIn);
+            throws CompressorException, IOException 
+    {
+        URI uri = URI.create (fileIn);
+        Configuration conf = new Configuration();
+        FileSystem file = FileSystem.get(uri, conf);
+        FSDataInputStream fin = file.open(new Path(uri));
+            
+        //FileInputStream fin = new FileInputStream(fileIn);
         BufferedInputStream bis = new BufferedInputStream(fin);
         CompressorInputStream input = new CompressorStreamFactory().createCompressorInputStream(bis);