Changeset 30937 for other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

Timestamp:

2016-10-26T13:44:38+13:00 (8 years ago)

Author:

davidb

Message:

Expanded set of ClusterFileIO methods

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java (modified) (4 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

@@ -1 +1 @@
 package org.hathitrust;
 
-import java.io.BufferedInputStream;
 import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
 import java.util.ArrayList;
 import java.util.Iterator;
 
-import org.apache.commons.compress.compressors.CompressorException;
-import org.apache.commons.compress.compressors.CompressorInputStream;
-import org.apache.commons.compress.compressors.CompressorStreamFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.spark.api.java.function.FlatMapFunction;
 import org.json.JSONArray;
@@ -42 +30 @@
     }
     
-    protected static BufferedInputStream getBufferedInputStream(String fileIn) 
-            throws IOException 
-    {
-        BufferedInputStream bis = null;
-        
-        if (fileIn.startsWith("hdfs://")) {
-            URI uri = URI.create (fileIn);
-            Configuration conf = new Configuration();
-            FileSystem file = FileSystem.get(uri, conf);
-            FSDataInputStream fin = file.open(new Path(uri));
-
-            bis = new BufferedInputStream(fin);
-        }
-        else {
-            // Trim 'file://' off the front
-            
-            String local_file_in = fileIn;
-            if (local_file_in.startsWith("file://")) {
-                local_file_in = fileIn.substring("file://".length());
-            }
-            FileInputStream fin = new FileInputStream(local_file_in);
-            bis = new BufferedInputStream(fin);
-        }
-
-        return bis;
-        
-    }
-    protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn) 
-            throws IOException, CompressorException 
-    {
-        BufferedInputStream bis = getBufferedInputStream(fileIn);
-        CompressorInputStream comp_input = new CompressorStreamFactory().createCompressorInputStream(bis);
-        BufferedReader br = new BufferedReader(new InputStreamReader(comp_input,"UTF8"));
-        return br;
-    }
-    
     protected JSONObject readJSONFile(String filename)
     {
@@ -87 +39 @@
 
             String str;
-            BufferedReader br = getBufferedReaderForCompressedFile(_input_dir + "/" + filename);
+            BufferedReader br = ClusterFileIO.getBufferedReaderForCompressedFile(_input_dir + "/" + filename);
             while ((str = br.readLine()) != null) {
                 sb.append(str);
@@ -120 +72 @@
     }
 
-    public Iterator<String> call(String s) 
+    public Iterator<String> call(String json_file_in) 
     { 
-        JSONObject extracted_feature_record = readJSONFile(s);
+        JSONObject extracted_feature_record = readJSONFile(json_file_in);
+        
+        // Check output directory for volume exists, and create it if not
+        
         
         String id = extracted_feature_record.getString("id");