Changeset 30941 for other-projects/hathitrust/solr-extracted-features

Timestamp:

2016-10-26T14:16:25+13:00 (7 years ago)

Author:

davidb

Message:

Moved to getFileSystemInstance() method to play nice on cluster

Location:

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust

Files:

• ClusterFileIO.java (modified) (8 diffs)
• PrepareForIngest.java (modified) (1 diff)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/ClusterFileIO.java

@@ -25 +25 @@
 public class ClusterFileIO {
 
-    protected static FileSystem _fs = null;
+    private static FileSystem __fs = null;
     
+    /*
     public static void initOLD(JavaSparkContext java_spark_context) 
     {
@@ -37 +38 @@
         }
     }
+    */
     
+    /*
     public static void init(String input_dir)
     {
@@ -53 +56 @@
         
     }
+    */
+    
+    public static FileSystem getFileSystemInstance(String input_dir)
+    {
+        if (__fs == null) {
+
+            try {
+                Configuration conf = new Configuration();
+                URI uri = new URI(input_dir);
+                __fs = FileSystem.get(uri, conf);
+            } 
+            catch (URISyntaxException e) {
+                e.printStackTrace();    
+            } 
+            catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        return __fs;
+    }
+    
     public static boolean isHDFS(String fileIn)
     {
@@ -60 +84 @@
     protected static boolean exists(String file) 
     {
+        FileSystem fs = getFileSystemInstance(file);
+        
         //Configuration conf = jsc.hadoopConfiguration();
         //FileSystem fs = org.apache.hadoop.fs.FileSystem.get(conf);
@@ -65 +91 @@
         
         try {
-            exists = _fs.exists(new Path(file));
+            exists = fs.exists(new Path(file));
         } catch (IllegalArgumentException e) {
             exists = false;
@@ -77 +103 @@
             throws IOException 
     {
+        FileSystem fs = getFileSystemInstance(fileIn);
+        
         BufferedInputStream bis = null;
         
@@ -86 +114 @@
             
             //FSDataInputStream fin = _fs.open(new Path(fileIn));
-            FSDataInputStream fin = _fs.open(new Path(uri));
+            FSDataInputStream fin = fs.open(new Path(uri));
             
             bis = new BufferedInputStream(fin);
@@ -101 +129 @@
             */
             
-            FSDataInputStream fin = _fs.open(new Path(fileIn));
+            FSDataInputStream fin = fs.open(new Path(fileIn));
             
             bis = new BufferedInputStream(fin);

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java

@@ -36 +36 @@
     {   
         String spark_app_name = "HathiTrust Extract Features: Prepare for Solr Ingest";
-        spark_app_name += "[" + _json_list_filename + "]";
+        spark_app_name += " [" + _json_list_filename + "]";
 
         SparkConf conf = new SparkConf().setAppName(spark_app_name);
         JavaSparkContext jsc = new JavaSparkContext(conf);
-        ClusterFileIO.init(_input_dir);
+        //ClusterFileIO.init(_input_dir);
         
         // Check output directory exists, and create it if not