Changeset 30945 for other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/ClusterFileIO.java

Timestamp:

2016-10-26T15:37:24+13:00 (7 years ago)

Author:

davidb

Message:

Getting closer to writing out JSON files

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/ClusterFileIO.java (modified) (7 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/ClusterFileIO.java

@@ -21 +21 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.spark.api.java.JavaSparkContext;
 
 public class ClusterFileIO {
@@ -58 +57 @@
     */
     
-    public static FileSystem getFileSystemInstance(String input_dir)
+    protected static FileSystem getFileSystemInstance(String input_dir)
     {
         if (__fs == null) {
@@ -82 +81 @@
     }
     
-    protected static boolean exists(String file) 
+    public static boolean exists(String file) 
     {
         FileSystem fs = getFileSystemInstance(file);
@@ -100 +99 @@
         return exists;
     }
-    protected static BufferedInputStream getBufferedInputStream(String fileIn) 
+    
+    public static String removeSuffix(String file,String suffix)
+    {
+        return file.substring(0,file.length() - suffix.length());
+    }
+    
+    public static boolean createDirectoryAll(String dir) 
+    {
+        FileSystem fs = getFileSystemInstance(dir);
+        boolean created_dir = false;
+
+        if (!exists(dir)) {
+            try {
+                URI uri = new URI(dir);
+                Path path = new Path(uri);
+                fs.mkdirs(path);
+                created_dir = true; 
+            } catch (URISyntaxException e) {
+                e.printStackTrace();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        
+        return created_dir;
+    }
+    
+    public static BufferedInputStream getBufferedInputStream(String fileIn) 
             throws IOException 
     {
@@ -137 +163 @@
     }
 
-    protected static BufferedOutputStream getBufferedOutputStream(String fileOut) 
+    public static BufferedOutputStream getBufferedOutputStream(String fileOut) 
             throws IOException 
     {
@@ -163 +189 @@
     }
     
-    protected static BufferedReader getBufferedReaderForCompressedFile(String fileIn) 
+    public static BufferedReader getBufferedReaderForCompressedFile(String fileIn) 
             throws IOException, CompressorException 
     {
@@ -172 +198 @@
     }
 
-    protected static BufferedWriter getBufferedWriterForCompressedFile(String fileOut) 
+    public static BufferedWriter getBufferedWriterForCompressedFile(String fileOut) 
             throws IOException, CompressorException 
     {