Changeset 32101

Timestamp:

2018-01-12T18:16:31+13:00 (6 years ago)

Author:

davidb

Message:

Tweaks to allow serial ingest to run

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• ClusterFileIO.java (modified) (2 diffs)
• ProcessForSerialSolrIngest.java (added)
• UniversalPOSLangMap.java (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ClusterFileIO.java

@@ -5 +5 @@
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -135 +136 @@
         }
         else {
+            
+            
+            // Trim 'file://' off the front
             /*
-            // Trim 'file://' off the front
             String local_file_in = fileIn;
             if (local_file_in.startsWith("file://")) {
                 local_file_in = fileIn.substring("file://".length());
             }
+            else if (local_file_in.startsWith("file:/")) {
+                local_file_in = fileIn.substring("file:/".length());
+            }
             FileInputStream fin = new FileInputStream(local_file_in);
             bis = new BufferedInputStream(fin);
             */
+            
+            
             URI uri = URI.create (fileIn);
             Path path = new Path(uri);
             
             FSDataInputStream fin = fs.open(path);
-            
             bis = new BufferedInputStream(fin);
+            
+            
         }
     

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/UniversalPOSLangMap.java

@@ -38 +38 @@
             e.printStackTrace();
         }
-        try (Stream<Path> stream_paths = Files.walk(Paths.get(langmap_directory_uri))) {
+            
+        Path langmap_directory_path = null;
+        try {
+            // Spark/Hadoop friendly
+            langmap_directory_path = Paths.get(langmap_directory_uri);
+        }
+        catch (Exception e) {
+            // Relative local file-system friendly
+            langmap_directory_path = Paths.get(langmap_directory_uri.getRawPath());
+        }
+        
+        
+        try (Stream<Path> stream_paths = Files.walk(langmap_directory_path)) {
             langmap_paths = stream_paths
                     .filter(Files::isRegularFile)