Changeset 31310 for other-projects/hathitrust/wcsa

Timestamp:

2017-01-21T21:18:02+13:00 (7 years ago)

Author:

davidb

Message:

Initial cut at files for working with MongoDB

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Files:

• pom.xml (modified) (1 diff)
• src/main/java/org/hathitrust/extractedfeatures/ClusterFileIO.java (modified) (2 diffs)
• src/main/java/org/hathitrust/extractedfeatures/PerVolumeMongoDBDocumentsMap.java (added)
• src/main/java/org/hathitrust/extractedfeatures/ProcessForMongoDBIngest.java (added)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/pom.xml

@@ -21 +21 @@
 
     <dependency>
+      <groupId>org.mongodb</groupId>
+      <artifactId>mongo-java-driver</artifactId>
+      <version>3.4.1</version>
+    </dependency>
+<!--
+    <dependency>
       <groupId>org.mongodb.spark</groupId>
       <artifactId>mongo-spark-connector_2.10</artifactId>
       <version>0.1</version>
     </dependency>
+-->
 
     <!--

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ClusterFileIO.java

@@ -21 +21 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.json.JSONObject;
 
 public class ClusterFileIO {
@@ -154 +155 @@
     }
 
+    protected static String readTextFile(String filename)
+    {
+        StringBuilder sb = new StringBuilder();
+        
+        try {   
+            BufferedReader br = ClusterFileIO.getBufferedReaderForCompressedFile(filename);
+
+            int cp;
+            while ((cp = br.read()) != -1) {
+                sb.append((char) cp);
+            }
+    
+            br.close();
+        } 
+        catch (Exception e) {
+            e.printStackTrace();
+        }
+        
+        return sb.toString();
+    }
+    
     public static BufferedOutputStream getBufferedOutputStream(String fileOut) 
             throws IOException