Changeset 31319 for other-projects/hathitrust

Timestamp:

2017-01-22T15:19:50+13:00 (7 years ago)

Author:

davidb

Message:

Changed to replace existing MongoDB entry. Fixed up printt statement

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerVolumeMongoDBDocumentsMap.java (modified) (4 diffs)
• ProcessForMongoDBIngest.java (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeMongoDBDocumentsMap.java

@@ -2 +2 @@
 
 import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.spark.api.java.function.Function;
 import org.apache.spark.util.DoubleAccumulator;
 import org.bson.Document;
-import org.json.JSONArray;
 import org.json.JSONObject;
 
@@ -57 +48 @@
         String[] key_array = key_set.toArray(new String[key_set.size()]);
         
-        
-        //Set<String> key_set = ef_count.keySet();
-        //for (String key : key_set) {
-            
-        //Iterator<String> key_iterator = ef_count.keySet().iterator();
-        //while (key_iterator.hasNext()) {
         for (int i=0; i<key_array.length; i++) {
             
@@ -69 +54 @@
             if (key.contains(".")) {
                 String new_key = key.replaceAll("\\.", "<PERIOD>");
-                System.out.println("**** old key:" + key + "=> new key:" + new_key);
+                //System.out.println("**** old key:" + key + "=> new key:" + new_key);
                 ef_count.put(new_key, ef_count.get(key));
                 ef_count.remove(key);
@@ -202 +187 @@
             fixup_volume(json_file_in,doc);
             
-            collection.insertOne(doc);      
+            collection.findOneAndReplace(Document.parse("{_id:" + doc.getString("_id") + "}"),doc);
+            
+            //collection.insertOne(doc);        
             
             /*

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForMongoDBIngest.java

@@ -106 +106 @@
         
         System.out.println("################");
-        System.out.println("# Total Page Count = " + total_page_count);
+        System.out.println("# Total Volume Count = " + total_page_count);
         System.out.println("################");
     }