Changeset 31312 for other-projects/hathitrust/wcsa/extracted-features-solr

Timestamp:

2017-01-21T23:57:09+13:00 (7 years ago)

Author:

davidb

Message:

MongoDB can't have 'period' and 'dollar' in key, as reserved characters

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeMongoDBDocumentsMap.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeMongoDBDocumentsMap.java

@@ -2 +2 @@
 
 import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.spark.api.java.function.Function;
 import org.apache.spark.util.DoubleAccumulator;
 import org.bson.Document;
+import org.json.JSONArray;
 import org.json.JSONObject;
 
@@ -39 +51 @@
     }
     
+    protected void fixup_section(Document ef_count)
+    {
+        Set<String> key_set = ef_count.keySet();
+        for (String key : key_set) {
+            if (key.matches("\\.")) {
+                String new_key = key.replaceAll("\\.", "<PERIOD>");
+                ef_count.put(new_key, ef_count.get(key));
+                ef_count.remove(key);
+                key = new_key;
+            }
+
+            if (key.matches("\\$")) {
+                String new_key = key.replaceAll("\\$", "<DOLLAR>");
+                ef_count.put(new_key, ef_count.get(key));
+                ef_count.remove(key);
+            }
+
+        }       
+    }
+    
+    protected  void fixup_page(String volume_id, String page_id, Document ef_page) 
+    {
+        if (ef_page != null) {
+            String[] zone_keys = { "header", "body", "footer" };
+            
+            for (String zone_key: zone_keys) {
+                Document ef_zone = (Document)ef_page.get(zone_key);
+                if (ef_zone != null) {
+                    String[] count_keys = { "beginCharCounts", "endCharCount", "tokenPosCount" };
+                    
+                    for (String count_key: count_keys) {
+                        Document ef_sub_section = (Document)ef_zone.get(count_key);
+                        if (ef_sub_section != null) {
+                            fixup_section(ef_sub_section);
+                            
+                            if (count_key.equals("tokenPostCount")) {
+                                Set<String> key_set = ef_sub_section.keySet();
+                                for (String key : key_set) {
+                                    Document token_section = (Document)ef_sub_section.get(key);
+                                    fixup_section(token_section);
+                                }
+                            }
+                        
+                            
+                        }
+                    }
+                }
+            }
+        }
+        else {
+            System.err.println("Warning: null page for '" + page_id + "'");
+        }
+
+    }
+    protected void fixup_volume(String json_file_in, Document extracted_feature_record)
+    {
+        String full_json_file_in = _input_dir + "/" + json_file_in;
+        
+        if (extracted_feature_record != null) {
+            String volume_id = extracted_feature_record.getString("id");
+            extracted_feature_record.put("_id",volume_id);
+            extracted_feature_record.remove("id");
+            
+            Document ef_features = (Document)extracted_feature_record.get("features");
+
+            int ef_page_count = ef_features.getInteger("pageCount");
+
+            if (_verbosity >= 1) {
+                System.out.println("Processing: " + json_file_in);
+                System.out.println("  pageCount = " + ef_page_count);
+            }
+
+            List<Document> ef_pages = (List<Document>)ef_features.get("pages");
+            int ef_num_pages = ef_pages.size();
+            if (ef_num_pages != ef_page_count) {
+                System.err.println("Warning: number of page elements in JSON (" + ef_num_pages + ")"
+                        +" does not match 'pageCount' metadata (" + ef_page_count + ")"); 
+            }
+    
+            if (_verbosity >= 2) {
+                System.out.print("  Pages: ");
+            }
+
+            for (int i = 0; i < ef_page_count; i++) {
+                String formatted_i = String.format("page-%06d", i);
+                String page_id = volume_id + "." + formatted_i;
+
+                if (_verbosity >= 2) {
+                    if (i>0) {
+                        System.out.print(", ");
+                    }
+                    System.out.print(page_id);
+                }
+
+                if (i==(ef_page_count-1)) {
+                    if (_verbosity >= 2) {
+                        System.out.println();
+                    }
+                }
+
+                Document ef_page = (Document)ef_pages.get(i);
+
+                if (ef_page != null) {
+                    
+                    fixup_page(volume_id, page_id, ef_page);                    
+                }
+                else {
+                    System.err.println("Skipping: " + page_id);
+                }
+            }
+        }
+        else {
+            // File did not exist, or could not be parsed
+            String mess = "Failed to read in bzipped JSON file '" + full_json_file_in + "'";
+        
+            System.err.println("Warning: " + mess);
+            System.out.println("Warning: " + mess);
+        
+        }
+    }
+    
     public Integer call(String json_file_in) throws IOException
     { 
@@ -53 +186 @@
             
             Document doc = Document.parse(extracted_feature_json_doc);
+            
+            fixup_volume(json_file_in,doc);
+            
             collection.insertOne(doc);