Changeset 31271 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumePOSStreamFlatmap.java

Timestamp:

2016-12-28T14:04:19+13:00 (7 years ago)

Author:

davidb

Message:

Updating of POS code to new files-per-partition paramater, plus some other related tweaks

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumePOSStreamFlatmap.java (modified) (6 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumePOSStreamFlatmap.java

@@ -41 +41 @@
         JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(full_json_file_in);
         
-        ArrayList<String> all_word_list = new ArrayList<String>();
+        ArrayList<String> all_pos_list = new ArrayList<String>();
         
         if (extracted_feature_record != null) {
@@ -62 +62 @@
             }
     
-            if (_verbosity >= 2) {
+            if (_verbosity >= 3) {
                 System.out.print("  Pages: ");
             }
@@ -70 +70 @@
                 String page_id = volume_id + "." + formatted_i;
 
-                if (_verbosity >= 2) {
+                if (_verbosity >= 3) {
                     if (i>0) {
                         System.out.print(", ");
@@ -78 +78 @@
 
                 if (i==(ef_page_count-1)) {
-                    if (_verbosity >= 2) {
+                    if (_verbosity >= 3) {
                         System.out.println();
                     }
@@ -87 +87 @@
                 if (ef_page != null) {
                     
-                    ArrayList<String> page_word_list = SolrDocJSON.generateTokenPosCountPOSLabels(volume_id, page_id, ef_page);                 
-                    all_word_list.addAll(page_word_list);
+                    ArrayList<String> page_pos_list = SolrDocJSON.generateTokenPosCountPOSLabels(volume_id, page_id, ef_page);                  
+                    all_pos_list.addAll(page_pos_list);
                 }
                 else {
@@ -109 +109 @@
         _progress_accum.add(_progress_step);
         
-        return all_word_list.iterator();
+        return all_pos_list.iterator();
     }