Changeset 31269 for other-projects/hathitrust/wcsa

Timestamp:

2016-12-28T10:30:08+13:00 (7 years ago)

Author:

davidb

Message:

Some variable name changes, and printing tidy up

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerVolumeJSON.java (modified) (1 diff)
• PerVolumeLangStreamFlatmap.java (modified) (3 diffs)
• ProcessForSolrIngest.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -113 +113 @@
                 ids.add(page_id); 
 
-                if (i==0) {
-                    System.out.println("Sample output JSON page file: " + output_json_bz2);
+                if (_verbosity >=2) {
+                    if (i==0) {
+                        System.out.println("Sample output JSON page file [i=0]: " + output_json_bz2);
+                    }
                 }
-
                 JSONObject ef_page = ef_pages.getJSONObject(i);
 

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeLangStreamFlatmap.java

@@ -41 +41 @@
         JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(full_json_file_in);
         
-        ArrayList<String> all_word_list = new ArrayList<String>();
+        ArrayList<String> all_lang_list = new ArrayList<String>();
         
         if (extracted_feature_record != null) {
@@ -87 +87 @@
                 if (ef_page != null) {
                     
-                    ArrayList<String> page_word_list = SolrDocJSON.generateTokenPosCountLangLabels(volume_id, page_id, ef_page);                    
-                    all_word_list.addAll(page_word_list);
+                    ArrayList<String> page_lang_labels = SolrDocJSON.generateTokenPosCountLangLabels(volume_id, page_id, ef_page);                  
+                    all_lang_list.addAll(page_lang_labels);
                 }
                 else {
@@ -109 +109 @@
         _progress_accum.add(_progress_step);
         
-        return all_word_list.iterator();
+        return all_lang_list.iterator();
     }
     

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -113 +113 @@
         DoubleAccumulator progress_accum = jsc.sc().doubleAccumulator("Progress Percent");
         
-        System.err.println();
-        System.err.println();
-        System.err.println();
-        System.err.println("****##### _input_dir =  " + _input_dir);
-        System.err.println();
-        System.err.println();
-        System.err.println();
-        
         boolean icu_tokenize = Boolean.getBoolean("wcsa-ef-ingest.icu-tokenize");
         boolean strict_file_io = Boolean.getBoolean("wcsa-ef-ingest.strict-file-io");
@@ -130 +122 @@
         //json_list_data_rp.foreach(per_vol_json);
         JavaRDD<String> per_page_ids = json_list_data_rp.flatMap(per_vol_json);
-        long num_page_ids = per_page_ids.count();
-        
-        long num_ids = num_volumes;
+        long num_page_ids = per_page_ids.count(); // trigger lazy eval of: flatmap:per-vol 
+        
+        //long num_ids = num_volumes;
         
         System.out.println("");