Changeset 31271 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForLangCount.java

Timestamp:

2016-12-28T14:04:19+13:00 (7 years ago)

Author:

davidb

Message:

Updating of POS code to new files-per-partition paramater, plus some other related tweaks

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForLangCount.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForLangCount.java

@@ -69 +69 @@
         
         //int num_partitions = Integer.getInteger("wcsa-ef-ingest.num-partitions", DEFAULT_NUM_PARTITIONS);
-        int files_per_partition = Integer.getInteger("wcsa-ef-ingest.num-partitions", DEFAULT_FILES_PER_PARTITION);
+        int files_per_partition = Integer.getInteger("wcsa-ef-ingest.files-per-partition", DEFAULT_FILES_PER_PARTITION);
         
-        
-        JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename,files_per_partition).cache();
+        JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename).cache();
         json_list_data.setName("JSON-file-list");
         
@@ -81 +80 @@
         
         JavaRDD<String> json_list_data_rp = json_list_data.repartition(num_partitions);
+        json_list_data_rp.setName("JSON-file-list--repartitioned");
         
         DoubleAccumulator per_vol_progress_accum = jsc.sc().doubleAccumulator("Per Volume Progress Percent");