Changeset 31276

Timestamp:

2017-01-06T00:04:53+13:00 (7 years ago)

Author:

davidb

Message:

Min num partition guard put in

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -28 +28 @@
     //   http://blog.cloudera.com/blog/2015/03/how-to-tune-your-apache-spark-jobs-part-2/
     
-    //protected static final int DEFAULT_NUM_CORES = 6;
-    //protected static final int DEFAULT_NUM_PARTITIONS = 3*DEFAULT_NUM_CORES; 
+    protected static final int DEFAULT_NUM_CORES = 10;
+    protected static final int MINIMUM_NUM_PARTITIONS = 4*DEFAULT_NUM_CORES; 
     
     protected static final int DEFAULT_FILES_PER_PARTITION = 3000;
@@ -108 +108 @@
         
         int num_partitions = (int)(num_volumes/files_per_partition)+1;
-        
+        if (num_partitions < MINIMUM_NUM_PARTITIONS) {
+            num_partitions = MINIMUM_NUM_PARTITIONS;
+        }
         JavaRDD<String> json_list_data_rp = json_list_data.repartition(num_partitions);