Changeset 31276

Show
Ignore:
Timestamp:
06.01.2017 00:04:53 (2 years ago)
Author:
davidb
Message:

Min num partition guard put in

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

    r31271 r31276  
    2828    //   http://blog.cloudera.com/blog/2015/03/how-to-tune-your-apache-spark-jobs-part-2/ 
    2929     
    30     //protected static final int DEFAULT_NUM_CORES = 6; 
    31     //protected static final int DEFAULT_NUM_PARTITIONS = 3*DEFAULT_NUM_CORES;  
     30    protected static final int DEFAULT_NUM_CORES = 10; 
     31    protected static final int MINIMUM_NUM_PARTITIONS = 4*DEFAULT_NUM_CORES;  
    3232     
    3333    protected static final int DEFAULT_FILES_PER_PARTITION = 3000; 
     
    108108         
    109109        int num_partitions = (int)(num_volumes/files_per_partition)+1; 
    110          
     110        if (num_partitions < MINIMUM_NUM_PARTITIONS) { 
     111            num_partitions = MINIMUM_NUM_PARTITIONS; 
     112        } 
    111113        JavaRDD<String> json_list_data_rp = json_list_data.repartition(num_partitions); 
    112114