Changeset 31276


Ignore:
Timestamp:
01/06/17 00:04:53 (4 years ago)
Author:
davidb
Message:

Min num partition guard put in

File:
1 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

    r31271 r31276  
    2828    //   http://blog.cloudera.com/blog/2015/03/how-to-tune-your-apache-spark-jobs-part-2/
    2929   
    30     //protected static final int DEFAULT_NUM_CORES = 6;
    31     //protected static final int DEFAULT_NUM_PARTITIONS = 3*DEFAULT_NUM_CORES;
     30    protected static final int DEFAULT_NUM_CORES = 10;
     31    protected static final int MINIMUM_NUM_PARTITIONS = 4*DEFAULT_NUM_CORES;
    3232   
    3333    protected static final int DEFAULT_FILES_PER_PARTITION = 3000;
     
    108108       
    109109        int num_partitions = (int)(num_volumes/files_per_partition)+1;
    110        
     110        if (num_partitions < MINIMUM_NUM_PARTITIONS) {
     111            num_partitions = MINIMUM_NUM_PARTITIONS;
     112        }
    111113        JavaRDD<String> json_list_data_rp = json_list_data.repartition(num_partitions);
    112114
Note: See TracChangeset for help on using the changeset viewer.