Changeset 31091 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk

Timestamp:

2016-11-10T16:15:30+13:00 (7 years ago)

Author:

davidb

Message:

Change of number of core for 'gsliscluster1' machine; commmented out experimental repartition added.

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -28 +28 @@
     //   http://blog.cloudera.com/blog/2015/03/how-to-tune-your-apache-spark-jobs-part-2/
     
-    public static final int NUM_CORES = 6;
+    public static final int NUM_CORES = 10*12;
     public static final int NUM_PARTITIONS = 2*NUM_CORES; // default would appear to be 2
     
@@ -103 +103 @@
         double per_vol = 100.0/(double)num_volumes;
         
+        //JavaRDD<String> json_list_data_rp = json_list_data.repartition((int)(num_volumes/100));
+
         DoubleAccumulator progress_accum = jsc.sc().doubleAccumulator("Progress Percent");
         
@@ -147 +149 @@
         double per_vol = 100.0/(double)num_volumes;
         
+        //JavaRDD<String> json_list_data_rp = json_list_data.repartition((int)(num_volumes/100));
+
         DoubleAccumulator per_vol_progress_accum = jsc.sc().doubleAccumulator("Per Volume Progress Percent");