Changeset 31095 for other-projects

Timestamp:

2016-11-10T18:58:06+13:00 (7 years ago)

Author:

davidb

Message:

Introduced num-partitions property

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Files:

• ef-solr.properties (modified) (1 diff)
• src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/ef-solr.properties

@@ -7 +7 @@
 wcsa-ef-ingest.strict-file-io = false
 
+# For guide on number of partitions to use, see "Parallelized collections" section of:
+#   https://spark.apache.org/docs/2.0.1/programming-guide.html
+# which suggests 2-4 * num_cores
+#
+# For a more detailed discussion see:
+#   http://blog.cloudera.com/blog/2015/03/how-to-tune-your-apache-spark-jobs-part-2/
+    
+# wcsa-ef-ingest.num-partitions = 12
+wcsa-ef-ingest.num-partitions = 120
+
+

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -28 +28 @@
     //   http://blog.cloudera.com/blog/2015/03/how-to-tune-your-apache-spark-jobs-part-2/
     
-    public static final int NUM_CORES = 10*12;
-    public static final int NUM_PARTITIONS = 2*NUM_CORES; // default would appear to be 2
+    protected static final int DEFAULT_NUM_CORES = 6;
+    protected static final int DEFAULT_NUM_PARTITIONS = 3*DEFAULT_NUM_CORES; 
     
     protected String _input_dir;
@@ -98 +98 @@
         }
                 
-        JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename,NUM_PARTITIONS).cache();
+        int num_partitions = Integer.getInteger("wcsa-ef-ingest.num-partitions", DEFAULT_NUM_PARTITIONS);
+        
+        JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename,num_partitions).cache();
 
         long num_volumes = json_list_data.count();
@@ -144 +146 @@
         }
                 
-        JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename,NUM_PARTITIONS).cache();
+        int num_partitions = Integer.getInteger("wcsa-ef-ingest.num-partitions", DEFAULT_NUM_PARTITIONS);
+        JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename,num_partitions).cache();
 
         long num_volumes = json_list_data.count();