Changeset 31267 for other-projects/hathitrust/wcsa

Timestamp:

2016-12-27T18:52:41+13:00 (7 years ago)

Author:

davidb

Message:

Values trialed on gsliscluster1. Rekindling idea of per-vol processing

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/ef-solr.properties (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/ef-solr.properties

@@ -1 +1 @@
 
-#wcsa-ef-ingest.process-ef-json-mode = per-volume
-wcsa-ef-ingest.process-ef-json-mode = per-page
+wcsa-ef-ingest.process-ef-json-mode = per-volume
+#wcsa-ef-ingest.process-ef-json-mode = per-page
+wcsa-ef-ingest.use-whitelist = true
+wcsa-ef-ingest.whitelist-filename = file:/hdfsd05/dbbridge/whitelist-peter1.txt
+#wcsa-ef-ingest.whitelist-filename = file:///home/dbbridge/extracted-features-solr/solr-ingest/whitelist-peter1.txt
+
 
 #wcsa-ef-ingest.solr-clode-nodes = 10.11.0.53:8983,10.11.0.54:8983,10.11.0.55:8983
 wcsa-ef-ingest.solr-cloud-nodes = gc0:8983,gc1:8983,gc2:8983,gc3:8983,gc4:8983,gc5:8983,gc6:8983,gc7:8983,gc8:8983,gc9:8983
+wcsa-ef-ingest.icu-tokenize = true
 wcsa-ef-ingest.strict-file-io = false
-wcsa-ef-ingest.icu-tokenize = true
+
 
 # For guide on number of partitions to use, see "Parallelized collections" section of:
@@ -16 +21 @@
     
 # wcsa-ef-ingest.num-partitions = 12
-wcsa-ef-ingest.num-partitions = 120
+#wcsa-ef-ingest.num-partitions = 110
+#wcsa-ef-ingest.num-partitions = 220
+#wcsa-ef-ingest.num-partitions = 400
+#wcsa-ef-ingest.num-partitions = 1000
+wcsa-ef-ingest.files-per-partition = 1300
 
-spark.executor.cores=11
+spark.executor.cores=10
+
+spark.driver.memory=50g
+spark.executor.memory=70g
+##spark.network.timeout=240s
+
+#spark.local.dir=/var/tmp