Changeset 30917 for other-projects/hathitrust/vagrant-spark-hdfs-cluster/trunk

Timestamp:

2016-10-25T14:20:26+13:00 (7 years ago)

Author:

davidb

Message:

Changes resulting from a fresh run at provisioning, which yielded the first successful run of RUN.bash on the cluster over HDFS. OS=Trusty as Xenial box doesn't appear to have puppet installed. JDK=1.7 because Trusty by default doesn't have apt-get for OpenJDK8.

Location:

other-projects/hathitrust/vagrant-spark-hdfs-cluster/trunk

Files:

• Vagrantfile (modified) (1 diff)
• manifests/base-hadoop.pp (modified) (1 diff)
• modules/hadoop/manifests/init.pp (modified) (1 diff)
• modules/hadoop/templates/hadoop-env.sh (modified) (1 diff)
• modules/hadoop/templates/setup-hadoop.bash (added)
• modules/spark (added)
• modules/spark/manifests (added)
• modules/spark/manifests/init.pp (added)
• modules/spark/templates (added)
• modules/spark/templates/setup-spark.bash (added)
• modules/spark/templates/slaves (added)

other-projects/hathitrust/vagrant-spark-hdfs-cluster/trunk/Vagrantfile

@@ -5 +5 @@
 
   config.vm.provider "virtualbox" do |v|
-    v.memory = 1024
+    v.memory = 2048
     v.cpus = 2
   end
 
   config.vm.box = "ubuntu/trusty64"
+#  config.vm.box = "ubuntu/xenial64"
 
   config.vm.provision :puppet do |puppet|

other-projects/hathitrust/vagrant-spark-hdfs-cluster/trunk/manifests/base-hadoop.pp

@@ -3 +3 @@
 $spark_home  = "${install_dir}/spark"
 
-$user = "ubuntu"
-$group = "ubuntu"
+$user = "htrc"
+$group = "hadoop"
 
 $hadoop_master = '10.10.0.52'

other-projects/hathitrust/vagrant-spark-hdfs-cluster/trunk/modules/hadoop/manifests/init.pp

@@ -2 +2 @@
 
 exec { "download_hadoop":
-  # Download from nearby mirror, otherwise task can time-out
-  command => "wget -O /tmp/hadoop-2.7.3.tar.gz http://apache.mirror.amaze.com.au/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz",
-  # command => "wget -O /tmp/hadoop-2.7.3.tar.gz http://www-eu.apache.org/dist/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz",
+  # Download from nearby mirror, otherwise task can time-out with default setting of '300'
+  # command => "wget -O /tmp/hadoop-2.7.3.tar.gz http://apache.mirror.amaze.com.au/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz",
+  command => "wget -O /tmp/hadoop-2.7.3.tar.gz   http://www-us.apache.org/dist/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz
+",
+  timeout => 0,
   path => $path,
-  user => $user,
-  group => $user,
+#  user => $user,
+#  group => $group,
   unless => "ls ${install_dir} | grep hadoop-2.7.3",
   require => Package["openjdk-7-jdk"]

other-projects/hathitrust/vagrant-spark-hdfs-cluster/trunk/modules/hadoop/templates/hadoop-env.sh

@@ -23 +23 @@
 
 # The java implementation to use.
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/
+export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
 
 # The jsvc implementation to use. Jsvc is required to run secure datanodes