Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/Vagrantfile
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/Vagrantfile (revision 31076)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/Vagrantfile (revision 31077)
@@ -17,29 +17,36 @@
puppet.module_path = "modules"
end
-
- config.vm.define :backup do |box|
+
+ config.vm.define :master do |box|
+ box.vm.network "private_network", ip: "10.10.0.52"
+ box.vm.host_name = "master"
+ end
+
+# config.vm.define :resman do |box|
+# box.vm.network "private_network", ip: "10.10.0.50"
+# box.vm.host_name = "resman"
+# end
+
+ config.vm.define :backupmaster do |box|
box.vm.network "private_network", ip: "10.10.0.51"
- box.vm.host_name = "backup"
+ box.vm.host_name = "backupmaster"
+ end
+
+ config.vm.define :hadoop1 do |box|
+ box.vm.network "private_network", ip: "10.10.0.53"
+ box.vm.host_name = "hadoop1"
end
- config.vm.define :hadoop1 do |hadoop1_config|
- hadoop1_config.vm.network "private_network", ip: "10.10.0.53"
- hadoop1_config.vm.host_name = "hadoop1"
+ config.vm.define :hadoop2 do |box|
+ box.vm.network "private_network", ip: "10.10.0.54"
+ box.vm.host_name = "hadoop2"
end
- config.vm.define :hadoop2 do |hadoop2_config|
- hadoop2_config.vm.network "private_network", ip: "10.10.0.54"
- hadoop2_config.vm.host_name = "hadoop2"
+ config.vm.define :hadoop3 do |box|
+ ip = "10.10.0.55"
+ box.vm.network "private_network", ip: ip
+ box.vm.host_name = "hadoop3"
end
- config.vm.define :hadoop3 do |hadoop3_config|
- hadoop3_config.vm.network "private_network", ip: "10.10.0.55"
- hadoop3_config.vm.host_name = "hadoop3"
- end
-
- config.vm.define :master do |master_config|
- master_config.vm.network "private_network", ip: "10.10.0.52"
- master_config.vm.host_name = "master"
- end
end
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/manifests/base-hadoop.pp
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/manifests/base-hadoop.pp (revision 31076)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/manifests/base-hadoop.pp (revision 31077)
@@ -7,5 +7,6 @@
$hadoop_master = '10.10.0.52'
-$hadoop_backup = '10.10.0.51'
+$hadoop_resman = '10.10.0.50'
+$hadoop_backupmaster = '10.10.0.51'
$hadoop_1 = '10.10.0.53'
$hadoop_2 = '10.10.0.54'
@@ -40,13 +41,21 @@
+ exec { 'add-apt-repository':
+ command => '/usr/bin/add-apt-repository ppa:webupd8team/java -y && \
+ echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \
+ echo debconf shared/accepted-oracle-license-v1-1 seen true | /usr/bin/debconf-set-selections'
+ }
-exec { 'apt-get update':
+
+ exec { 'apt-get update':
command => '/usr/bin/apt-get update',
-}
+ require => [ Exec['add-apt-repository'], File["/home/${user}"] ]
+ }
-package { "openjdk-7-jdk" :
+ package { "oracle-java8-installer" :
ensure => present,
require => [ Exec['apt-get update'], File["/home/${user}"] ]
-}
+ }
+
#package { "subversion" :
@@ -106,4 +115,16 @@
}
+host { 'master':
+ ip => "${hadoop_master}",
+}
+
+host { 'resman':
+ ip => "${hadoop_resman}",
+}
+
+host { 'backupmaster':
+ ip => "${hadoop_backupmaster}",
+}
+
host { 'hadoop1':
ip => "${hadoop_1}",
@@ -118,9 +139,2 @@
}
-host { 'master':
- ip => "${hadoop_master}",
-}
-
-host { 'backup':
- ip => "${hadoop_backup}",
-}
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/manifests/init.pp
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/manifests/init.pp (revision 31076)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/manifests/init.pp (revision 31077)
@@ -11,5 +11,5 @@
# group => $group,
unless => "ls ${install_dir} | grep hadoop-2.7.3",
- require => Package["openjdk-7-jdk"]
+ require => Package["oracle-java8-installer"]
}
@@ -79,5 +79,14 @@
require => Exec["chown_hadoop"]
}
-
+
+ file {
+ "${hadoop_home}/etc/hadoop/yarn-site.xml":
+ content => template('hadoop/yarn-site.xml'),
+ mode => 644,
+ owner => $user,
+ group => $group,
+ require => Exec["chown_hadoop"]
+ }
+
file {
"${hadoop_home}/etc/hadoop/hadoop-env.sh":
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hadoop-env.sh
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hadoop-env.sh (revision 31076)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hadoop-env.sh (revision 31077)
@@ -23,5 +23,5 @@
# The java implementation to use.
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+export JAVA_HOME=/usr/lib/jvm/java-8-oracle
# The jsvc implementation to use. Jsvc is required to run secure datanodes
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hdfs-site.xml
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hdfs-site.xml (revision 31076)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hdfs-site.xml (revision 31077)
@@ -7,4 +7,5 @@
The actual number of replications can be specified when the file is created.
+
-
- dfs.namenode.name.dir
- file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode
-
-
- dfs.datanode.data.dir
- file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode
-
+
+
+ dfs.namenode.name.dir
+ file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode
+
+
+ dfs.datanode.data.dir
+ file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode
+
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/masters
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/masters (revision 31076)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/masters (revision 31077)
@@ -1,1 +1,1 @@
-<%= @hadoop_backup %>
+<%= @hadoop_backupmaster %>
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/setup-hadoop.bash
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/setup-hadoop.bash (revision 31076)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/setup-hadoop.bash (revision 31077)
@@ -1,4 +1,4 @@
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+export JAVA_HOME=/usr/lib/jvm/java-8-oracle
export PATH="$JAVA_HOME/bin:$PATH"
@@ -12,2 +12,5 @@
. "$HADOOP_HOME/etc/hadoop/hadoop-env.sh"
+
+export HADOOP_YARN_USER=htrc
+. "$HADOOP_HOME/etc/hadoop/yarn-env.sh"
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/spark/manifests/init.pp
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/spark/manifests/init.pp (revision 31076)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/spark/manifests/init.pp (revision 31077)
@@ -7,5 +7,5 @@
# group => $user,
unless => "ls ${install_dir} | grep spark-2.0.1",
- require => Package["openjdk-7-jdk"]
+ require => Package["oracle-java8-installer"]
}