Changeset 31077 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop

Timestamp:

2016-11-07T22:34:31+13:00 (7 years ago)

Author:

davidb

Message:

Move up to JDK1.8. Tidy up of Vagrant machine names. Support for YARN. Some XML whitespace indenting

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop

Files:

• manifests/init.pp (modified) (2 diffs)
• templates/hadoop-env.sh (modified) (1 diff)
• templates/hdfs-site.xml (modified) (2 diffs)
• templates/masters (modified) (1 diff)
• templates/setup-hadoop.bash (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/manifests/init.pp

@@ -11 +11 @@
 #  group => $group,
   unless => "ls ${install_dir} | grep hadoop-2.7.3",
-  require => Package["openjdk-7-jdk"]
+  require => Package["oracle-java8-installer"]
  }
 
@@ -79 +79 @@
   require => Exec["chown_hadoop"]
  }
- 
+
+ file {
+  "${hadoop_home}/etc/hadoop/yarn-site.xml":
+  content => template('hadoop/yarn-site.xml'),
+  mode => 644,
+  owner => $user,
+  group => $group,
+  require => Exec["chown_hadoop"]
+ }
+
 file {
   "${hadoop_home}/etc/hadoop/hadoop-env.sh":

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hadoop-env.sh

@@ -23 +23 @@
 
 # The java implementation to use.
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+export JAVA_HOME=/usr/lib/jvm/java-8-oracle
 
 # The jsvc implementation to use. Jsvc is required to run secure datanodes

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hdfs-site.xml

@@ -7 +7 @@
     <description>The actual number of replications can be specified when the file is created.</description>
   </property>
+
   <!--
   <property>
@@ -17 +18 @@
     </property>
   -->
-     <property>
-       <name>dfs.namenode.name.dir</name>
-       <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value>
-     </property>
-     <property>
-       <name>dfs.datanode.data.dir</name>
-       <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value>
-     </property>
+
+  <property>
+    <name>dfs.namenode.name.dir</name>
+    <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value>
+  </property>
+  <property>
+    <name>dfs.datanode.data.dir</name>
+    <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value>
+  </property>
 </configuration>

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/masters

@@ -1 @@
-<%= @hadoop_backup %>
+<%= @hadoop_backupmaster %>

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/setup-hadoop.bash

@@ -1 +1 @@
 
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+export JAVA_HOME=/usr/lib/jvm/java-8-oracle
 export PATH="$JAVA_HOME/bin:$PATH"
 
@@ -12 +12 @@
 
 . "$HADOOP_HOME/etc/hadoop/hadoop-env.sh"
+
+export HADOOP_YARN_USER=htrc
+. "$HADOOP_HOME/etc/hadoop/yarn-env.sh"