Changeset 31077 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk

Timestamp:

2016-11-07T22:34:31+13:00 (7 years ago)

Author:

davidb

Message:

Move up to JDK1.8. Tidy up of Vagrant machine names. Support for YARN. Some XML whitespace indenting

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster

Files:

• Vagrantfile (modified) (1 diff)
• manifests/base-hadoop.pp (modified) (4 diffs)
• modules/hadoop/manifests/init.pp (modified) (2 diffs)
• modules/hadoop/templates/hadoop-env.sh (modified) (1 diff)
• modules/hadoop/templates/hdfs-site.xml (modified) (2 diffs)
• modules/hadoop/templates/masters (modified) (1 diff)
• modules/hadoop/templates/setup-hadoop.bash (modified) (2 diffs)
• modules/spark/manifests/init.pp (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/Vagrantfile

@@ -17 +17 @@
      puppet.module_path = "modules"
   end
-  
-  config.vm.define :backup do |box|
+
+  config.vm.define :master do |box|
+    box.vm.network "private_network", ip: "10.10.0.52"
+    box.vm.host_name = "master"
+  end
+
+#  config.vm.define :resman do |box|
+#    box.vm.network "private_network", ip: "10.10.0.50"
+#    box.vm.host_name = "resman"  
+#  end
+
+  config.vm.define :backupmaster do |box|
     box.vm.network "private_network", ip: "10.10.0.51"
-    box.vm.host_name = "backup"  
+    box.vm.host_name = "backupmaster"  
+  end
+
+  config.vm.define :hadoop1 do |box|
+    box.vm.network "private_network", ip: "10.10.0.53"
+    box.vm.host_name = "hadoop1"
   end
   
-  config.vm.define :hadoop1 do |hadoop1_config|
-    hadoop1_config.vm.network "private_network", ip: "10.10.0.53"
-    hadoop1_config.vm.host_name = "hadoop1"
+  config.vm.define :hadoop2 do |box|
+    box.vm.network "private_network", ip: "10.10.0.54"
+    box.vm.host_name = "hadoop2"
   end
   
-  config.vm.define :hadoop2 do |hadoop2_config|
-    hadoop2_config.vm.network "private_network", ip: "10.10.0.54"
-    hadoop2_config.vm.host_name = "hadoop2"
+  config.vm.define :hadoop3 do |box|
+    ip = "10.10.0.55"
+    box.vm.network "private_network", ip: ip
+    box.vm.host_name = "hadoop3"
   end
   
-  config.vm.define :hadoop3 do |hadoop3_config|
-    hadoop3_config.vm.network "private_network", ip: "10.10.0.55"
-    hadoop3_config.vm.host_name = "hadoop3"
-  end
-  
-   config.vm.define :master do |master_config|
-    master_config.vm.network "private_network", ip: "10.10.0.52"
-    master_config.vm.host_name = "master"
-  end
 
 end

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/manifests/base-hadoop.pp

@@ -7 +7 @@
 
 $hadoop_master = '10.10.0.52'
-$hadoop_backup = '10.10.0.51'
+$hadoop_resman = '10.10.0.50'
+$hadoop_backupmaster = '10.10.0.51'
 $hadoop_1 = '10.10.0.53'
 $hadoop_2 = '10.10.0.54'
@@ -40 +41 @@
     
 
+ exec { 'add-apt-repository':
+    command => '/usr/bin/add-apt-repository ppa:webupd8team/java -y && \
+                echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \
+                echo debconf shared/accepted-oracle-license-v1-1 seen true | /usr/bin/debconf-set-selections'
+ }
 
-exec { 'apt-get update':
+                
+ exec { 'apt-get update':
     command => '/usr/bin/apt-get update',
-}
+    require => [ Exec['add-apt-repository'], File["/home/${user}"] ]
+ }
 
-package { "openjdk-7-jdk" :
+ package { "oracle-java8-installer" :
    ensure => present,
   require => [ Exec['apt-get update'], File["/home/${user}"] ]
-}
+ }
+
 
 #package { "subversion" :
@@ -106 +115 @@
 }
 
+host { 'master':
+    ip => "${hadoop_master}",
+}
+
+host { 'resman':
+    ip => "${hadoop_resman}",
+}
+
+host { 'backupmaster':
+    ip => "${hadoop_backupmaster}",
+}
+
 host { 'hadoop1':
     ip => "${hadoop_1}",
@@ -118 +139 @@
 }
 
-host { 'master':
-    ip => "${hadoop_master}",
-}
-
-host { 'backup':
-    ip => "${hadoop_backup}",
-}

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/manifests/init.pp

@@ -11 +11 @@
 #  group => $group,
   unless => "ls ${install_dir} | grep hadoop-2.7.3",
-  require => Package["openjdk-7-jdk"]
+  require => Package["oracle-java8-installer"]
  }
 
@@ -79 +79 @@
   require => Exec["chown_hadoop"]
  }
- 
+
+ file {
+  "${hadoop_home}/etc/hadoop/yarn-site.xml":
+  content => template('hadoop/yarn-site.xml'),
+  mode => 644,
+  owner => $user,
+  group => $group,
+  require => Exec["chown_hadoop"]
+ }
+
 file {
   "${hadoop_home}/etc/hadoop/hadoop-env.sh":

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hadoop-env.sh

@@ -23 +23 @@
 
 # The java implementation to use.
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+export JAVA_HOME=/usr/lib/jvm/java-8-oracle
 
 # The jsvc implementation to use. Jsvc is required to run secure datanodes

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hdfs-site.xml

@@ -7 +7 @@
     <description>The actual number of replications can be specified when the file is created.</description>
   </property>
+
   <!--
   <property>
@@ -17 +18 @@
     </property>
   -->
-     <property>
-       <name>dfs.namenode.name.dir</name>
-       <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value>
-     </property>
-     <property>
-       <name>dfs.datanode.data.dir</name>
-       <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value>
-     </property>
+
+  <property>
+    <name>dfs.namenode.name.dir</name>
+    <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value>
+  </property>
+  <property>
+    <name>dfs.datanode.data.dir</name>
+    <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value>
+  </property>
 </configuration>

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/masters

@@ -1 @@
-<%= @hadoop_backup %>
+<%= @hadoop_backupmaster %>

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/setup-hadoop.bash

@@ -1 +1 @@
 
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+export JAVA_HOME=/usr/lib/jvm/java-8-oracle
 export PATH="$JAVA_HOME/bin:$PATH"
 
@@ -12 +12 @@
 
 . "$HADOOP_HOME/etc/hadoop/hadoop-env.sh"
+
+export HADOOP_YARN_USER=htrc
+. "$HADOOP_HOME/etc/hadoop/yarn-env.sh"

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/spark/manifests/init.pp

@@ -7 +7 @@
 #  group => $user,
   unless => "ls ${install_dir} | grep spark-2.0.1",
-  require => Package["openjdk-7-jdk"]
+  require => Package["oracle-java8-installer"]
  }