Changeset 31077

Show
Ignore:
Timestamp:
07.11.2016 22:34:31 (3 years ago)
Author:
davidb
Message:

Move up to JDK1.8. Tidy up of Vagrant machine names. Support for YARN. Some XML whitespace indenting

Location:
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster
Files:
8 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/Vagrantfile

    r30917 r31077  
    1717     puppet.module_path = "modules" 
    1818  end 
    19    
    20   config.vm.define :backup do |box| 
     19 
     20  config.vm.define :master do |box| 
     21    box.vm.network "private_network", ip: "10.10.0.52" 
     22    box.vm.host_name = "master" 
     23  end 
     24 
     25#  config.vm.define :resman do |box| 
     26#    box.vm.network "private_network", ip: "10.10.0.50" 
     27#    box.vm.host_name = "resman"   
     28#  end 
     29 
     30  config.vm.define :backupmaster do |box| 
    2131    box.vm.network "private_network", ip: "10.10.0.51" 
    22     box.vm.host_name = "backup"   
     32    box.vm.host_name = "backupmaster"   
     33  end 
     34 
     35  config.vm.define :hadoop1 do |box| 
     36    box.vm.network "private_network", ip: "10.10.0.53" 
     37    box.vm.host_name = "hadoop1" 
    2338  end 
    2439   
    25   config.vm.define :hadoop1 do |hadoop1_config| 
    26     hadoop1_config.vm.network "private_network", ip: "10.10.0.53" 
    27     hadoop1_config.vm.host_name = "hadoop1" 
     40  config.vm.define :hadoop2 do |box| 
     41    box.vm.network "private_network", ip: "10.10.0.54" 
     42    box.vm.host_name = "hadoop2" 
    2843  end 
    2944   
    30   config.vm.define :hadoop2 do |hadoop2_config| 
    31     hadoop2_config.vm.network "private_network", ip: "10.10.0.54" 
    32     hadoop2_config.vm.host_name = "hadoop2" 
     45  config.vm.define :hadoop3 do |box| 
     46    ip = "10.10.0.55" 
     47    box.vm.network "private_network", ip: ip 
     48    box.vm.host_name = "hadoop3" 
    3349  end 
    3450   
    35   config.vm.define :hadoop3 do |hadoop3_config| 
    36     hadoop3_config.vm.network "private_network", ip: "10.10.0.55" 
    37     hadoop3_config.vm.host_name = "hadoop3" 
    38   end 
    39    
    40    config.vm.define :master do |master_config| 
    41     master_config.vm.network "private_network", ip: "10.10.0.52" 
    42     master_config.vm.host_name = "master" 
    43   end 
    4451 
    4552end 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/manifests/base-hadoop.pp

    r30917 r31077  
    77 
    88$hadoop_master = '10.10.0.52' 
    9 $hadoop_backup = '10.10.0.51' 
     9$hadoop_resman = '10.10.0.50' 
     10$hadoop_backupmaster = '10.10.0.51' 
    1011$hadoop_1 = '10.10.0.53' 
    1112$hadoop_2 = '10.10.0.54' 
     
    4041     
    4142 
     43 exec { 'add-apt-repository': 
     44    command => '/usr/bin/add-apt-repository ppa:webupd8team/java -y && \ 
     45                echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \ 
     46                echo debconf shared/accepted-oracle-license-v1-1 seen true | /usr/bin/debconf-set-selections' 
     47 } 
    4248 
    43 exec { 'apt-get update': 
     49                 
     50 exec { 'apt-get update': 
    4451    command => '/usr/bin/apt-get update', 
    45 } 
     52    require => [ Exec['add-apt-repository'], File["/home/${user}"] ] 
     53 } 
    4654 
    47 package { "openjdk-7-jdk" : 
     55 package { "oracle-java8-installer" : 
    4856   ensure => present, 
    4957  require => [ Exec['apt-get update'], File["/home/${user}"] ] 
    50 } 
     58 } 
     59 
    5160 
    5261#package { "subversion" : 
     
    106115} 
    107116 
     117host { 'master': 
     118    ip => "${hadoop_master}", 
     119} 
     120 
     121host { 'resman': 
     122    ip => "${hadoop_resman}", 
     123} 
     124 
     125host { 'backupmaster': 
     126    ip => "${hadoop_backupmaster}", 
     127} 
     128 
    108129host { 'hadoop1': 
    109130    ip => "${hadoop_1}", 
     
    118139} 
    119140 
    120 host { 'master': 
    121     ip => "${hadoop_master}", 
    122 } 
    123  
    124 host { 'backup': 
    125     ip => "${hadoop_backup}", 
    126 } 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/manifests/init.pp

    r30917 r31077  
    1111#  group => $group, 
    1212  unless => "ls ${install_dir} | grep hadoop-2.7.3", 
    13   require => Package["openjdk-7-jdk"] 
     13  require => Package["oracle-java8-installer"] 
    1414 } 
    1515 
     
    7979  require => Exec["chown_hadoop"] 
    8080 } 
    81   
     81 
     82 file { 
     83  "${hadoop_home}/etc/hadoop/yarn-site.xml": 
     84  content => template('hadoop/yarn-site.xml'), 
     85  mode => 644, 
     86  owner => $user, 
     87  group => $group, 
     88  require => Exec["chown_hadoop"] 
     89 } 
     90 
    8291file { 
    8392  "${hadoop_home}/etc/hadoop/hadoop-env.sh": 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hadoop-env.sh

    r30917 r31077  
    2323 
    2424# The java implementation to use. 
    25 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 
     25export JAVA_HOME=/usr/lib/jvm/java-8-oracle 
    2626 
    2727# The jsvc implementation to use. Jsvc is required to run secure datanodes 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hdfs-site.xml

    r30903 r31077  
    77    <description>The actual number of replications can be specified when the file is created.</description> 
    88  </property> 
     9 
    910  <!-- 
    1011  <property> 
     
    1718    </property> 
    1819  --> 
    19      <property> 
    20        <name>dfs.namenode.name.dir</name> 
    21        <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value> 
    22      </property> 
    23      <property> 
    24        <name>dfs.datanode.data.dir</name> 
    25        <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value> 
    26      </property> 
     20 
     21  <property> 
     22    <name>dfs.namenode.name.dir</name> 
     23    <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value> 
     24  </property> 
     25  <property> 
     26    <name>dfs.datanode.data.dir</name> 
     27    <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value> 
     28  </property> 
    2729</configuration> 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/masters

    r30903 r31077  
    1 <%= @hadoop_backup %> 
     1<%= @hadoop_backupmaster %> 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/setup-hadoop.bash

    r30930 r31077  
    11 
    2 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 
     2export JAVA_HOME=/usr/lib/jvm/java-8-oracle 
    33export PATH="$JAVA_HOME/bin:$PATH" 
    44 
     
    1212 
    1313. "$HADOOP_HOME/etc/hadoop/hadoop-env.sh" 
     14 
     15export HADOOP_YARN_USER=htrc 
     16. "$HADOOP_HOME/etc/hadoop/yarn-env.sh" 
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/spark/manifests/init.pp

    r30917 r31077  
    77#  group => $user, 
    88  unless => "ls ${install_dir} | grep spark-2.0.1", 
    9   require => Package["openjdk-7-jdk"] 
     9  require => Package["oracle-java8-installer"] 
    1010 } 
    1111