Ignore:
Timestamp:
11/07/16 22:34:31 (4 years ago)
Author:
davidb
Message:

Move up to JDK1.8. Tidy up of Vagrant machine names. Support for YARN. Some XML whitespace indenting

Location:
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster
Files:
8 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/Vagrantfile

    r30917 r31077  
    1717     puppet.module_path = "modules"
    1818  end
    19  
    20   config.vm.define :backup do |box|
     19
     20  config.vm.define :master do |box|
     21    box.vm.network "private_network", ip: "10.10.0.52"
     22    box.vm.host_name = "master"
     23  end
     24
     25#  config.vm.define :resman do |box|
     26#    box.vm.network "private_network", ip: "10.10.0.50"
     27#    box.vm.host_name = "resman" 
     28#  end
     29
     30  config.vm.define :backupmaster do |box|
    2131    box.vm.network "private_network", ip: "10.10.0.51"
    22     box.vm.host_name = "backup" 
     32    box.vm.host_name = "backupmaster" 
     33  end
     34
     35  config.vm.define :hadoop1 do |box|
     36    box.vm.network "private_network", ip: "10.10.0.53"
     37    box.vm.host_name = "hadoop1"
    2338  end
    2439 
    25   config.vm.define :hadoop1 do |hadoop1_config|
    26     hadoop1_config.vm.network "private_network", ip: "10.10.0.53"
    27     hadoop1_config.vm.host_name = "hadoop1"
     40  config.vm.define :hadoop2 do |box|
     41    box.vm.network "private_network", ip: "10.10.0.54"
     42    box.vm.host_name = "hadoop2"
    2843  end
    2944 
    30   config.vm.define :hadoop2 do |hadoop2_config|
    31     hadoop2_config.vm.network "private_network", ip: "10.10.0.54"
    32     hadoop2_config.vm.host_name = "hadoop2"
     45  config.vm.define :hadoop3 do |box|
     46    ip = "10.10.0.55"
     47    box.vm.network "private_network", ip: ip
     48    box.vm.host_name = "hadoop3"
    3349  end
    3450 
    35   config.vm.define :hadoop3 do |hadoop3_config|
    36     hadoop3_config.vm.network "private_network", ip: "10.10.0.55"
    37     hadoop3_config.vm.host_name = "hadoop3"
    38   end
    39  
    40    config.vm.define :master do |master_config|
    41     master_config.vm.network "private_network", ip: "10.10.0.52"
    42     master_config.vm.host_name = "master"
    43   end
    4451
    4552end
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/manifests/base-hadoop.pp

    r30917 r31077  
    77
    88$hadoop_master = '10.10.0.52'
    9 $hadoop_backup = '10.10.0.51'
     9$hadoop_resman = '10.10.0.50'
     10$hadoop_backupmaster = '10.10.0.51'
    1011$hadoop_1 = '10.10.0.53'
    1112$hadoop_2 = '10.10.0.54'
     
    4041   
    4142
     43 exec { 'add-apt-repository':
     44    command => '/usr/bin/add-apt-repository ppa:webupd8team/java -y && \
     45                echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \
     46                echo debconf shared/accepted-oracle-license-v1-1 seen true | /usr/bin/debconf-set-selections'
     47 }
    4248
    43 exec { 'apt-get update':
     49               
     50 exec { 'apt-get update':
    4451    command => '/usr/bin/apt-get update',
    45 }
     52    require => [ Exec['add-apt-repository'], File["/home/${user}"] ]
     53 }
    4654
    47 package { "openjdk-7-jdk" :
     55 package { "oracle-java8-installer" :
    4856   ensure => present,
    4957  require => [ Exec['apt-get update'], File["/home/${user}"] ]
    50 }
     58 }
     59
    5160
    5261#package { "subversion" :
     
    106115}
    107116
     117host { 'master':
     118    ip => "${hadoop_master}",
     119}
     120
     121host { 'resman':
     122    ip => "${hadoop_resman}",
     123}
     124
     125host { 'backupmaster':
     126    ip => "${hadoop_backupmaster}",
     127}
     128
    108129host { 'hadoop1':
    109130    ip => "${hadoop_1}",
     
    118139}
    119140
    120 host { 'master':
    121     ip => "${hadoop_master}",
    122 }
    123 
    124 host { 'backup':
    125     ip => "${hadoop_backup}",
    126 }
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/manifests/init.pp

    r30917 r31077  
    1111#  group => $group,
    1212  unless => "ls ${install_dir} | grep hadoop-2.7.3",
    13   require => Package["openjdk-7-jdk"]
     13  require => Package["oracle-java8-installer"]
    1414 }
    1515
     
    7979  require => Exec["chown_hadoop"]
    8080 }
    81  
     81
     82 file {
     83  "${hadoop_home}/etc/hadoop/yarn-site.xml":
     84  content => template('hadoop/yarn-site.xml'),
     85  mode => 644,
     86  owner => $user,
     87  group => $group,
     88  require => Exec["chown_hadoop"]
     89 }
     90
    8291file {
    8392  "${hadoop_home}/etc/hadoop/hadoop-env.sh":
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hadoop-env.sh

    r30917 r31077  
    2323
    2424# The java implementation to use.
    25 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
     25export JAVA_HOME=/usr/lib/jvm/java-8-oracle
    2626
    2727# The jsvc implementation to use. Jsvc is required to run secure datanodes
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hdfs-site.xml

    r30903 r31077  
    77    <description>The actual number of replications can be specified when the file is created.</description>
    88  </property>
     9
    910  <!--
    1011  <property>
     
    1718    </property>
    1819  -->
    19      <property>
    20        <name>dfs.namenode.name.dir</name>
    21        <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value>
    22      </property>
    23      <property>
    24        <name>dfs.datanode.data.dir</name>
    25        <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value>
    26      </property>
     20
     21  <property>
     22    <name>dfs.namenode.name.dir</name>
     23    <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value>
     24  </property>
     25  <property>
     26    <name>dfs.datanode.data.dir</name>
     27    <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value>
     28  </property>
    2729</configuration>
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/masters

    r30903 r31077  
    1 <%= @hadoop_backup %>
     1<%= @hadoop_backupmaster %>
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/setup-hadoop.bash

    r30930 r31077  
    11
    2 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
     2export JAVA_HOME=/usr/lib/jvm/java-8-oracle
    33export PATH="$JAVA_HOME/bin:$PATH"
    44
     
    1212
    1313. "$HADOOP_HOME/etc/hadoop/hadoop-env.sh"
     14
     15export HADOOP_YARN_USER=htrc
     16. "$HADOOP_HOME/etc/hadoop/yarn-env.sh"
  • other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/spark/manifests/init.pp

    r30917 r31077  
    77#  group => $user,
    88  unless => "ls ${install_dir} | grep spark-2.0.1",
    9   require => Package["openjdk-7-jdk"]
     9  require => Package["oracle-java8-installer"]
    1010 }
    1111
Note: See TracChangeset for help on using the changeset viewer.