Changeset 31077
- Timestamp:
- 2016-11-07T22:34:31+13:00 (7 years ago)
- Location:
- other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster
- Files:
-
- 8 edited
Legend:
- Unmodified
- Added
- Removed
-
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/Vagrantfile
r30917 r31077 17 17 puppet.module_path = "modules" 18 18 end 19 20 config.vm.define :backup do |box| 19 20 config.vm.define :master do |box| 21 box.vm.network "private_network", ip: "10.10.0.52" 22 box.vm.host_name = "master" 23 end 24 25 # config.vm.define :resman do |box| 26 # box.vm.network "private_network", ip: "10.10.0.50" 27 # box.vm.host_name = "resman" 28 # end 29 30 config.vm.define :backupmaster do |box| 21 31 box.vm.network "private_network", ip: "10.10.0.51" 22 box.vm.host_name = "backup" 32 box.vm.host_name = "backupmaster" 33 end 34 35 config.vm.define :hadoop1 do |box| 36 box.vm.network "private_network", ip: "10.10.0.53" 37 box.vm.host_name = "hadoop1" 23 38 end 24 39 25 config.vm.define :hadoop 1 do |hadoop1_config|26 hadoop1_config.vm.network "private_network", ip: "10.10.0.53"27 hadoop1_config.vm.host_name = "hadoop1"40 config.vm.define :hadoop2 do |box| 41 box.vm.network "private_network", ip: "10.10.0.54" 42 box.vm.host_name = "hadoop2" 28 43 end 29 44 30 config.vm.define :hadoop2 do |hadoop2_config| 31 hadoop2_config.vm.network "private_network", ip: "10.10.0.54" 32 hadoop2_config.vm.host_name = "hadoop2" 45 config.vm.define :hadoop3 do |box| 46 ip = "10.10.0.55" 47 box.vm.network "private_network", ip: ip 48 box.vm.host_name = "hadoop3" 33 49 end 34 50 35 config.vm.define :hadoop3 do |hadoop3_config|36 hadoop3_config.vm.network "private_network", ip: "10.10.0.55"37 hadoop3_config.vm.host_name = "hadoop3"38 end39 40 config.vm.define :master do |master_config|41 master_config.vm.network "private_network", ip: "10.10.0.52"42 master_config.vm.host_name = "master"43 end44 51 45 52 end -
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/manifests/base-hadoop.pp
r30917 r31077 7 7 8 8 $hadoop_master = '10.10.0.52' 9 $hadoop_backup = '10.10.0.51' 9 $hadoop_resman = '10.10.0.50' 10 $hadoop_backupmaster = '10.10.0.51' 10 11 $hadoop_1 = '10.10.0.53' 11 12 $hadoop_2 = '10.10.0.54' … … 40 41 41 42 43 exec { 'add-apt-repository': 44 command => '/usr/bin/add-apt-repository ppa:webupd8team/java -y && \ 45 echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \ 46 echo debconf shared/accepted-oracle-license-v1-1 seen true | /usr/bin/debconf-set-selections' 47 } 42 48 43 exec { 'apt-get update': 49 50 exec { 'apt-get update': 44 51 command => '/usr/bin/apt-get update', 45 } 52 require => [ Exec['add-apt-repository'], File["/home/${user}"] ] 53 } 46 54 47 package { "openjdk-7-jdk" :55 package { "oracle-java8-installer" : 48 56 ensure => present, 49 57 require => [ Exec['apt-get update'], File["/home/${user}"] ] 50 } 58 } 59 51 60 52 61 #package { "subversion" : … … 106 115 } 107 116 117 host { 'master': 118 ip => "${hadoop_master}", 119 } 120 121 host { 'resman': 122 ip => "${hadoop_resman}", 123 } 124 125 host { 'backupmaster': 126 ip => "${hadoop_backupmaster}", 127 } 128 108 129 host { 'hadoop1': 109 130 ip => "${hadoop_1}", … … 118 139 } 119 140 120 host { 'master':121 ip => "${hadoop_master}",122 }123 124 host { 'backup':125 ip => "${hadoop_backup}",126 } -
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/manifests/init.pp
r30917 r31077 11 11 # group => $group, 12 12 unless => "ls ${install_dir} | grep hadoop-2.7.3", 13 require => Package["o penjdk-7-jdk"]13 require => Package["oracle-java8-installer"] 14 14 } 15 15 … … 79 79 require => Exec["chown_hadoop"] 80 80 } 81 81 82 file { 83 "${hadoop_home}/etc/hadoop/yarn-site.xml": 84 content => template('hadoop/yarn-site.xml'), 85 mode => 644, 86 owner => $user, 87 group => $group, 88 require => Exec["chown_hadoop"] 89 } 90 82 91 file { 83 92 "${hadoop_home}/etc/hadoop/hadoop-env.sh": -
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hadoop-env.sh
r30917 r31077 23 23 24 24 # The java implementation to use. 25 export JAVA_HOME=/usr/lib/jvm/java- 7-openjdk-amd6425 export JAVA_HOME=/usr/lib/jvm/java-8-oracle 26 26 27 27 # The jsvc implementation to use. Jsvc is required to run secure datanodes -
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/hdfs-site.xml
r30903 r31077 7 7 <description>The actual number of replications can be specified when the file is created.</description> 8 8 </property> 9 9 10 <!-- 10 11 <property> … … 17 18 </property> 18 19 --> 19 <property> 20 <name>dfs.namenode.name.dir</name> 21 <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value> 22 </property> 23 <property> 24 <name>dfs.datanode.data.dir</name> 25 <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value> 26 </property> 20 21 <property> 22 <name>dfs.namenode.name.dir</name> 23 <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/namenode</value> 24 </property> 25 <property> 26 <name>dfs.datanode.data.dir</name> 27 <value>file:/opt/hadoop-2.7.3/hadoop_store/hdfs/datanode</value> 28 </property> 27 29 </configuration> -
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/masters
r30903 r31077 1 <%= @hadoop_backup %>1 <%= @hadoop_backupmaster %> -
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/templates/setup-hadoop.bash
r30930 r31077 1 1 2 export JAVA_HOME=/usr/lib/jvm/java- 7-openjdk-amd642 export JAVA_HOME=/usr/lib/jvm/java-8-oracle 3 3 export PATH="$JAVA_HOME/bin:$PATH" 4 4 … … 12 12 13 13 . "$HADOOP_HOME/etc/hadoop/hadoop-env.sh" 14 15 export HADOOP_YARN_USER=htrc 16 . "$HADOOP_HOME/etc/hadoop/yarn-env.sh" -
other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/spark/manifests/init.pp
r30917 r31077 7 7 # group => $user, 8 8 unless => "ls ${install_dir} | grep spark-2.0.1", 9 require => Package["o penjdk-7-jdk"]9 require => Package["oracle-java8-installer"] 10 10 } 11 11
Note:
See TracChangeset
for help on using the changeset viewer.