root/other-projects/hathitrust/wcsa/extracted-features-solr/trunk/vagrant-spark-hdfs-cluster/modules/hadoop/manifests/init.pp @ 31077

Revision 31077, 3.1 KB (checked in by davidb, 3 years ago)

Move up to JDK1.8. Tidy up of Vagrant machine names. Support for YARN. Some XML whitespace indenting

Line 
1class hadoop {
2
3exec { "download_hadoop":
4  # Download from nearby mirror, otherwise task can time-out with default setting of '300'
5  # command => "wget -O /tmp/hadoop-2.7.3.tar.gz http://apache.mirror.amaze.com.au/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz",
6  command => "wget -O /tmp/hadoop-2.7.3.tar.gz   http://www-us.apache.org/dist/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz
7",
8  timeout => 0,
9  path => $path,
10#  user => $user,
11#  group => $group,
12  unless => "ls ${install_dir} | grep hadoop-2.7.3",
13  require => Package["oracle-java8-installer"]
14 }
15
16exec { "unpack_hadoop" :
17  command => "tar -zxf /tmp/hadoop-2.7.3.tar.gz -C ${install_dir}",
18  path => $path,
19  creates => "${hadoop_home}-2.7.3",
20  require => Exec["download_hadoop"]
21 }
22
23exec { "rename_hadoop" :
24  command => "ln -s ${install_dir}/hadoop-2.7.3 ${install_dir}/hadoop",
25  path => $path,
26  creates => "${hadoop_home}",
27  require => Exec["unpack_hadoop"]
28 }
29
30exec { 'chown_hadoop':
31  command => "/bin/chown -R ${user}:${group} ${hadoop_home}-2.7.3",
32  path => '/bin',
33  user => 'root',
34  require => Exec["rename_hadoop"]
35 }
36
37file {
38  "${hadoop_home}/etc/hadoop/slaves":
39  content => template('hadoop/slaves'),
40  mode => 644,
41  owner => $user,
42  group => $group,
43  require => Exec["chown_hadoop"]
44 }
45 
46file {
47  "${hadoop_home}/etc/hadoop/masters":
48  content => template('hadoop/masters'),
49  mode => 644,
50  owner => $user,
51  group => $group,
52  require => Exec["chown_hadoop"]
53 }
54
55file {
56  "${hadoop_home}/etc/hadoop/core-site.xml":
57  content => template('hadoop/core-site.xml'),
58  mode => 644,
59  owner => $user,
60  group => $group,
61  require => Exec["chown_hadoop"]
62 }
63 
64file {
65  "${hadoop_home}/etc/hadoop/mapred-site.xml":
66  content => template('hadoop/mapred-site.xml'),
67  mode => 644,
68  owner => $user,
69  group => $group,
70  require => Exec["chown_hadoop"]
71 }
72 
73 file {
74  "${hadoop_home}/etc/hadoop/hdfs-site.xml":
75  content => template('hadoop/hdfs-site.xml'),
76  mode => 644,
77  owner => $user,
78  group => $group,
79  require => Exec["chown_hadoop"]
80 }
81
82 file {
83  "${hadoop_home}/etc/hadoop/yarn-site.xml":
84  content => template('hadoop/yarn-site.xml'),
85  mode => 644,
86  owner => $user,
87  group => $group,
88  require => Exec["chown_hadoop"]
89 }
90
91file {
92  "${hadoop_home}/etc/hadoop/hadoop-env.sh":
93  content => template('hadoop/hadoop-env.sh'),
94  mode => 644,
95  owner => $user,
96  group => $group,
97  require => Exec["chown_hadoop"]
98 }
99
100file {
101   [ "${hadoop_home}/hadoop_store",
102     "${hadoop_home}/hadoop_store/hdfs",
103     "${hadoop_home}/hadoop_store/hdfs/namenode",
104     "${hadoop_home}/hadoop_store/hdfs/datanode"]:
105   ensure => 'directory',
106   owner => "${user}",
107   group => "${group}",
108   mode => 755,
109   require => Exec["chown_hadoop"]
110  }
111
112file {
113  "/home/${user}/.bashrc-setup-hadoop":
114  content => template('hadoop/setup-hadoop.bash'),
115  mode => 644,
116  owner => $user,
117  group => $group,
118  require => [  Exec["unpack_hadoop"], File["/home/${user}"] ]
119 }
120
121file_line { "setup_hadoop_home":
122    ensure => present,
123    path => "/home/${user}/.bashrc",
124    line => ". .bashrc-setup-hadoop",
125    require => [  Exec["unpack_hadoop"], File["/home/${user}"] ]   
126 }
127
128
129
130}
131
Note: See TracBrowser for help on using the browser.