Changeset 30913 for other-projects/hathitrust/vagrant-spark-hdfs-cluster

Timestamp:

2016-10-25T10:02:58+13:00 (8 years ago)

Author:

davidb

Message:

Renaming to better represent what the cluster is designed for

Location:

other-projects/hathitrust/vagrant-spark-hdfs-cluster

Files:

• . (moved) (moved from other-projects/hathitrust/vagrant-hadoop-cluster )
• trunk/README.txt (modified) (2 diffs)
• trunk/manifests/base-hadoop.pp (modified) (3 diffs)
• trunk/modules/hadoop/manifests/init.pp (modified) (1 diff)

other-projects/hathitrust/vagrant-spark-hdfs-cluster/trunk/README.txt

@@ -1 +1 @@
 
-Vargrant provisioning files to spin up a modest (4 node) Hadoop
-cluster for experiments processing HTRC Extracted Feature JSON files
-suitable for ingesting into Solr.
+Vargrant provisioning files to spin up a modest Spark cluster (master
++ 3 slaves + backup) for experiments processing HTRC Extracted Feature
+JSON files suitable for ingesting into Solr.
+
+To aid parallelism, code is designed to read JSON files from HDFS, so
+the provision of the cluster includes Hadoop core in addition to Spark
 
 
-Top-level code Apache Spark, processing HDFS stored JSON files, hence
-the need for an underlying Hadoop cluster.
+Provisioning uses Puppet scripting, based on the following on-line
+resources, but updated to use newer versions of Ubuntu, Java,
+and Hadoop.  Spark is then added in on top of that.
 
-Provisioning based on the following online resources, but updated to
-use newer versions of Ubuntu, Java, and Hadoop.
 
   http://cscarioni.blogspot.co.nz/2012/09/setting-up-hadoop-virtual-cluster-with.html
 
   https://github.com/calo81/vagrant-hadoop-cluster
+
+To get everything setup, type:
+
+  vargrant up
+
+Then log in to the master node, and swithc to 'ubuntu' user
+
+  vargrant ssh master
+  sudo su - ubuntu
+
+If the first time, you need to format an HDFS area to use:
+  hdfs namenode -format
+
+Otherwise start up HDFS and Spark deamon processes:
+
+  start-dfs.sh
+  spark-start-all.sh
+
+You can visit the Spark cluster monitoring page at:
+
+  http://10.10.0.52:8080/
+
 
 
@@ -60 +84 @@
 http://kvz.io/blog/2013/01/16/vagrant-tip-keep-virtualbox-guest-additions-in-sync/
 
+----
+SecondaryNode
+----
+
+http://stackoverflow.com/questions/23581425/hadoop-how-to-start-secondary-namenode-on-other-node
+
+<property>
+  <name>dfs.namenode.secondary.http-address</name>
+  <value>ec2-54-187-222-213.us-west-2.compute.amazonaws.com:50090</value>
+</property>
+
+----
+Spark Cluster
+----
+
+http://spark.apache.org/docs/latest/spark-standalone.html

other-projects/hathitrust/vagrant-spark-hdfs-cluster/trunk/manifests/base-hadoop.pp

@@ -1 +1 @@
 $install_dir = "/opt"
 $hadoop_home = "${install_dir}/hadoop"
+$spark_home  = "${install_dir}/spark"
+
 $user = "ubuntu"
 $group = "ubuntu"
+
 $hadoop_master = '10.10.0.52'
 $hadoop_backup = '10.10.0.51'
@@ -10 +13 @@
 
 include hadoop
+include spark
 
 file { 
@@ -46 +50 @@
 }
 
-  
+#package { "subversion" :
+#   ensure => present,
+#  require => [ Exec['apt-get update'], File["/home/${user}"] ]
+#}
+
+#package { "maven" :
+#   ensure => present,
+#  require => [ Exec['apt-get update'], File["/home/${user}"] ]
+#}
+
 file { 
     "/home/${user}/.ssh":
     ensure => "directory",
     owner  => "${user}",
-    group  => "${user}",
+    group  => "${group}",
     mode   => 750,
     require => [ Exec['apt-get update'], File["/home/${user}"] ]

other-projects/hathitrust/vagrant-spark-hdfs-cluster/trunk/modules/hadoop/manifests/init.pp

@@ -2 +2 @@
 
 exec { "download_hadoop":
-# Download from nearby mirror, otherwise task can time-out
-command => "wget -O /tmp/hadoop.tar.gz http://apache.mirror.amaze.com.au/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz",
-# command => "wget -O /tmp/hadoop.tar.gz http://www-eu.apache.org/dist/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz",
-path => $path,
-user => $user,
-group => $user,
-unless => "ls ${install_dir} | grep hadoop-2.7.3",
-require => Package["openjdk-7-jdk"]
-}
+  # Download from nearby mirror, otherwise task can time-out
+  command => "wget -O /tmp/hadoop-2.7.3.tar.gz http://apache.mirror.amaze.com.au/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz",
+  # command => "wget -O /tmp/hadoop-2.7.3.tar.gz http://www-eu.apache.org/dist/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz",
+  path => $path,
+  user => $user,
+  group => $user,
+  unless => "ls ${install_dir} | grep hadoop-2.7.3",
+  require => Package["openjdk-7-jdk"]
+ }
 
 exec { "unpack_hadoop" :
-  command => "tar -zxf /tmp/hadoop.tar.gz -C ${install_dir}",
+  command => "tar -zxf /tmp/hadoop-2.7.3.tar.gz -C ${install_dir}",
   path => $path,
   creates => "${hadoop_home}-2.7.3",
   require => Exec["download_hadoop"]
-}
+ }
 
-exec { 'chown':
-  command => "/bin/chown -R ${user}:${group} /opt/hadoop-2.7.3",
+exec { "rename_hadoop" :
+  command => "ln -s ${install_dir}/hadoop-2.7.3 ${install_dir}/hadoop",
+  path => $path,
+  creates => "${hadoop_home}",
+  require => Exec["unpack_hadoop"]
+ }
+
+exec { 'chown_hadoop':
+  command => "/bin/chown -R ${user}:${group} ${hadoop_home}-2.7.3",
   path => '/bin',
   user => 'root',
-  require => Exec["unpack_hadoop"]
-}
+  require => Exec["rename_hadoop"]
+ }
 
-file {
-  "${hadoop_home}-2.7.3/etc/hadoop/slaves":
+file { 
+  "${hadoop_home}/etc/hadoop/slaves":
   content => template('hadoop/slaves'),
   mode => 644,
   owner => $user,
   group => $group,
-  require => Exec["unpack_hadoop"]
+  require => Exec["chown_hadoop"]
  }
  
-file {
-  "${hadoop_home}-2.7.3/etc/hadoop/masters":
+file { 
+  "${hadoop_home}/etc/hadoop/masters":
   content => template('hadoop/masters'),
   mode => 644,
   owner => $user,
   group => $group,
-  require => Exec["unpack_hadoop"]
+  require => Exec["chown_hadoop"]
  }
 
 file {
-  "${hadoop_home}-2.7.3/etc/hadoop/core-site.xml":
+  "${hadoop_home}/etc/hadoop/core-site.xml":
   content => template('hadoop/core-site.xml'),
   mode => 644,
   owner => $user,
   group => $group,
-  require => Exec["unpack_hadoop"]
+  require => Exec["chown_hadoop"]
  }
  
 file {
-  "${hadoop_home}-2.7.3/etc/hadoop/mapred-site.xml":
+  "${hadoop_home}/etc/hadoop/mapred-site.xml":
   content => template('hadoop/mapred-site.xml'),
   mode => 644,
   owner => $user,
   group => $group,
-  require => Exec["unpack_hadoop"]
+  require => Exec["chown_hadoop"]
  }
  
  file {
-  "${hadoop_home}-2.7.3/etc/hadoop/hdfs-site.xml":
+  "${hadoop_home}/etc/hadoop/hdfs-site.xml":
   content => template('hadoop/hdfs-site.xml'),
   mode => 644,
   owner => $user,
   group => $group,
-  require => Exec["unpack_hadoop"]
+  require => Exec["chown_hadoop"]
  }
  
 file {
-  "${hadoop_home}-2.7.3/etc/hadoop/hadoop-env.sh":
+  "${hadoop_home}/etc/hadoop/hadoop-env.sh":
   content => template('hadoop/hadoop-env.sh'),
   mode => 644,
   owner => $user,
   group => $group,
-  require => Exec["unpack_hadoop"]
+  require => Exec["chown_hadoop"]
  }
 
-file{
-   [ "${hadoop_home}-2.7.3/hadoop_store",
-     "${hadoop_home}-2.7.3/hadoop_store/hdfs",
-     "${hadoop_home}-2.7.3/hadoop_store/hdfs/namenode",
-     "${hadoop_home}-2.7.3/hadoop_store/hdfs/datanode"]:
+file {
+   [ "${hadoop_home}/hadoop_store",
+     "${hadoop_home}/hadoop_store/hdfs",
+     "${hadoop_home}/hadoop_store/hdfs/namenode",
+     "${hadoop_home}/hadoop_store/hdfs/datanode"]:
    ensure => 'directory',
    owner => "${user}",
    group => "${group}",
    mode => 755,
-   require => Exec["unpack_hadoop"]
+   require => Exec["chown_hadoop"]
   }
 
-file_line { "add_hadoop_home":
+file {
+  "/home/${user}/.bashrc-setup-hadoop":
+  content => template('hadoop/setup-hadoop.bash'),
+  mode => 644,
+  owner => $user,
+  group => $group,
+  require => [  Exec["unpack_hadoop"], File["/home/${user}"] ]
+ }
+
+file_line { "setup_hadoop_home":
     ensure => present,
     path => "/home/${user}/.bashrc",
-    line => "export HADOOP_HOME=\"${hadoop_home}-2.7.3\"",
+    line => ". .bashrc-setup-hadoop",
     require => [  Exec["unpack_hadoop"], File["/home/${user}"] ]    
-}
+ }
 
-file_line { "add_hadoop_confdir":
-    ensure => present,
-    path => "/home/${user}/.bashrc",
-    line => 'export HADOOP_CONF_DIR="$HADOOP_HOME/etc/hadoop"',
-    require => [  Exec["unpack_hadoop"], File["/home/${user}"] ]    
-}
 
-file_line { "add_hadoop_setup":
-    ensure => present,
-    path => "/home/${user}/.bashrc",
-    line => 'source "$HADOOP_HOME/etc/hadoop/hadoop-env.sh"',
-    require => [  Exec["unpack_hadoop"], File["/home/${user}"] ]
-}
-
-file_line { "add_hadoop_path":
-    ensure => present,
-    path => "/home/${user}/.bashrc",
-    line => 'export PATH="$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin"',
-    require => [  Exec["unpack_hadoop"], File["/home/${user}"] ]    
-}
 
 }