Hadoop2 installation script
#! / bin/bash
#
# Install Hadoop 2 using pdsh/pdcp where possible.
#
# Command can be interactive or file-based. This script sets up
# a Hadoop 2 cluster with basic configuration. Modify data, log, and pid
# directories as desired. Further configure your cluster with. / conf-hadoop2.sh
#
#
# Basic environment variables. Edit as necessary
HADOOP_VERSION=2.0.5-alpha
HADOOP_HOME= "/ opt/hadoop-$ {HADOOP_VERSION}"
NN_DATA_DIR=/var/data/hadoop/hdfs/nn
SNN_DATA_DIR=/var/data/hadoop/hdfs/snn
DN_DATA_DIR=/var/data/hadoop/hdfs/dn
YARN_LOG_DIR=/var/log/hadoop/yarn
HADOOP_LOG_DIR=/var/log/hadoop/hdfs
HADOOP_MAPRED_LOG_DIR=/var/log/hadoop/mapred
YARN_PID_DIR=/var/run/hadoop/yarn
HADOOP_PID_DIR=/var/run/hadoop/hdfs
HADOOP_MAPRED_PID_DIR=/var/run/hadoop/mapred
HTTP_STATIC_USER=hdfs
YARN_PROXY_PORT=8081
Source hadoop-xml-conf.sh
CMD_OPTIONS=$ (getopt-n "$0"-o hif-- long "help,interactive,file"-- "$@")
# Take care of bad options in the command
If [$?-ne 0]
Then
Exit 1
Fi
Eval set-"$CMD_OPTIONS"
All_hosts= "all_hosts"
Nn_host= "nn_host"
Snn_host= "snn_host"
Dn_hosts= "dn_hosts"
Rm_host= "rm_host"
Nm_hosts= "nm_hosts"
Mr_history_host= "mr_history_host"
Yarn_proxy_host= "yarn_proxy_host"
Install ()
{
Echo "Copying Hadoop $HADOOP_VERSION to all hosts..."
Pdcp-w ^ all_hosts hadoop- "$HADOOP_VERSION" .tar.gz / opt
Echo "Copying JDK 1.6.031 to all hosts..."
Pdcp-w ^ all_hosts jdk-6u31-linux-x64-rpm.bin / opt
Echo "Installing JDK 1.6.031 on all hosts..."
Pdsh-w ^ all_hosts chmod adepx / opt/jdk-6u31-linux-x64-rpm.bin
Pdsh-w ^ all_hosts / opt/jdk-6u31-linux-x64-rpm.bin-noregister 1 > &-2 > &-
Echo "Setting JAVA_HOME and HADOOP_HOME environment variables on all hosts..."
Pdsh-w ^ all_hosts' echo export JAVA_HOME=/usr/java/jdk1.6.0_31 > / etc/profile.d/java.sh'
Pdsh-w ^ all_hosts "source / etc/profile.d/java.sh"
Pdsh-w ^ all_hosts "echo export HADOOP_HOME=$HADOOP_HOME > / etc/profile.d/hadoop.sh"
Pdsh-w ^ all_hosts' echo export HADOOP_PREFIX=$HADOOP_HOME > > / etc/profile.d/hadoop.sh'
Pdsh-w ^ all_hosts "source / etc/profile.d/hadoop.sh"
Echo "Extracting Hadoop $HADOOP_VERSION distribution on all hosts..."
Pdsh-w ^ all_hosts tar-zxf / opt/hadoop- "$HADOOP_VERSION" .tar.gz-C / opt
Echo "Creating system accounts and groups on all hosts..."
Pdsh-w ^ all_hosts groupadd hadoop
Pdsh-w ^ all_hosts useradd-g hadoop yarn
Pdsh-w ^ all_hosts useradd-g hadoop hdfs
Pdsh-w ^ all_hosts useradd-g hadoop mapred
Echo "Creating HDFS data directories on NameNode host, Secondary NameNode host, and DataNode hosts..."
Pdsh-w ^ nn_host "mkdir-p $NN_DATA_DIR & & chown hdfs:hadoop $NN_DATA_DIR"
Pdsh-w ^ snn_host "mkdir-p $SNN_DATA_DIR & & chown hdfs:hadoop $SNN_DATA_DIR"
Pdsh-w ^ dn_hosts "mkdir-p $DN_DATA_DIR & & chown hdfs:hadoop $DN_DATA_DIR"
Echo "Creating log directories on all hosts..."
Pdsh-w ^ all_hosts "mkdir-p $YARN_LOG_DIR & & chown yarn:hadoop $YARN_LOG_DIR"
Pdsh-w ^ all_hosts "mkdir-p $HADOOP_LOG_DIR & & chown hdfs:hadoop $HADOOP_LOG_DIR"
Pdsh-w ^ all_hosts "mkdir-p $HADOOP_MAPRED_LOG_DIR & & chown mapred:hadoop $HADOOP_MAPRED_LOG_DIR"
Echo "Creating pid directories on all hosts..."
Pdsh-w ^ all_hosts "mkdir-p $YARN_PID_DIR & & chown yarn:hadoop $YARN_PID_DIR"
Pdsh-w ^ all_hosts "mkdir-p $HADOOP_PID_DIR & & chown hdfs:hadoop $HADOOP_PID_DIR"
Pdsh-w ^ all_hosts "mkdir-p $HADOOP_MAPRED_PID_DIR & & chown mapred:hadoop $HADOOP_MAPRED_PID_DIR"
Echo "Editing Hadoop environment scripts for log directories on all hosts..."
Pdsh-w ^ all_hosts echo "export HADOOP_LOG_DIR=$HADOOP_LOG_DIR > > $HADOOP_HOME/etc/hadoop/hadoop-env.sh"
Pdsh-w ^ all_hosts echo "export YARN_LOG_DIR=$YARN_LOG_DIR > > $HADOOP_HOME/etc/hadoop/yarn-env.sh"
Pdsh-w ^ all_hosts echo "export HADOOP_MAPRED_LOG_DIR=$HADOOP_MAPRED_LOG_DIR > > $HADOOP_HOME/etc/hadoop/mapred-env.sh"
Echo "Editing Hadoop environment scripts for pid directories on all hosts..."
Pdsh-w ^ all_hosts echo "export HADOOP_PID_DIR=$HADOOP_PID_DIR > > $HADOOP_HOME/etc/hadoop/hadoop-env.sh"
Pdsh-w ^ all_hosts echo "export YARN_PID_DIR=$YARN_PID_DIR > > $HADOOP_HOME/etc/hadoop/yarn-env.sh"
Pdsh-w ^ all_hosts echo "export HADOOP_MAPRED_PID_DIR=$HADOOP_MAPRED_PID_DIR > > $HADOOP_HOME/etc/hadoop/mapred-env.sh"
Echo "Creating base Hadoop XML config files..."
Create_config-file core-site.xml
Put_config-file core-site.xml-property fs.default.name-value "hdfs://$nn:9000"
Put_config-file core-site.xml-property hadoop.http.staticuser.user-value "$HTTP_STATIC_USER"
Create_config-file hdfs-site.xml
Put_config-file hdfs-site.xml-property dfs.namenode.name.dir-value "$NN_DATA_DIR"
Put_config-file hdfs-site.xml-property fs.checkpoint.dir-value "$SNN_DATA_DIR"
Put_config-file hdfs-site.xml-property fs.checkpoint.edits.dir-value "$SNN_DATA_DIR"
Put_config-file hdfs-site.xml-property dfs.datanode.data.dir-value "$DN_DATA_DIR"
Put_config-file hdfs-site.xml-property dfs.namenode.http-address-value "$nn:50070"
Put_config-file hdfs-site.xml-property dfs.namenode.secondary.http-address-value "$snn:50090"
Create_config-file mapred-site.xml
Put_config-file mapred-site.xml-property mapreduce.framework.name-value yarn
Put_config-file mapred-site.xml-property mapreduce.jobhistory.address-value "$mr_hist:10020"
Put_config-file mapred-site.xml-property mapreduce.jobhistory.webapp.address-value "$mr_hist:19888"
Put_config-file mapred-site.xml-property yarn.app.mapreduce.am.staging-dir-value / mapred
Create_config-file yarn-site.xml
Put_config-file yarn-site.xml-property yarn.nodemanager.aux-services-value mapreduce.shuffle
Put_config-file yarn-site.xml-property yarn.nodemanager.aux-services.mapreduce.shuffle.class-value org.apache.hadoop.mapred.ShuffleHandler
Put_config-file yarn-site.xml-property yarn.web-proxy.address-value "$yarn_proxy:$YARN_PROXY_PORT"
Put_config-file yarn-site.xml-property yarn.resourcemanager.scheduler.address-value "$rmgr:8030"
Put_config-file yarn-site.xml-property yarn.resourcemanager.resource-tracker.address-value "$rmgr:8031"
Put_config-file yarn-site.xml-property yarn.resourcemanager.address-value "$rmgr:8032"
Put_config-file yarn-site.xml-property yarn.resourcemanager.admin.address-value "$rmgr:8033"
Put_config-file yarn-site.xml-property yarn.resourcemanager.webapp.address-value "$rmgr:8088"
Echo "Copying base Hadoop XML config files to all hosts..."
Pdcp-w ^ all_hosts core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml $HADOOP_HOME/etc/hadoop/
Echo "Creating configuration, command, and script links on all hosts..."
Pdsh-w ^ all_hosts "ln-s $HADOOP_HOME/etc/hadoop / etc/hadoop"
Pdsh-w ^ all_hosts "ln-s $HADOOP_HOME/bin/* / usr/bin"
Pdsh-w ^ all_hosts "ln-s $HADOOP_HOME/libexec/* / usr/libexec"
Echo "Formatting the NameNode..."
Pdsh-w ^ nn_host "su-hdfs-c'$HADOOP_HOME/bin/hdfs namenode-format'"
Echo "Copying startup scripts to all hosts..."
Pdcp-w ^ nn_host hadoop-namenode / etc/init.d/
Pdcp-w ^ snn_host hadoop-secondarynamenode / etc/init.d/
Pdcp-w ^ dn_hosts hadoop-datanode / etc/init.d/
Pdcp-w ^ rm_host hadoop-resourcemanager / etc/init.d/
Pdcp-w ^ nm_hosts hadoop-nodemanager / etc/init.d/
Pdcp-w ^ mr_history_host hadoop-historyserver / etc/init.d/
Pdcp-w ^ yarn_proxy_host hadoop-proxyserver / etc/init.d/
Echo "Starting Hadoop $HADOOP_VERSION services on all hosts..."
Pdsh-w ^ nn_host "chmod 755 / etc/init.d/hadoop-namenode & & chkconfig hadoop-namenode on & & service hadoop-namenode start"
Pdsh-w ^ snn_host "chmod 755 / etc/init.d/hadoop-secondarynamenode & & chkconfig hadoop-secondarynamenode on & & service hadoop-secondarynamenode start"
Pdsh-w ^ dn_hosts "chmod 755 / etc/init.d/hadoop-datanode & & chkconfig hadoop-datanode on & & service hadoop-datanode start"
Pdsh-w ^ rm_host "chmod 755 / etc/init.d/hadoop-resourcemanager & & chkconfig hadoop-resourcemanager on & & service hadoop-resourcemanager start"
Pdsh-w ^ nm_hosts "chmod 755 / etc/init.d/hadoop-nodemanager & & chkconfig hadoop-nodemanager on & & service hadoop-nodemanager start"
Pdsh-w ^ yarn_proxy_host "chmod 755 / etc/init.d/hadoop-proxyserver & & chkconfig hadoop-proxyserver on & & service hadoop-proxyserver start"
Echo "Creating MapReduce Job History directories..."
Su-hdfs-c "hadoop fs-mkdir-p / mapred/history/done_intermediate"
Su-hdfs-c "hadoop fs-chown-R mapred:hadoop / mapred"
Su-hdfs-c "hadoop fs-chmod-R g+rwx / mapred"
Pdsh-w ^ mr_history_host "chmod 755 / etc/init.d/hadoop-historyserver & & chkconfig hadoop-historyserver on & & service hadoop-historyserver start"
Echo "Running YARN smoke test..."
Pdsh-w ^ all_hosts "usermod-a-G hadoop $(whoami)"
Su-hdfs-c "hadoop fs-mkdir-p / user/$ (whoami)"
Su-hdfs-c "hadoop fs-chown $(whoami): $(whoami) / user/$ (whoami)"
Source / etc/profile.d/java.sh
Source / etc/profile.d/hadoop.sh
Source / etc/hadoop/hadoop-env.sh
Source / etc/hadoop/yarn-env.sh
Hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-$HADOOP_VERSION.jar pi-Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory-libjars $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-$HADOOP_VERSION.jar 16 10000
}
Interactive ()
{
Echo-n "Enter NameNode hostname:"
Read nn
Echo-n "Enter Secondary NameNode hostname:"
Read snn
Echo-n "Enter ResourceManager hostname:"
Read rmgr
Echo-n "Enter Job History Server hostname:"
Read mr_hist
Echo-n "Enter YARN Proxy hostname:"
Read yarn_proxy
Echo-n "Enter DataNode hostnames (comma separated or hostlist syntax):"
Read dns
Echo-n "Enter NodeManager hostnames (comma separated or hostlist syntax):"
Read nms
Echo "$nn" > "$nn_host"
Echo "$snn" > "$snn_host"
Echo "$rmgr" > "$rm_host"
Echo "$mr_hist" > "$mr_history_host"
Echo "$yarn_proxy" > "$yarn_proxy_host"
Dn_hosts_var=$ (sed's /\, /\ nUniverse g'"$all_hosts"
}
Help ()
{
Cat
