Install SPARK and SCALA
1. Download spark
Http://mirrors.cnnic.cn/apache/spark/spark-1.3.0/spark-1.3.0-bin-hadoop2.3.tgz
2. Download scala
Http://www.scala-lang.org/download/2.10.5.html
3. Install scala
Mkdir / usr/lib/scala
Tar-zxvf scala-2.10.5.tgz
Mv scala-2.10.5 / usr/lib/scala
4. Set the scala path
Vim / etc/bashrc
Export SCALA_HOME=/usr/lib/scala/scala-2.10.5
Export PATH=$SCALA_HOME/bin:$PATH
Source / etc/bashrc
Scala-version
5. Distribution
Scp-r / usr/lib/scala/ hd2:/usr/lib/scala
Scp-r / usr/lib/scala/ hd3:/usr/lib/scala
Scp-r / usr/lib/scala/ hd4:/usr/lib/scala
Scp-r / usr/lib/scala/ hd5:/usr/lib/scala
Scp / etc/bashrc hd2:/etc/bashrc
Scp / etc/bashrc hd3:/etc/bashrc
Scp / etc/bashrc hd4:/etc/bashrc
Scp / etc/bashrc hd5:/etc/bashrc
6. Install spark
Tar-zxvf spark-1.3.0-bin-hadoop2.3.tgz
Mkdir / usr/local/spark
Mv spark-1.3.0-bin-hadoop2.3 / usr/local/spark
Vim / etc/bashrc
Export SPARK_HOME=/usr/local/spark/spark-1.3.0-bin-hadoop2.3
Export PATH=$SCALA_HOME/bin:$SPARK_HOME/bin:$PATH
Source / etc/bashrc
Cd / usr/local/spark/spark-1.3.0-bin-hadoop2.3/conf/
Cp spark-env.sh.template spark-env.sh
Vim spark-env.sh
Export JAVA_HOME=/java
Export SCALA_HOME=/usr/lib/scala/scala-2.10.5
Export SPARK_HOME=/usr/local/spark/spark-1.3.0-bin-hadoop2.3
Export SPARK_MASTER_IP=192.168.137.101
Export SPARK_WORKER_MEMORY=10g
Export SPARK_DRIVER_MEMORY=9g
Export HADOOP_CONF_DIR=/home/hadoop/hadoop/etc/hadoop
Export SPARK_LIBRARY_PATH=$SPARK_HOME/lib
Export SCALA_LIBRARY_PATH=$SPARK_LIBRARY_PATH
Cp slaves.template slaves
Vim slaves
Hd1
Hd2
Hd3
Hd4
Hd5
7. Distribution
Scp / etc/bashrc hd2:/etc
Scp / etc/bashrc hd3:/etc
Scp / etc/bashrc hd4:/etc
Scp / etc/bashrc hd5:/etc
Scp-r / usr/local/spark/spark-1.3.0-bin-hadoop2.3 hd2:/usr/local/spark/
Scp-r / usr/local/spark/spark-1.3.0-bin-hadoop2.3 hd3:/usr/local/spark/
Scp-r / usr/local/spark/spark-1.3.0-bin-hadoop2.3 hd4:/usr/local/spark/
Scp-r / usr/local/spark/spark-1.3.0-bin-hadoop2.3 hd5:/usr/local/spark/
7. Start
In hd1, start
Cd $SPARK_HOME/sbin
. / start-all.sh
