56 Hadoop Advanced
01 hadoop pseudo-distributed model
Configure the environment
Node1: 192.168.1.131 CentOS Linux release 7.2
1. Install hadoop
# installation
[root@node1 ~] # vim / etc/profile.d/java.sh
Export JAVA_HOME=/usr
[root@node1 ~] # yum install java-1.7.0-openjdk-devel.x86_64
[root@node1 ~] # mkdir / bdapps
[root@node1] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/
[root@node1 ~] # cd / bdapps/
[root@node1 bdapps] # ll
Total 0
Lrwxrwxrwx 1 root root 13 Jan 7 09:24 hadoop- > hadoop-2.6.2/
Drwxr-xr-x 9 10011 10011 139 Oct 22 2015 hadoop-2.6.2
[root@node1 hadoop] # vim / etc/profile.d/hadoop.sh
Export HADOOP_PREFIX=/bdapps/hadoop
Export PATH=$PATH:$ {HADOOP_PREFIX} / bin:$ {HADOOP_PREFIX} / sbin
Export HADOOP_YARN_HOME=$ {HADOOP_PREFIX}
Export HADOOP_MAPPERD_HOME=$ {HADOOP_PREFIX}
Export HADOOP_COMMON_HOME=$ {HADOOP_PREFIX}
Export HADOOP_HDFS_HOME=$ {HADOOP_PREFIX}
[root@node1 hadoop] #. / etc/profile.d/hadoop.sh
[root@node1 hadoop] # pwd
/ bdapps/hadoop
[root@node1 hadoop] # ls
Bin etc include lib libexec LICENSE.txt NOTICE.txt README.txt sbin share
[root@node1 hadoop] # ls bin/
Container-executor hdfs mapred.cmd yarn
Hadoop hdfs.cmd rcc yarn.cmd
Hadoop.cmd mapred test-container-executor
[root@node1 hadoop] # ls include/
Hdfs.h Pipes.hh SerialUtils.hh StringUtils.hh TemplateFactory.hh
[root@node1 hadoop] # ls lib
Native
[root@node1 hadoop] # ls libexec/
Hadoop-config.cmd hdfs-config.sh mapred-config.cmd yarn-config.sh
Hadoop-config.sh httpfs-config.sh mapred-config.sh
Hdfs-config.cmd kms-config.sh yarn-config.cmd
[root@node1 hadoop] # ls sbin/
Distribute-exclude.sh start-all.cmd stop-balancer.sh
Hadoop-daemon.sh start-all.sh stop-dfs.cmd
Hadoop-daemons.sh start-balancer.sh stop-dfs.sh
Hdfs-config.cmd start-dfs.cmd stop-secure-dns.sh
Hdfs-config.sh start-dfs.sh stop-yarn.cmd
Httpfs.sh start-secure-dns.sh stop-yarn.sh
Kms.sh start-yarn.cmd yarn-daemon.sh
Mr-jobhistory-daemon.sh start-yarn.sh yarn-daemons.sh
Refresh-namenodes.sh stop-all.cmd
Slaves.sh stop-all.sh
[root@node1 hadoop] # ls etc/hadoop/
Capacity-scheduler.xml httpfs-env.sh mapred-env.sh
Configuration.xsl httpfs-log4j.properties mapred-queues.xml.template
Container-executor.cfg httpfs-signature.secret mapred-site.xml.template
Core-site.xml httpfs-site.xml slaves
Hadoop-env.cmd kms-acls.xml ssl-client.xml.example
Hadoop-env.sh kms-env.sh ssl-server.xml.example
Hadoop-metrics2.properties kms-log4j.properties yarn-env.cmd
Hadoop-metrics.properties kms-site.xml yarn-env.sh
Hadoop-policy.xml log4j.properties yarn-site.xml
Hdfs-site.xml mapred-env.cmd
# create users and related directories running Hadoop processes
[root@node1 hadoop] # groupadd hadoop
[root@node1 hadoop] # useradd-g hadoop yarn
[root@node1 hadoop] # useradd-g hadoop hdfs
[root@node1 hadoop] # useradd-g hadoop mapred
# create data and log directories
[root@node1 hadoop] # mkdir-p / data/hadoop/hdfs/ {nn,snn,dn}
[root@node1 hadoop] # chown-R hdfs:hadoop / data/hadoop/hdfs/
[root@node1 hadoop] # ll / data/hadoop/hdfs/
Total 0
Drwxr-xr-x 2 hdfs hadoop 6 Jan 7 09:48 dn
Drwxr-xr-x 2 hdfs hadoop 6 Jan 7 09:48 nn
Drwxr-xr-x 2 hdfs hadoop 6 Jan 7 09:48 snn
[root@node1 hadoop] # mkdir logs
[root@node1 hadoop] # chmod Grouw logs
[root@node1 hadoop] # chown-R yarn:hadoop. / *
[root@node1 hadoop] # ll
Total 36
Drwxr-xr-x 2 yarn hadoop 4096 Oct 22 2015 bin
Drwxr-xr-x 3 yarn hadoop 19 Oct 22 2015 etc
Drwxr-xr-x 2 yarn hadoop 101 Oct 22 2015 include
Drwxr-xr-x 3 yarn hadoop 19 Oct 22 2015 lib
Drwxr-xr-x 2 yarn hadoop 4096 Oct 22 2015 libexec
-rw-r--r-- 1 yarn hadoop 15429 Oct 22 2015 LICENSE.txt
Drwxrwxr-x 2 yarn hadoop 6 Jan 7 09:51 logs
-rw-r--r-- 1 yarn hadoop 101 Oct 22 2015 NOTICE.txt
-rw-r--r-- 1 yarn hadoop 1366 Oct 22 2015 README.txt
Drwxr-xr-x 2 yarn hadoop 4096 Oct 22 2015 sbin
Drwxr-xr-x 4 yarn hadoop 29 Oct 22 2015 share
# configure hadoop
[root@node1 hadoop] # cd etc/hadoop/
[root@node1 hadoop] # vim core-site.xml
Add at the end:
Fs.defaultFS
Hdfs://localhost:8020
True
[root@node1 hadoop] # vim hdfs-site.xml
Add at the end:
Dfs.replication
one
Dfs.namenode.name.dir
File:///data/hadoop/hdfs/nn
Dfs.datanode.data.dir
File:///data/hadoop/hdfs/dn
Dfs.checkpoint.dir
File:///data/hadoop/hdfs/snn
Fs.checkpoint.edits.dir
File:///data/hadoop/hdfs/snn
[root@node1 hadoop] # cp mapred-site.xml.template mapred-site.xml
[root@node1 hadoop] # vim mapred-site.xml
Add at the end:
Mapreduce.framework.name
Yarn
[root@node1 hadoop] # vim yarn-site.xml
Yarn.resourcemanager.address
Localhost:8032
Yarn.resourcemanager.schedul.address
Localhost:8030
Yarn.resourcemanager.resource-tracker.address
Localhost:8031
Yarn.resourcemanager.admin.address
Localhost:8033
Yarn.resourcemanager.webapp.address
Localhost:8088
Yarn.nodemanager.aux-services
Mapreduce_shuffle
Yarn.nodemanager.auxservices.mapreduce_shuffle.class
Org.apache.hadoop.mapred.ShuffleHandler
Yarn.resourcemanager.scheduler.class
Org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
# formatting HDFS
[root@node1 hadoop] # su-hdfs
[hdfs@node1 ~] $hdfs namenode-format
[hdfs@node1 ~] $ls / data/hadoop/hdfs/nn/current/
Fsp_w_picpath_0000000000000000000 fsp_w_picpath_0000000000000000000.md5 seen_txid VERSION
# start Hadoop
[hdfs@node1 ~] $hadoop-daemon.sh start namenode
Starting namenode, logging to / bdapps/hadoop/logs/hadoop-hdfs-namenode-node1.out
[hdfs@node1 ~] $ls / bdapps/hadoop/logs/
Hadoop-hdfs-namenode-node1.log SecurityAuth-hdfs.audit
Hadoop-hdfs-namenode-node1.out
[hdfs@node1 ~] $jps
41937 NameNode
42025 Jps
[hdfs@node1 ~] $hadoop-daemon.sh start secondarynamenode
Starting secondarynamenode, logging to / bdapps/hadoop/logs/hadoop-hdfs-secondarynamenode-node1.out
[hdfs@node1 ~] $jps
41937 NameNode
42090 SecondaryNameNode
42133 Jps
[hdfs@node1 ~] $hadoop-daemon.sh start datanode
Starting datanode, logging to / bdapps/hadoop/logs/hadoop-hdfs-datanode-node1.out
[hdfs@node1 ~] $jps
41937 NameNode
42242 Jps
42166 DataNode
42090 SecondaryNameNode
[hdfs@node1 ~] $hdfs dfs-mkdir / test
[hdfs@node1 ~] $hdfs dfs-ls /
Found 1 items
Drwxr-xr-x-hdfs supergroup 0 2017-01-07 11:08 / test
[hdfs@node1 ~] $hdfs dfs-put / etc/fstab / test/fstab
[hdfs@node1 ~] $hdfs dfs-ls / test
Found 1 items
-rw-r--r-- 1 hdfs supergroup 465 2017-01-07 11:11 / test/fstab
[hdfs@node1 ~] $hdfs dfs-lsr /
Lsr: DEPRECATED: Please use'ls-R 'instead.
Drwxr-xr-x-hdfs supergroup 0 2017-01-07 11:11 / test
-rw-r--r-- 1 hdfs supergroup 465 2017-01-07 11:11 / test/fstab
[hdfs@node1] $ls / data/hadoop/hdfs/dn/current/BP-1017498243-192.168.1.131-1483757917078/current/finalized/subdir0/subdir0/
Blk_1073741825 blk_1073741825_1001.meta
[hdfs@node1] $file / data/hadoop/hdfs/dn/current/BP-1017498243-192.168.1.131-1483757917078/current/finalized/subdir0/subdir0/blk_1073741825
/ data/hadoop/hdfs/dn/current/BP-1017498243-192.168.1.131-1483757917078/current/finalized/subdir0/subdir0/blk_1073741825: ASCII text
[hdfs@node1] $cat / data/hadoop/hdfs/dn/current/BP-1017498243-192.168.1.131-1483757917078/current/finalized/subdir0/subdir0/blk_1073741825
#
# / etc/fstab
# Created by anaconda on Fri Sep 2 00:27:27 2016
#
# Accessible filesystems, by reference, are maintained under'/ dev/disk'
# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info
#
/ dev/mapper/centos-root / xfs defaults 0 0
UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0
/ dev/mapper/centos-swap swap swap defaults 0 0
[hdfs@node1 ~] $hdfs dfs-cat / test/fstab
#
# / etc/fstab
# Created by anaconda on Fri Sep 2 00:27:27 2016
#
# Accessible filesystems, by reference, are maintained under'/ dev/disk'
# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info
#
/ dev/mapper/centos-root / xfs defaults 0 0
UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0
/ dev/mapper/centos-swap swap swap defaults 0 0
[hdfs@node1 ~] $exit
[root@node1 hadoop] # su-yarn
[yarn@node1 ~] $yarn-daemon.sh start resourcemanager
Starting resourcemanager, logging to / bdapps/hadoop/logs/yarn-yarn-resourcemanager-node1.out
[yarn@node1 ~] $jps
42932 ResourceManager
43161 Jps
[yarn@node1 ~] $yarn-daemon.sh start nodemanager
Starting nodemanager, logging to / bdapps/hadoop/logs/yarn-yarn-nodemanager-node1.out
[root@node1 hadoop] # pwd
/ bdapps/hadoop
[root@node1 hadoop] # cd share/hadoop/mapreduce/
[root@node1 mapreduce] # ls
Hadoop-mapreduce-client-app-2.6.2.jar
Hadoop-mapreduce-client-common-2.6.2.jar
Hadoop-mapreduce-client-core-2.6.2.jar
Hadoop-mapreduce-client-hs-2.6.2.jar
Hadoop-mapreduce-client-hs-plugins-2.6.2.jar
Hadoop-mapreduce-client-jobclient-2.6.2.jar
Hadoop-mapreduce-client-jobclient-2.6.2-tests.jar
Hadoop-mapreduce-client-shuffle-2.6.2.jar
Hadoop-mapreduce-examples-2.6.2.jar
Lib
Lib-examples
Sources
[root@node1 mapreduce] # su-hdfs
Last login: Sat Jan 7 10:40:42 CST 2017 on pts/0
[hdfs@node1 ~] $yarn jar / bdapps/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.2.jar
An example program must be given as the first argument.
Valid program names are:
Aggregatewordcount: An Aggregate based map/reduce program that counts the words in the input files.
Aggregatewordhist: An Aggregate based map/reduce program that computes the histogram of the words in the input files.
Bbp: A map/reduce program that uses Bailey-Borwein-Plouffe to compute exact digits of Pi.
Dbcount: An example job that count the pageview counts from a database.
Distbbp: A map/reduce program that uses a BBP-type formula to compute exact bits of Pi.
Grep: A map/reduce program that counts the matches of a regex in the input.
Join: A job that effects a join over sorted, equally partitioned datasets
Multifilewc: A job that counts words from several files.
Pentomino: A map/reduce tile laying program to find solutions to pentomino problems.
Pi: A map/reduce program that estimates Pi using a quasi-Monte Carlo method.
Randomtextwriter: A map/reduce program that writes 10GB of random textual data per node.
Randomwriter: A map/reduce program that writes 10GB of random data per node.
Secondarysort: An example defining a secondarysort to the reduce.
Sort: A map/reduce program that sorts the data written by the random writer.
Sudoku: A sudoku solver.
Teragen: Generate data for the terasort
Terasort: Run the terasort
Teravalidate: Checking results of terasort
Wordcount: A map/reduce program that counts the words in the input files.
Wordmean: A map/reduce program that counts the average length of the words in the input files.
Wordmedian: A map/reduce program that counts the median length of the words in the input files.
Wordstandarddeviation: A map/reduce program that counts the standarddeviation of the length of the words in the input files.
[hdfs@node1 ~] $yarn jar / bdapps/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.2.jar wordcount / test/fstab / test/fstab.out
17-01-07 11:43:42 INFO client.RMProxy: Connecting to ResourceManager at localhost/127.0.0.1:8032
17-01-07 11:43:43 INFO input.FileInputFormat: Total input paths to process: 1
17-01-07 11:43:43 INFO mapreduce.JobSubmitter: number of splits:1
17-01-07 11:43:44 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1483759378332_0001
17-01-07 11:43:45 INFO impl.YarnClientImpl: Submitted application application_1483759378332_0001
11:43:45 on 17-01-07 INFO mapreduce.Job: The url to track the job: http://node1:8088/proxy/application_1483759378332_0001/
17-01-07 11:43:45 INFO mapreduce.Job: Running job: job_1483759378332_0001
17-01-07 11:43:55 INFO mapreduce.Job: Job job_1483759378332_0001 running in uber mode: false
17-01-07 11:43:55 INFO mapreduce.Job: map 0 reduce 0
17-01-07 11:44:03 INFO mapreduce.Job: map 100% reduce 0
17-01-07 11:44:10 INFO mapreduce.Job: map 100 reduce 100%
17-01-07 11:44:11 INFO mapreduce.Job: Job job_1483759378332_0001 completed successfully
17-01-07 11:44:11 INFO mapreduce.Job: Counters: 49
File System Counters
FILE: Number of bytes read=554
FILE: Number of bytes written=214131
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=562
HDFS: Number of bytes written=396
HDFS: Number of read operations=6
HDFS: Number of large read operations=0
HDFS: Number of write operations=2
Job Counters
Launched map tasks=1
Launched reduce tasks=1
Data-local map tasks=1
Total time spent by all maps in occupied slots (ms) = 5413
Total time spent by all reduces in occupied slots (ms) = 4521
Total time spent by all map tasks (ms) = 5413
Total time spent by all reduce tasks (ms) = 4521
Total vcore-seconds taken by all map tasks=5413
Total vcore-seconds taken by all reduce tasks=4521
Total megabyte-seconds taken by all map tasks=5542912
Total megabyte-seconds taken by all reduce tasks=4629504
Map-Reduce Framework
Map input records=11
Map output records=54
Map output bytes=588
Map output materialized bytes=554
Input split bytes=97
Combine input records=54
Combine output records=38
Reduce input groups=38
Reduce shuffle bytes=554
Reduce input records=38
Reduce output records=38
Spilled Records=76
Shuffled Maps = 1
Failed Shuffles=0
Merged Map outputs=1
GC time elapsed (ms) = 258
CPU time spent (ms) = 3020
Physical memory (bytes) snapshot=429760512
Virtual memory (bytes) snapshot=4256686080
Total committed heap usage (bytes) = 301465600
Shuffle Errors
BAD_ID=0
CONNECTION=0
IO_ERROR=0
WRONG_LENGTH=0
WRONG_MAP=0
WRONG_REDUCE=0
File Input Format Counters
Bytes Read=465
File Output Format Counters
Bytes Written=396
[hdfs@node1 ~] $hdfs dfs-ls / test/fstab.out
Found 2 items
-rw-r--r-- 1 hdfs supergroup 0 2017-01-07 11:44 / test/fstab.out/_SUCCESS
-rw-r--r-- 1 hdfs supergroup 396 2017-01-07 11:44 / test/fstab.out/part-r-00000
[hdfs@node1 ~] $hdfs dfs-cat / test/fstab.out/part-r-00000
# 7
'/ dev/disk' 1
/ 1
/ boot 1
/ dev/mapper/centos-root 1
/ dev/mapper/centos-swap 1
/ etc/fstab 1
0 6
00:27:27 1
2 1
2016 1
Accessible 1
Created 1
Fri 1
See 1
Sep 1
UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 1
Anaconda 1
And/or 1
Are 1
Blkid (8) 1
By 2
Defaults 3
Filesystems, 1
Findfs (8), 1
For 1
Fstab (5), 1
Info 1
Maintained 1
Man 1
More 1
Mount (8) 1
On 1
Pages 1
Reference, 1
Swap 2
Under 1
Xfs 2
02 Hadoop 2 distributed cluster
Configure the environment
Node1: 192.168.1.130 CentOS Linux release 7.2
Node2: 192.168.1.131 CentOS Linux release 7.2
Node3: 192.168.1.132 CentOS Linux release 7.2
Node4: 192.168.1.133 CentOS Linux release 7.2
[root@node1 ~] # vim / etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
:: 1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.130 node1 master
192.168.1.131 node2
192.168.1.132 node3
192.168.1.133 node4
[root@node1 ~] # scp / etc/hosts node2:/etc/
[root@node1 ~] # scp / etc/hosts node3:/etc/
[root@node1 ~] # scp / etc/hosts node4:/etc/
[root@node1 ~] # vim / etc/profile.d/java.sh
Export JAVA_HOME=/usr
[root@node1 ~] # yum makecache
[root@node1 ~] # yum install java-1.7.0-openjdk-devel.x86_64-y
[root@node1 ~] # java-version
Openjdk version "1.8.0mm 101"
OpenJDK Runtime Environment (build 1.8.0_101-b13)
OpenJDK 64-Bit Server VM (build 25.101-b13, mixed mode)
[root@node1 ~] # scp / etc/profile.d/java.sh node2:/etc/profile.d/
[root@node1 ~] # scp / etc/profile.d/java.sh node3:/etc/profile.d/
[root@node1 ~] # scp / etc/profile.d/java.sh node4:/etc/profile.d/
[root@node1 ~] # groupadd hadoop
[root@node1 ~] # useradd-g hadoop hadoop
[root@node1 ~] # echo hadoop:mageedu | chpasswd
[root@node2 ~] # yum install java-1.7.0-openjdk-devel.x86_64-y
[root@node2 ~] # useradd hadoop
[root@node2 ~] # echo hadoop:mageedu | chpasswd
[root@node3 ~] # yum install java-1.7.0-openjdk-devel.x86_64-y
[root@node3 ~] # useradd hadoop
[root@node3 ~] # echo hadoop:mageedu | chpasswd
[root@node4 ~] # yum install java-1.7.0-openjdk-devel.x86_64-y
[root@node4 ~] # useradd hadoop
[root@node4 ~] # echo hadoop:mageedu | chpasswd
[hadoop@node1 ~] $ssh-keygen-t rsa-P''
Generating public/private rsa key pair.
Enter file in which to save the key (/ home/hadoop/.ssh/id_rsa):
Created directory'/ home/hadoop/.ssh'.
Your identification has been saved in / home/hadoop/.ssh/id_rsa.
Your public key has been saved in / home/hadoop/.ssh/id_rsa.pub.
The key fingerprint is:
2b:fc:62:6d:4e:ff:0d:01:6a:9e:4b:9f:55:a7:af:53 hadoop@node1
The key's randomart p_w_picpath is:
+-[RSA 2048]-+
| | |
| | |
|. | |
|. . | |
| S. . . |
|. O o o oE |
| o.* o. | |
| | oroomo + o o.. | | |
|. + +. Oo. |
+-+
[hadoop@node1] $for i in 2 34 X do ssh-copy-id-I. ssh / id_rsa.pub hadoop@node$ {I}; done
[hadoop@node1 ~] $exit
Logout
[root@node1] # mkdir-p / bdapps / data/hadoop/hdfs/ {nn,snn,dn}
[root@node1] # chown-R hadoop.hadoop / data/hadoop/hdfs/
[root@node1] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/
[root@node1 ~] # cd / bdapps/
[root@node1 bdapps] # ls
Hadoop-2.6.2
[root@node1 bdapps] # ln-s hadoop-2.6.2/ hadoop
[root@node1 bdapps] # cd hadoop
[root@node1 hadoop] # mkdir logs
[root@node1 hadoop] # chmod Grouw logs
[root@node1 hadoop] # chown-R hadoop.hadoop. / *
[root@node1 hadoop] # ll
Total 36
Drwxr-xr-x 2 hadoop hadoop 4096 Oct 22 2015 bin
Drwxr-xr-x 3 hadoop hadoop 19 Oct 22 2015 etc
Drwxr-xr-x 2 hadoop hadoop 101 Oct 22 2015 include
Drwxr-xr-x 3 hadoop hadoop 19 Oct 22 2015 lib
Drwxr-xr-x 2 hadoop hadoop 4096 Oct 22 2015 libexec
-rw-r--r-- 1 hadoop hadoop 15429 Oct 22 2015 LICENSE.txt
Drwxrwxr-x 2 hadoop hadoop 6 Jan 7 14:21 logs
-rw-r--r-- 1 hadoop hadoop 101 Oct 22 2015 NOTICE.txt
-rw-r--r-- 1 hadoop hadoop 1366 Oct 22 2015 README.txt
Drwxr-xr-x 2 hadoop hadoop 4096 Oct 22 2015 sbin
Drwxr-xr-x 4 hadoop hadoop 29 Oct 22 2015 share
[root@node1 hadoop] # cd etc/hadoop/
[root@node1 hadoop] # vim core-site.xml
Last line add
Fs.defaultFS
Hdfs://master:8020
True
[root@node1 hadoop] # vim yarn-site.xml
Add at the end
Yarn.resourcemanager.address
Master:8032
Yarn.resourcemanager.schedul.address
Master:8030
Yarn.resourcemanager.resource-tracker.address
Master:8031
Yarn.resourcemanager.admin.address
Master:8033
Yarn.resourcemanager.webapp.address
Master:8088
Yarn.nodemanager.aux-services
Mapreduce_shuffle
Yarn.nodemanager.auxservices.mapreduce_shuffle.class
Org.apache.hadoop.mapred.ShuffleHandler
Yarn.resourcemanager.scheduler.class
Org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
[root@node1 hadoop] # vim hdfs-site.xml
Add at the end
Dfs.replication
two
Dfs.namenode.name.dir
File:///data/hadoop/hdfs/nn
Dfs.datanode.data.dir
File:///data/hadoop/hdfs/dn
Dfs.checkpoint.dir
File:///data/hadoop/hdfs/snn
Fs.checkpoint.edits.dir
File:///data/hadoop/hdfs/snn
[root@node1 hadoop] # vim slaves
Node2
Node3
Node4
[root@node2] # mkdir-p / bdapps / data/hadoop/hdfs/ {nn,snn,dn}
[root@node2] # chown-R hadoop.hadoop / data/hadoop/hdfs/
[root@node3] # mkdir-p / bdapps / data/hadoop/hdfs/ {nn,snn,dn}
[root@node3] # chown-R hadoop.hadoop / data/hadoop/hdfs/
[root@node4] # mkdir-p / bdapps / data/hadoop/hdfs/ {nn,snn,dn}
[root@node4] # chown-R hadoop.hadoop / data/hadoop/hdfs/
[root@node2] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/
[root@node2 ~] # cd / bdapps/
[root@node2 bdapps] # ln-s hadoop-2.6.2/ hadoop
[root@node2 bdapps] # cd hadoop
[root@node2 hadoop] # mkdir logs
[root@node2 hadoop] # chmod Grouw logs/
[root@node2 hadoop] # chown hadoop.hadoop-R. / *
[root@node3] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/
[root@node3 ~] # cd / bdapps/
[root@node3 bdapps] # ln-s hadoop-2.6.2/ hadoop
[root@node3 bdapps] # cd hadoop
[root@node3 hadoop] # mkdir logs
[root@node3 hadoop] # chmod Grouw logs
[root@node3 hadoop] # chown hadoop.hadoop-R. / *
[root@node4] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/
[root@node4 ~] # cd / bdapps/
[root@node4 bdapps] # ln-s hadoop-2.6.2/ hadoop
[root@node4 bdapps] # cd hadoop
[root@node4 hadoop] # mkdir logs
[root@node4 hadoop] # chmod Grouw logs/
[root@node4 hadoop] # chown hadoop.hadoop-R. / *
[root@node1 ~] # su-hadoop
[hadoop@node1 ~] $scp / bdapps/hadoop/etc/hadoop/* node2:/bdapps/hadoop/etc/hadoop/
[hadoop@node1 ~] $scp / bdapps/hadoop/etc/hadoop/* node3:/bdapps/hadoop/etc/hadoop/
[hadoop@node1 ~] $scp / bdapps/hadoop/etc/hadoop/* node4:/bdapps/hadoop/etc/hadoop/
[hadoop@node1 ~] $exit p/etc/hadoop/
Logout
[root@node1 ~] # vim / etc/profile.d/hadoop.sh
Export HADOOP_PREFIX=/bdapps/hadoop
Export PATH=$PATH:$ {HADOOP_PREFIX} / bin:$ {HADOOP_PREFIX} / sbin
Export HADOOP_COMMON_HOME=$ {HADOOP_PREFIX}
Export HADOOP_YARN_HOME=$ {HADOOP_PREFIX}
Export HADOOP_HDFS_HOME=$ {HADOOP_PREFIX}
Export HADOOP_MAPRED_HOME=$ {HADOOP_PREFIX}
[root@node1 ~] # scp / etc/profile.d/hadoop.sh node2:/etc/profile.d/
[root@node1 ~] # scp / etc/profile.d/hadoop.sh node3:/etc/profile.d/
[root@node1 ~] # scp / etc/profile.d/hadoop.sh node4:/etc/profile.d/
# formatting HDFS
[root@node1 ~] # su-hadoop
[hadoop@node1 ~] $hdfs namenode-format
[hadoop@node1 ~] $ls / data/hadoop/hdfs/nn/
Current
# start the service
[hadoop@node1 ~] $start-dfs.sh
[hadoop@node1 ~] $jps
35957 NameNode
36153 SecondaryNameNode
36292 Jps
# stop the service
[hadoop@node1 ~] $stop-dfs.sh
[root@node2 hadoop] # su-hadoop
[hadoop@node2 ~] $jps
35527 DataNode
35639 Jps
[root@node3 hadoop] # su-hadoop
[hadoop@node3 ~] $jps
35113 DataNode
35241 Jps
[root@node4 hadoop] # su-hadoop
[hadoop@node4 ~] $jps
35113 DataNode
35242 Jps
# create a directory
[hadoop@node1 ~] $hdfs dfs-mkdir / test
# upload a file to hadoop
[hadoop@node1 ~] $hdfs dfs-put / etc/fstab / test/fstab
# View uploaded files
[hadoop@node1 ~] $hdfs dfs-lsr / test
Lsr: DEPRECATED: Please use'ls-R 'instead.
-rw-r--r-- 2 hadoop supergroup 465 2017-01-07 15:16 / test/fstab
# View the contents of the uploaded file
[hadoop@node1 ~] $hdfs dfs-cat / test/fstab
#
# / etc/fstab
# Created by anaconda on Fri Sep 2 00:27:27 2016
#
# Accessible filesystems, by reference, are maintained under'/ dev/disk'
# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info
#
/ dev/mapper/centos-root / xfs defaults 0 0
UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0
/ dev/mapper/centos-swap swap swap defaults 0 0
# Storage location of uploaded files in hadoop
[hadoop@node2] $cat / data/hadoop/hdfs/dn/current/BP-1026073846-192.168.1.130-1483772708588/current/finalized/subdir0/subdir0/blk_1073741825
#
# / etc/fstab
# Created by anaconda on Fri Sep 2 00:27:27 2016
#
# Accessible filesystems, by reference, are maintained under'/ dev/disk'
# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info
#
/ dev/mapper/centos-root / xfs defaults 0 0
UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0
/ dev/mapper/centos-swap swap swap defaults 0 0
[hadoop@node3] $cat / data/hadoop/hdfs/dn/current/BP-1026073846-192.168.1.130-1483772708588/current/finalized/subdir0/subdir0/blk_1073741825
#
# / etc/fstab
# Created by anaconda on Fri Sep 2 00:27:27 2016
#
# Accessible filesystems, by reference, are maintained under'/ dev/disk'
# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info
#
/ dev/mapper/centos-root / xfs defaults 0 0
UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0
/ dev/mapper/centos-swap swap swap defaults 0 0
[hadoop@node4] $ls / data/hadoop/hdfs/dn/current/BP-1026073846-192.168.1.130-1483772708588/current/finalized/
[hadoop@node1 ~] $start-yarn.sh
Starting yarn daemons
Starting resourcemanager, logging to / bdapps/hadoop/logs/yarn-hadoop-resourcemanager-node1.out
Node4: starting nodemanager, logging to / bdapps/hadoop/logs/yarn-hadoop-nodemanager-node4.out
Node3: starting nodemanager, logging to / bdapps/hadoop/logs/yarn-hadoop-nodemanager-node3.out
Node2: starting nodemanager, logging to / bdapps/hadoop/logs/yarn-hadoop-nodemanager-node2.out
[hadoop@node1 ~] $jps
35957 NameNode
36153 SecondaryNameNode
36980 Jps
36715 ResourceManager
[hadoop@node2 ~] $jps
35996 Jps
35527 DataNode
35880 NodeManager
[hadoop@node3 ~] $jps
35113 DataNode
35474 NodeManager
35591 Jps
[hadoop@node4 ~] $jps
35477 NodeManager
35113 DataNode
35600 Jps
[root@node1 ~] # ss-tnl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 *: 50090 *: *
LISTEN 0 128 192.168.1.130 8020 *: *
LISTEN 0 5 192.168.122.1 LISTEN 53 *: *
LISTEN 0 128 *: 50070 *: *
LISTEN 0 128 *: 22 *: *
LISTEN 0 128 127.0.0.1 631 *: *
LISTEN 0 100 127.0.0.1 purl 25 *: *
LISTEN 0 128:: 22: *
LISTEN 0 128:: 1 LISTEN:: *
LISTEN 0 128:: ffff:192.168.1.130:8088: *
LISTEN 0 100:: 1:25: *
LISTEN 0 128:: 8030: *
LISTEN 0 128:: ffff:192.168.1.130:8031: *
LISTEN 0 128:: ffff:192.168.1.130:8032: *
LISTEN 0 128:: ffff:192.168.1.130:8033: *
[hadoop@node1 ~] $hdfs dfs-put / etc/rc.d/init.d/functions / test
[hadoop@node1 ~] $hdfs dfs-ls / test
Found 2 items
-rw-r--r-- 2 hadoop supergroup 465 2017-01-07 15:16 / test/fstab
-rw-r--r-- 2 hadoop supergroup 13948 2017-01-07 15:35 / test/functions
[hadoop@node1 ~] $hdfs dfs-put hadoop-2.6.2.tar.gz / test/
# Statistical text file
[hadoop@node1 ~] $yarn jar / bdapps/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.2.jar wordcount / test/fstab / test/functions / test/wc
[hadoop@node1 ~] $hdfs-ls / test/wc
[hadoop@node1 ~] $hdfs dfs-cat / test/wc/part-r-00000
Hadoop application
Http://192.168.1.130:8088
Manage hadoop
Http://192.168.1.130:50070
2. YARN cluster management command
[hadoop@node1 ~] $yarn application-list
17-01-07 17:11:25 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8032
Total number of applications (application-types: [] and states: [SUBMITTED, ACCEPTED, RUNNING]): 0
Application-Id Application-Name Application-Type User Queue State Final-State Progress Tracking-URL
[hadoop@node1 ~] $yarn application-list-appStates=all
17-01-07 17:12:13 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8032
Total number of applications (application-types: [] and states: [NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED]): 0
Application-Id Application-Name Application-Type User Queue State Final-State Progress Tracking-URL
[hadoop@node1 ~] $yarn application-status
[hadoop@node1 ~] $yarn node-list
17-01-07 17:16:47 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8032
Total Nodes:3
Node-Id Node-State Node-Http-Address Number-of-Running-Containers
Node2:40860 RUNNING node2:8042 0
Node3:47737 RUNNING node3:8042 0
Node4:45637 RUNNING node4:8042 0
[hadoop@node1 ~] $yarn node-status node3:47737
17-01-07 17:20:41 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8032
Node Report:
Node-Id: node3:47737
Rack: / default-rack
Node-State: RUNNING
Node-Http-Address: node3:8042
Last-Health-Update: Sat 07/Jan/17 05:18:54:160CST
Health-Report:
Containers: 0
Memory-Used: 0MB
Memory-Capacity: 8192MB
CPU-Used: 0 vcores
CPU-Capacity: 8 vcores
Node-Labels:
[hadoop@node1 ~] $yarn logs-applicationId
[hadoop@node1 ~] $yarn rmadmin-refreshNodes
17-01-07 17:27:20 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8033