Hadoop分布式集群部署

简介: Hadoop分布式集群部署

环境准备

IP HOSTNAME SYSTEM
192.168.131.129 hadoop-master CentOS 7.6

192.168.131.135 hadoop-slave1 CentOS 7.6
192.168.131.137 hadoop-slave2 CentOS 7.6
[root@localhost ~]# cat /etc/redhat-release
CentOS Linux release 7.6.1810 (Core)
[root@localhost ~]# sestatus
SELinux status:                 disabled
[root@localhost ~]# systemctl status firewalld.service
● firewalld.service - firewalld - dynamic firewall daemon
   Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
   Active: inactive (dead)
     Docs: man:firewalld(1)
[root@localhost ~]# hostnamectl set-hostname --static hadoop-master
[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave1
[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave2

配置免密

[root@hadoop-master ~]# cat >> /etc/hosts <<EOF
192.168.131.129 hadoop-master
192.168.131.135 hadoop-slave1
192.168.131.137 hadoop-slave2
EOF
[root@hadoop-master ~]# ssh-keygen
[root@hadoop-master ~]# ssh-copy-id hadoop-slave1
[root@hadoop-master ~]# ssh-copy-id hadoop-slave2
[root@hadoop-master ~]# scp /etc/hosts hadoop-slave1:/etc/hosts
hosts                                                              100%  248   151.9KB/s   00:00
[root@hadoop-master ~]# scp /etc/hosts hadoop-slave2:/etc/hosts
hosts                                                              100%  248   220.9KB/s   00:00

配置java环境

[root@hadoop-master ~]# tar xf jdk-8u211-linux-x64.tar.gz -C /usr/local/
[root@hadoop-master ~]# ln -s /usr/local/jdk1.8.0_211/ /usr/local/java
[root@hadoop-master ~]# ll /usr/local/
total 0
drwxr-xr-x. 2 root root   6 Apr 11  2018 bin
drwxr-xr-x. 2 root root   6 Apr 11  2018 etc
drwxr-xr-x. 2 root root   6 Apr 11  2018 games
drwxr-xr-x. 2 root root   6 Apr 11  2018 include
lrwxrwxrwx  1 root root  24 Sep  7 14:49 java -> /usr/local/jdk1.8.0_211/
drwxr-xr-x  7   10  143 245 Apr  2  2019 jdk1.8.0_211
drwxr-xr-x. 2 root root   6 Apr 11  2018 lib
drwxr-xr-x. 2 root root   6 Apr 11  2018 lib64
drwxr-xr-x. 2 root root   6 Apr 11  2018 libexec
drwxr-xr-x. 2 root root   6 Apr 11  2018 sbin
drwxr-xr-x. 5 root root  49 Mar 30  2019 share
drwxr-xr-x. 2 root root   6 Apr 11  2018 src
[root@hadoop-master ~]# vim /etc/profile
export JAVA_HOME=/usr/local/java/
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@hadoop-master ~]# source /etc/profile
[root@hadoop-master ~]# java -version
java version "1.8.0_211"
Java(TM) SE Runtime Environment (build 1.8.0_211-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.211-b12, mixed mode)

部署hadoop

[root@hadoop-master ~]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz
[root@hadoop-master ~]# tar xf hadoop-2.7.7.tar.gz -C /opt/
[root@hadoop-master ~]# vim /etc/profile
export HADOOP_HOME=/opt/hadoop-2.7.7
export PATH=$HADOOP_HOME/bin:$PATH
[root@hadoop-master ~]# source /etc/profile
[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/
[root@hadoop-master hadoop-2.7.7]# ls
bin  etc  include  lib  libexec  LICENSE.txt  NOTICE.txt  README.txt  sbin  share
[root@hadoop-master hadoop-2.7.7]# cd etc/hadoop/
[root@hadoop-master hadoop]# ll
total 152
-rw-r--r-- 1 1000 ftp  4436 Jul 19  2018 capacity-scheduler.xml
-rw-r--r-- 1 1000 ftp  1335 Jul 19  2018 configuration.xsl
-rw-r--r-- 1 1000 ftp   318 Jul 19  2018 container-executor.cfg
-rw-r--r-- 1 1000 ftp   774 Jul 19  2018 core-site.xml
-rw-r--r-- 1 1000 ftp  3670 Jul 19  2018 hadoop-env.cmd
-rw-r--r-- 1 1000 ftp  4224 Jul 19  2018 hadoop-env.sh
-rw-r--r-- 1 1000 ftp  2598 Jul 19  2018 hadoop-metrics2.properties
-rw-r--r-- 1 1000 ftp  2490 Jul 19  2018 hadoop-metrics.properties
-rw-r--r-- 1 1000 ftp  9683 Jul 19  2018 hadoop-policy.xml
-rw-r--r-- 1 1000 ftp   775 Jul 19  2018 hdfs-site.xml
-rw-r--r-- 1 1000 ftp  1449 Jul 19  2018 httpfs-env.sh
-rw-r--r-- 1 1000 ftp  1657 Jul 19  2018 httpfs-log4j.properties
-rw-r--r-- 1 1000 ftp    21 Jul 19  2018 httpfs-signature.secret
-rw-r--r-- 1 1000 ftp   620 Jul 19  2018 httpfs-site.xml
-rw-r--r-- 1 1000 ftp  3518 Jul 19  2018 kms-acls.xml
-rw-r--r-- 1 1000 ftp  1527 Jul 19  2018 kms-env.sh
-rw-r--r-- 1 1000 ftp  1631 Jul 19  2018 kms-log4j.properties
-rw-r--r-- 1 1000 ftp  5540 Jul 19  2018 kms-site.xml
-rw-r--r-- 1 1000 ftp 11801 Jul 19  2018 log4j.properties
-rw-r--r-- 1 1000 ftp   951 Jul 19  2018 mapred-env.cmd
-rw-r--r-- 1 1000 ftp  1383 Jul 19  2018 mapred-env.sh
-rw-r--r-- 1 1000 ftp  4113 Jul 19  2018 mapred-queues.xml.template
-rw-r--r-- 1 1000 ftp   758 Jul 19  2018 mapred-site.xml.template
-rw-r--r-- 1 1000 ftp    10 Jul 19  2018 slaves
-rw-r--r-- 1 1000 ftp  2316 Jul 19  2018 ssl-client.xml.example
-rw-r--r-- 1 1000 ftp  2697 Jul 19  2018 ssl-server.xml.example
-rw-r--r-- 1 1000 ftp  2250 Jul 19  2018 yarn-env.cmd
-rw-r--r-- 1 1000 ftp  4567 Jul 19  2018 yarn-env.sh
-rw-r--r-- 1 1000 ftp   690 Jul 19  2018 yarn-site.xml

部署hadoop

[root@hadoop-master ~]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz
[root@hadoop-master ~]# tar xf hadoop-2.7.7.tar.gz -C /opt/
[root@hadoop-master ~]# vim /etc/profile
export HADOOP_HOME=/opt/hadoop-2.7.7
export PATH=$HADOOP_HOME/bin:$PATH
[root@hadoop-master ~]# source /etc/profile
[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/
[root@hadoop-master hadoop-2.7.7]# ls
bin  etc  include  lib  libexec  LICENSE.txt  NOTICE.txt  README.txt  sbin  share
[root@hadoop-master hadoop-2.7.7]# cd etc/hadoop/
[root@hadoop-master hadoop]# ll
total 152
-rw-r--r-- 1 1000 ftp  4436 Jul 19  2018 capacity-scheduler.xml
-rw-r--r-- 1 1000 ftp  1335 Jul 19  2018 configuration.xsl
-rw-r--r-- 1 1000 ftp   318 Jul 19  2018 container-executor.cfg
-rw-r--r-- 1 1000 ftp   774 Jul 19  2018 core-site.xml
-rw-r--r-- 1 1000 ftp  3670 Jul 19  2018 hadoop-env.cmd
-rw-r--r-- 1 1000 ftp  4224 Jul 19  2018 hadoop-env.sh
-rw-r--r-- 1 1000 ftp  2598 Jul 19  2018 hadoop-metrics2.properties
-rw-r--r-- 1 1000 ftp  2490 Jul 19  2018 hadoop-metrics.properties
-rw-r--r-- 1 1000 ftp  9683 Jul 19  2018 hadoop-policy.xml
-rw-r--r-- 1 1000 ftp   775 Jul 19  2018 hdfs-site.xml
-rw-r--r-- 1 1000 ftp  1449 Jul 19  2018 httpfs-env.sh
-rw-r--r-- 1 1000 ftp  1657 Jul 19  2018 httpfs-log4j.properties
-rw-r--r-- 1 1000 ftp    21 Jul 19  2018 httpfs-signature.secret
-rw-r--r-- 1 1000 ftp   620 Jul 19  2018 httpfs-site.xml
-rw-r--r-- 1 1000 ftp  3518 Jul 19  2018 kms-acls.xml
-rw-r--r-- 1 1000 ftp  1527 Jul 19  2018 kms-env.sh
-rw-r--r-- 1 1000 ftp  1631 Jul 19  2018 kms-log4j.properties
-rw-r--r-- 1 1000 ftp  5540 Jul 19  2018 kms-site.xml
-rw-r--r-- 1 1000 ftp 11801 Jul 19  2018 log4j.properties
-rw-r--r-- 1 1000 ftp   951 Jul 19  2018 mapred-env.cmd
-rw-r--r-- 1 1000 ftp  1383 Jul 19  2018 mapred-env.sh
-rw-r--r-- 1 1000 ftp  4113 Jul 19  2018 mapred-queues.xml.template
-rw-r--r-- 1 1000 ftp   758 Jul 19  2018 mapred-site.xml.template
-rw-r--r-- 1 1000 ftp    10 Jul 19  2018 slaves
-rw-r--r-- 1 1000 ftp  2316 Jul 19  2018 ssl-client.xml.example
-rw-r--r-- 1 1000 ftp  2697 Jul 19  2018 ssl-server.xml.example
-rw-r--r-- 1 1000 ftp  2250 Jul 19  2018 yarn-env.cmd
-rw-r--r-- 1 1000 ftp  4567 Jul 19  2018 yarn-env.sh
-rw-r--r-- 1 1000 ftp   690 Jul 19  2018 yarn-site.xml
修改hadoop-env.sh和yarn-env.sh中JAVA_HOME参数
[root@hadoop-master hadoop]# sed -i 's#export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' hadoop-env.sh
[root@hadoop-master hadoop]# sed -i 's#.*export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' yarn-env.sh
[root@hadoop-master hadoop]# grep "export JAVA_HOME" hadoop-env.sh
export JAVA_HOME=/usr/local/java
[root@hadoop-master hadoop]# grep "export JAVA_HOME" yarn-env.sh
export JAVA_HOME=/usr/local/java
编辑core-site.xml
[root@hadoop-master hadoop]# vim core-site.xml
<configuration>
   <property>
      <name>fs.default.name</name>
      <value>hdfs://192.168.131.129:9000</value>
   </property>
   <property>
      <name>hadoop.tmp.dir</name>
      <value>/opt/hadoop/tmp</value>
   </property>
</configuration>
编辑hdfs-site.xml
[root@hadoop-master hadoop]# vim hdfs-site.xml
<configuration>
   <property>
      <name>dfs.replication</name>
      <value>2</value>
   </property>
   <property>
      <name>dfs.namenode.name.dir</name>
      <value>file:/opt/hadoop/dfs/name</value>
   </property>
   <property>
      <name>dfs.datanode.data.dir</name>
      <value>file:/opt/hadoop/dfs/data</value>
   </property>
</configuration>
编辑mapred-site.xml
[root@hadoop-master hadoop]# cp mapred-site.xml.template mapred-site.xml
[root@hadoop-master hadoop]# vim mapred-site.xml
<configuration>
   <property>
      <name>mapreduce.framework.name</name>
      <value>yarn</value>
   </property>
   <property>
      <name>mapreduce.jobhistory.address</name>
      <value>192.168.131.129:10020</value>
   </property>
   <property>
      <name>mapreduce.jobhistory.webapp.address</name>
      <value>192.168.131.129:19888</value>
   </property>
</configuration>
编辑yarn-site.xml
[root@hadoop-master hadoop]# vim yarn-site.xml
<configuration>
<!-- Site specific YARN configuration properties -->
   <property>
      <name>yarn.nodemanager.aux-services</name>
      <value>mapreduce_shuffle</value>
   </property>
   <property>
      <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
      <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>
   <property>
      <name>yarn.resourcemanager.address</name>
      <value>192.168.131.129:8032</value>
   </property>
   <property>
      <name>yarn.resourcemanager.scheduler.address</name>
      <value>192.168.131.129:8030</value>
   </property>
   <property>
      <name>yarn.resourcemanager.resource-tracker.address</name>
      <value>192.168.131.129:8031</value>
   </property>
   <property>
      <name>yarn.resourcemanager.admin.address</name>
      <value>192.168.131.129:8033</value>
   </property>
   <property>
      <name>yarn.resourcemanager.webapp.address</name>
      <value>192.168.131.129:8088</value>
   </property>
</configuration>
编辑slaves
[root@hadoop-master hadoop]# vim slaves
hadoop-slave1
hadoop-slave2
把配置分发到各个节点
[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave1:/opt/
[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave2:/opt/
在主节点启动hadoop
初始化namenode
[root@hadoop-master ~]# hdfs namenode -format
启动HDFS
[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/sbin/
[root@hadoop-master sbin]# ./start-dfs.sh
启动YARN
[root@hadoop-master sbin]# ./start-yarn.sh

测试

浏览器访问
YARN WEB地址:http://192.168.131.129:8088/
HDFS WEB地址:http://192.168.131.129:50070/


目录
相关文章
|
4月前
|
SQL 分布式计算 Hadoop
大数据行业部署实战1:Hadoop伪分布式部署
大数据行业部署实战1:Hadoop伪分布式部署
96 0
|
3月前
|
分布式计算 资源调度 Hadoop
Hadoop【部署 02】hadoop-3.1.3 单机版YARN(配置、启动停止shell脚本修改及服务验证)
Hadoop【部署 02】hadoop-3.1.3 单机版YARN(配置、启动停止shell脚本修改及服务验证)
28 0
|
3月前
|
分布式计算 Hadoop Java
Hadoop【部署 01】腾讯云Linux环境CentOS Linux release 7.5.1804单机版hadoop-3.1.3详细安装步骤(安装+配置+初始化+启动脚本+验证)
Hadoop【部署 01】腾讯云Linux环境CentOS Linux release 7.5.1804单机版hadoop-3.1.3详细安装步骤(安装+配置+初始化+启动脚本+验证)
59 0
|
3月前
|
分布式计算 Hadoop 大数据
大数据成长之路-- hadoop集群的部署(4)退役旧数据节点
大数据成长之路-- hadoop集群的部署(4)退役旧数据节点
40 0
|
3月前
|
分布式计算 Hadoop 大数据
大数据成长之路-- hadoop集群的部署(3)HDFS新增节点
大数据成长之路-- hadoop集群的部署(3)HDFS新增节点
38 0
|
3月前
|
分布式计算 Hadoop 大数据
大数据成长之路------hadoop集群的部署(2)Hadoop集群安装部署
大数据成长之路------hadoop集群的部署(2)Hadoop集群安装部署
41 0
|
3月前
|
分布式计算 网络协议 Hadoop
大数据成长之路------hadoop集群的部署 配置系统网络(静态) 新增集群(三台)
大数据成长之路------hadoop集群的部署 配置系统网络(静态) 新增集群(三台)
21 0
|
3月前
|
SQL 分布式计算 Hadoop
Hadoop学习笔记(HDP)-Part.08 部署Ambari集群
01 关于HDP 02 核心组件原理 03 资源规划 04 基础环境配置 05 Yum源配置 06 安装OracleJDK 07 安装MySQL 08 部署Ambari集群 09 安装OpenLDAP 10 创建集群 11 安装Kerberos 12 安装HDFS 13 安装Ranger 14 安装YARN+MR 15 安装HIVE 16 安装HBase 17 安装Spark2 18 安装Flink 19 安装Kafka 20 安装Flume
58 0
Hadoop学习笔记(HDP)-Part.08 部署Ambari集群
|
4月前
|
分布式计算 Hadoop 大数据
:大数据行业部署实战3:基于Hadoop的Web版的云盘
:大数据行业部署实战3:基于Hadoop的Web版的云盘
74 0
|
3月前
|
消息中间件 分布式计算 大数据
【大数据技术Hadoop+Spark】Flume、Kafka的简介及安装(图文解释 超详细)
【大数据技术Hadoop+Spark】Flume、Kafka的简介及安装(图文解释 超详细)
44 0

相关实验场景

更多