docker 安装
curl -sSL https://get.daocloud.io/docker | sh ##换源 curl -sSL https://get.daocloud.io/daotools/set_mirror.sh | sh -s http://67e93489.m.daocloud.io systemctl restart docker
拉取镜像
docker pull docker pull centos # 查看镜像是否拉取成功 docker images
创建容器
#设置固定IP docker network create --subnet=172.16.0.0/16 huan-net #创建容器 #cluster-master docker run -d --privileged -ti -v /sys/fs/cgroup:/sys/fs/cgroup --name cluster-master -h cluster-master --net netgroup --ip 172.16.0.2 daocloud.io/library/centos /usr/sbin/init #cluster-slaves docker run -d --privileged -ti -v /sys/fs/cgroup:/sys/fs/cgroup --name cluster-slave1 -h cluster-slave1 --net netgroup --ip 172.16.0.3 daocloud.io/library/centos /usr/sbin/init docker run -d --privileged -ti -v /sys/fs/cgroup:/sys/fs/cgroup --name cluster-slave2 -h cluster-slave2 --net netgroup --ip 172.16.0.4 daocloud.io/library/centos /usr/sbin/init docker run -d --privileged -ti -v /sys/fs/cgroup:/sys/fs/cgroup --name cluster-slave3 -h cluster-slave3 --net netgroup --ip 172.16.0.5 daocloud.io/library/centos /usr/sbin/init
更换YUM源
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
部署Openssh
master
yum -y install openssh openssh-server openssh-clients systemctl start sshd vi /etc/ssh/ssh_config #设置StrictHostKeyChecking为no systemctl restart sshd
slave1
yum -y install openssh openssh-server openssh-clients systemctl start sshd
slave2
yum -y install openssh openssh-server openssh-clients systemctl start sshd
slave3
yum -y install openssh openssh-server openssh-clients systemctl start sshd
公钥分发
ssh-keygen -t rsa ssh-copy-id -i ~/.ssh/id_rsa.pub <romte_ip>
zookeeper配置
master
将文件从主机复制到docker容器
sudo docker cp apache-zookeeper-3.5.5.tar.gz cluster-slave1:/root/tar
创建目录放置解压文件
mkdir /opt/zookeeper
解压
tar -xivf apache-zookeeper-3.5.5.tar.gz -C /opt/zookeeper/
创建zoo.cfg配置文件
cp /opt/zookeeper/apache-zookeeper-3.5.5/conf/zoo.cfg /opt/zookeeper/apache-zookeeper-3.5.5/conf/zoo.cfg
编辑zoo配置文件
vim /opt/zookeeper/apache-zookeeper-3.5.5/conf/zoo.cfg
添加结点参数
server.1=172.16.0.2:2888:3888 server.2=172.16.0.3:2899:3899 server.3=172.16.0.4:2877:3877
创建myid文件
echo 1 > /tmp/zookeeper/myid
编辑环境变量配置文件
vim ~/.bash_profile
添加环境变量
ZK_HOME=/opt/zookeeper/apache-zookeeper-3.5.5 PATH=$PATH:$ZK_HOME/bin
使配置生效
source ~/.bash_profile
将配置文件发送到slave1,slave2
scp -r ~/.bash_profile cluster-slave1:~/
scp -r ~/.bash_profile cluster-slave2:~/
scp -r /tmp/zookeeper/myid cluster-slave1:/tmp/zookeeper/
scp -r /tmp/zookeeper/myid cluster-slave2:/tmp/zookeeper/
scp -r zoo.cfg cluster-slave1:/opt/zookeeper/apache-zookeeper-3.5.5/conf/
scp -r zoo.cfg cluster-slave2:/opt/zookeeper/apache-zookeeper-3.5.5/conf/
slave1
使配置生效
source ~/.bash_profile
修改myid文件为2
slave2
修改myid文件为3
启动
zhServer.sh start
Hadoop配置
core-site.xml
<configuration> <property> <name>fs.defaultFS</name> <value>hdfs://jinbill</value> </property> <property> <name>ha.zookeeper.quorum</name> <value>192.168.11.46:12181</value> </property> <property> <name>hadoop.tmp.dir</name> <value>/opt/hadoop</value> </property> </configuration>
hdfs-site.xml
<configuration> <property> <name>dfs.nameservices</name> <value>jinbill</value> </property> <property> <name>dfs.ha.namenodes.jinbill</name> <value>nn1,nn2</value> </property> <property> <name>dfs.namenode.rpc-address.jinbill.nn1</name> <value>cluster-master:8020</value> </property> <property> <name>dfs.namenode.rpc-address.jinbill.nn2</name> <value>cluster-slave1:8020</value> </property> <property> <name>dfs.namenode.http-address.shsxt.nn1</name> <value>cluster-master:50070</value> </property> <property> <name>dfs.namenode.http-address.shsxt.nn2</name> <value>cluster-slave1:50070</value> </property> <property> <!-- 指定 namenode 元数据存储在 journalnode 中的路径 --> <name>dfs.namenode.shared.edits.dir</name> <value>qjournal://cluster-slave1:8485;cluster-slave2:8485;cluster-slave3:8485/jinbill</value> </property> <property> <!-- 指定 HDFS 客户端连接 active namenode 的 java 类 --> <name>dfs.client.failover.proxy.provider.jinbill</name> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider </value> </property> <property> <!-- 配置隔离机制为 ssh 防止裂 脑裂 --> <name>dfs.ha.fencing.methods</name> <value>sshfence</value> </property> <property> <!-- 指定秘钥的位置 --> <name>dfs.ha.fencing.ssh.private-key-files</name> <value>/root/.ssh/id_rsa</value> </property> <property> <!-- 指定 journalnode 日志文件存储的路径 --> <name>dfs.journalnode.edits.dir</name> <value>/opt/hadoop/data</value> </property> <property> <!-- 开启自动故障转移 --> <name>dfs.ha.automatic-failover.enabled</name> <value>true</value> </property> </configuration>
yarn-site.xml
<configuration> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.resourcemanager.ha.enabled</name> <value>true</value> </property> <property> <name>yarn.resourcemanager.cluster-id</name> <value>mr_jinbill</value> </property> <property> <name>yarn.resourcemanager.ha.rm-ids</name> <value>rm1,rm2</value> </property> <property> <name>yarn.resourcemanager.hostname.rm1</name> <value>cluster-slave2</value> </property> <property> <name>yarn.resourcemanager.hostname.rm2</name> <value>cluster-slave3</value> </property> <property> <name>yarn.resourcemanager.zk-address</name> <value>192.168.11.46:12181</value> </property> </configuration>
mapred-site.xml
<configuration> <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> </configuration>
slaves
cluster-slave1 cluster-slave2 cluster-slave3
启动三个JornalNode
hadoop-daemon.sh start journalnode
在其中一个NameNode上格式化
hdfs namenode -format
把格式化后的元数据拷贝到另一个NameNode上
元数据初始化后会存放在
hadoop.tmp.dir这个参数配置的地址下的叫dfs的目录里面，将这个目录复制到另一个NN的同目录下
启动刚刚格式化的NameNode
hadoop-daemon.sh start namenode
在没有格式化的NameNode上执行
hdfs namenode -bootstrapStandby
启动第二个NameNode
hadoop-daemon.sh start namenode
在其中一个NameNode上初始化zkfc
hdfs zkfc –formatZK
停止所有结点
stop-dfs.sh
全面启动
start-dfs.sh
start-yarn.sh
访问地址 50070端口和8088端口
因为我是跨域访问，所以用cmd添加一个路由
route add 172.16.0.0 mask 255.255.0.0 192.168.11.38 -p
- 网段地址
- 子网掩码
- 虚拟机地址，docker容器部署在该虚拟机上
效果展示
