四、kafka 集群部署
Kafka 依赖Zookeeper环境运行，所以需要先搭建zk集群
1、Zookeeper集群部署
# 下载安装包 wget https://dlcdn.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0-bin.tar.gz --no-check-certificate # 上传到其他节点 scp apache-zookeeper-3.7.0-bin.tar.gz 10.10.8.165:/root scp apache-zookeeper-3.7.0-bin.tar.gz 10.10.8.166:/root # 安装 tar xf apache-zookeeper-3.7.0-bin.tar.gz mv apache-zookeeper-3.7.0-bin /usr/local/zookeeper # 配置 vim /usr/local/zookeeper/conf/zoo.cfg tickTime=2000 initLimit=10 syncLimit=5 dataDir=/usr/local/zookeeper/data clientPort=2181 autopurge.snapRetainCount=3 autopurge.purgeInterval=1 server.1=10.10.8.164:2888:3888 server.2=10.10.8.165:2888:3888 server.3=10.10.8.166:2888:3888 # 创建数据目录并指定myid mkdir /usr/local/zookeeper/data echo 1 >/usr/local/zookeeper/data/myid # 每个机器的id不同 可以 echo为 1 2 3 # 启动zk /usr/local/zookeeper/bin/zkServer.sh start # 查看状态 /usr/local/zookeeper/bin/zkServer.sh status //一个节点显示leader，其他节点显示follower则成功 Mode: leader Mode: follower # 验证 /usr/local/zookeeper/bin/zkCli.sh # 连接zk create /elk # 创建索引 ls / # 查看索引
2、kafka 集群部署
# 下载安装包 wget https://downloads.apache.org/kafka/3.1.0/kafka_2.12-3.1.0.tgz # 上传到其他节点 scp kafka_2.12-3.1.0.tgz 10.10.8.165:/root scp kafka_2.12-3.1.0.tgz 10.10.8.166:/root # 安装 tar xf kafka_2.12-3.1.0.tgz mv kafka_2.12-3.1.0 /usr/local/kafka # 配置 vim /usr/local/kafka/bin/kafka-server-start.sh export KAFKA_HEAP_OPTS="-Xmx256M -Xms256M" # 修改最大内存占用 vim /usr/local/kafka/config/server.properties # kafka 本机监听地址 listeners=PLAINTEXT://10.10.8.164:9092 num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 log.dirs=/tmp/kafka-logs # 日志保留时间 num.partitions=1 num.recovery.threads.per.data.dir=1 offsets.topic.replication.factor=1 transaction.state.log.replication.factor=1 transaction.state.log.min.isr=1 log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 # zk 连接信息 zookeeper.connect=10.10.8.164:2181,10.10.8.165:2181,10.10.8.166:2181 zookeeper.connection.timeout.ms=60000 group.initial.rebalance.delay.ms=0 # 使用systemctl管理 vim /usr/lib/systemd/system/kafka.service [Unit] Description=kafka After=network.target [Service] Type=simple ExecStart=/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties User=root [Install] WantedBy=multi-user.target # Kafka启动 systemctl enable kafka systemctl start kafka # 验证 netstat -lntp|grep 9092 ps -ef|grep kafka # 创建topic，创建成功说明kafka集群搭建成功 /usr/local/kafka/bin/kafka-topics.sh --bootstrap-server 10.10.8.164:9092 --create --replication-factor 2 --partitions 2 --topic elktest # 验证 /usr/local/kafka/bin/kafka-topics.sh --describe --bootstrap-server 10.10.8.165:9092 --topic elktest
