Zookeeper 3.5.8 & Kafka 2.4.0 安装与调优

$ cd /opt
$ tar zxvf apache-zookeeper-3.5.8-bin.tar.gz
$ cd /opt/apache-zookeeper-3.5.8-bin/conf

# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=5
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=2
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/tmp/zookeeper/data
dataLogDir=/tmp/zookeeper/datalog
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
# Setting this to 0 entirely removes the limit on concurrent connections.
maxClientCnxns=120
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
autopurge.purgeInterval=24
## Metrics Providers
#
# https://prometheus.io Metrics Exporter
#metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider
#metricsProvider.httpPort=7000
#metricsProvider.exportJvmInfo=true
# Disable AdminServer
admin.enableServer=false
#server.1=zoo1:2888:3888
#server.2=zoo2:2888:3888
#server.3=zoo3:2888:3888

#-------------------------------------------------------------------
# zookeeper-env.sh
#--------------------------------------------------------------------
# Configure Log
ZOO_LOG_DIR="/opt/apache-zookeeper-3.5.8-bin/logs"
ZOO_LOG4J_PROP="INFO,DAILYROLLINGFILE"
# Maximum Heap Size(m)
ZK_SERVER_HEAP=2048
# Configure JVM GC log
ZOO_GC_LOG_DIR="/opt/apache-zookeeper-3.5.8-bin/logs"
SERVER_JVMFLAGS="-Xms2048m -Xmx2048m -XX:+UseG1GC -XX:MaxGCPauseMillis=50 -XX:+AlwaysPreTouch -Djute.maxbuffer=4194304"
# Enable GC log
#SERVER_JVMFLAGS="-Xms2048m -Xmx2048m -XX:+UseG1GC -XX:MaxGCPauseMillis=50 -XX:+AlwaysPreTouch -Djute.maxbuffer=4194304 -verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M -Xloggc:$ZOO_GC_LOG_DIR/zookeeper.gc"
# Configure JMX (enabled by default for local monitoring by PID)
# define JMXPORT allows remote monitoring
JMXDISABLE=true
#JMXLOCALONLY=true
#JMXPORT=10900
# Backup log GC at startup
if [ "x$1" = "xstart" ]; then
    for f in $ZOO_GC_LOG_DIR/zookeeper.gc*; do
        ## Check if the glob gets expanded to existing files.
        ## If not, f here will be exactly the pattern above
        ## and the exists test will evaluate to false.
        if [ -e "$f" ] ; then
            echo "GC log files found - backing up"
            d=$PWD && cd $ZOO_GC_LOG_DIR && tar czf zookeeper.gc.$(date +%Y%m%d-%H%M%S).tgz zookeeper.gc* && cd $d
        else
            echo "No GC log files found"
        fi
        ## This is all we needed to know, so we can break after the first iteration
        break
    done
fi

log4j.appender.DAILYROLLINGFILE=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DAILYROLLINGFILE.DatePattern='.'yyyy-MM-dd
log4j.appender.DAILYROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file}
log4j.appender.DAILYROLLINGFILE.layout=org.apache.log4j.PatternLayout
log4j.appender.DAILYROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n

$ cd /opt
$ tar zxvf kafka_2.12-2.4.0.tgz
$ cd /opt/kafka_2.12-2.4.0

$ cd /opt/kafka_2.12-2.4.0/bin 

#!/bin/bash
#-------------------------------------------------------------------
# kafka-server-start-tuned.sh
#--------------------------------------------------------------------
base_dir=$(dirname $0)
export LOG_DIR="/tmp/kafka/logs"
export KAFKA_OPTS="-Djute.maxbuffer=4194304 -Dlog4j.rootLogger=INFO,kafkaAppender"
export KAFKA_HEAP_OPTS="-Xmx6g -Xms6g -XX:+AlwaysPreTouch"
export KAFKA_JVM_PERFORMANCE_OPTS="-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1HeapRegionSize=16M -Djava.awt.headless=true"
export KAFKA_GC_LOG_OPTS=" "
export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=59999"
#export KAFKA_LOG4J_OPTS
exec $base_dir/kafka-server-start.sh -daemon $base_dir/../config/server.properties
#exec $base_dir/kafka-server-start.sh -daemon -loggc $base_dir/../config/server.properties

$ chmod +x kafka-server-start-tuned.sh 

$ cd /opt/kafka_2.12-2.4.0/config