集群规划:

三台服务器各部署一个kafka集群节点、一个zookeeper集群节点。

服务器

zookeeper角色

kafka角色

ip

node1

master

broker:0

192.168.146.199

node2

follower

broker:1

192.168.146.200

node3

follower

broker:2

192.168.146.201



PS:命令行开头为#表示在所有主机上操作,命令行开后为具体主机名则表示在该主机上操作。


1、三台服务器基础设置

设置主机名:

# hostnamectl --static set-hostname node1/node2/node3
# vi /etc/hosts
192.168.146.199 node1
192.168.146.200 node2
192.168.146.201 node3


时区调整,时间校准:

# date -R
# timedatectl set-timezone Asia/Shanghai
# yum -y install ntp
# ntpdate ntp1.aliyun.com


安装wget:

$ yum install -y wget


关闭selinux:

# vi /etc/sysconfig/selinux
SELINUX=disabled


重启服务器并验证:

# getenforce
Disabled


设置防火墙

开放zookeeper相关端口2181,2888,3888

# firewall-cmd --zone=public --add-port=2181/tcp --permanent
success
# firewall-cmd --zone=public --add-port=2888/tcp --permanent
success
# firewall-cmd --zone=public --add-port=3888/tcp --permanent
success


开放kafka相关端口9092

# firewall-cmd --zone=public --add-port=9092/tcp --permanent
success


reload 防火墙配置使其生效

# firewall-cmd --reload
success



2、基础环境安装

安装JDK环境

# yum install java-1.8.0-openjdk
# java -version
openjdk version "1.8.0_262"
OpenJDK Runtime Environment (build 1.8.0_262-b10)
OpenJDK 64-Bit Server VM (build 25.262-b10, mixed mode)


创建目录与文件

# mkdir -p /var/zookeeper/log
# touch /var/zookeeper/myid
# mkdir -p /var/log/kafka



3、安装zookeeper

# cd /opt
# wget http://archive.apache.org/dist/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz
# tar -zxvf apache-zookeeper-3.6.1-bin.tar.gz
# mv apache-zookeeper-3.6.1-bin zookeeper-3.6.1


复制模板

# mv /opt/zookeeper-3.6.1/conf/zoo_sample.cfg /opt/zookeeper-3.6.1/conf/zoo.cfg


修改配置文件

# vi /opt/zookeeper-3.6.1/conf/zoo.cfg
dataDir=/var/zookeeper
server.1=node1:2888:3888
server.2=node2:2888:3888
server.3=node3:2888:3888

[root@node1 opt]# echo 1 > /var/zookeeper/myid
[root@node2 opt]# echo 2 > /var/zookeeper/myid
[root@node3 opt]# echo 3 > /var/zookeeper/myid



4、安装kafka

# cd /opt
# wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.5.0/kafka_2.13-2.5.0.tgz
# tar -zxvf kafka_2.13-2.5.0.tgz


修改配置文件

[root@node1 opt]# vi /opt/kafka_2.13-2.5.0/config/server.properties
broker.id=0
log.dirs=/var/log/kafka
zookeeper.connect=node1:2181,node2:2181,node3:2181

[root@node2 opt]# vi /opt/kafka_2.13-2.5.0/config/server.properties
broker.id=1
log.dirs=/var/log/kafka
zookeeper.connect=node1:2181,node2:2181,node3:2181

[root@node3 opt]# vi /opt/kafka_2.13-2.5.0/config/server.properties
broker.id=2
log.dirs=/var/log/kafka
zookeeper.connect=node1:2181,node2:2181,node3:2181



5、启动和关闭zookeeper

# /opt/zookeeper-3.6.1/bin/zkServer.sh start
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED

[root@node1 opt]# /opt/zookeeper-3.6.1/bin/zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower

[root@node2 opt]# /opt/zookeeper-3.6.1/bin/zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower

[root@node3 opt]# /opt/zookeeper-3.6.1/bin/zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: leader


关闭方法

# /opt/zookeeper-3.6.1/bin/zkServer.sh stop


查看日志

# tail -f /opt/zookeeper-3.6.1/logs/zookeeper-root-server-node{myid}.out



6、启动和关闭kafka

# /opt/kafka_2.13-2.5.0/bin/kafka-server-start.sh /opt/kafka_2.13-2.5.0/config/server.properties 1>/dev/null 2>&1 &
# tail -f /opt/kafka_2.13-2.5.0/logs/server.log
关闭方法
# /opt/kafka_2.13-2.5.0/bin/kafka-server-stop.sh



7、测试

# cd /opt/kafka_2.13-2.5.0


创建topic

[root@node1 kafka_2.13-2.5.0]# bin/kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181 --replication-factor 3 --partitions 3 --topic gxm-test
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
Created topic gxm-test.


查看topic

[root@node2 kafka_2.13-2.5.0]# bin/kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181 --topic gxm-test
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
Topic: gxm-test    PartitionCount: 3    ReplicationFactor: 3    Configs:
    Topic: gxm-test    Partition: 0    Leader: 1    Replicas: 1,2,0    Isr: 1,2,0
    Topic: gxm-test    Partition: 1    Leader: 2    Replicas: 2,0,1    Isr: 2,0,1
    Topic: gxm-test    Partition: 2    Leader: 0    Replicas: 0,1,2    Isr: 0,1,2


列出topic

[root@node3 kafka_2.13-2.5.0]# bin/kafka-topics.sh --list --zookeeper node1:2181,node2:2181,node3:2181
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
gxm-test


在node1创建 producer测试

[root@node1 kafka_2.13-2.5.0]# bin/kafka-console-producer.sh --broker-list node1:9092 -topic gxm-test


在node2创建 consumer测试

[root@node2 kafka_2.13-2.5.0]# bin/kafka-console-consumer.sh --bootstrap-server node2:9092 -topic gxm-test --from-beginning


在node3上创建 consumer测试

[root@node3 kafka_2.13-2.5.0]# bin/kafka-console-consumer.sh --bootstrap-server node2:9092 -topic gxm-test --from-beginning


删除topic

[root@node1 kafka_2.13-2.5.0]#  bin/kafka-topics.sh --delete --zookeeper node1:2181,node2:2181,node3:2181 --topic gxm-test