centos7搭建Kafka集群并实现自启动

精选原创

喵来个鱼 2021-04-25 11:34:49 博主文章分类：系统 ©著作权

文章标签 kafka zookeeper centos7 linux jdk 文章分类 运维

©著作权归作者所有：来自51CTO博客作者喵来个鱼的原创作品，请联系作者获取转载授权，否则将追究法律责任

搭建Kafka集群

zookeep集群已经搭建了，这里直接跳过,确保已经安装了jdk

Kafka集群节点>=2时便可对外提供高可用服务

下载分发

mkdir -p /data/ansible/kafka
cd /data/ansible/kafka
wget https://www.apache.org/dyn/closer.cgi?path=/kafka/2.6.1/kafka_2.13-2.6.1.tgz
tar -zxf kafka_2.13-2.6.1.tgz
mv kafka_2.13-2.6.1 kafka
ansible kafka -m file -a "path=/usr/local/kafka state=absent"
ansible kafka -m copy -a "src=/data/ansible/kafka/kafka dest=/usr/local/"
ansible kafka -m shell -a "ls -l /usr/local/kafka"

配置

目录/usr/local/kafka

kafka节点都执行

cat >> /etc/hosts <<EOF
10.11.19.8 zk1
10.11.19.9 zk2
10.11.19.10 zk3
EOF

cat <<EOF> /etc/profile.d/kafka_home.sh
export PATH=/usr/local/kafka/bin:$PATH
EOF
source /etc/profile.d/kafka_home.sh

10.11.19.8

cat > /usr/local/kafka/config/server.properties <<EOF
broker.id=0
listeners=PLAINTEXT://zk1:9092
advertised.listeners=PLAINTEXT://zk1:9092
num.network.threads=3
.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
=300000
zookeeper.connect=zk1:2181,zk2:2181,zk3:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
delete.topic.enable=true
EOF

10.11.19.9

cat > /usr/local/kafka/config/server.properties <<EOF
broker.id=1
listeners=PLAINTEXT://zk2:9092
advertised.listeners=PLAINTEXT://zk2:9092
num.network.threads=3
.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
=300000
zookeeper.connect=zk1:2181,zk2:2181,zk3:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
delete.topic.enable=true
EOF

10.11.19.10

cat > /usr/local/kafka/config/server.properties <<EOF
broker.id=2
listeners=PLAINTEXT://zk3:9092
advertised.listeners=PLAINTEXT://zk3:9092
num.network.threads=3
.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
=300000
zookeeper.connect=zk1:2181,zk2:2181,zk3:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
delete.topic.enable=true
EOF

启动

三台都要执行

chmod -R +x /usr/local/kafka/bin/
 /usr/local/kafka/config/server.properties &

 -daemon /usr/local/kafka/config/server.properties

ansible kafka -m shell -a "/usr/local/kafka/bin/ -daemon /usr/local/kafka/config/server.properties"

停止

ansible kafka -m shell -a "/usr/local/kafka/bin/"

Kafka应用实践

Topic管理

创建`topic`

kafka-topics.sh --create \
--zookeeper zk1:2181,zk2:2181,zk3:2181 \
--replication-factor 3 \
--partitions 3 \
--topic topic_1

WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
Created topic topic_1.

--replication-factor 3：副本集数量，不能大于 broker 节点数量，多了也没用，1个节点放>=2个副本挂了都完蛋。--partitions 3`：分区数

查看`topic`列表

kafka-topics.sh \
--zookeeper zk1:2181,zk2:2181,zk3:2181 --list

topic_1

查看Topic详情

可以描述Topic分区数/副本数/副本Leader/副本ISR等信息：

kafka-topics.sh \
--zookeeper zk1:2181,zk2:2181,zk3:2181 \
--describe --topic topic_1
Topic: topic_1	PartitionCount: 3	ReplicationFactor: 3	Configs:
	Topic: topic_1	Partition: 0	Leader: 1	Replicas: 1,0,2	Isr: 1,0,2
	Topic: topic_1	Partition: 1	Leader: 2	Replicas: 2,1,0	Isr: 2,1,0
	Topic: topic_1	Partition: 2	Leader: 0	Replicas: 0,2,1	Isr: 0,2,1

删除Topic

注意，只是删除Topic在zk的元数据，日志数据仍需手动删除。

kafka-topics.sh \
--zookeeper zk1:2181,zk2:2181,zk3:2181 \
--delete --topic topic_1

#Topic topic_2 is marked for deletion.
#Note: This will have no impact if delete.topic.enable is not set to true.

#再查看topic列表
kafka-topics.sh \
--zookeeper zk1:2181,zk2:2181,zk3:2181 --list


Topic topic_1 is marked for deletion.
Note: This will have no impact if delete.topic.enable is not set to true.

生产者

 \
--broker-list zk1:9092,zk2:9092,zk3:9092 \
--topic topic_1
# 进入 cli 输入消息回车发送
# hello kafka [enter]
# send message [enter]

#  \
--broker-list zk1:9092,zk2:9092,zk3:9092 \
--topic topic_2
>hello kafka
>u
>f
>kk
>kafka
>

消费者

新模式，offset存储在borker --new-consumer Use new consumer. This is the default. --bootstrap-server <server to connectto> REQUIRED (unless old consumer is used): The server to connect to. 老消费模式，offset存储在zk --zookeeper <urls> REQUIRED (only when using old consumer): The connection string for the zookeeper connection in the form host:port. Multiple URLS can be given to allow fail-over.

创建消费者

kafka-console-consumer.sh \
--new-consumer \
--bootstrap-server zk1:9092,zk2:9092,zk3:9092 \
--from-beginning \
--topic topic_1

kafka-console-consumer.sh --zookeeper zk1:2181,zk2:2181,zk3:2181 --topic topic_2 --from-beginning
 hello word

2.60版的消费命令，上面两个命令无效

kafka-console-consumer.sh --bootstrap-server zk1:9092,zk2:9092,zk3:9092 --topic topic_2 --group test_grp --consumer-property enable.auto.commit=true --from-beginning

  kafka-console-consumer.sh --bootstrap-server zk1:9092,zk2:9092,zk3:9092 --topic topic_1 --group test_grp --consumer-property enable.auto.commit=true --from-beginning

kafka-console-consumer.sh --bootstrap-server zk1:9092,zk2:9092,zk3:9092 --topic topic_2 --group test_grp --consumer-property enable.auto.commit=true --from-beginning
hello kafka
u
f
kk
kafka

可以尝试创建多个不同消费组的消费者（这里的sh脚本创建的都是不同消费组的），订阅同一个topic来实现发布订阅模式。

查看消费组/消费者

kafka-consumer-groups.sh \
--bootstrap-server zk1:9092,zk2:9092,zk3:9092 \
--list
#这里有两个消费组的消费者

查看消费详情

可以查看到消费的订阅的 topic，负责的 partition，消费进度 offset, 积压的消息LAG。

kafka-consumer-groups.sh \
--bootstrap-server zk1:9092,zk2:9092,zk3:9092 \
--group test_grp \
--describe

GROUP           TOPIC           PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG             CONSUMER-ID                                              HOST            CLIENT-ID
test_grp        topic_2         0          3               3               0               consumer-test_grp-1-7012d8ed-4d1a-45c7-9f2f-090606bba3aa /10.11.19.6     consumer-test_grp-1
test_grp        topic_2         1          1               1               0               consumer-test_grp-1-7012d8ed-4d1a-45c7-9f2f-090606bba3aa /10.11.19.6     consumer-test_grp-1
test_grp        topic_2         2          1               1               0               consumer-test_grp-1-7012d8ed-4d1a-45c7-9f2f-090606bba3aa /10.11.19.6     consumer-test_grp-1

自启动

注意标点符号，引号，不要是中文

cat > /usr/lib/systemd/system/kafka.service <<EOF
[Unit]
Description=Apache Kafka server (broker)
After=network.target zookeeper.service

[Service]
Type=simple
Environment="PATH=/usr/local/kafka/bin:$PATH"
User=root
Group=root
ExecStart=/usr/local/kafka/bin/ /usr/local/kafka/config/server.properties
ExecStop=/usr/local/kafka/bin/
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kafka
systemctl start kafka
systemctl status kafka