kafka 集群搭建

原创

潇凝子潇 2021-09-23 09:44:03 ©著作权

文章标签 集群 kafka 文章分类 大数据

©著作权归作者所有：来自51CTO博客作者潇凝子潇的原创作品，请联系作者获取转载授权，否则将追究法律责任

kafka 集群搭建

1. 安装scala 环境

root@node148:/data# tar -zxvf scala-2.11.4.tgz 
scala-2.11.4/
scala-2.11.4/bin/
scala-2.11.4/bin/scala.bat
scala-2.11.4/bin/scalap.bat
scala-2.11.4/bin/scala
scala-2.11.4/bin/fsc.bat
scala-2.11.4/bin/scaladoc.bat
scala-2.11.4/bin/fsc
scala-2.11.4/bin/scaladoc
scala-2.11.4/bin/scalac.bat
scala-2.11.4/bin/scalac
scala-2.11.4/bin/scalap
scala-2.11.4/man/
scala-2.11.4/man/man1/
scala-2.11.4/man/man1/scalap.1
scala-2.11.4/man/man1/fsc.1
scala-2.11.4/man/man1/scalac.1
scala-2.11.4/man/man1/scaladoc.1
scala-2.11.4/man/man1/scala.1
scala-2.11.4/lib/
scala-2.11.4/lib/scala-continuations-plugin_2.11.4-1.0.2.jar
scala-2.11.4/lib/scala-reflect.jar
scala-2.11.4/lib/scala-library.jar
scala-2.11.4/lib/scala-swing_2.11-1.0.1.jar
scala-2.11.4/lib/config-1.2.1.jar
scala-2.11.4/lib/jline-2.12.jar
scala-2.11.4/lib/scala-xml_2.11-1.0.2.jar
scala-2.11.4/lib/scalap-2.11.4.jar
scala-2.11.4/lib/akka-actor_2.11-2.3.4.jar
scala-2.11.4/lib/scala-compiler.jar
scala-2.11.4/lib/scala-actors-migration_2.11-1.1.0.jar
scala-2.11.4/lib/scala-actors-2.11.0.jar
scala-2.11.4/lib/scala-parser-combinators_2.11-1.0.2.jar
scala-2.11.4/lib/scala-continuations-library_2.11-1.0.2.jar
scala-2.11.4/doc/
scala-2.11.4/doc/licenses/
scala-2.11.4/doc/licenses/bsd_asm.txt
scala-2.11.4/doc/licenses/mit_tools.tooltip.txt
scala-2.11.4/doc/licenses/bsd_jline.txt
scala-2.11.4/doc/licenses/mit_jquery-ui.txt
scala-2.11.4/doc/licenses/mit_sizzle.txt
scala-2.11.4/doc/licenses/mit_jquery-layout.txt
scala-2.11.4/doc/licenses/apache_jansi.txt
scala-2.11.4/doc/licenses/mit_jquery.txt
scala-2.11.4/doc/tools/
scala-2.11.4/doc/tools/images/
scala-2.11.4/doc/tools/images/scala_logo.png
scala-2.11.4/doc/tools/images/external.gif
scala-2.11.4/doc/tools/index.html
scala-2.11.4/doc/tools/scala.html
scala-2.11.4/doc/tools/fsc.html
scala-2.11.4/doc/tools/scaladoc.html
scala-2.11.4/doc/tools/scalac.html
scala-2.11.4/doc/tools/scalap.html
scala-2.11.4/doc/tools/css/
scala-2.11.4/doc/tools/css/style.css
scala-2.11.4/doc/License.rtf
scala-2.11.4/doc/LICENSE.md
scala-2.11.4/doc/README
root@node148:/data# vim ~/.bashrc 
JAVA_HOME=/usr/lib/jvm/java-8-openjdk-mips64el
zookpeer_home=/data/zookeeper-3.4.14
SCALA_HOME=/data/scala-2.11.4
PATH=$JAVA_HOME/bin:$zookpeer/bin:$SCALA_HOME/bin:$PATH
export JAVA_HOME zookpeer_home SCALA_HOME PATH
ulimit -n 65535
ulimit -u 65535
root@node148:/data# source ~/.bashrc 
root@node148:/data# scala -version
Scala code runner version 2.11.4 -- Copyright 2002-2013, LAMP/EPFL

1.配置kafka

# kafka安装
root@node148:/data# cd kafka_2.9.2-0.8.1/config/
root@node148:/data/kafka_2.9.2-0.8.1/config# vim server.properties
#依次增长的整数，0、1、2，集群中Broker的唯一id
broker.id=0
#zk集群地址
zookeeper.connect=10.173.203.14:2181,10.173.203.148:2181,10.173.203.158:2181

#
#安装slf4j
#将课程提供的slf4j-1.7.6.zip上传到/usr/local目录下
#unzip slf4j-1.7.6.zip
#把slf4j中的slf4j-nop-1.7.6.jar复制到kafka的libs目录下面

#解决kafka Unrecognized VM option 'UseCompressedOops'问题

#vi /usr/local/kafka/bin/kafka-run-class.sh 

#if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then
#  KAFKA_JVM_PERFORMANCE_OPTS="-server  -XX:+UseCompressedOops -XX:+UseParNewGC -# #XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:+CMSScavengeBeforeRemark -# #XX:+DisableExplicitGC -Djava.awt.headless=true"
#fi

#去掉-XX:+UseCompressedOops即可
root@node148:/data/kafka_2.9.2-0.8.1/config# cd ..
root@node148:/data/kafka_2.9.2-0.8.1# cd bin
root@node148:/data/kafka_2.9.2-0.8.1/bin# ls
kafka-console-consumer.sh	     kafka-producer-perf-test.sh   kafka-server-start.sh	       kafka-topics.sh		  zookeeper-shell.sh
kafka-console-producer.sh	     kafka-reassign-partitions.sh  kafka-server-stop.sh		       windows
kafka-consumer-perf-test.sh	     kafka-replay-log-producer.sh  kafka-simple-consumer-perf-test.sh  zookeeper-server-start.sh
kafka-preferred-replica-election.sh  kafka-run-class.sh		   kafka-simple-consumer-shell.sh      zookeeper-server-stop.sh
root@node148:/data/kafka_2.9.2-0.8.1/bin# cd ..
# 启动
root@node148:/data/kafka_2.9.2-0.8.1# nohup bin/kafka-server-start.sh config/server.properties &
[1] 5961
# 查看进程
root@node148:/data/kafka_2.9.2-0.8.1# jps
6017 Jps
27220 NodeAgentMain
5653 org.eclipse.equinox.launcher_1.3.0.v20161111-0305.jar
5961 Kafka
5210 QuorumPeerMain
4154 NodeAgentMain

bin/kafka-topics.sh --zookeeper 10.173.203.14:2181,10.173.203.148:2181,10.173.203.158:2181 --topic test --replication-factor 1 --partitions 1 --create

bin/kafka-console-producer.sh --broker-list 10.173.203.14:9092,10.173.203.148:9092,10.173.203.158:9092 --topic test

bin/kafka-console-consumer.sh --zookeeper 10.173.203.14:2181,10.173.203.148:2181,10.173.203.158:2181 --topic test --from-beginning