java kafka消息队列 java消费kafka消息

转载

数据小香 2023-05-26 15:40:01

文章标签 kafka java apache 文章分类 Java 后端开发

@[TOC](java实现Kafka消息生产与消费功能实现(This is very easy example))
1、通过eclipse或者IntelliJIDEA创建一个Maven工程。
2、在pom.xml中添加kafka的依赖，如下：

<project xmlns="http://maven.apache.org/POM/4.0.0"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>com.wongoing</groupId>
	<artifactId>kafka</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<name>kafkademo01</name>

	<properties>
		<java.version>1.8</java.version>
		<kafka.version>2.6.0</kafka.version>
	</properties>
	
	<dependencies>
		<dependency>
			<groupId>jdk.tools</groupId>
			<artifactId>jdk.tools</artifactId>
			<version>1.8</version>
			<scope>system</scope>
			<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
		</dependency>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>4.13.1</version>
			<scope>test</scope>
		</dependency>
		<dependency>
		    <groupId>org.apache.kafka</groupId>
		    <artifactId>kafka-clients</artifactId>
		    <version>${kafka.version}</version>
		</dependency>
		<dependency>
		    <groupId>org.apache.kafka</groupId>
		    <artifactId>kafka-streams</artifactId>
		    <version>${kafka.version}</version>
		</dependency>
		
	</dependencies>
</project>

3、创建一个单元测试类KafkaProducerTest.java，实现消息生产测试功能，代码如下：

package com.wongoing.kafka.test;

import java.util.Properties;
import java.util.concurrent.ExecutionException;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

/**
 * 功能说明：消息生产者测试类
 * 修改说明：
 * @author zheng
 * @date 2020-12-10 10:17:57
 * @version 0.1
 */
public class KafkaProducerTest {
	private final static String TOPIC = "qingdao";			//此处指定已经创建好的一个topic
	private static KafkaProducer<String, String> producer;		//定义消息生产者对象
	
	/**
	 * 功能说明：初始化方法
	 * 修改说明：
	 * @author zheng
	 * @date 2020-12-10 10:20:35
	 */
	@BeforeClass
	public static void init() {
		Properties props = new Properties();
		props.put("bootstrap.servers", "172.16.1.153:9092");		//kafka集群服务器IP:端口号列表
		props.put("acks", "all");	//所有broker都响应了才认为消息提交成功，即"committed"
		props.put("retries", 0);	//retries = MAX 无限重试，直到你意识到出现了问题
		props.put("batch.size", 16384);		//producer将试图批处理消息记录，以减少请求的次数，默认的批量处理消息字节数 //batch.size当批量的数据大小达到设定值后，就会立即发送，不顾下面的linger.ms
		props.put("linger.ms", 1);		//延迟1ms发送，这项设置将通过增加小的延迟来完成--即，不是立即发送一条记录，producer将会等待给定的延迟时间以允许其他消息记录发送，这些消息记录可以批量处理
		props.put("buffer.memory", 33554432);		//producer可以用来缓存数据的内存大小
		props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
		props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
		
		producer = new KafkaProducer<String, String>(props);		//实例化消息生产者对象
	}
	
	/**
	 * 功能说明：生产消息测试方法
	 * 修改说明：
	 * @author zheng
	 * @throws ExecutionException 
	 * @throws InterruptedException 
	 * @date 2020-12-10 10:33:56
	 */
	@Test
	public void testProduceMessage() throws InterruptedException, ExecutionException {
		int messageNo = 1;
		int count = 50;
		while(messageNo < count) {
			String key = String.valueOf(messageNo);
			String data = String.format("This is Kafka Producer message : %s", key);
			
			producer.send(new ProducerRecord<String, String>(TOPIC, data)).get();
			System.out.println("send message complete " + messageNo);
			
			messageNo++;
		}
	}
	
	/**
	 * 功能说明：销毁方法
	 * 修改说明：
	 * @author zheng
	 * @date 2020-12-10 10:38:23
	 */
	@AfterClass
	public static void destroy() {
		System.out.println("正在关闭producer...");
		producer.close();
		System.out.println("producer已关闭...");
	}
}

3、在创建一个单元测试类KafkaConsumerTest.java，实现消息消费测试功能，代码如下：

package com.wongoing.kafka.test;

import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

/**
 * 功能说明：消息消费者测试类
 * 修改说明：
 * @author zheng
 * @date 2020-12-10 10:18:12
 * @version 0.1
 */
public class KafkaConsumerTest {
	private final static String TOPIC = "qingdao";			//此处指定已经创建好的一个topic
	private static KafkaConsumer<String, String> consumer;	//kafka消息消费对象
	
	/**
	 * 功能说明：初始化方法
	 * 修改说明：
	 * @author zheng
	 * @date 2020-12-10 10:42:09
	 */
	@BeforeClass
	public static void init() {
		Properties props = new Properties();
		props.put("bootstrap.servers", "172.16.1.153:9092,172.16.1.152:9092,172.16.1.154:9092");		//kafka集群服务器IP:端口号列表
		props.put("group.id", "jd-group");
		props.put("enable.auto.commit", true);			//是否开启自动提交
		props.put("auto.commit.interval.ms", "1000");	//自动提交间隔时间
		props.put("auto.offset.reset", "latest");		
		props.put("session.timeout.ms", "30000");		//会话超时时间
		props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
		props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
		
		consumer = new KafkaConsumer<String, String>(props);
		consumer.subscribe(Arrays.asList(TOPIC));	//消费者订阅一个或多个topic
	}
	
	/**
	 * 功能说明：消息消费测试方法
	 * 修改说明：
	 * @author zheng
	 * @date 2020-12-10 10:49:48
	 */
	@Test
	public void testConsumeMessage() {
		while(true ) {
			ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
			for(ConsumerRecord<String, String> record : records) {
				System.out.println("-----------------------------------");
				System.out.printf("offset = %d, value = %s", record.offset(), record.value());
				System.out.println();
			}
		}
	}
	
	/**
	 * 功能说明：销毁方法
	 * 修改说明：
	 * @author zheng
	 * @date 2020-12-10 10:48:06
	 */
	@AfterClass
	public static void destroy() {
		consumer.close();
	}
}

4、运行方式，先启动消息消费的测试方法，同时新建一个控制台，再执行消息生产的测试方法。
输出如下，在消息生产的控制台输出如下：

log4j:WARN No appenders could be found for logger (org.apache.kafka.clients.producer.ProducerConfig).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
send message complete 1
send message complete 2
send message complete 3
send message complete 4
send message complete 5
send message complete 6
send message complete 7
send message complete 8
send message complete 9
send message complete 10
send message complete 11
send message complete 12
send message complete 13
send message complete 14
send message complete 15
send message complete 16
send message complete 17
send message complete 18
send message complete 19
send message complete 20
send message complete 21
send message complete 22
send message complete 23
send message complete 24
send message complete 25
send message complete 26
send message complete 27
send message complete 28
send message complete 29
send message complete 30
send message complete 31
send message complete 32
send message complete 33
send message complete 34
send message complete 35
send message complete 36
send message complete 37
send message complete 38
send message complete 39
send message complete 40
send message complete 41
send message complete 42
send message complete 43
send message complete 44
send message complete 45
send message complete 46
send message complete 47
send message complete 48
send message complete 49
正在关闭producer...
producer已关闭...

在消息消费的控制台输出如下：

log4j:WARN No appenders could be found for logger (org.apache.kafka.clients.consumer.ConsumerConfig).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
-----------------------------------
offset = 7, value = This is Kafka Producer message : 1
-----------------------------------
offset = 9, value = This is Kafka Producer message : 2
-----------------------------------
offset = 5, value = This is Kafka Producer message : 3
-----------------------------------
offset = 8, value = This is Kafka Producer message : 4
-----------------------------------
offset = 10, value = This is Kafka Producer message : 1
-----------------------------------
offset = 9, value = This is Kafka Producer message : 2
-----------------------------------
offset = 6, value = This is Kafka Producer message : 3
-----------------------------------
offset = 11, value = This is Kafka Producer message : 4
-----------------------------------
offset = 10, value = This is Kafka Producer message : 5
-----------------------------------
offset = 12, value = This is Kafka Producer message : 6
-----------------------------------
offset = 7, value = This is Kafka Producer message : 7
-----------------------------------
offset = 13, value = This is Kafka Producer message : 8
-----------------------------------
offset = 11, value = This is Kafka Producer message : 9
-----------------------------------
offset = 14, value = This is Kafka Producer message : 10
-----------------------------------
offset = 8, value = This is Kafka Producer message : 11
-----------------------------------
offset = 12, value = This is Kafka Producer message : 12
-----------------------------------
offset = 9, value = This is Kafka Producer message : 13
-----------------------------------
offset = 15, value = This is Kafka Producer message : 14
-----------------------------------
offset = 10, value = This is Kafka Producer message : 15
-----------------------------------
offset = 16, value = This is Kafka Producer message : 16
-----------------------------------
offset = 13, value = This is Kafka Producer message : 17
-----------------------------------
offset = 11, value = This is Kafka Producer message : 18
-----------------------------------
offset = 14, value = This is Kafka Producer message : 19
-----------------------------------
offset = 12, value = This is Kafka Producer message : 20
-----------------------------------
offset = 17, value = This is Kafka Producer message : 21
-----------------------------------
offset = 13, value = This is Kafka Producer message : 22
-----------------------------------
offset = 18, value = This is Kafka Producer message : 23
-----------------------------------
offset = 15, value = This is Kafka Producer message : 24
-----------------------------------
offset = 19, value = This is Kafka Producer message : 25
-----------------------------------
offset = 14, value = This is Kafka Producer message : 26
-----------------------------------
offset = 20, value = This is Kafka Producer message : 27
-----------------------------------
offset = 16, value = This is Kafka Producer message : 28
-----------------------------------
offset = 15, value = This is Kafka Producer message : 29
-----------------------------------
offset = 17, value = This is Kafka Producer message : 30
-----------------------------------
offset = 16, value = This is Kafka Producer message : 31
-----------------------------------
offset = 21, value = This is Kafka Producer message : 32
-----------------------------------
offset = 17, value = This is Kafka Producer message : 33
-----------------------------------
offset = 22, value = This is Kafka Producer message : 34
-----------------------------------
offset = 18, value = This is Kafka Producer message : 35
-----------------------------------
offset = 23, value = This is Kafka Producer message : 36
-----------------------------------
offset = 19, value = This is Kafka Producer message : 37
-----------------------------------
offset = 24, value = This is Kafka Producer message : 38
-----------------------------------
offset = 20, value = This is Kafka Producer message : 39
-----------------------------------
offset = 25, value = This is Kafka Producer message : 40
-----------------------------------
offset = 21, value = This is Kafka Producer message : 41
-----------------------------------
offset = 26, value = This is Kafka Producer message : 42
-----------------------------------
offset = 22, value = This is Kafka Producer message : 43
-----------------------------------
offset = 18, value = This is Kafka Producer message : 44
-----------------------------------
offset = 27, value = This is Kafka Producer message : 45
-----------------------------------
offset = 23, value = This is Kafka Producer message : 46
-----------------------------------
offset = 19, value = This is Kafka Producer message : 47
-----------------------------------
offset = 28, value = This is Kafka Producer message : 48
-----------------------------------
offset = 20, value = This is Kafka Producer message : 49

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。