最近的业务接触了一下消息插件kafka ,其他客户端负责发送,我方负责接收消费。功能很简单,但是!!!!我们要求把消息入库,而且数据量每天达到了千万级别,这就日了苟啊!

 

废话不多说,上代码!

 

由于消息需要入库,我使用的是手动提交消息,如果不入库,不要求准确定,使用自动提交就ok

 

消费者配置类

package com.asiainfo.hsop.server.kafka;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.AbstractMessageListenerContainer;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.listener.config.ContainerProperties;

import java.util.HashMap;
import java.util.Map;
/**
 * kafka消息队列消费者 配置类
 *
 * @author cyc
 * @version 1.0.0
 * @date 2021-07-02
 *
 * */
@Configuration
@EnableKafka
public class KafkaConsumerConfig {
    /*
    *
    *启动项目之前 需要修改 etc/hosts 文件,把 kafka实例名称和主机ip对应关系填好,否则会报错 ,
    *
    *报错信息如下:
    *           Discovered group coordinator......
    *           (Re-)joining group......
    *           Marking the coordinator......
    *尝试升级kafka-clients ,但是没有啥卵用.......最好的解决方案 就是修改系统hosts文件
    *
    *   主机ip 和 实例名称 对应关系
    *   11.251.10.111 sss-nn-01
    *   
    *
    *  */
    public Map<String, Object> consumerConfig(String consumerGroupId) {
        Map<String, Object> props = new HashMap<>();
        //kafka服务地址
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.253.6.194:6667,10.253.6.195:6667,10.253.6.196:6667,10.253.6.197:6667,10.253.6.198:6667,10.253.6.199:6667,10.253.6.200:6667,10.253.6.201:6667");
        //消费后是否自动提交 当前为false
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        //获取消息后提交偏移量的最大时间
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100");
        //超时时间,服务端没有收到心跳就会认为当前消费者失效
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
        //默认消费组
        props.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroupId);
        //earliest  从头开始消费、latest获取最新消息 、none 没理解
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
        //序列化
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        return props;
    }

    public ConsumerFactory<String, String> consumerFactory(String consumerGroupId) {
        return new DefaultKafkaConsumerFactory<>(consumerConfig(consumerGroupId));
    }

    /*有多个消费组监听,直接复制改方法,
    * 修改 @Bean(name = "kafkaListenerContainerFactory") name重新命名
    *
    * factory.setConsumerFactory(consumerFactory("etctu_tradeinfolist_test")); 改下消费组 名称
    *
    * 在调用的时候containerFactory="kafkaListenerContainerFactory" 改一下就ok
    *
    * */
    @Bean(name = "kafkaListenerContainerFactory")
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory("etctu_tradeinfolist_test"));
     //监听的线程数量,多个线程入库,数据库的id是自增的话,可能导致死锁,建议使用UUID factory.setConcurrency(1); /*消费者有两种消费模式,一种是kafka实例主动推送push模式,推送速度由kafka决定,很有可能导致消费端阻塞, *另一种就是 消费者主动拉取,poll模式 * */ factory.getContainerProperties().setPollTimeout(3000); //当使用手动提交时必须设置ackMode为MANUAL,否则会报错No Acknowledgment available as an argument, the listener container must have a MANUAL AckMode to populate the Acknowledgment. // factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL); // 下面两个条件哪个先满足,就会先使用那个 factory.getContainerProperties().setAckCount(10);//达到10条数据的时候提交一次 factory.getContainerProperties().setAckTime(10000);//10s提交一次 return factory; } @Bean(name = "kafkaListenerContainerFactory2") public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory2() { ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory("exit_toll_host_test")); factory.setConcurrency(1); factory.getContainerProperties().setPollTimeout(3000); factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL); factory.getContainerProperties().setAckCount(10); factory.getContainerProperties().setAckTime(10000); return factory; } @Bean(name = "kafkaListenerContainerFactory3") public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory3() { ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory("entryinfo_test")); factory.setConcurrency(1); factory.getContainerProperties().setPollTimeout(3000); //当使用手动提交时必须设置ackMode为MANUAL,否则会报错No Acknowledgment available as an argument, the listener container must have a MANUAL AckMode to populate the Acknowledgment. factory.getContainerProperties().setAckMode(AbstractMessageListenerContainer.AckMode.MANUAL); factory.getContainerProperties().setAckCount(10); factory.getContainerProperties().setAckTime(10000); return factory; } }

 

 

消费者类,里面包含了Hbase 引入的包,没啥用,删掉就行

package com.asiainfo.hsop.server.kafka;

import com.alibaba.fastjson.JSON;
import com.asiainfo.hsop.base.po.data.kafka.HospKafkaDetail;

import com.asiainfo.hsop.common.utils.HBaseUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.math.BigDecimal;
import java.text.ParseException;
import java.util.*;

/**
 *
 * @author cyc
 *
 * @date 2021-07-02
 *
 *
 *
 *
 * */
@Component
public class Consumer {

    private Logger logger= LoggerFactory.getLogger(Consumer.class);




        @KafkaListener(topics = "highwayetctu",containerFactory="kafkaListenerContainerFactory")
        public void consumer1(ConsumerRecord<?, ?> record, Acknowledgment ack) throws InterruptedException, ParseException, IOException {

            //具体业务

        //手动提交 ack.acknowledge(); } @KafkaListener(topics = "exitList_0706",containerFactory="kafkaListenerContainerFactory2") public void consumer2(ConsumerRecord<?, ?> record,Acknowledgment ack ) throws IOException, ParseException {         //具体业务
        //手动提交 ack.acknowledge(); } @KafkaListener(topics = "entryinfo",containerFactory="kafkaListenerContainerFactory3") public void consumer3(ConsumerRecord<?, ?> record,Acknowledgment ack ) throws IOException, ParseException {   //具体业务
      
        //手动提交 ack.acknowledge(); } }