java rocketMQ消息消费 rocketmq 消息顺序消费

转载

mob64ca14079fb3 2023-11-09 08:09:21

文章标签 java rocketMQ消息消费队列 spring jar apache 文章分类 Java 后端开发

什么事顺序消息

消息顺序（Message Order）有两种：顺序消费（Orderly）和并行消费（Concurrently）。顺序消费表示消息消费的顺序同生产者为每个消息队列发送的顺序一致，所以如果正在处理全局顺序是强制性的场景，需要确保使用的主题只有一个消息队列。并行消费不再保证消息顺序，消费的最大并行数量受每个消费者客户端指定的线程池限制。

要保证顺序消息，需满足以下三点：

生产者写入有序
消息中间件内部有序
消费者消费有序

与kafka类似，rocketmq内部同一个topic有多个queue，默认4个，每个queue中的数据是有序的。

首先我们要保证顺序写入，就要使用单线程有序的写入到topic的某个queue中。rocketmq提供了MessageQueueSelector来根据业务规则选择queue，比如根据用户id，相同用户的数据写入到同一个队列中，保证单一用户有序。

默认有SelectMessageQueueByHash,SelectMessageQueueByRandom, SelectMessageQueueByMachineRoom三种实现。但是如上述需求默认的选择权无法满足，我们可以自定义选择器。

再来说消费者有序消费，在生产者有序生产消息并且写入到queue中以后，消费者只需要保证单线程消费，也就是单线程消费某个topic的一个queue的数据。但是通常消费者是配置了线城池的，会有多个线程一起消费，因此需要使用特殊的MessageListenerOrderly监听器来消费消息。

实战

编写代码来实战一下顺序消息。

生产者顺序生产消息

这里我们用了两个线程发送消息，每个线程消息中的orderID相同。这样就保证了orderid级别的有序。
然后我们自定义了MessageQueueSelector，通过id % mqs.size()orderID%队列长度来决定数据应该插入哪个队列。

public static void main(String[] args) throws Exception {
        //Instantiate with a producer group name.
        DefaultMQProducer producer = new DefaultMQProducer("example_group_name");
        producer.setNamesrvAddr("node1:9876");
        //Launch the instance.
        producer.start();

        //顺序发送100条编号为0到99的，orderId为1 的消息
        new Thread(() -> {
            Integer orderId = 1;
            sendMessage(producer, orderId);
        }).start();
        //顺序发送100条编号为0到99的，orderId为2 的消息
        new Thread(() -> {
            Integer orderId = 2;
            sendMessage(producer, orderId);
        }).start();

        TimeUnit.SECONDS.sleep(5);

        producer.shutdown();
    }

    private static void sendMessage(MQProducer producer, Integer orderId) {
        for (int i = 0; i < 30; i++) {
            try {
                Message msg =
                        new Message("TopicTest", "TagA", i + "",
                                (orderId + "").getBytes(RemotingHelper.DEFAULT_CHARSET));
                SendResult sendResult = producer.send(msg, (mqs, msg1, arg) -> {
                    Integer id = (Integer) arg;
                    int index = id % mqs.size();
                    return mqs.get(index);
                }, orderId);
                System.out.println("message send,orderId:" + orderId);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

消费者顺序消费

消费者这里我们采用了MessageListenerConcurrently和支持顺序消费的MessageListenerOrderly来进行对比。

public static void main(String[] args) throws Exception {
        normal();//普通消费

//        order();//顺序消费
    }

    private static void normal() throws MQClientException {
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("example_group_name");
        consumer.setNamesrvAddr("node1:9876");

        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

        consumer.subscribe("TopicTest", "*");
        consumer.setConsumeThreadMin(3);
        consumer.setConsumeThreadMax(6);

        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {
            for (MessageExt msg : msgs) {
//                    System.out.println("收到消息," + new String(msg.getBody()));
                System.out.println("queueId:"+msg.getQueueId()+",orderId:"+new String(msg.getBody())+",i:"+msg.getKeys());
            }
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        });

        consumer.start();

        System.out.printf("Consumer Started.%n");
    }


    private static void order() throws MQClientException {
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("example_group_name");
        consumer.setNamesrvAddr("node1:9876");

        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

        consumer.subscribe("TopicTest", "*");
        //消费者并行消费
        consumer.setConsumeThreadMin(3);
        consumer.setConsumeThreadMax(6);

        consumer.registerMessageListener((MessageListenerOrderly) (msgs, context) -> {
//                context.setAutoCommit(false);
            for (MessageExt msg : msgs) {
                System.out.println("queueId:"+msg.getQueueId()+",orderId:"+new String(msg.getBody())+",i:"+msg.getKeys());
            }
            return ConsumeOrderlyStatus.SUCCESS;
        });

        consumer.start();

        System.out.printf("Consumer Started.%n");
    }

当执行normal();//普通消费方法时，输出结果如下，可以看到相同orderID的数据并没有有序打印：

java rocketMQ消息消费 rocketmq 消息顺序消费_队列

当执行order();//顺序消费有序消费时，可以看到相同orderID的数据是严格按照顺序打印的：

"C:\Program Files\Java\jdk-11.0.2\bin\java.exe" "-javaagent:C:\Program Files\JetBrains\IntelliJ IDEA Community Edition 2019.3.1\lib\idea_rt.jar=59377:C:\Program Files\JetBrains\IntelliJ IDEA Community Edition 2019.3.1\bin" -Dfile.encoding=UTF-8 -classpath D:\IdeaProjects\springboot2_demo\springboot-rocketmq\target\classes;D:\repository\org\springframework\boot\spring-boot-starter-web\2.1.1.RELEASE\spring-boot-starter-web-2.1.1.RELEASE.jar;D:\repository\org\springframework\boot\spring-boot-starter\2.1.1.RELEASE\spring-boot-starter-2.1.1.RELEASE.jar;D:\repository\org\springframework\boot\spring-boot\2.1.1.RELEASE\spring-boot-2.1.1.RELEASE.jar;D:\repository\org\springframework\boot\spring-boot-autoconfigure\2.1.1.RELEASE\spring-boot-autoconfigure-2.1.1.RELEASE.jar;D:\repository\org\springframework\boot\spring-boot-starter-logging\2.1.1.RELEASE\spring-boot-starter-logging-2.1.1.RELEASE.jar;D:\repository\ch\qos\logback\logback-classic\1.2.3\logback-classic-1.2.3.jar;D:\repository\ch\qos\logback\logback-core\1.2.3\logback-core-1.2.3.jar;D:\repository\org\apache\logging\log4j\log4j-to-slf4j\2.11.1\log4j-to-slf4j-2.11.1.jar;D:\repository\org\apache\logging\log4j\log4j-api\2.11.1\log4j-api-2.11.1.jar;D:\repository\org\slf4j\jul-to-slf4j\1.7.25\jul-to-slf4j-1.7.25.jar;D:\repository\javax\annotation\javax.annotation-api\1.3.2\javax.annotation-api-1.3.2.jar;D:\repository\org\springframework\boot\spring-boot-starter-json\2.1.1.RELEASE\spring-boot-starter-json-2.1.1.RELEASE.jar;D:\repository\com\fasterxml\jackson\core\jackson-databind\2.9.7\jackson-databind-2.9.7.jar;D:\repository\com\fasterxml\jackson\core\jackson-annotations\2.9.0\jackson-annotations-2.9.0.jar;D:\repository\com\fasterxml\jackson\core\jackson-core\2.9.7\jackson-core-2.9.7.jar;D:\repository\com\fasterxml\jackson\datatype\jackson-datatype-jdk8\2.9.7\jackson-datatype-jdk8-2.9.7.jar;D:\repository\com\fasterxml\jackson\datatype\jackson-datatype-jsr310\2.9.7\jackson-datatype-jsr310-2.9.7.jar;D:\repository\com\fasterxml\jackson\module\jackson-module-parameter-names\2.9.7\jackson-module-parameter-names-2.9.7.jar;D:\repository\org\springframework\boot\spring-boot-starter-tomcat\2.1.1.RELEASE\spring-boot-starter-tomcat-2.1.1.RELEASE.jar;D:\repository\org\apache\tomcat\embed\tomcat-embed-core\9.0.13\tomcat-embed-core-9.0.13.jar;D:\repository\org\apache\tomcat\embed\tomcat-embed-el\9.0.13\tomcat-embed-el-9.0.13.jar;D:\repository\org\apache\tomcat\embed\tomcat-embed-websocket\9.0.13\tomcat-embed-websocket-9.0.13.jar;D:\repository\org\hibernate\validator\hibernate-validator\6.0.13.Final\hibernate-validator-6.0.13.Final.jar;D:\repository\javax\validation\validation-api\2.0.1.Final\validation-api-2.0.1.Final.jar;D:\repository\org\jboss\logging\jboss-logging\3.3.2.Final\jboss-logging-3.3.2.Final.jar;D:\repository\com\fasterxml\classmate\1.4.0\classmate-1.4.0.jar;D:\repository\org\springframework\spring-web\5.1.3.RELEASE\spring-web-5.1.3.RELEASE.jar;D:\repository\org\springframework\spring-beans\5.1.3.RELEASE\spring-beans-5.1.3.RELEASE.jar;D:\repository\org\springframework\spring-webmvc\5.1.3.RELEASE\spring-webmvc-5.1.3.RELEASE.jar;D:\repository\org\springframework\spring-aop\5.1.3.RELEASE\spring-aop-5.1.3.RELEASE.jar;D:\repository\org\springframework\spring-context\5.1.3.RELEASE\spring-context-5.1.3.RELEASE.jar;D:\repository\org\springframework\spring-expression\5.1.3.RELEASE\spring-expression-5.1.3.RELEASE.jar;D:\repository\org\apache\rocketmq\rocketmq-spring-boot-starter\2.0.4\rocketmq-spring-boot-starter-2.0.4.jar;D:\repository\org\apache\rocketmq\rocketmq-spring-boot\2.0.4\rocketmq-spring-boot-2.0.4.jar;D:\repository\org\slf4j\slf4j-api\1.7.25\slf4j-api-1.7.25.jar;D:\repository\org\apache\rocketmq\rocketmq-client\4.5.2\rocketmq-client-4.5.2.jar;D:\repository\org\apache\rocketmq\rocketmq-common\4.5.2\rocketmq-common-4.5.2.jar;D:\repository\org\apache\commons\commons-lang3\3.8.1\commons-lang3-3.8.1.jar;D:\repository\org\apache\rocketmq\rocketmq-acl\4.5.2\rocketmq-acl-4.5.2.jar;D:\repository\org\apache\rocketmq\rocketmq-remoting\4.5.2\rocketmq-remoting-4.5.2.jar;D:\repository\io\netty\netty-all\4.1.31.Final\netty-all-4.1.31.Final.jar;D:\repository\io\netty\netty-tcnative-boringssl-static\2.0.20.Final\netty-tcnative-boringssl-static-2.0.20.Final.jar;D:\repository\org\apache\rocketmq\rocketmq-logging\4.5.2\rocketmq-logging-4.5.2.jar;D:\repository\org\apache\rocketmq\rocketmq-srvutil\4.5.2\rocketmq-srvutil-4.5.2.jar;D:\repository\commons-cli\commons-cli\1.2\commons-cli-1.2.jar;D:\repository\commons-codec\commons-codec\1.11\commons-codec-1.11.jar;D:\repository\org\springframework\spring-messaging\5.1.3.RELEASE\spring-messaging-5.1.3.RELEASE.jar;D:\repository\org\springframework\boot\spring-boot-starter-validation\2.1.1.RELEASE\spring-boot-starter-validation-2.1.1.RELEASE.jar;D:\repository\com\alibaba\fastjson\1.2.54\fastjson-1.2.54.jar;D:\repository\org\springframework\spring-core\5.1.3.RELEASE\spring-core-5.1.3.RELEASE.jar;D:\repository\org\springframework\spring-jcl\5.1.3.RELEASE\spring-jcl-5.1.3.RELEASE.jar;D:\repository\org\yaml\snakeyaml\1.23\snakeyaml-1.23.jar com.example.service.Test1
Consumer Started.
queueId:2,orderId:2,i:0
queueId:1,orderId:1,i:0
queueId:1,orderId:1,i:1
queueId:1,orderId:1,i:2
queueId:2,orderId:2,i:1
queueId:1,orderId:1,i:3
queueId:2,orderId:2,i:2
queueId:2,orderId:2,i:3
queueId:2,orderId:2,i:4
queueId:2,orderId:2,i:5
queueId:2,orderId:2,i:6
queueId:2,orderId:2,i:7
queueId:2,orderId:2,i:8
queueId:1,orderId:1,i:4
queueId:2,orderId:2,i:9
queueId:1,orderId:1,i:5
queueId:2,orderId:2,i:10
queueId:1,orderId:1,i:6
queueId:2,orderId:2,i:11
queueId:1,orderId:1,i:7
queueId:2,orderId:2,i:12
queueId:1,orderId:1,i:8
queueId:2,orderId:2,i:13
queueId:2,orderId:2,i:14
queueId:1,orderId:1,i:9
queueId:2,orderId:2,i:15
queueId:1,orderId:1,i:10
queueId:2,orderId:2,i:16
queueId:1,orderId:1,i:11
queueId:2,orderId:2,i:17
queueId:1,orderId:1,i:12
queueId:2,orderId:2,i:18
queueId:1,orderId:1,i:13
queueId:1,orderId:1,i:14
queueId:1,orderId:1,i:15
queueId:1,orderId:1,i:16
queueId:2,orderId:2,i:19
queueId:1,orderId:1,i:17
queueId:2,orderId:2,i:20
queueId:1,orderId:1,i:18
queueId:2,orderId:2,i:21
queueId:1,orderId:1,i:19
queueId:2,orderId:2,i:22
queueId:1,orderId:1,i:20
queueId:2,orderId:2,i:23
queueId:1,orderId:1,i:21
queueId:2,orderId:2,i:24
queueId:1,orderId:1,i:22
queueId:2,orderId:2,i:25
queueId:1,orderId:1,i:23
queueId:2,orderId:2,i:26
queueId:1,orderId:1,i:24
queueId:2,orderId:2,i:27
queueId:1,orderId:1,i:25
queueId:2,orderId:2,i:28
queueId:1,orderId:1,i:26
queueId:2,orderId:2,i:29
queueId:1,orderId:1,i:27
queueId:1,orderId:1,i:28
queueId:1,orderId:1,i:29

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。