紧接上一篇
三、消息消费
1. DEMO
public class RocketMQConsumerDemo {
public static void main(String[] args) throws Exception{
//创建消费者
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("group-java");
//设置 nameServer 地址
consumer.setNamesrvAddr("192.168.66.66:9876");
//设置订阅的消息主题
consumer.subscribe("topicA", "*");
//监听模式
//有序消费MessageListenerOrderly
//并发消费MessageListenerConcurrently
consumer.registerMessageListener(new MessageListenerConcurrently() {
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext consumeConcurrentlyContext) {
for (MessageExt messageExt : list) {
System.out.println("接受消息：队列ID==>"+messageExt.getQueueId() + " 消息Id:" + messageExt.getMsgId());
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
});
consumer.start();
//System.in.read();
}
}注意： 订阅消息消费者订阅的主题必须存在。
2. 问题探究
问题1、一个消费者组不同的消费者是否可以同时消费多个主题？？（试验）
创建2个主题，各自生产20个消息：
同一个消费者组，启动两个线程分别消费主题A和主题B：
查看消费状态：
答案：同一个消费者组中多个消费者不能同时消费多个主题topic,如果出现此现象，会导致部分消息无法消费。
什么原因导致这样现象呢？？
答案：负载均衡算法问题导致了这一现象。（平均分配算法，环形平均分配算法…）
负载算法：
- 1、根据组名，把主题下的队列平均分配给同一个消费者组下的消费者
- 2、topicA主题：4个队列先平均分配，分配给A 2个队列，B 2个队列，但是B没有订阅TopocA，因此就会导致2,3队列无法被消费。
- 3、topicB主题：4个队列先平均分配，分配给A 2个队列，B 2个队列，但是A没有订阅topicB主题，因此就会导致0,1无法被消费。
问题2：不同的消费者组，是否可以消费不同的topic ?? 可以的
问题3：不同的消费者组，是否可以消费相同的topic ?? 可以的
3. 消费模型
1）一对多 （topic 多个消费者组）
2）一对一
注意：默认消费模式就是 一个消费者组消费一个主题（使用负载均衡算法进行分配）
4. 源码初探 - PUSH模型消费流程
推拉模型的区别参考：
push本质就是pull，其实都是拉取，只不过push是系统自动拉，而pull是用户自己拉
看start入口方法：
//DefaultMQPushConsumer#start
public void start() throws MQClientException {
this.defaultMQPushConsumerImpl.start();
}
//DefaultMQPushConsumerImpl#start
public void start() throws MQClientException {
//根据serviceState状态有不同操作
switch (this.serviceState) {
case CREATE_JUST:
//刚刚创建
log.info("the consumer [{}] start beginning. messageModel={}, isUnitMode={}", this.defaultMQPushConsumer.getConsumerGroup(),
this.defaultMQPushConsumer.getMessageModel(), this.defaultMQPushConsumer.isUnitMode());
this.serviceState = ServiceState.START_FAILED;//先默认给个启动失败状态
// 检查配置，做一些校验
this.checkConfig();
// Rebalance负载均衡 复制订阅数据
// 当集群消费时，加入当前消费分组重试消息的订阅
this.copySubscription();
//如果消费模式是集群模式
if (this.defaultMQPushConsumer.getMessageModel() == MessageModel.CLUSTERING) {
// 设置instanceName,为一个字符串化的数字,比如10072
this.defaultMQPushConsumer.changeInstanceNameToPID();
}
// 获取MQClient对象，clientId为ip@instanceName，比如192.168.0.1@10072
this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook);
// 负载均衡器设置消费者组
this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup());
//设置消费模式
//默认消费模式为集群模式,每条消息被同一组的消费者中的一个消费
//还可以设置为广播模式,每条消息被同一个组的所有消费者都消费一次
this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel());
//设置策略，默认是AllocateMessageQueueAveragely,均分策略
this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer.getAllocateMessageQueueStrategy());
this.rebalanceImpl.setmQClientFactory(this.mQClientFactory);
// 拉取API装饰增强
this.pullAPIWrapper = new PullAPIWrapper(mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode());
this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList);
//生成消费进度处理器,集群模式下消费进度保存在Broker上，因为同一组内的消费者要共享进度;广播模式下进度保存在消费者端
if (this.defaultMQPushConsumer.getOffsetStore() != null) {
this.offsetStore = this.defaultMQPushConsumer.getOffsetStore();
} else {
switch (this.defaultMQPushConsumer.getMessageModel()) {
case BROADCASTING:
this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
break;
case CLUSTERING:
this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
break;
default:
break;
}
}
//若是广播模式,加载本地的消费进度文件
//若是集群模式，从broker中获取进度
this.offsetStore.load();
// 根据监听是顺序模式还是并发模式来生成相应的ConsumerService
if (this.getMessageListenerInner() instanceof MessageListenerOrderly) {
this.consumeOrderly = true;
this.consumeMessageService = new ConsumeMessageOrderlyService(this, (MessageListenerOrderly)this.getMessageListenerInner());
} else if (this.getMessageListenerInner() instanceof MessageListenerConcurrently) {
this.consumeOrderly = false;
this.consumeMessageService = new ConsumeMessageConcurrentlyService(this, (MessageListenerConcurrently)this.getMessageListenerInner());
}
this.consumeMessageService.start();
// 设置MQClient对象（本地注册）
boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this);
if (!registerOK) {
this.serviceState = ServiceState.CREATE_JUST;
this.consumeMessageService.shutdown();
throw new MQClientException("The consumer group[" + this.defaultMQPushConsumer.getConsumerGroup()
+ "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),
null);
}
//开启消费
mQClientFactory.start();
log.info("the consumer [{}] start OK.", this.defaultMQPushConsumer.getConsumerGroup());
// 设置服务状态
this.serviceState = ServiceState.RUNNING;
break;
case RUNNING:
case START_FAILED:
case SHUTDOWN_ALREADY:
throw new MQClientException("The PushConsumer service state not OK, maybe started once, "//
+ this.serviceState//
+ FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),
null);
default:
break;
}
// 从Namesrv获取TopicRouteData,更新TopicPublishInfo和MessageQueue （在Consumer start时马上调用，之后每隔一段时间调用一次）
this.updateTopicSubscribeInfoWhenSubscriptionChanged();
// 向TopicRouteData里的所有Broker发送心跳,注册Consumer/Producer信息到Broker上 （在Consumer start时马上调用，之后每隔一段时间调用一次）
this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();
// 唤醒MessageQueue均衡服务,负载均衡后马上开启第一次拉取消息
this.mQClientFactory.rebalanceImmediately();
}4.1 开始执行消息拉取
继续看mQClientFactory.start()：
//MQClientInstance#start
public void start() throws MQClientException {
synchronized (this) {
switch (this.serviceState) {
case CREATE_JUST:
this.serviceState = ServiceState.START_FAILED;
// If not specified,looking address from name server
if (null == this.clientConfig.getNamesrvAddr()) {
//如果url未指定，可以通过Http请求从其他处获取
this.mQClientAPIImpl.fetchNameServerAddr();
}
// Start request-response channel
this.mQClientAPIImpl.start();
// 启动多个定时任务
this.startScheduledTask();
// Start pull service
// 拉取消息服务
this.pullMessageService.start();
// 消费者负载均衡服务
// Start Consumer rebalance service
this.rebalanceService.start();
//启动内部默认的生产者,用于消费者SendMessageBack,但不会执行MQClientInstance.start(),也就是当前方法不会被执行
this.defaultMQProducer.getDefaultMQProducerImpl().start(false);
log.info("the client factory [{}] start OK", this.clientId);
this.serviceState = ServiceState.RUNNING;
break;
case RUNNING:
break;
case SHUTDOWN_ALREADY:
break;
case START_FAILED:
throw new MQClientException("The Factory object[" + this.getClientId() + "] has been created before, and failed.", null);
default:
break;
}
}
}4.2 消息拉取服务
看一下拉取消息服务：
//PullMessageService#run
public void run() {
log.info(this.getServiceName() + " service started");
while (!this.isStopped()) {
try {
//使用BlockingQueue阻塞队列,当提交了消息拉取请求后,马上执行
//take：检索并删除此队列的头，必要时等待，直到某个元素变为可用。
PullRequest pullRequest = this.pullRequestQueue.take();
if (pullRequest != null) {
//拉取消息
this.pullMessage(pullRequest);
}
} catch (InterruptedException e) {
} catch (Exception e) {
log.error("Pull Message Service Run Method exception", e);
}
}
log.info(this.getServiceName() + " service end");
}
//PullMessageService#pullMessage
/**
* 拉取消息
*
* @param pullRequest 拉取消息请求
*/
private void pullMessage(final PullRequest pullRequest) {
// 通过消费者组 组名获取消费者
final MQConsumerInner consumer = this.mQClientFactory.selectConsumer(pullRequest.getConsumerGroup());
if (consumer != null) {
DefaultMQPushConsumerImpl impl = (DefaultMQPushConsumerImpl) consumer;
impl.pullMessage(pullRequest);
} else {
log.warn("No matched consumer for the PullRequest {}, drop it", pullRequest);
}
}//DefaultMQPushConsumerImpl#pullMessage
/**
* 拉取消息
*
* @param pullRequest 拉取消息请求
*/
public void pullMessage(final PullRequest pullRequest) {
final ProcessQueue processQueue = pullRequest.getProcessQueue();
if (processQueue.isDropped()) {
log.info("the pull request[{}] is dropped.", pullRequest.toString());
return;
}
// 设置队列最后一次拉取消息时间
pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis());
// 判断consumer状态是否运行中。如果不是，则延迟拉取消息。
try {
this.makeSureStateOK();
} catch (MQClientException e) {
log.warn("pullMessage exception, consumer state not ok", e);
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
return;
}
// 判断是否暂停中。暂停就延迟拉取消息。
if (this.isPause()) {
log.warn("consumer was paused, execute pull request later. instanceName={}, group={}", this.defaultMQPushConsumer.getInstanceName(),
this.defaultMQPushConsumer.getConsumerGroup());
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND);
return;
}
// 判断是否超过最大持有消息数量。默认最大值为1000
// 超过则延迟50毫秒放入请求队列。从而达到消费限流作用。
long size = processQueue.getMsgCount().get();
if (size > this.defaultMQPushConsumer.getPullThresholdForQueue()) {
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); // 提交延迟消息拉取请求。50ms。
if ((flowControlTimes1++ % 1000) == 0) {
log.warn(
"the consumer message buffer is full, so do flow control, minOffset={}, maxOffset={}, size={}, pullRequest={}, flowControlTimes={}",
processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), size, pullRequest, flowControlTimes1);
}
return;
}
if (!this.consumeOrderly) { //并发消费
// 判断消息Offset跨度是否过大 > 2000
if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) {
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); // 提交延迟消息拉取请求。50ms。
if ((flowControlTimes2++ % 1000) == 0) {
log.warn(
"the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}",
processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(),
pullRequest, flowControlTimes2);
}
return;
}
} else { //顺序消费
//顺序消费流程跟并发消费最大的区别在于，对要处理的队列加锁，
//确保同一队列，同一时间，只允许一个消费线程处理。
if (processQueue.isLocked()) { //broker端的队列锁
if (!pullRequest.isLockedFirst()) {
//计算从哪拉取
final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue());
//如果获取的offset 小于 下一个offset,默认消费方式CONSUME_FROM_LAST_OFFSET
boolean brokerBusy = offset < pullRequest.getNextOffset();
log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}",
pullRequest, offset, brokerBusy);
if (brokerBusy) {
log.info(
"[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: "
+ "{}",
pullRequest, offset);
}
pullRequest.setLockedFirst(true);
pullRequest.setNextOffset(offset);
}
} else {
//延迟消费3000ms
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
log.info("pull message later because not locked in broker, {}", pullRequest);
return;
}
}
// 获取Topic 对应的订阅信息。若不存在，则延迟拉取消息
final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());
if (null == subscriptionData) {
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
log.warn("find the consumer's subscription failed, {}", pullRequest);
return;
}
final long beginTimestamp = System.currentTimeMillis();
//创建拉取的回调
PullCallback pullCallback = new PullCallback() {
@Override
public void onSuccess(PullResult pullResult) {//拉取成功的处理
if (pullResult != null) {
//提取ByteBuffer生成List<MessageExt>
pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult, subscriptionData);
switch (pullResult.getPullStatus()) {//根据不同拉取结果处理
//有新消息的情况
case FOUND:
// 设置下次拉取消息队列位置
long prevRequestOffset = pullRequest.getNextOffset();
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
// 统计消息消费的响应时间
long pullRT = System.currentTimeMillis() - beginTimestamp;
DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(),
pullRequest.getMessageQueue().getTopic(), pullRT);
long firstMsgOffset = Long.MAX_VALUE;
if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) {
//如果这次请求没有获取到消息，马上进行另一次拉取
//就是把请求放入一开始看到的pullRequestQueue队列
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
} else {
firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();
// 统计
DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(),
pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size());
// 提交拉取到的消息到ProcessQueue的TreeMap中
//返回 true : 上一批次的消息已经消费完了
//返回 false: 上一批次的消息还没消费完
boolean dispathToConsume = processQueue.putMessage(pullResult.getMsgFoundList());
// 在有序消费模式下,仅当 dispathToConsume=true 时提交消费请求,也就是上一批次的消息消费完了才提交消费请求
// 在并发消费模式下,dispathToConsume不起作用,直接提交消费请求
// 所谓提交消费请求，就会触发我们实现的监听器处理业务逻辑
DefaultMQPushConsumerImpl.this.consumeMessageService
.submitConsumeRequest(pullResult.getMsgFoundList(), processQueue, pullRequest.getMessageQueue(), dispathToConsume);
// 提交下次拉取消息请求
if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) {
DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest,
DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval());
} else {
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
}
}
// 下次拉取消费队列位置小于上次拉取消息队列位置 或者 第一条消息的消费队列位置小于上次拉取消息队列位置，则判定为BUG，输出log
if (pullResult.getNextBeginOffset() < prevRequestOffset || firstMsgOffset < prevRequestOffset) {
log.warn(
"[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}",
pullResult.getNextBeginOffset(),
firstMsgOffset,
prevRequestOffset);
}
break;
//没有新消息的情况
case NO_NEW_MSG:
// 设置下次拉取消息队列位置
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
// 持久化消费进度
DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
// 立即提交拉取消息请求
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
break;
//没有匹配的消息的情况
case NO_MATCHED_MSG:
// 设置下次拉取消息队列位置
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
// 持久化消费进度
DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
// 提交立即拉取消息请求
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
break;
case OFFSET_ILLEGAL:
log.warn("the pull request offset illegal, {} {}", //
pullRequest.toString(), pullResult.toString());
// 设置下次拉取消息队列位置
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
// 设置消息处理队列为dropped
pullRequest.getProcessQueue().setDropped(true);
// 提交延迟任务，进行消费处理队列移除
DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() {
@Override
public void run() {
try {
// 更新消费进度，同步消费进度到Broker
DefaultMQPushConsumerImpl.this.offsetStore.updateOffset(pullRequest.getMessageQueue(),
pullRequest.getNextOffset(), false);
DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue());
// 移除消费处理队列
DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue());
log.warn("fix the pull request offset, {}", pullRequest);
} catch (Throwable e) {
log.error("executeTaskLater Exception", e);
}
}
}, 10000);
break;
default:
break;
}
}
}
@Override
public void onException(Throwable e) {
if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
log.warn("execute the pull request exception", e);
}
// 提交延迟拉取消息请求
DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
}
};
// 集群消息模型下，计算提交的消费进度。
boolean commitOffsetEnable = false;
long commitOffsetValue = 0L;
if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) {
commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY);
if (commitOffsetValue > 0) {
commitOffsetEnable = true;
}
}
// 计算请求的 订阅表达式 和 是否进行filtersrv过滤消息
String subExpression = null;
boolean classFilter = false;
SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());
if (sd != null) {
if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) { //默认否
subExpression = sd.getSubString();
}
classFilter = sd.isClassFilterMode();
}
// 计算拉取消息系统标识
int sysFlag = PullSysFlag.buildSysFlag(commitOffsetEnable, true, subExpression != null, classFilter);
// 执行拉取。如果拉取请求发生异常时，提交延迟拉取消息请求。
// 这里是真正拉取的逻辑
try {
this.pullAPIWrapper.pullKernelImpl(//
pullRequest.getMessageQueue(), // 1
subExpression, // 2
subscriptionData.getSubVersion(), // 3
pullRequest.getNextOffset(), // 4
this.defaultMQPushConsumer.getPullBatchSize(), // 5
sysFlag, // 6
commitOffsetValue, // 7
BROKER_SUSPEND_MAX_TIME_MILLIS, // 8
CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND, // 9
CommunicationMode.ASYNC, // 10
pullCallback// 11
);
} catch (Exception e) {
log.error("pullKernelImpl exception", e);
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
}
}关于有序消费和并发消费，参考博文：
继续看真正拉取的逻辑：
//PullAPIWrapper#pullKernelImpl
public PullResult pullKernelImpl(
final MessageQueue mq,
final String subExpression,
final String expressionType,
final long subVersion,
final long offset,
final int maxNums,
final int sysFlag,
final long commitOffset,
final long brokerSuspendMaxTimeMillis,
final long timeoutMillis,
final CommunicationMode communicationMode,
final PullCallback pullCallback
) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
//获取broker地址
FindBrokerResult findBrokerResult =
this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(),
this.recalculatePullFromWhichNode(mq), false);
if (null == findBrokerResult) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
findBrokerResult =
this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(),
this.recalculatePullFromWhichNode(mq), false);
}
//不为空的情况下拉取，否则抛异常
if (findBrokerResult != null) {
{
// check version
// 校验broker是否支持过滤消息
if (!ExpressionType.isTagType(expressionType)
&& findBrokerResult.getBrokerVersion() < MQVersion.Version.V4_1_0_SNAPSHOT.ordinal()) {
throw new MQClientException("The broker[" + mq.getBrokerName() + ", "
+ findBrokerResult.getBrokerVersion() + "] does not upgrade to support for filter message by " + expressionType, null);
}
}
int sysFlagInner = sysFlag;
//判断是否是从 从broker获取，是的话要清楚提交的消费偏移量
if (findBrokerResult.isSlave()) {
sysFlagInner = PullSysFlag.clearCommitOffsetFlag(sysFlagInner);
}
//设置请求头
PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();
requestHeader.setConsumerGroup(this.consumerGroup);
requestHeader.setTopic(mq.getTopic());
requestHeader.setQueueId(mq.getQueueId());
requestHeader.setQueueOffset(offset);
requestHeader.setMaxMsgNums(maxNums);
requestHeader.setSysFlag(sysFlagInner);
requestHeader.setCommitOffset(commitOffset);
requestHeader.setSuspendTimeoutMillis(brokerSuspendMaxTimeMillis);
requestHeader.setSubscription(subExpression);
requestHeader.setSubVersion(subVersion);
requestHeader.setExpressionType(expressionType);
String brokerAddr = findBrokerResult.getBrokerAddr();
if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) {
brokerAddr = computPullFromWhichFilterServer(mq.getTopic(), brokerAddr);
}
//发起拉取消息请求
PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage(
brokerAddr,
requestHeader,
timeoutMillis,
communicationMode,
pullCallback);
return pullResult;
}
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
//MQClientAPIImpl#pullMessage
public PullResult pullMessage(
final String addr,
final PullMessageRequestHeader requestHeader,
final long timeoutMillis,
final CommunicationMode communicationMode,
final PullCallback pullCallback
) throws RemotingException, MQBrokerException, InterruptedException {
//构建请求命令
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);
//底层不看了，是netty
switch (communicationMode) {
case ONEWAY:
assert false;
return null;
case ASYNC:
this.pullMessageAsync(addr, request, timeoutMillis, pullCallback);
return null;
case SYNC:
return this.pullMessageSync(addr, request, timeoutMillis);
default:
assert false;
break;
}
return null;
}回到DefaultMQPushConsumerImpl#pullMessage，看整个流程中任意一个立即拉取的方法：
//DefaultMQPushConsumerImpl#executePullRequestImmediately
public void executePullRequestImmediately(final PullRequest pullRequest) {
this.mQClientFactory.getPullMessageService().executePullRequestImmediately(pullRequest);
}
//PullMessageService#executePullRequestImmediately
/**
* 执行立即拉取消息请求
*
* @param pullRequest 拉取消息请求
*/
public void executePullRequestImmediately(final PullRequest pullRequest) {
try {
//看到就是往队列里面塞一个拉取请求
this.pullRequestQueue.put(pullRequest);
} catch (InterruptedException e) {
log.error("executePullRequestImmediately pullRequestQueue.put", e);
}
}5. 负载均衡
5.1 流程分析
看另一个负载均衡服务：
//MQClientInstance#doRebalance
/**
* 消费者进行平衡
*/
public void doRebalance() {
for (Map.Entry<String, MQConsumerInner> entry : this.consumerTable.entrySet()) {
//获取内部注册的消费者
MQConsumerInner impl = entry.getValue();
if (impl != null) {
try {
//进行负载均衡重分配
impl.doRebalance();
} catch (Throwable e) {
log.error("doRebalance exception", e);
}
}
}
}看push模型的实现：
//DefaultMQPushConsumerImpl#doRebalance
/**
* 进行平衡
*/
@Override
public void doRebalance() {
if (!this.pause) {
//isConsumeOrderly是否是顺序消费，否则是并发消费
this.rebalanceImpl.doRebalance(this.isConsumeOrderly());
}
}
//RebalanceImpl#doRebalance
/**
* 执行分配消费队列
*
* @param isOrder 是否顺序消息
*/
public void doRebalance(final boolean isOrder) {
// 分配每个 topic 的消息队列
// 获取内部的订阅信息，key是主题
Map<String, SubscriptionData> subTable = this.getSubscriptionInner();
if (subTable != null) {
for (final Map.Entry<String, SubscriptionData> entry : subTable.entrySet()) {
final String topic = entry.getKey();
try {
//核心在这，为当前消费者，指定的主题重新分配消费队列
this.rebalanceByTopic(topic, isOrder);
} catch (Throwable e) {
if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
log.warn("rebalanceByTopic Exception", e);
}
}
}
}
// 移除未订阅的topic对应的消息队列
this.truncateMessageQueueNotMyTopic();
}//RebalanceImpl#rebalanceByTopic
/**
* 消费者对 单个Topic 重新进行平衡
*
* @param topic Topic
* @param isOrder 是否顺序
*/
private void rebalanceByTopic(final String topic, final boolean isOrder) {
switch (messageModel) {
case BROADCASTING: { //广播模式,每条消息被同一组的所有消费者均消费
Set<MessageQueue> mqSet = this.topicSubscribeInfoTable.get(topic);
if (mqSet != null) {
boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, isOrder);
if (changed) {
this.messageQueueChanged(topic, mqSet, mqSet);
log.info("messageQueueChanged {} {} {} {}", //
consumerGroup, //
topic, //
mqSet, //
mqSet);
}
} else {
log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic);
}
break;
}
case CLUSTERING: { //默认是集群模式,每条消息被同一消费者组的一个消费,
// 获取 topic 对应的所有队列 和 consumer信息
Set<MessageQueue> mqSet = this.topicSubscribeInfoTable.get(topic);
//获取该主题、消费者组下的所有消费者id
List<String> cidAll = this.mQClientFactory.findConsumerIdList(topic, consumerGroup);
if (null == mqSet) {
if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic);
}
}
if (null == cidAll) {
log.warn("doRebalance, {} {}, get consumer id list failed", consumerGroup, topic);
}
if (mqSet != null && cidAll != null) {
// 排序 消费队列 和 消费者数组。因为是在Client进行分配队列，排序后，各Client的顺序才能保持一致。
List<MessageQueue> mqAll = new ArrayList<>();
mqAll.addAll(mqSet);
Collections.sort(mqAll);
Collections.sort(cidAll);
//获取分配策略!
AllocateMessageQueueStrategy strategy = this.allocateMessageQueueStrategy;
//AllocateMessageQueueAveragely
// 根据 队列分配策略 分配消费队列
List<MessageQueue> allocateResult;
try {
//分配结果
allocateResult = strategy.allocate(this.consumerGroup, this.mQClientFactory.getClientId(), mqAll, cidAll);
} catch (Throwable e) {
log.error("AllocateMessageQueueStrategy.allocate Exception. allocateMessageQueueStrategyName={}", strategy.getName(), e);
return;
}
Set<MessageQueue> allocateResultSet = new HashSet<>();
if (allocateResult != null) {
allocateResultSet.addAll(allocateResult);
}
// 更新消费队列
boolean changed = this.updateProcessQueueTableInRebalance(topic, allocateResultSet, isOrder);
if (changed) {
log.info(
"rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, clientId={}, mqAllSize={}, cidAllSize={}, "
+ "rebalanceResultSize={}, rebalanceResultSet={}",
strategy.getName(), consumerGroup, topic, this.mQClientFactory.getClientId(), mqSet.size(), cidAll.size(),
allocateResultSet.size(), allocateResultSet);
this.messageQueueChanged(topic, mqSet, allocateResultSet);
}
}
break;
}
default:
break;
}
}主要关注分配策略的接口：
5.1平均分配算法
/**
* Average Hashing queue algorithm
* 队列分配策略 - 平均分配
* 如果 队列数 和 消费者数量 相除有余数时，余数按照顺序"1"个"1"个分配消费者。
* 例如，5个队列，3个消费者时，分配如下：
* - 消费者0：[0, 1] 2个队列
* - 消费者1：[2, 3] 2个队列
* - 消费者2：[4, 4] 1个队列
*
* 代码块 (mod > 0 && index < mod) 判断即在处理相除有余数的情况。
*/
public class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy {
private final Logger log = ClientLogger.getLog();
@Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll, List<String> cidAll) {
//consumerGroup：消费者组
//currentCID：当前消费者id
//mqAll：当前主题下所有队列
//cidAll：所有消费者集合
// 校验参数是否正确
if (currentCID == null || currentCID.length() < 1) {
throw new IllegalArgumentException("currentCID is empty");
}
if (mqAll == null || mqAll.isEmpty()) {
throw new IllegalArgumentException("mqAll is null or mqAll empty");
}
if (cidAll == null || cidAll.isEmpty()) {
throw new IllegalArgumentException("cidAll is null or cidAll empty");
}
List<MessageQueue> result = new ArrayList<>();
if (!cidAll.contains(currentCID)) {
log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}",
consumerGroup,
currentCID,
cidAll);
return result;
}
// 平均分配
int index = cidAll.indexOf(currentCID); // 第几个consumer。
int mod = mqAll.size() % cidAll.size(); // 余数，即多少消息队列无法平均分配。
//队列总数 <= 消费者总数时，分配当前消费者1个队列
//不能均分 && 当前消费者序号(从0开始) < 余下的队列数 ，分配当前消费者 mqAll / cidAll +1 个队列
//不能均分 && 当前消费者序号(从0开始) >= 余下的队列数 ,分配当前消费者 mqAll / cidAll 个队列
int averageSize = mqAll.size() <= cidAll.size() ? 1 : (mod > 0 && index < mod ? mqAll.size() / cidAll.size() + 1 : mqAll.size() / cidAll.size());
// 有余数的情况下，[0, mod) 平分余数，即每consumer多分配一个节点；
// 第index开始，跳过前mod余数。
int startIndex = (mod > 0 && index < mod) ? index * averageSize
: index * averageSize + mod;
// 分配队列数量。之所以要Math.min()的原因是，
// mqAll.size() <= cidAll.size()，部分consumer分配不到消费队列。
int range = Math.min(averageSize, mqAll.size() - startIndex);
for (int i = 0; i < range; i++) {
result.add(mqAll.get((startIndex + i) % mqAll.size()));
}
return result;
}
@Override
public String getName() {
return "AVG";
}
}5.2 环形平均分配
/**
* Cycle average Hashing queue algorithm
* 队列分配策略 - 环状分配
*/
public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy {
private final Logger log = ClientLogger.getLog();
@Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
List<String> cidAll) {
// 校验参数是否正确
if (currentCID == null || currentCID.length() < 1) {
throw new IllegalArgumentException("currentCID is empty");
}
if (mqAll == null || mqAll.isEmpty()) {
throw new IllegalArgumentException("mqAll is null or mqAll empty");
}
if (cidAll == null || cidAll.isEmpty()) {
throw new IllegalArgumentException("cidAll is null or cidAll empty");
}
List<MessageQueue> result = new ArrayList<MessageQueue>();
if (!cidAll.contains(currentCID)) {
log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}",
consumerGroup,
currentCID,
cidAll);
return result;
}
// 环状分配
int index = cidAll.indexOf(currentCID);
for (int i = index; i < mqAll.size(); i++) {
if (i % cidAll.size() == index) {
result.add(mqAll.get(i));
}
}
return result;
}
@Override
public String getName() {
return "AVG_BY_CIRCLE";
}
}5.3 机房broker健康算法
/**
* Computer room Hashing queue algorithm, such as Alipay logic room
* 分配策略 - 首先筛选可消费Broker对应的消费队列，筛选后的消费队列平均分配。
* TODO 疑问：Broker、Consumer如何配置
*/
public class AllocateMessageQueueByMachineRoom implements AllocateMessageQueueStrategy {
/**
* 消费者消费brokerName集合
*/
private Set<String> consumeridcs;
@Override
public List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
List<String> cidAll) {
// 参数校验
List<MessageQueue> result = new ArrayList<MessageQueue>();
int currentIndex = cidAll.indexOf(currentCID);
if (currentIndex < 0) {
return result;
}
// 计算符合当前配置的消费者数组('consumeridcs')对应的消费队列
List<MessageQueue> premqAll = new ArrayList<MessageQueue>();
for (MessageQueue mq : mqAll) {
String[] temp = mq.getBrokerName().split("@");
if (temp.length == 2 && consumeridcs.contains(temp[0])) {
premqAll.add(mq);
}
}
// 平均分配
int mod = premqAll.size() / cidAll.size();
int rem = premqAll.size() % cidAll.size();
int startIndex = mod * currentIndex;
int endIndex = startIndex + mod;
for (int i = startIndex; i < endIndex; i++) {
result.add(mqAll.get(i));
}
if (rem > currentIndex) {
result.add(premqAll.get(currentIndex + mod * cidAll.size()));
}
return result;
}
@Override
public String getName() {
return "MACHINE_ROOM";
}
public Set<String> getConsumeridcs() {
return consumeridcs;
}
public void setConsumeridcs(Set<String> consumeridcs) {
this.consumeridcs = consumeridcs;
}
}
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
