Consumer

Consumer

Consumer与Name Server集群中的其中一个节点（随机选择）建立长连接，定期从Name Server取Topic路由信息，并向提供Topic服务的Master、Slave建立长连接，且定时向Master、Slave发送心跳。Consumer既可以从Master订阅消息，也可以从Slave订阅消息，订阅规则由Broker配置决定。 Consumer支持顺序消费和无序消费。

Consumer Group用来表示一组消费消息应用，一个Consumer Group下包含多个Consumer实例，可以是多台机器，也可以是多个进程，或者是一个进程的多个Consumer对象。一个Consumer Group下的多个Consumer以均摊方式消费消息，如果设置为广播方式，那么这个Consumer Group下的每个实例都消费全量数据。

DefaultMQPushConsumer

启动

DefaultMQPushConsumerImpl.start()

public synchronized void start() throws MQClientException {
    switch (this.serviceState) {
        case CREATE_JUST:
            log.info("the consumer [{}] start beginning. messageModel={}, isUnitMode={}", this.defaultMQPushConsumer.getConsumerGroup(),
                this.defaultMQPushConsumer.getMessageModel(), this.defaultMQPushConsumer.isUnitMode());
            this.serviceState = ServiceState.START_FAILED;

            this.checkConfig();

            this.copySubscription();

            if (this.defaultMQPushConsumer.getMessageModel() == MessageModel.CLUSTERING) {
                this.defaultMQPushConsumer.changeInstanceNameToPID();
            }
            //创建客户端实例，注册请求处理类ClientRemotingProcessor
            this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook);

            this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup());
            this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel());
            this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer.getAllocateMessageQueueStrategy());
            this.rebalanceImpl.setmQClientFactory(this.mQClientFactory);

            this.pullAPIWrapper = new PullAPIWrapper(
                mQClientFactory,
                this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode());
            this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList);

            if (this.defaultMQPushConsumer.getOffsetStore() != null) {
                this.offsetStore = this.defaultMQPushConsumer.getOffsetStore();
            } else {
                switch (this.defaultMQPushConsumer.getMessageModel()) {
                    case BROADCASTING:
                        //将storePath目录下的offsets.json文件加载到内存
                        //也会将保存在内存中的偏移量数据持久化到offsets.json文件中
                        this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
                        break;
                    case CLUSTERING:
                        //根据topic将偏移量数据同步给指定broker，主从节点当中随机同步一个，默认
                        this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
                        break;
                    default:
                        break;
                }
                this.defaultMQPushConsumer.setOffsetStore(this.offsetStore);
            }
            //加载偏移量
            this.offsetStore.load();

            if (this.getMessageListenerInner() instanceof MessageListenerOrderly) { //顺序消费
                this.consumeOrderly = true;
                this.consumeMessageService =
                    new ConsumeMessageOrderlyService(this, (MessageListenerOrderly) this.getMessageListenerInner());
            } else if (this.getMessageListenerInner() instanceof MessageListenerConcurrently) { //立即消费，不保证顺序
                this.consumeOrderly = false;
                this.consumeMessageService =
                    new ConsumeMessageConcurrentlyService(this, (MessageListenerConcurrently) this.getMessageListenerInner());
            }
            //启动消费服务，实现消费消息
            //ConsumeMessageConcurrentlyService:启动定时器，定期从队列中删除过期的消息
            //ConsumeMessageOrderlyService:只能在集群模式启动，定期向broker主节点发送请求，锁住消息队列
            this.consumeMessageService.start();
            //将消费者信息保存到consumerTable
            boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this);
            if (!registerOK) {
                this.serviceState = ServiceState.CREATE_JUST;
                this.consumeMessageService.shutdown();
                throw new MQClientException("The consumer group[" + this.defaultMQPushConsumer.getConsumerGroup()
                    + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),
                    null);
            }

            mQClientFactory.start();
            log.info("the consumer [{}] start OK.", this.defaultMQPushConsumer.getConsumerGroup());
            this.serviceState = ServiceState.RUNNING;
            break;
        case RUNNING:
        case START_FAILED:
        case SHUTDOWN_ALREADY:
            throw new MQClientException("The PushConsumer service state not OK, maybe started once, "
                + this.serviceState
                + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),
                null);
        default:
            break;
    }
    //在name server中更新topic路由信息
    this.updateTopicSubscribeInfoWhenSubscriptionChanged();
    this.mQClientFactory.checkClientInBroker();
    this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();
    this.mQClientFactory.rebalanceImmediately();
}

客户端处理类

NettyClientHandler处理请求和响应。然后调用ClientRemotingProcessor执行具体操作。

ClientRemotingProcessor.processRequest()

public RemotingCommand processRequest(ChannelHandlerContext ctx,
    RemotingCommand request) throws RemotingCommandException {
    switch (request.getCode()) {
        case RequestCode.CHECK_TRANSACTION_STATE:   //执行自定义监听，向broker发送请求，结束事务
            return this.checkTransactionState(ctx, request);
        case RequestCode.NOTIFY_CONSUMER_IDS_CHANGED:
            return this.notifyConsumerIdsChanged(ctx, request);
        case RequestCode.RESET_CONSUMER_CLIENT_OFFSET:
            return this.resetOffset(ctx, request);
        case RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT:
            return this.getConsumeStatus(ctx, request);

        case RequestCode.GET_CONSUMER_RUNNING_INFO:
            return this.getConsumerRunningInfo(ctx, request);

        case RequestCode.CONSUME_MESSAGE_DIRECTLY:  //根据不同的消费模式（顺序消费与非顺序消费），消费消息，执行consumeMessage，返回消费状态
            return this.consumeMessageDirectly(ctx, request);
        default:
            break;
    }
    return null;
}

无序消费

ConsumeMessageConcurrentlyService.consumeMessageDirectly()

public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) {
    ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult();
    //无序
    result.setOrder(false);
    result.setAutoCommit(true);

    List<MessageExt> msgs = new ArrayList<MessageExt>();
    msgs.add(msg);
    MessageQueue mq = new MessageQueue();
    mq.setBrokerName(brokerName);
    mq.setTopic(msg.getTopic());
    mq.setQueueId(msg.getQueueId());

    ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(mq);

    this.resetRetryTopic(msgs);

    final long beginTime = System.currentTimeMillis();

    log.info("consumeMessageDirectly receive new message: {}", msg);

    try {
        //消费消息
        ConsumeConcurrentlyStatus status = this.messageListener.consumeMessage(msgs, context);
        if (status != null) {
            switch (status) {
                case CONSUME_SUCCESS:
                    result.setConsumeResult(CMResult.CR_SUCCESS);
                    break;
                case RECONSUME_LATER:   //延迟消费
                    result.setConsumeResult(CMResult.CR_LATER);
                    break;
                default:
                    break;
            }
        } else {
            result.setConsumeResult(CMResult.CR_RETURN_NULL);
        }
    } catch (Throwable e) {
        result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);
        result.setRemark(RemotingHelper.exceptionSimpleDesc(e));

        log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s",
            RemotingHelper.exceptionSimpleDesc(e),
            ConsumeMessageConcurrentlyService.this.consumerGroup,
            msgs,
            mq), e);
    }

    result.setSpentTimeMills(System.currentTimeMillis() - beginTime);

    log.info("consumeMessageDirectly Result: {}", result);

    return result;
}

有序消费

ConsumeMessageOrderlyService.consumeMessageDirectly()

public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) {
    ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult();
    //有序
    result.setOrder(true);

    List<MessageExt> msgs = new ArrayList<MessageExt>();
    msgs.add(msg);
    MessageQueue mq = new MessageQueue();
    mq.setBrokerName(brokerName);
    mq.setTopic(msg.getTopic());
    mq.setQueueId(msg.getQueueId());

    ConsumeOrderlyContext context = new ConsumeOrderlyContext(mq);

    final long beginTime = System.currentTimeMillis();

    log.info("consumeMessageDirectly receive new message: {}", msg);

    try {
        //消费消息
        ConsumeOrderlyStatus status = this.messageListener.consumeMessage(msgs, context);
        if (status != null) {
            switch (status) {
                case COMMIT:
                    result.setConsumeResult(CMResult.CR_COMMIT);
                    break;
                case ROLLBACK:
                    result.setConsumeResult(CMResult.CR_ROLLBACK);
                    break;
                case SUCCESS:
                    result.setConsumeResult(CMResult.CR_SUCCESS);
                    break;
                case SUSPEND_CURRENT_QUEUE_A_MOMENT:
                    result.setConsumeResult(CMResult.CR_LATER);
                    break;
                default:
                    break;
            }
        } else {
            result.setConsumeResult(CMResult.CR_RETURN_NULL);
        }
    } catch (Throwable e) {
        result.setConsumeResult(CMResult.CR_THROW_EXCEPTION);
        result.setRemark(RemotingHelper.exceptionSimpleDesc(e));

        log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s",
            RemotingHelper.exceptionSimpleDesc(e),
            ConsumeMessageOrderlyService.this.consumerGroup,
            msgs,
            mq), e);
    }

    result.setAutoCommit(context.isAutoCommit());
    result.setSpentTimeMills(System.currentTimeMillis() - beginTime);

    log.info("consumeMessageDirectly Result: {}", result);

    return result;
}