Kafka+SparkStreaming+Zookeeper(ZK存储Offset,解决checkpoint问题)

创建一个topic

./kafka-topics.sh --create --zookeeper 192.168.1.244:2181,192.168.1.245:2181,192.168.1.246:2181 --replication-factor 1
--partitions 1 --topic topic_test_zk_minOffset_zkGroup

查看topic列表

./kafka-topics.sh --list --zookeeper 192.168.1.244:2181,192.168.1.245:2181,192.168.1.246:2181

producer 代码如下

package com.kafka.test;

import java.util.Properties;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

/**
* @author:FengZhen
* @create:2018年8月9日
*/
public class Producer_zk {

  public static void main(String[] args) {
    Properties props = new Properties();
    props.put("bootstrap.servers", "192.168.1.244:6667,192.168.1.247:6667");
    //props.put("zookeeper.connect", "192.168.1.244:2181,192.168.1.245:2181,192.168.1.246:2181");
      props.put("acks", "all");
      props.put("retries", 0);
      props.put("batch.size", 16384);
      props.put("linger.ms", 1);
      props.put("buffer.memory", 33554432);
      props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
      props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

      KafkaProducer<String, String> producer = new KafkaProducer<String, String>(props);
      for (int i = 30; i < 40; i++)
          producer.send(new ProducerRecord<String, String>("topic_test_zk_minOffset_zkGroup", Integer.toString(i), "中文测试-"+Integer.toString(i)));

      producer.close();
  }
  
}

Streaming代码如下

package streaming

import kafka.api.{OffsetRequest, PartitionOffsetRequestInfo, TopicMetadataRequest}
import kafka.common.TopicAndPartition
import kafka.consumer.SimpleConsumer
import kafka.message.MessageAndMetadata
import kafka.serializer.StringDecoder
import kafka.utils.{ZKGroupTopicDirs, ZkUtils}
import org.I0Itec.zkclient.ZkClient
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka.{HasOffsetRanges, KafkaUtils, OffsetRange}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}

object KafkaLog_local_zk_minOffset_zkGroup {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("KafkaLog_local_zk_minOffset_zkGroup").setMaster("local[2]")
    val sc = new SparkContext(conf)
    sc.setLogLevel("WARN")
    val ssc = new StreamingContext(sc, Seconds(5))

    val broker_servers = "192.168.1.244:6667,192.168.1.247:6667"
    val zk_host = "192.168.1.244:2181,192.168.1.245:2181,192.168.1.246:2181"
    //消费的 topic 名字
    val topic : String = "topic_test_zk_minOffset_zkGroup"
    //创建 stream 时使用的 topic 名字集合
    val topics : Set[String] = Set(topic)

    var kafkaParam:Map[String,String] = Map()
    kafkaParam += ("bootstrap.servers" -> broker_servers)
    kafkaParam += ("group.id" -> "test")
    kafkaParam += ("enable.auto.commit" -> "true")
    kafkaParam += ("auto.commit.interval.ms" -> "100")

    //创建一个 ZKGroupTopicDirs 对象，对保存
    val topicDirs = new ZKGroupTopicDirs("topic_test_zk_minOffset_zkGroup_group", topic)

    //获取 zookeeper 中的路径，这里会变成 /consumers/test_spark_streaming_group/offsets/topic_name
    // /consumers/topic_test_zk_minOffset_zkGroup_group/offsets/topic_test_zk_minOffset_zkGroup/0
    val zkTopicPath = s"${topicDirs.consumerOffsetDir}"

    //zookeeper 的host 和 ip，创建一个 client
    val zkClient = new ZkClient(zk_host)
    //查询该路径下是否字节点（默认有字节点为我们自己保存不同 partition 时生成的）
    val children = zkClient.countChildren(zkTopicPath)

    var kafkaStream : InputDStream[(String, String)] = null

    //如果 zookeeper 中有保存 offset，我们会利用这个 offset 作为 kafkaStream 的起始位置
    var fromOffsets: Map[TopicAndPartition, Long] = Map()

    //如果保存过 offset，这里更好的做法，还应该和  kafka 上最小的 offset 做对比，不然会报 OutOfRange 的错误
    if (children > 0) {
      for (i <- 0 until children) {
        val topic2 = List(topic)
        val req = new TopicMetadataRequest(topic2, 0)
        // 第一个参数是 kafka broker 的host，第二个是 port
        val getLeaderConsumer = new SimpleConsumer("192.168.1.244", 6667, 10000, 10000, "OffsetLookup")
        val res = getLeaderConsumer.send(req)
        val topicMetaOption = res.topicsMetadata.headOption
        val partitions = topicMetaOption match {
          // 将结果转化为 partition -> leader 的映射关系
          case Some(tm) =>
            tm.partitionsMetadata.map(pm => (pm.partitionId, pm.leader.get.host)).toMap[Int, String]
          case None =>
            Map[Int, String]()
        }
        //去出分片对应的leader host
        val brokerLeaderHost = partitions.get(i).toString.replace("Some(", "").replace(")","")

        val partitionOffset = zkClient.readData[String](s"${zkTopicPath}/${i}")
        val tp = TopicAndPartition(topic, i)

        val requestMin = OffsetRequest(Map(tp -> PartitionOffsetRequestInfo(OffsetRequest.EarliestTime, 1)))
        val consumerMin = new SimpleConsumer(brokerLeaderHost, 6667, 10000, 10000, "getMinOffset")
        val curOffsets = consumerMin.getOffsetsBefore(requestMin).partitionErrorAndOffsets(tp).offsets
        var nextOffset = partitionOffset.toLong
        // 通过比较从 kafka 上该 partition 的最小 offset 和 zk 上保存的 offset，进行选择
        if (curOffsets.length > 0 && nextOffset < curOffsets.head) {
          nextOffset = curOffsets.head
        }
        //设置正确的 offset，这里将 nextOffset 设置为 0（0 只是一个特殊值），可以观察到 offset 过期的想想
        fromOffsets += (tp -> nextOffset)
        println("@@@@@@ topic[" + topic + "] partition[" + i + "] offset[" + partitionOffset + "] @@@@@@")
      }

      //这个会将 kafka 的消息进行 transform，最终 kafak 的数据都会变成 (topic_name, message) 这样的 tuple
      val messageHandler = (mmd : MessageAndMetadata[String, String]) => (mmd.topic, mmd.message())
      kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder, (String, String)](ssc, kafkaParam, fromOffsets, messageHandler)
    }
    else {
      //如果未保存，根据 kafkaParam 的配置使用最新或者最旧的 offset
      kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParam, topics)
    }

    var offsetRanges = Array[OffsetRange]()
    //得到该 rdd 对应 kafka 的消息的 offset
    kafkaStream.transform{ rdd =>
      offsetRanges = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
      rdd
    }.foreachRDD { rdd => //.map(msg => Utils.msgDecode(msg))
      for (o <- offsetRanges) {
        val zkPath = s"${zkTopicPath}/${o.partition}"
        //将该 partition 的 offset 保存到 zookeeper
        ZkUtils.updatePersistentPath(zkClient, zkPath, o.fromOffset.toString)
        println(s"@@@@@@ topic  ${o.topic}  partition ${o.partition}  fromoffset ${o.fromOffset}  untiloffset ${o.untilOffset} #######")
      }
      rdd.foreachPartition(
        message => {
          while(message.hasNext) {
            println(s"@^_^@   [" + message.next() + "] @^_^@")
          }
        }
      )
    }
    //开启流式计算
    ssc.start()
    //一直会阻塞，等待退出
    ssc.awaitTermination()
  }
}

 

出现的问题

使用simpleConsumer时报错

Exception in thread "main" java.nio.channels.ClosedChannelException
  at kafka.network.BlockingChannel.send(BlockingChannel.scala:100)
  at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:78)
  at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:68)
  at kafka.consumer.SimpleConsumer.getOffsetsBefore(SimpleConsumer.scala:127)
  at streaming.KafkaLog_local_zk_minOffset$$anonfun$main$1.apply$mcVI$sp(KafkaLog_local_zk_minOffset.scala:64)
  at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:160)
  at streaming.KafkaLog_local_zk_minOffset$.main(KafkaLog_local_zk_minOffset.scala:44)
  at streaming.KafkaLog_local_zk_minOffset.main(KafkaLog_local_zk_minOffset.scala)

解决将Kafka config下的server.properties的参数修改下

num.network.threads=3
zookeeper.connection.timeout.ms=6000

　再次尝试即可.