文章目录


依赖

<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-scala_2.11</artifactId>
    <version>1.7.0</version>
</dependency>

<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-streaming-scala_2.11</artifactId>
    <version>1.7.0</version>
</dependency>
<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-connector-kafka-0.11_2.11</artifactId>
    <version>1.7.0</version>
</dependency>

kafka的source

package kafka

import java.util.Properties

import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.connectors.kafka.{FlinkKafka011ErrorCode, FlinkKafkaConsumer011}

object KafKaUtil {

  val prop = new Properties()
  prop.setProperty("bootstrap.servers", "note01:9092,note02:9092,note03:9092")
  prop.setProperty("group.id", "test")

  def getConsumer(topic: String): FlinkKafkaConsumer011[String] ={
    val myKafkaConsumer: FlinkKafkaConsumer011[String] = new FlinkKafkaConsumer011[String](topic,new SimpleStringSchema(),prop)
    myKafkaConsumer
  }
}

kafka的消费

package kafka

import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.api.scala._
object ConsumerApp {
  def main(args: Array[String]): Unit = {
      val environment = StreamExecutionEnvironment.getExecutionEnvironment
      val kafkaConsumer = KafKaUtil.getConsumer("test")

    //import org.apache.flink.api.scala._ 这里要加入隐式转换
    val dstream = environment.addSource(kafkaConsumer)

    dstream.print()
    environment.execute()
  }
}

Flink+kafka是如何实现exactly-once语义的

Flink通过checkpoint来保存数据是否处理完成的状态

由JobManager协调各个TaskManager进行checkpoint存储，checkpoint保存在 StateBackend中，默认StateBackend是内存级的，也可以改为文件级的进行持久化保存。

执行过程实际上是一个两段式提交，每个算子执行完成，会进行“预提交”，直到执行完sink操作，会发起“确认提交”，如果执行失败，预提交会放弃掉。

如果宕机需要通过StateBackend进行恢复，只能恢复所有确认提交的操作。(9)flink的kafka的结合_kafka