GO kafka sarama demo

引用Golang客户端库Sarama

go get github.com/Shopify/sarama@latest 

• 消费者，Sarama客户提供了AsyncProducer和SyncProdcer两种类型的生产者，这里以SyncProducer为例

package main

import (
	"fmt"
	"github.com/Shopify/sarama"
)

func main() {
	config := sarama.NewConfig()
	// 发送完数据需要leader和follow都确认
  //kafka提供了三种模式
  //NoResponse RequiredAcks = 0：这个代表的就是数据推出的成功与否都与我无关了
  //WaitForLocal RequiredAcks = 1：当local(leader)确认接收成功后，就可以返回了
  //WaitForAll RequiredAcks = -1：当所有的leader和follower都接收成功时，才会返回
	//在生产环境中我们可以选择模式2或者模式3来保证消息的可靠性，具体需要根据业务场景来进行选择，在乎吞吐量就选择模式2，不在乎吞吐量，就选择模式3，要想完全保证数据不丢失就选择模式3是最可靠的。
  config.Producer.RequiredAcks = sarama.WaitForAll
	// 新选出一个partition
	config.Producer.Partitioner = sarama.NewRandomPartitioner
	// 成功交付的消息将在success channel返回
	config.Producer.Return.Successes = true
	// 构造一个消息
	msg := &sarama.ProducerMessage{}
	msg.Topic = "go-chat-message"
	msg.Value = sarama.StringEncoder("测试")
	// 连接kafka
	client, err := sarama.NewSyncProducer([]string{"127.0.0.1:9092"}, config)
	if err != nil {
		fmt.Println("producer closed, err:", err)
		return
	}
	defer client.Close()
	// 发送消息
	pid, offset, err := client.SendMessage(msg)
	if err != nil {
		fmt.Println("send msg failed, err:", err)
		return
	}
	fmt.Printf("pid:%v offset:%v\n", pid, offset)
}

• 消费者，代码示例如下：

package main

import (
	"fmt"
	"github.com/Shopify/sarama"
)

// kafka consumer

func main() {
	consumer, err := sarama.NewConsumer([]string{"127.0.0.1:9092"}, nil)
	if err != nil {
		fmt.Printf("fail to start consumer, err:%v\\n", err)
		return
	}
	// 根据topic取到所有的分区
	partitionList, err := consumer.Partitions("go-chat-message")
	if err != nil {
		fmt.Printf("fail to get list of partition:err%v\\n", err)
		return
	}
	fmt.Printf("get partitions:%v\\n", partitionList)
   var wg sync.WaitGroup
   wg.Add(1)
	// 遍历所有的分区
	for partition := range partitionList {
		// 针对每个分区创建一个对应的分区消费者
		fmt.Printf("create consumer for %v \\n", partition)
		pc, err := consumer.ConsumePartition("go-chat-message", int32(partition), sarama.OffsetNewest)
		if err != nil {
			fmt.Printf("failed to start consumer for partition %d,err:%v\\n", partition, err)
			return
		}
		defer pc.AsyncClose()
		// 异步从每个分区消费信息
		fmt.Printf("star to get message from %v \\n", partition)
	  	go func(sarama.PartitionConsumer) {
      	defer wg.Done()
   		for msg := range pc.Messages() {
            fmt.Printf("Partition:%d Offset:%d Key:%v Value:%v",
               msg.Partition, msg.Offset, msg.Key, msg.Value)
			}
		}(pc)
	}
   wg.Wait()
}