<dependency>
<groupId>com.aliyun.datahub</groupId>
<artifactId>aliyun-sdk-datahub</artifactId>
<version>2.17.1-public</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.aliyun.datahub</groupId>
<artifactId>datahub-client-library</artifactId>
<version>1.1.12-public</version>
</dependency>
协同写入数据
写入工具类
package com.peony.common.util;
import com.alibaba.fastjson.JSON;
import lombok.Data;
import lombok.experimental.Accessors;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
/**
* @projectName: server
* @package: com.peony.common.util
* @className: DatahubUtil
* @author: zy
* @description:
* @date: 2022-04-27 15:57
*/
@Data
@Accessors(chain = true)
public class DatahubUtil {
/**
* 目前只支持两级继承关系，否则属性丢失
* @param o
* @return
*/
public static List<PutOne> objToPutOneList(Object o){
ArrayList<PutOne> list = new ArrayList<>();
Field[] fields = o.getClass().getDeclaredFields();
if(fields.length>0){
for(Field f:fields){
try{
PutOne putOne = new PutOne();
f.setAccessible(true);
String name = f.getName();
putOne.setKey(name);
Object value = f.get(o);
putOne.setValue(value);
list.add(putOne);
}catch (Exception e){
throw new RuntimeException("数据获取错误！"+ JSON.toJSONString(o));
}
}
}
//判断父类
Class<?> aClass = o.getClass().getSuperclass();
if(!aClass.equals(Object.class)){
Field[] fieldsSuper = aClass.getDeclaredFields();
if(fieldsSuper.length>0){
for(Field f:fieldsSuper){
try{
PutOne putOne = new PutOne();
f.setAccessible(true);
String name = f.getName();
putOne.setKey(name);
Object value = f.get(o);
putOne.setValue(value);
list.add(putOne);
}catch (Exception e){
throw new RuntimeException("数据获取错误！"+ JSON.toJSONString(o));
}
}
}
}
return list;
}
@Data
@Accessors(chain = true)
public static class PutOne{
private String key;
private Object value;
}
}
客户端对象配置
@Bean
public DatahubClient datahubClient(){
String endpoint = "xxx";
String accessId = "xxx";
String accessKey = "xxx";
DatahubClient datahubClient = DatahubClientBuilder.newBuilder().setDatahubConfig(
new DatahubConfig(endpoint, new AliyunAccount(accessId, accessKey),
true)).setHttpConfig(new HttpConfig()
.setCompressType(HttpConfig.CompressType.LZ4)
.setConnTimeout(10000))
.build();
return datahubClient;
}
协同写入案例
private static int pushId = 0;
private static int totalCount = 5;
public boolean putDataDisposeInfo(PutDataDisposeInfo info){
return putData(DatahubUtil.objToPutOneList(info));
}
public boolean putData( List<DatahubUtil.PutOne> putOnes) {
String project ="hb_ryjj_test";
String topic ="dispose_info";
// 获取schema
RecordSchema recordSchema = datahubClient.getTopic(project,topic ).getRecordSchema();
List<RecordEntry> recordEntries = new ArrayList<>();
RecordEntry recordEntry = new RecordEntry();
TupleRecordData data = new TupleRecordData(recordSchema);
Optional.ofNullable(putOnes).orElseThrow(()->new SupportPersonException("发送数据不能为空！")).forEach(m->{
data.setField(m.getKey(),m.getValue());
});
recordEntry.setRecordData(data);
recordEntries.add(recordEntry);
String pushIdForString = pushId % totalCount + "";
try {
datahubClient.putRecordsByShard(project, topic, pushIdForString, recordEntries);
pushId ++ ;
if(pushId > Integer.MAX_VALUE - 1) {
pushId = 0;
}
return true;
} catch (InvalidParameterException e) {
throw new SupportPersonException("无效参数，请检查您的参数");
} catch (AuthorizationFailureException e) {
throw new SupportPersonException("AK错误，请检查您的accessId和accessKey");
} catch (ResourceNotFoundException e) {
throw new SupportPersonException("project、 topic未找到");
} catch (ShardSealedException e) {
throw new SupportPersonException("状态已关闭，无法写入");
} catch (DatahubClientException e) {
throw new SupportPersonException("其他异常");
}
}
读取数据案例
@Async
public void example() {
String endpoint = "xxx";
String accessId = "xxx";
String accessKey = "xxx";
String projectName ="hb_ryjj_test";
String topicName = "warn_info";
String subId = "xxxx";
// 1. 使用协同消费，subId
ConsumerConfig config = new ConsumerConfig(endpoint, accessId, accessKey);
Consumer consumer = new Consumer(projectName, topicName, subId, config);
int maxRetry = 3;
boolean stop = false;
try {
while (!stop) {
try {
while (true) {
RecordEntry record = consumer.read(maxRetry);
// 处理数据
if (record != null) {
TupleRecordData data = (TupleRecordData) record.getRecordData();
Object field0 = data.getField("id");
log.info(String.valueOf(field0));
Object field1 = data.getField("service_id");
log.info(String.valueOf(field1));
Object field2 = data.getField("warn_type");
log.info(String.valueOf(field2));
Object value = data.getField("value");
if(value!=null){
WarnInfo warnInfo = JSON.parseObject(value.toString(), WarnInfo.class);
log.info("value="+value);
}
record.getKey().ack();
} else {
log.info("read null");
}
}
} catch (SubscriptionOffsetResetException e) {
// 点位被重置，重新初始化consumer
try {
consumer.close();
consumer = new Consumer(projectName, topicName, subId, config);
} catch (DatahubClientException e1) {
log.info("create consumer failed");
throw e;
}
} catch (InvalidParameterException |
SubscriptionOfflineException |
SubscriptionSessionInvalidException |
AuthorizationFailureException |
NoPermissionException e) {
log.info("read failed请求参数非法/订阅被下线/订阅下相同shard被其他客户端占用/签名不正确/没有权限");
throw e;
} catch (DatahubClientException e) {
// 基类异常，包含网络问题等，可以选择重试
log.info("read failed, retry");
}
}
} catch (Throwable e) {
e.fillInStackTrace();
} finally {
log.info("确保资源正确释放");
consumer.close();
}
}
项目启动调用读取方法
@Component
public class RunConfig {
@Resource
private DatahubServiceImpl datahubService;
@PostConstruct
public void init() {
try{
datahubService.example();
}catch (Exception e){
e.getMessage();
}
}
}
