之前例子中存在的问题
在之前的例子中,使用了checkpoint,虽然异常重启后可以继续失败前sum值继续运算,但是数据源的消费位置是从头开始,这不是我们想要的,要实现重启后消费位置offset是继重启前的offset,那么需要source端有CheckpointedFunction 的支持
MySource
public class MySource implements SourceFunction<Tuple3<String, Long, Long>> , CheckpointedFunction {
private static final Logger logger = LoggerFactory.getLogger(MySource.class);
// 存消费位置offset的状态值
// private ValueState<Long> offsetState;//不能是ValueState<Long>吗,只能ListState?ValueState属于keyState,针对的是业务的
private ListState<Long> offsetState;
// offset的状态名
private static final String OFFSET_STATE_NAME = "offset-state";
private long index;//这里是long类型,初始值是0
private volatile boolean running = true;
@Override
public void run(SourceContext<Tuple3<String, Long, Long>> sourceContext) throws Exception {
while (true) {
sourceContext.collect(new Tuple3<>("key", ++index, System.currentTimeMillis()));
Thread.sleep(100);
}
}
@Override
public void cancel() {}
@Override
public void snapshotState(FunctionSnapshotContext functionSnapshotContext) throws Exception {
// 快照,checkpoint会进来
// 这里要记录消费的offset
if(running){
//清空并设置状态
this.offsetState.clear();
// this.offsetState.update(index);
this.offsetState.add(index);
}else{
logger.error("停止的source被调用snapshotState方法");
}
}
@Override
public void initializeState(FunctionInitializationContext ctx) throws Exception {
// job第一次或者异常恢复都会进入,设置状态
this.offsetState = ctx.getOperatorStateStore().getListState(new ListStateDescriptor<Long>(OFFSET_STATE_NAME, Types.LONG));
// ctx.getKeyedStateStore()为空
// ValueState<Long> state = ctx.getKeyedStateStore().getState(new ValueStateDescriptor<>(OFFSET_STATE_NAME, Types.LONG));
Iterable<Long> longs = this.offsetState.get();
for (Long offsetTemp : longs){
//其实只有一个元素
this.index = offsetTemp;
}
}
}main
public class RestartWithSourceOffect {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment streamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
// 1 固定允许重启次数,延迟重启
streamEnv.setRestartStrategy(RestartStrategies.fixedDelayRestart(2, Time.seconds(3)));
// 2 启动checkpoint,默认存在JM内存中,升级时,如果JM退出,则checkpoint也没了,所以可以通过设置StateBackend,并设置job取消后不删除checkpoint
streamEnv.enableCheckpointing(20);
streamEnv.setParallelism(1);
//source
DataStreamSource<Tuple3<String, Long, Long>> sourceData = streamEnv.addSource(new MySource());
sourceData.map(new MapFunction<Tuple3<String, Long, Long>, Tuple3<String, Long, Long>>() {
@Override
public Tuple3<String, Long, Long> map(Tuple3<String, Long, Long> item) throws Exception {
//根据一定条件抛出异常
if(item.f1 % 5 == 0){
throw new Exception(String.format("数据%d出错",item.f1));
}
return item;
}
}).keyBy(new KeySelector<Tuple3<String, Long, Long>, String>() {
@Override
public String getKey(Tuple3<String, Long, Long> s) throws Exception {
return s.f0;
}
}).sum(1)
.print();
streamEnv.execute();
}
}上面的例子
在checkpoint的时候会触发CheckpointedFunction 的snapshotState方法,进入存储offset状态值,
在重启恢复的时候触发CheckpointedFunction 的initializeState方法,取出状态offset,再生产出数据
所以结果是类似如下
(key,1,1599707659444)
(key,3,1599707659444)
(key,6,1599707659444)
(key,10,1599707659444)
java.lang.Exception: 数据5出错
java.lang.Exception: 数据5出错
java.lang.Exception: 数据5出错 // 上面代码只允许出错两次,否则不再重启
重启后还是在offset为5的位置开始生产数据
为了演示效果,可以在initializeState中,跳过错误数据,5 10 ..
在initializeState方法中增加跳过错误数据逻辑
@Override
public void initializeState(FunctionInitializationContext ctx) throws Exception {
// job第一次或者异常恢复都会进入,设置状态
this.offsetState = ctx.getOperatorStateStore().getListState(new ListStateDescriptor<Long>(OFFSET_STATE_NAME, Types.LONG));
// ctx.getKeyedStateStore()为空
// ValueState<Long> state = ctx.getKeyedStateStore().getState(new ValueStateDescriptor<>(OFFSET_STATE_NAME, Types.LONG));
Iterable<Long> longs = this.offsetState.get();
for (Long offsetTemp : longs){
//其实只有一个元素
this.index = offsetTemp;
if(this.index == 4 || this.index == 9){
this.index++;
}
}
}改动后的结果
(key,1,1599710569964)
(key,3,1599710569964)
(key,6,1599710569964)
(key,10,1599710569964)
java.lang.Exception: 数据5出错
(key,16,1599710569964)
(key,23,1599710569964)
(key,31,1599710569964)
(key,40,1599710569964)
java.lang.Exception: 数据10出错
(key,51,1599710569964)
(key,63,1599710569964)
(key,76,1599710569964)
(key,90,1599710569964)
java.lang.Exception: 数据15出错
java.lang.Exception: 数据15出错
java.lang.Exception: 数据15出错
多并发
如果修改代码
streamEnv.setParallelism(2);
则source需要多并发支持
主要改变地方
1 继承RichParallelSourceFunction
2 增加indexOfTask变量
3 并发不同的slot的run方法中生成数据速度不一样
4 重启的时候打印出task编号和对应的消费offset
public class MySourceParallel
extends RichParallelSourceFunction<Tuple3<String, Long, Long>> //改成支持多并发的SourceFunction
implements CheckpointedFunction {
private static final Logger logger = LoggerFactory.getLogger(MySourceParallel.class);
// 存消费位置offset的状态值
// private ValueState<Long> offsetState;//不能是ValueState<Long>吗,只能ListState?ValueState属于keyState,针对的是业务的
private ListState<Long> offsetState;
// offset的状态名
private static final String OFFSET_STATE_NAME = "offset-state";
private long index;//这里是long类型,初始值是0
private volatile boolean running = true;
// 当前任务实例的编号
private int indexOfTask;
@Override
public void run(SourceContext<Tuple3<String, Long, Long>> sourceContext) throws Exception {
while (true) {
sourceContext.collect(new Tuple3<>("key", ++index, System.currentTimeMillis()));
Thread.sleep(100*(indexOfTask+1));
}
}
@Override
public void cancel() {}
@Override
public void snapshotState(FunctionSnapshotContext functionSnapshotContext) throws Exception {
// 快照,checkpoint会进来
// 这里要记录消费的offset
if(running){
//清空并设置状态
this.offsetState.clear();
// this.offsetState.update(index);
this.offsetState.add(index);
}else{
logger.error("停止的source被调用snapshotState方法");
}
}
@Override
public void initializeState(FunctionInitializationContext ctx) throws Exception {
//设置当前task实例编号
this.indexOfTask = getRuntimeContext().getIndexOfThisSubtask();
// job第一次或者异常恢复都会进入,设置状态
this.offsetState = ctx.getOperatorStateStore().getListState(new ListStateDescriptor<Long>(OFFSET_STATE_NAME, Types.LONG));
// ctx.getKeyedStateStore()为空
// ValueState<Long> state = ctx.getKeyedStateStore().getState(new ValueStateDescriptor<>(OFFSET_STATE_NAME, Types.LONG));
Iterable<Long> longs = this.offsetState.get();
for (Long offsetTemp : longs){
//其实只有一个元素
this.index = offsetTemp;
if(this.index == 4 || this.index == 9){
this.index++;
}
logger.error(String.format("重启task编号:%d offset:%d",this.indexOfTask,this.index));
}
}
}结果:
2> (key,1,1599718871857)
2> (key,2,1599718871857)
2> (key,4,1599718871857)
2> (key,7,1599718871857)
2> (key,9,1599718871857)
2> (key,13,1599718871857)
重启task编号:0 offset:5
重启task编号:1 offset:2
2> (key,19,1599718871857)
2> (key,22,1599718871857)
2> (key,29,1599718871857)
2> (key,37,1599718871857)
2> (key,41,1599718871857)
2> (key,50,1599718871857)
重启task编号:1 offset:5
重启task编号:0 offset:10
2> (key,61,1599718871857)
2> (key,67,1599718871857)
2> (key,79,1599718871857)
2> (key,92,1599718871857)
2> (key,99,1599718871857)
2> (key,113,1599718871857)
重启task编号:0 offset:14
重启task编号:1 offset:7
2> (key,121,1599718871857)
