flink run \
-m yarn-cluster \
-ys 2 \
-yjm 2g \
-ytm 4g \
-c com.xxxxx.flink.app.incr.TradeOrderBinlogResolveApp \
-d \
/opt/tools/flink-1.12.0/xxxxx-realtime-etl-1.0-SNAPSHOT.jar
package com.xxxxx.flink.app.incr;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.xxxxx.flink.contranst.TopicPattern;
import com.xxxxx.flink.executions.TradeOrderExecutions;
import com.xxxxx.flink.function.RowsFlatMapFunction;
import com.xxxxx.flink.schema.FlatMessageSchema;
import com.xxxxx.flink.utils.ConfigUtils;
import com.xxxxx.flink.utils.TableResolveUtils;
import com.xxxxx.flink.watermark.RowWatermarkAssigner;
import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.java.typeutils.RowTypeInfo;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.SqlDialect;
import org.apache.flink.table.api.StatementSet;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.catalog.hive.HiveCatalog;
import org.apache.flink.types.Row;
import java.time.Duration;
public class TradeOrderBinlogResolveApp {
public static void main(String[] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(8);
env.enableCheckpointing(60000);
env.getConfig().setAutoWatermarkInterval(200);
EnvironmentSettings tableEnvSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build();
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,tableEnvSettings);
tableEnv.getConfig().getConfiguration().set(ExecutionCheckpointingOptions.CHECKPOINTING_MODE, CheckpointingMode.EXACTLY_ONCE);
tableEnv.getConfig().getConfiguration().set(ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL,
Duration.ofMinutes(1));
String catalogName = "devHive";
HiveCatalog hiveCatalog = new HiveCatalog(catalogName,
"default",
ConfigUtils.HIVE_CONF_DIR,
ConfigUtils.HADOOP_CONF_DIR,
ConfigUtils.HIVE_VERSION
);
tableEnv.registerCatalog(catalogName,hiveCatalog);
tableEnv.useCatalog(catalogName);
RowTypeInfo tradeOrderTypes = TableResolveUtils.getRowTypeinformations("ods.trade_order_incr",tableEnv);
RowTypeInfo tradeOrderItemTypes = TableResolveUtils.getRowTypeinformations("ods.trade_order_item_incr",tableEnv);
RowTypeInfo tradeRealDeliveryTypes = TableResolveUtils.getRowTypeinformations("ods.trade_real_delivery_incr",tableEnv);
RowTypeInfo tradeSteelItemTypes = TableResolveUtils.getRowTypeinformations("ods.trade_steel_item_incr",tableEnv);
FlinkKafkaConsumerBase<FlatMessage> messages = new FlinkKafkaConsumer<>(TopicPattern.TRADE_PATTERN,
new FlatMessageSchema(),
ConfigUtils.getKafkaConfig())
.setStartFromEarliest();
FlinkKafkaConsumerBase<FlatMessage> messagesWaters = messages.assignTimestampsAndWatermarks(
WatermarkStrategy.<FlatMessage>forBoundedOutOfOrderness(Duration.ofSeconds(5))
.withTimestampAssigner(
new SerializableTimestampAssigner<FlatMessage>() {
@Override
public long extractTimestamp(FlatMessage element, long recordTimestamp) {
return element.getEs();
}
}
)
);
DataStreamSource<FlatMessage> messageSources = env.addSource(messagesWaters);
SingleOutputStreamOperator<Row> tradeOrder = messageSources.filter(x -> "trade_order".equalsIgnoreCase(x.getTable()))
.flatMap(new RowsFlatMapFunction(), tradeOrderTypes);
SingleOutputStreamOperator<Row> tradeOrderItem =
messageSources.filter(x ->
"trade_order_item".equalsIgnoreCase(x.getTable()))
.flatMap(new RowsFlatMapFunction(), tradeOrderItemTypes);
SingleOutputStreamOperator<Row> tradeRealDelivery =
messageSources.filter(x ->
"trade_real_delivery".equalsIgnoreCase(x.getTable()))
.flatMap(new RowsFlatMapFunction(), tradeRealDeliveryTypes);
SingleOutputStreamOperator<Row> tradeSteelItem =
messageSources.filter(x ->
"trade_steel_item".equalsIgnoreCase(x.getTable()))
.flatMap(new RowsFlatMapFunction(), tradeSteelItemTypes);
tableEnv.createTemporaryView("trade_order_tmp",tradeOrder);
tableEnv.createTemporaryView("trade_order_item_tmp",tradeOrderItem);
tableEnv.createTemporaryView("trade_real_delivery_tmp",tradeRealDelivery);
tableEnv.createTemporaryView("trade_steel_item_tmp",tradeSteelItem);
StatementSet statementSql =
TradeOrderExecutions.getStatementSql(tableEnv);
TableResult execute = statementSql.execute();
try {
env.execute();
} catch (Exception e) {
e.printStackTrace();
}
}
}