flatmap
package pers.aishuang.flink.streaming.function.flatmap;
import org.apache.flink.streaming.api.functions.co.RichCoFlatMapFunction;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pers.aishuang.flink.streaming.entity.OnlineDataObj;
import pers.aishuang.flink.streaming.entity.VehicleInfoModel;
import pers.aishuang.flink.streaming.utils.DateUtil;
import java.util.HashMap;
public class VehicleInfoMapMysqlFunction extends RichCoFlatMapFunction<
OnlineDataObj, HashMap<String,VehicleInfoModel>, OnlineDataObj> {
//创建日志打印器
private static final Logger logger = LoggerFactory.getLogger(VehicleInfoMapMysqlFunction.class);
//定义广播变量
private static HashMap<String, VehicleInfoModel> vehicleInfoModelConfig = null;
@Override
public void flatMap1(OnlineDataObj value, Collector<OnlineDataObj> out) throws Exception {
//1. 通过vin 获取到车辆基础信息
String vin = value.getVin();
VehicleInfoModel vehicleInfoModel = vehicleInfoModelConfig.getOrDefault(vin, null);
//2. 如果车辆基础信息不为空
if(vehicleInfoModel != null) {
//--将车系seriesName、车型modelName、车限liveTime、销售日期saleDate，车辆类型carType封装到onlineDataObj对象中
value.setModelName(vehicleInfoModel.getModelName());
value.setSeriesName(vehicleInfoModel.getSeriesName());
value.setLiveTime(
(System.currentTimeMillis() - DateUtil.convertStringToDateTime(vehicleInfoModel.getSalesDate())
.getTime()) / 1000 / 3600 / 24 / 30 + ""
);
value.setSalesDate(vehicleInfoModel.getSalesDate());
value.setCarType(vehicleInfoModel.getCarType());
//-- 将onlineDataObj收集返回
out.collect(value);
//-- 打印输出，基础信息不存在
}else {
logger.error("当前上报车辆，未在库里记录");
}
}
@Override
public void flatMap2(HashMap<String, VehicleInfoModel> value, Collector<OnlineDataObj> out) throws Exception {
vehicleInfoModelConfig = value;
}
}map
package pers.aishuang.flink.streaming.function.map;
import com.alibaba.fastjson.JSON;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.hadoop.hbase.util.Bytes;
import pers.aishuang.flink.streaming.entity.ItcastDataPartObj;
import pers.aishuang.flink.streaming.utils.GeoHashUtil;
import pers.aishuang.flink.streaming.utils.RedisUtil;
import pers.aishuang.flink.streaming.entity.VehicleLocationModel;
/**
* 经纬度坐标只能表示一块区域，实际生活中不存在点，因为点的大小可以无穷小
* geoHash算法原理：
* 将经度和纬度值转化为2进制数字，以纬度在奇数位，经度在偶数位，将二者穿插合并成一个2进制数字
* 再基于base32编码将其转成字符串，字符串越长，位置越精准
*/
public class LocactionInfoReidsFunction extends RichMapFunction<ItcastDataPartObj,ItcastDataPartObj> {
@Override
public ItcastDataPartObj map(ItcastDataPartObj value) throws Exception {
//1. 获取车辆数据的经度和纬度生成 geohash（经度，纬度）-> geohash字符串 -> 地理详细位置
Double lng = value.getLng();
Double lat = value.getLat();
String geohash = GeoHashUtil.encode(lat, lng);
//2. 根据geohash 从redis中获取value值（geohash在redis中是作为主键存在）
byte[] locationDetailArr = RedisUtil.get(Bytes.toBytes(geohash));
//3. 如果查询出来的值不为空，将其通过JSON对象转换成VehicleLocationModel对象，否则置为null
if(locationDetailArr != null && locationDetailArr.length > 0) {
String vehicleLocationJson = Bytes.toString(locationDetailArr);
VehicleLocationModel model = JSON.parseObject(vehicleLocationJson,VehicleLocationModel.class);
//4. 如果当前对象不为空，将国家，省市区地址赋给itcastDataPartObj,否则置为null
if(model != null) {
value.setCountry(model.getCountry());
value.setProvince(model.getProvince());
value.setCity(model.getCity());
value.setDistrict(model.getDistrict());
value.setAddress(model.getAddress());
} else {
value.setCountry(null);
value.setProvince(null);
value.setCity(null);
value.setDistrict(null);
value.setAddress(null);
}
}else {
value.setCountry(null);
value.setProvince(null);
value.setCity(null);
value.setDistrict(null);
value.setAddress(null);
}
//5. 返回数据
return value;
}
}window
package pers.aishuang.flink.streaming.function.window;
import com.clearspring.analytics.util.Lists;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.flink.api.common.state.MapState;
import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.state.StateTtlConfig;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pers.aishuang.flink.streaming.entity.ElectricFenceModel;
import java.util.Collections;
import java.util.List;
/**
* 考虑问题：时间滚动窗口，有的窗口没有数据，flink是怎么处理的
*/
public class ElectricFenceWindowFunction extends RichWindowFunction<ElectricFenceModel, ElectricFenceModel,
String , TimeWindow> {
//创建logger
private static final Logger logger = LoggerFactory.getLogger(ElectricFenceWindowFunction.class);
//1.定义存储历史电子围栏数据的state，<vin，是否在电子围栏内0：内， 1：外> MapState<String, Integer>
MapState<String, Byte> lastState = null;
@Override
public void open(Configuration parameters) throws Exception {
//1. 定义mapState的描述器，就是定义state的结构
MapStateDescriptor<String, Byte> lastStateDesc = new MapStateDescriptor<String, Byte>("lastState",String.class,Byte.class);
//2. 获取parameterTool，用来读取配置文件参数
StateTtlConfig stateTtlConfig = StateTtlConfig
//设置状态有效时间
.newBuilder(Time.seconds(180))
//设置状态更新类型：更新
.setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
//设置已过期但还未被清理的状态如何处理
.setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)
//过期对象的清理策略
.cleanupFullSnapshot()
.build();
//3. 设置state的ttl
lastStateDesc.enableTimeToLive(stateTtlConfig);
//4. 获取当前运行环境 所描述的那个MapState
lastState = getRuntimeContext().getMapState(lastStateDesc);
}
@Override
public void close() throws Exception {
super.close();
}
@Override
public void apply(String key, TimeWindow window, Iterable<ElectricFenceModel> input, Collector<ElectricFenceModel> out) throws Exception {
//1. 创建返回对象
ElectricFenceModel electricFenceModel = new ElectricFenceModel();
//2. 对窗口内的数据进行排序(按“终端时间戳”属性进行升序排序)
List<ElectricFenceModel> electricFenceModelList = Lists.newArrayList(input);
Collections.sort(electricFenceModelList);
Collections.reverse(electricFenceModelList);
//3. 从state中获取车辆vin对应的上一次窗口电子围栏lastStateValue标记（车辆上一次窗口是否在电子围栏中）0：电子围栏内，1：电子围栏外
Byte lastStateValue = lastState.get(key);
//4. 如果上次状态为空，初始化赋值
if(lastStateValue == null ){
lastStateValue = -1;
}
//5. 判断当前处于电子围栏内还是电子围栏外
//-- 定义当前车辆电子围栏圈内出现的次数
long inElectricFence = electricFenceModelList.stream()
.filter(model -> model.getNowStatus() == 0)
.count();
//-- 定义当前车辆电子围栏圈外出现的次数
long outElectricFence = electricFenceModelList.stream()
.filter(model -> model.getNowStatus() == 1)
.count();
//6. 定义当前窗口的电子围栏状态，默认让当前状态为1，在电子围栏外
byte currentState = 1;
//7. 90s内车辆出现在电子围栏内的次数多于出现在电子围栏外的次数，则认为当前处于电子围栏内
if(inElectricFence >= outElectricFence){
currentState = 0;
}
//8. 将当前窗口的电子围栏状态写入到state中，供下次判断
lastState.put(key, currentState);
//9. 如果当前电子围栏状态与上一次电子围栏状态不同
//-- 如果上一次窗口处于电子围栏外，而本次是电子围栏内，则将进入电子围栏的时间写入到数据库中
if(lastStateValue == 1 && currentState == 0) {
//-- 进入栅栏，找到最后一条在外面的数据
//-- 出去栅栏，找到最后一条在里面的数据
ElectricFenceModel lastOutEleModel = electricFenceModelList
.stream()
.filter(model -> model.getNowStatus() == 1)
.findFirst()
.get();
//-- 拷贝属性给 electricFenceModel 并将进围栏终端时间赋值，并且将状态告警字段赋值为1 0：出围栏 1：进围栏，将数据collect返回
BeanUtils.copyProperties(electricFenceModel, lastOutEleModel);
electricFenceModel.setInEleTime(lastOutEleModel.getGpsTime());
electricFenceModel.setStatusAlarm(1);
out.collect(electricFenceModel);
} else if(lastStateValue == 0 && currentState == 1) {
//如果上一次窗口处于电子围栏内，而本次是电子围栏外
//-- 进入栅栏，找到最后一条在外面的数据
//-- 出去栅栏，找到最后一条在里面的数据
ElectricFenceModel lastInEleModel = electricFenceModelList
.stream()
.filter(model -> model.getNowStatus() == 0)
.findFirst()
.get();
//-- 拷贝属性给electricFenceModel 并将出栅栏终端时间赋值，并且将状态告警 0：出围栏，1：进围栏，将数据collect返回
BeanUtils.copyProperties(electricFenceModel,lastInEleModel);
electricFenceModel.setOutEleTime(lastInEleModel.getGpsTime());
electricFenceModel.setStatusAlarm(0);
out.collect(electricFenceModel);
}
}
}package pers.aishuang.flink.streaming.function.window;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;
import pers.aishuang.flink.streaming.entity.ItcastDataPartObj;
import pers.aishuang.flink.streaming.entity.OnlineDataObj;
import pers.aishuang.flink.streaming.utils.DateUtil;
import java.util.ArrayList;
import java.util.Collections;
/**
* 对30s窗口内的数据生成一个新的对象，包含拉宽数据地理信息，实时上报数据
* 数据库中的静态数据 OnlineDataObj
* 开发步骤：
* 1、对数据进行排序
* 2、获取每一条数据
* 3、对每条数据判断是否有告警信息，alarm=1 说明是告警信息
* 4、封装成对象OnlineDataObj
* 5、返回
*/
//http请求超时时间和 redis会话超时都要小于水印乱序时间
public class OnlineStatisticsWindowFunction extends RichWindowFunction<
ItcastDataPartObj, OnlineDataObj,
String, TimeWindow> {
@Override
public void apply(String key, TimeWindow window, Iterable<ItcastDataPartObj> input, Collector<OnlineDataObj> out) throws Exception {
OnlineDataObj onlineDataObj = null;
//1. 对当前的数据集合进行升序排序
ArrayList<ItcastDataPartObj> itcastDataPartObjList = Lists.newArrayList(input);//guava的Lists工具类
Collections.sort(itcastDataPartObjList);
//第二种写法：
//Collections.sort(itcastDataPartObjList, Comparator.comparingLong(ItcastDataPartObj::getTerminalTimeStamp));
//要想降序的话，直接再反转就行：Collections.reverse(itcastDataPartObjList);
//2. 获取集合中第一条数据
ItcastDataPartObj firstItcastData = itcastDataPartObjList.get(0);
//3. 循环遍历每条数据，将集合中存在异常的数据拼接到指定属性中
for(ItcastDataPartObj itcastDataPartObj : itcastDataPartObjList) {
//30s窗口最多6条数据，每条数据需要检测19个字段，如果出现异常字段就进行 字符串拼接
//-- 过滤没有各种警告的信息，调用setOnlineDataObj 将每一条对象和每条对象和标识0 返回到OnlineDataObj，并收集这个对象
if(filterNoAlarm(itcastDataPartObj)){ //true：没有告警：0
onlineDataObj = setOnlineDataObj(firstItcastData, itcastDataPartObj, 0);
}
//否则 调用setOnlineDataObj 将第一条对象和每条对象和标识1 返回到OnlineDataObj,并收集这个对象
else {
onlineDataObj = setOnlineDataObj(firstItcastData,itcastDataPartObj,1);
}
out.collect(onlineDataObj);
}
}
/**
*
* @param firstItcastData
* @param itcastDataPartObj
* @param alarmFlag
* @return
*/
private OnlineDataObj setOnlineDataObj(ItcastDataPartObj firstItcastData, ItcastDataPartObj itcastDataPartObj, int alarmFlag) {
//1. 定义OnlineDataObj
OnlineDataObj onlineDataObj = new OnlineDataObj();
try {
//2. 将每条的对象属性拷贝到定义OnlineDataObj
BeanUtils.copyProperties(onlineDataObj,itcastDataPartObj);
//3. 将每条对象中表显历程赋值给mileage(已经有了，其实不用写）
onlineDataObj.setMileage(itcastDataPartObj.getTotalOdometer());
//4. 将告警信号赋值给isAlarm
onlineDataObj.setIsAlarm(alarmFlag);
//5. 将每个对象通过addAlarmNameList生成告警list,拼接成字符串赋值给alarmName,通过字符串join
onlineDataObj.setAlarmName(String.join("~",addAlarmNameList(itcastDataPartObj)));
//6. 将窗口内第一条数据告警时间赋值给earliestTime
onlineDataObj.setEarliestTime(firstItcastData.getTerminalTime());
//7. 将获取每条记录的充电状态通过getChargeState返回充电标识赋值给充电标记
onlineDataObj.setChargeFlag(getChargeState(itcastDataPartObj.getChargeStatus()));
//8. 将当前时间赋值给处理时间
onlineDataObj.setProcessTime(DateUtil.getCurrentDateTime());
} catch (Exception e) {
e.printStackTrace();
}
return onlineDataObj;
}
/**
* 根据充电状态返回充电标记
* @param chargeState
* @return
*/
private Integer getChargeState(Integer chargeState) {
int chargeFlag = -999999; //充电状态的初始值
//充电状态：0x01:停车充电、0x02:行车充电
if(chargeState == 1 || chargeState == 2 ){
chargeFlag = 1;
}
//0x04：充电完成 0x03：未充电
else if(chargeState == 4 || chargeState ==3){
chargeFlag = 0;
} else{
chargeFlag = 2;
}
return chargeFlag;
}
/**
* 将每条数据的故障名称追加到故障名称列表中
* @param itcastDataPartObj
* @return
*/
private ArrayList<String> addAlarmNameList(ItcastDataPartObj itcastDataPartObj) {
//定义故障名称列表对象
ArrayList<String> alarmNameList = new ArrayList<>();
//电池高温报警
if(itcastDataPartObj.getBatteryAlarm() == 1) {
alarmNameList.add("电池高温报警");
}
//单体电池高压报警
if(itcastDataPartObj.getSingleBatteryOverVoltageAlarm() == 1) {
alarmNameList.add("单体电池高压报警");
}
//电池单体一致性差报警
if(itcastDataPartObj.getBatteryConsistencyDifferenceAlarm() == 1) {
alarmNameList.add("电池单体一致性差报警");
}
//绝缘报警
if(itcastDataPartObj.getInsulationAlarm() == 1) {
alarmNameList.add("绝缘报警");
}
//高压互锁状态报警
if(itcastDataPartObj.getHighVoltageInterlockStateAlarm() == 1) {
alarmNameList.add("高压互锁状态报警");
}
//SOC跳变报警
if(itcastDataPartObj.getSocJumpAlarm() == 1) {
alarmNameList.add("SOC跳变报警");
}
//驱动电机控制器温度报警
if(itcastDataPartObj.getDriveMotorControllerTemperatureAlarm() == 1) {
alarmNameList.add("驱动电机控制器温度报警");
}
//DC-DC温度报警（dc-dc可以理解为车辆动力智能系统转换器）
if(itcastDataPartObj.getDcdcTemperatureAlarm() == 1) {
alarmNameList.add("DC-DC温度报警");
}
//SOC过高报警
if(itcastDataPartObj.getSocHighAlarm() == 1) {
alarmNameList.add("SOC过高报警");
}
//SOC低报警
if(itcastDataPartObj.getSocLowAlarm() == 1) {
alarmNameList.add("SOC低报警");
}
//温度差异报警
if(itcastDataPartObj.getTemperatureDifferenceAlarm() == 1) {
alarmNameList.add("温度差异报警");
}
//车载储能装置欠压报警
if(itcastDataPartObj.getVehicleStorageDeviceUndervoltageAlarm() == 1) {
alarmNameList.add("车载储能装置欠压报警");
}
//DC-DC状态报警
if(itcastDataPartObj.getDcdcStatusAlarm() == 1) {
alarmNameList.add("DC-DC状态报警");
}
//单体电池欠压报警
if(itcastDataPartObj.getSingleBatteryUnderVoltageAlarm() == 1) {
alarmNameList.add("单体电池欠压报警");
}
//可充电储能系统不匹配报警
if(itcastDataPartObj.getRechargeableStorageDeviceMismatchAlarm() == 1) {
alarmNameList.add("可充电储能系统不匹配报警");
}
//车载储能装置过压报警
if(itcastDataPartObj.getVehicleStorageDeviceOvervoltageAlarm() == 1) {
alarmNameList.add("车载储能装置过压报警");
}
//制动系统报警
if(itcastDataPartObj.getBrakeSystemAlarm() == 1) {
alarmNameList.add("制动系统报警");
}
//驱动电机温度报警
if(itcastDataPartObj.getDriveMotorTemperatureAlarm() == 1) {
alarmNameList.add("驱动电机温度报警");
}
//车载储能装置类型过充报警
if(itcastDataPartObj.getVehiclePureDeviceTypeOvercharge() == 1) {
alarmNameList.add("车载储能装置类型过充报警");
}
return alarmNameList;
}
/**
* 判断是否存在报警的字段
* @param itcastDataPartObj
* @return
*/
private boolean filterNoAlarm(ItcastDataPartObj itcastDataPartObj) {
//正常：0 ， 异常：1
if(
//电池高温报警
itcastDataPartObj.getBatteryAlarm() == 1 ||
//单体电池高压报警
itcastDataPartObj.getSingleBatteryOverVoltageAlarm() == 1 ||
//电池单体一致性差报警
itcastDataPartObj.getBatteryConsistencyDifferenceAlarm() == 1 ||
//绝缘报警
itcastDataPartObj.getInsulationAlarm() == 1 ||
//高压互锁状态报警
itcastDataPartObj.getHighVoltageInterlockStateAlarm() == 1 ||
//SOC跳变报警
itcastDataPartObj.getSocJumpAlarm() == 1 ||
//驱动电机控制器温度报警
itcastDataPartObj.getDriveMotorControllerTemperatureAlarm() == 1 ||
//DC-DC温度报警（dc-dc可以理解为车辆动力智能系统转换器）
itcastDataPartObj.getDcdcTemperatureAlarm() ==1 ||
//SOC过高报警
itcastDataPartObj.getSocHighAlarm() == 1||
//SOC低报警
itcastDataPartObj.getSocLowAlarm() == 1 ||
//温度差异报警
itcastDataPartObj.getTemperatureDifferenceAlarm() == 1||
//车载储能装置欠压报警
itcastDataPartObj.getVehicleStorageDeviceUndervoltageAlarm() == 1||
//DC-DC状态报警
itcastDataPartObj.getDcdcStatusAlarm() == 1||
//单体电池欠压报警
itcastDataPartObj.getSingleBatteryUnderVoltageAlarm() == 1||
//可充电储能系统不匹配报警
itcastDataPartObj.getRechargeableStorageDeviceMismatchAlarm() == 1||
//车载储能装置过压报警
itcastDataPartObj.getVehicleStorageDeviceOvervoltageAlarm() == 1||
//制动系统报警
itcastDataPartObj.getBrakeSystemAlarm() == 1 ||
//驱动电机温度报警
itcastDataPartObj.getDriveMotorTemperatureAlarm() == 1 ||
//车载储能装置类型过充报警
itcastDataPartObj.getVehiclePureDeviceTypeOvercharge() == 1
){
return false;
}else {
return true;
}
}
}
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
