5 Flink 流处理API

Enviroment
package env;

import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

/**
 * @author wangkai
 */
public class Env {
    public static void main(String[] args) {
        /**
         * 流式执行环境
         * 创建一个执行环境，表示当前执行程序的上下文。 如果程序是独立调用的，则
         * 此方法返回本地执行环境；如果从命令行客户端调用程序以提交到集群，则此方法
         * 返回此集群的执行环境，也就是说，getExecutionEnvironment 会根据查询运行的方
         * 式决定返回什么样的运行环境，是最常用的一种创建执行环境的方式
         * */
        StreamExecutionEnvironment StreamEnvironment = StreamExecutionEnvironment.getExecutionEnvironment();

        /**
         * 批处理执行环境
         * */
        ExecutionEnvironment BatchEnvironment1 = ExecutionEnvironment.getExecutionEnvironment();

        /**
         *
         *创建本地执行环境
         * */
        LocalStreamEnvironment localEnvironment = StreamExecutionEnvironment.createLocalEnvironment();


        /**
         * 创建远程执行环境
         * */
        StreamExecutionEnvironment remoteEnvironment = StreamExecutionEnvironment.createRemoteEnvironment("xxx", 1234, "xxx.jar");

    }
}
Source
package source;

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;

import java.util.Arrays;
import java.util.Properties;

/**
 * @author wangkai 
 */
public class Source {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        /**
         * 1 从集合读取
         * */
        DataStreamSource<String> source = env.fromCollection(Arrays.asList("zs", "ls", "wmz"));

        /**
         * 2 从文件读取数据
         * */
        DataStreamSource<String> source1 = env.readTextFile("path");

        /**
         * 3 从kafka读取数据
         * */
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers","xxx");
        properties.setProperty("group.id","xxx");

        DataStreamSource<String> source2 = env.addSource(new FlinkKafkaConsumer011<String>("topic", new SimpleStringSchema(), properties));
        
        /**
         *
         * 4 自定义source
         * */
        DataStreamSource<String> source3 = env.addSource(new MySourceFunction());

    }

    public static class MySourceFunction implements SourceFunction<String> {

        private boolean isRunning = true;

        public void run(SourceContext<String> ctx) throws Exception {
            while(isRunning){
                System.out.println("自定义数据源");
            }

        }

        public void cancel() {
            isRunning = false;

        }
    }
}
Transform
  • 转换算子
package transform;

import com.alibaba.fastjson.JSONObject;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

/**
 * @author wangkai 
 */
public class TransForm {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        DataStreamSource<String> source = env.socketTextStream("", 7777);


        source.map(new MapFunction<String, String>() {
            /**
             * map算子,接受一个元素并产生一个元素
             * */
                       public String map(String value) throws Exception {
                           Object parse = JSONObject.parse(value);
                           return parse.toString();
                       }
                   }
       );


        SingleOutputStreamOperator<Tuple2<String, Integer>> flatMapSource = source.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            /**
             * flatMap 算子:接受一个元素并产生零个、一个或多个元素
             *
             * */
            public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
                String[] s = value.split(" ");
                for (int i = 0; i < s.length; i++) {
                    out.collect(new Tuple2<String, Integer>(s[i], 1));
                }

            }
        });

        source.filter(new FilterFunction<String>() {
            /**
             * filter算子: 对每个元素计算一个布尔函数，并保留该函数返回true的那些元素
             * */
            public boolean filter(String value) throws Exception {
                return value == "filter";
            }
        });


        KeyedStream<Tuple2<String, Integer>, String> keyedStream = flatMapSource.keyBy(new KeySelector<Tuple2<String, Integer>, String>() {
            public String getKey(Tuple2<String, Integer> value) throws Exception {
                return value.f0;
            }
        });

    }

}
  • 滚动聚合算子
  • sum
  • max
  • min
  • minBy
  • maxBy
package transform;


import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

/**
 * @author wangkai
 */
public class RollingAggregation {
    public static void main(String[] args) throws Exception{
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        /**
         * 滚动聚合算子：sum max min maxBy minBy
         *
         * */
        
        
        /**
         * 
         * sensor1 123456789 35
         * sensor2 234567890 36
         * sensor3 456962456 24
         * sensor1 123456789 20
         * 
         * 
         * */

        DataStreamSource<String> source = env.readTextFile("D:\\git\\csdn-flink\\csdn-flink-1\\src\\main\\resources\\sensor");

        SingleOutputStreamOperator<Tuple3<String, String, Long>> stream = source.flatMap(new FlatMapFunction<String, Tuple3<String, String, Long>>() {
            public void flatMap(String value, Collector<Tuple3<String, String, Long>> out) throws Exception {
                String[] s = value.split(" ");
                out.collect(new Tuple3<String, String, Long>(s[0], s[1], Long.parseLong(s[2])));
            }
        });

        /**
         * sum
         * */

        SingleOutputStreamOperator<Tuple3<String, String, Long>> sum = stream
                .keyBy(0)
                .sum(2);

        sum.print("sum");

        /**
         * max
         * */
        SingleOutputStreamOperator<Tuple3<String, String, Long>> max = stream
                .keyBy(0)
                .max(2);
        max.print("max");

        /**
         * min
         * */
        SingleOutputStreamOperator<Tuple3<String, String, Long>> min = stream
                .keyBy(0)
                .min(2);
        min.print("min");

        /**
         * maxBy
         * */
        SingleOutputStreamOperator<Tuple3<String, String, Long>> maxBy = stream
                .keyBy(0)
                .maxBy(2);
        maxBy.print("maxby");


        /**
         * minBy
         * */
        SingleOutputStreamOperator<Tuple3<String, String, Long>> minBy = stream
                .keyBy(0)
                .minBy(2);
        minBy.print("minby");


        env.execute("rolling aggregate");
        
        /**
         * max> (sensor1,123456789,35)
         * min> (sensor1,123456789,35)
         * maxby> (sensor1,123456789,35)
         * minby> (sensor1,123456789,35)
         * sum> (sensor1,123456789,35)
         * maxby> (sensor2,234567890,36)
         * min> (sensor2,234567890,36)
         * max> (sensor2,234567890,36)
         * maxby> (sensor3,456962456,24)
         * sum> (sensor2,234567890,36)
         * min> (sensor3,456962456,24)
         * minby> (sensor2,234567890,36)
         * sum> (sensor3,456962456,24)
         * min> (sensor1,123456789,20)
         * maxby> (sensor1,123456789,35)
         * max> (sensor3,456962456,24)
         * sum> (sensor1,123456789,55)
         * max> (sensor1,123456789,35)
         * minby> (sensor3,456962456,24)
         * minby> (sensor1,123456789,20)
         * */
    }


}
  • Reduce
package transform;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

/**
 * @author wangkai 

 */
public class Reduce {
    public static void main(String[] args) throws Exception{
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        /**
         *
         * KeyedStream → DataStream：一个分组数据流的聚合操作，合并当前的元素
         * 和上次聚合的结果，产生一个新的值，返回的流中包含每一次聚合的结果，而不是
         * 只返回最后一次聚合的最终结果
         * 
         * sensor1 123456789 35
         * sensor2 234567890 36
         * sensor3 456962456 24
         * sensor1 123456789 20
         * */
        DataStreamSource<String> source = env.readTextFile("D:\\git\\csdn-flink\\csdn-flink-1\\src\\main\\resources\\sensor");

        SingleOutputStreamOperator<Tuple3<String, String, Long>> stream = source.flatMap(new FlatMapFunction<String, Tuple3<String, String, Long>>() {
            public void flatMap(String value, Collector<Tuple3<String, String, Long>> out) throws Exception {
                String[] s = value.split(" ");
                out.collect(new Tuple3<String, String, Long>(s[0], s[1], Long.parseLong(s[2])));
            }
        });


        SingleOutputStreamOperator<Tuple3<String, String, Long>> reduce = stream
                .keyBy(0)
                .reduce(new ReduceFunction<Tuple3<String, String, Long>>() {
                    public Tuple3<String, String, Long> reduce(Tuple3<String, String, Long> value1, Tuple3<String, String, Long> value2) throws Exception {
                        return new Tuple3<String, String, Long>(value1.f0, value1.f1, Math.max(value1.f2, value2.f2));
                    }
                });

        
        reduce.print("reduce");

        env.execute("reduce");
        
        
        /**
         * reduce> (sensor1,123456789,35)
         * reduce> (sensor2,234567890,36)
         * reduce> (sensor3,456962456,24)
         * reduce> (sensor1,123456789,35)
         * */
    }
}
  • Split和select
package transform;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.datastream.SplitStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

import java.util.Collections;

/**
 * @author 
 */
public class SplitAndSelect {
    public static void main(String[] args) throws Exception{
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        /**
         *split:DataStream → SplitStream：根据某些特征把一个 DataStream 拆分成两个或者多个DataStream。
         *select:SplitStream→DataStream：从一个 SplitStream 中获取一个或者多个DataStream
         *
         * sensor1 123456789 35
         * sensor2 234567890 36
         * sensor3 456962456 24
         * sensor1 123456789 20
         * */
        DataStreamSource<String> source = env.readTextFile("D:\\git\\csdn-flink\\csdn-flink-1\\src\\main\\resources\\sensor");

        SingleOutputStreamOperator<Tuple3<String, String, Long>> stream = source.flatMap(new FlatMapFunction<String, Tuple3<String, String, Long>>() {
            public void flatMap(String value, Collector<Tuple3<String, String, Long>> out) throws Exception {
                String[] s = value.split(" ");
                out.collect(new Tuple3<String, String, Long>(s[0], s[1], Long.parseLong(s[2])));
            }
        });


        SplitStream<Tuple3<String, String, Long>> split = stream.split(new OutputSelector<Tuple3<String, String, Long>>() {
            public Iterable<String> select(Tuple3<String, String, Long> value) {
                return value.f2 > 30 ? Collections.singletonList("high") : Collections.singletonList("low");
            }
        });

        split.print("split");

        DataStream<Tuple3<String, String, Long>> high = split.select("high");
        DataStream<Tuple3<String, String, Long>> low = split.select("low");

        high.print("high");
        low.print("low");


        env.execute("split and select");
        
        
        /**
         * 
         * split> (sensor1,123456789,35)
         * high> (sensor1,123456789,35)
         * split> (sensor2,234567890,36)
         * high> (sensor2,234567890,36)
         * split> (sensor3,456962456,24)
         * low> (sensor3,456962456,24)
         * split> (sensor1,123456789,20)
         * low> (sensor1,123456789,20)
         * 
         * */

    }
}
  • connect和CoMap
DataStream<Tuple3<String, String, Long>> high = split.select("high");
        DataStream<Tuple3<String, String, Long>> low = split.select("low");

        high.print("high");
        low.print("low");


        ConnectedStreams<Tuple3<String, String, Long>, Tuple3<String, String, Long>> connect = high.connect(low);
        connect.flatMap(new CoFlatMapFunction<Tuple3<String, String, Long>, Tuple3<String, String, Long>, Object>() {
            public void flatMap1(Tuple3<String, String, Long> value, Collector<Object> out) throws Exception {
                out.collect(value);

            }

            public void flatMap2(Tuple3<String, String, Long> value, Collector<Object> out) throws Exception {
                out.collect(value);

            }
        });
  • union
DataStream<Tuple3<String, String, Long>> union = high.union(low);

  Connect 与 Union 区别：
   1． Union 之前两个流的类型必须是一样，Connect 可以不一样，在之后的 coMap中再去调整成为一样的。
   2. Connect 只能操作两个流，Union 可以操作多个。
支持的数据类型

Flink 流应用程序处理的是以数据对象表示的事件流。所以在 Flink 内部，我们 需要能够处理这些对象。它们需要被序列化和反序列化，以便通过网络传送它们； 或者从状态后端、检查点和保存点读取它们。为了有效地做到这一点，Flink 需要明 确知道应用程序所处理的数据类型。Flink 使用类型信息的概念来表示数据类型，并 为每个数据类型生成特定的序列化器、反序列化器和比较器。 Flink 还具有一个类型提取系统，该系统分析函数的输入和返回类型，以自动获 取类型信息，从而获得序列化器和反序列化器。但是，在某些情况下，例如 lambda 函数或泛型类型，需要显式地提供类型信息，才能使应用程序正常工作或提高其性 能。

自定义udf函数
  • 函数类
  • 匿名函数
  • 富函数
    “富函数”是 DataStream API 提供的一个函数类的接口，所有 Flink 函数类都 有其 Rich 版本。它与常规函数的不同在于，可以获取运行环境的上下文，并拥有一 些生命周期方法，所以可以实现更复杂的功能。 有一个生命周期的概念。典型的生命周期方法有： open()方法是 rich function 的初始化方法，当一个算子例如 map 或者 filter 被调用之前 open()会被调用。 close()方法是生命周期中的最后一个调用的方法，做一些清理工作。 getRuntimeContext()方法提供了函数的 RuntimeContext 的一些信息，例如函 数执行的并行度，任务的名字，以及 state 状态
Sink
  • 自定义sink函数
package sink;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;

public class MyJdbcSink extends RichSinkFunction<String> {
    Connection conn = null;
    PreparedStatement insertStmt = null;


    @Override
    public void open(Configuration parameters) throws Exception {
        conn = DriverManager.getConnection("url", "username", "password");
        insertStmt = conn.prepareStatement("insert into xxx (a) values (?)");
    }

    /**
     * 调用连接，执行sql
     */

    @Override
    public void invoke(String value, Context context) throws Exception {
        insertStmt.setString(1,"test");
        insertStmt.execute();

    }

    @Override
    public void close() throws Exception {
        insertStmt.close();
        conn.close();
    }
}