spark存入clickhouse自动建表

转载

数码悟透 2024-10-01 22:28:50

文章标签 flink java servlet List json 文章分类 Spark 大数据

简介

关键词搜索实时分析，技术要点，自定义FlinkSql函数，使用聚合功能把结果输出到clickhourse

前置知识

ik分词

<dependency>
            <groupId>com.janeluo</groupId>
            <artifactId>ikanalyzer</artifactId>
            <version>2012_u6</version>
        </dependency>

测试

public class KeywordUtil {
    public static List<String> analyze(String text){
        List<String> keywordList = new ArrayList<>();
        StringReader reader = new StringReader(text);
        IKSegmenter ikSegmenter = new IKSegmenter(reader,true);
        try {
            Lexeme lexeme = null;
            while((lexeme = ikSegmenter.next())!=null){
                String keyword = lexeme.getLexemeText();
                keywordList.add(keyword);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return keywordList;
    }

    public static void main(String[] args) {
        List<String> list = analyze("Apple iPhoneXSMax (A2104) 256GB 深空灰色 移动联通电信4G手机 双卡双待");
        System.out.println(list);
    }
}

结果

[apple, iphonexsmax, a2104, 256gb, 深, 空, 灰色, 移动, 联通, 电信, 4g, 手机, 双, 卡, 双, 待]

实战

原始数据

{
    "common": {
        "ar": "110000",
        "uid": "282",
        "os": "Android 8.1",
        "ch": "huawei",
        "is_new": "0",
        "md": "Xiaomi 9",
        "mid": "mid_710062",
        "vc": "v2.1.132",
        "ba": "Xiaomi"
    },
    "page": {
        "page_id": "home",
        "during_time": 6539
    },
    "ts": 1592101568000
}

依赖

<dependency>
            <groupId>com.janeluo</groupId>
            <artifactId>ikanalyzer</artifactId>
            <version>2012_u6</version>
        </dependency>

<!--        <dependency>-->
<!--            <groupId>ru.yandex.clickhouse</groupId>-->
<!--            <artifactId>clickhouse-jdbc</artifactId>-->
<!--            <version>0.3.0</version>-->
<!--            <exclusions>-->
<!--                <exclusion>-->
<!--                    <groupId>com.fasterxml.jackson.core</groupId>-->
<!--                    <artifactId>jackson-databind</artifactId>-->
<!--                </exclusion>-->
<!--                <exclusion>-->
<!--                    <groupId>com.fasterxml.jackson.core</groupId>-->
<!--                    <artifactId>jackson-core</artifactId>-->
<!--                </exclusion>-->
<!--            </exclusions>-->
<!--        </dependency>-->
        <!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.41</version>
        </dependency>

工具类

自定义UTTF函数

@FunctionHint(output = @DataTypeHint("ROW<word STRING>"))
public class KeywordUDTF extends TableFunction<Row> {
    public void eval（String text) {
        for (String keyword : KeywordUtil.analyze(text)) {
            collect(Row.of(keyword));
        }
    }
}

分词工具类

public class KeywordUtil {
    public static List<String> analyze(String text){
        List<String> keywordList = new ArrayList<>();
        StringReader reader = new StringReader(text);
        IKSegmenter ikSegmenter = new IKSegmenter(reader,true);
        try {
            Lexeme lexeme = null;
            while((lexeme = ikSegmenter.next())!=null){
                String keyword = lexeme.getLexemeText();
                keywordList.add(keyword);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return keywordList;
    }

    public static void main(String[] args) {
        List<String> list = analyze("Apple iPhoneXSMax (A2104) 256GB 深空灰色 移动联通电信4G手机 双卡双待");
        System.out.println(list);
    }
}

将kafka里面的数据装换成表

public class KafkaUtil {
    private final static String BOOTSTRAP_SERVERS="master:9092";

    /**
     * Kafka-Source DDL 语句
     *
     * @param topic   数据源主题
     * @param groupId 消费者组
     * @return 拼接好的 Kafka 数据源 DDL 语句
     */
    public static String getKafkaDDL(String topic, String groupId) {

        return " with ('connector' = 'kafka', " +
                " 'topic' = '" + topic + "'," +
                " 'properties.bootstrap.servers' = '" + BOOTSTRAP_SERVERS + "', " +
                " 'properties.group.id' = '" + groupId + "', " +
                " 'format' = 'json', " +
                " 'scan.startup.mode' = 'group-offsets')";
    }

    /**
     * Kafka-Sink DDL 语句
     *
     * @param topic 输出到 Kafka 的目标主题
     * @return 拼接好的 Kafka-Sink DDL 语句
     */
    public static String getUpsertKafkaDDL(String topic) {

        return "WITH ( " +
                "  'connector' = 'upsert-kafka', " +
                "  'topic' = '" + topic + "', " +
                "  'properties.bootstrap.servers' = '" + BOOTSTRAP_SERVERS + "', " +
                "  'key.format' = 'json', " +
                "  'value.format' = 'json' " +
                ")";
    }
}

常量类

public class GmallConstant {
    // 10 单据状态
    public static final String ORDER_STATUS_UNPAID="1001";  //未支付
    public static final String ORDER_STATUS_PAID="1002"; //已支付
    public static final String ORDER_STATUS_CANCEL="1003";//已取消
    public static final String ORDER_STATUS_FINISH="1004";//已完成
    public static final String ORDER_STATUS_REFUND="1005";//退款中
    public static final String ORDER_STATUS_REFUND_DONE="1006";//退款完成


    // 11 支付状态
    public static final String PAYMENT_TYPE_ALIPAY="1101";//支付宝
    public static final String PAYMENT_TYPE_WECHAT="1102";//微信
    public static final String PAYMENT_TYPE_UNION="1103";//银联

    // 12 评价
    public static final String APPRAISE_GOOD="1201";// 好评
    public static final String APPRAISE_SOSO="1202";// 中评
    public static final String APPRAISE_BAD="1203";//  差评
    public static final String APPRAISE_AUTO="1204";// 自动

    // 13 退货原因
    public static final String REFUND_REASON_BAD_GOODS="1301";// 质量问题
    public static final String REFUND_REASON_WRONG_DESC="1302";// 商品描述与实际描述不一致
    public static final String REFUND_REASON_SALE_OUT="1303";//   缺货
    public static final String REFUND_REASON_SIZE_ISSUE="1304";//  号码不合适
    public static final String REFUND_REASON_MISTAKE="1305";//  拍错
    public static final String REFUND_REASON_NO_REASON="1306";//  不想买了
    public static final String REFUND_REASON_OTHER="1307";//    其他

    // 14 购物券状态
    public static final String COUPON_STATUS_UNUSED="1401";//    未使用
    public static final String COUPON_STATUS_USING="1402";//     使用中
    public static final String COUPON_STATUS_USED="1403";//       已使用

    // 15退款类型
    public static final String REFUND_TYPE_ONLY_MONEY="1501";//   仅退款
    public static final String REFUND_TYPE_WITH_GOODS="1502";//    退货退款

    // 24来源类型
    public static final String SOURCE_TYPE_QUREY="2401";//   用户查询
    public static final String SOURCE_TYPE_PROMOTION="2402";//   商品推广
    public static final String SOURCE_TYPE_AUTO_RECOMMEND="2403";//   智能推荐
    public static final String SOURCE_TYPE_ACTIVITY="2404";//   促销活动


    // 购物券范围
    public static final String COUPON_RANGE_TYPE_CATEGORY3="3301";//
    public static final String COUPON_RANGE_TYPE_TRADEMARK="3302";//
    public static final String COUPON_RANGE_TYPE_SPU="3303";//

    //购物券类型
    public static final String COUPON_TYPE_MJ="3201";//满减
    public static final String COUPON_TYPE_DZ="3202";// 满量打折
    public static final String COUPON_TYPE_DJ="3203";//  代金券

    public static final String ACTIVITY_RULE_TYPE_MJ="3101";
    public static final String ACTIVITY_RULE_TYPE_DZ ="3102";
    public static final String ACTIVITY_RULE_TYPE_ZK="3103";


    public static final String KEYWORD_SEARCH="SEARCH";
    public static final String KEYWORD_CLICK="CLICK";
    public static final String KEYWORD_CART="CART";
    public static final String KEYWORD_ORDER="ORDER";

}

表装换成对象流的bean

@Data
public class KeywordBean {
    // 窗口起始时间
    private String stt;

    // 窗口闭合时间
    private String edt;

    // 关键词来源
    private String source;

    // 关键词
    private String keyword;

    // 关键词出现频次
    private Long keyword_count;

    // 时间戳
    private Long ts;
}

应用实现模板

public class DwsTrafficSourceKeywordPageViewWindow {
    public static void main(String[] args) throws Exception {

        // TODO 1. 基本环境准备
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);
        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
        // 注册自定义函数
        tableEnv.createTemporarySystemFunction("ik_analyze", KeywordUDTF.class);

        // TODO 2. 检查点设置
//        env.enableCheckpointing(3000L, CheckpointingMode.EXACTLY_ONCE);
//        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(3000L);
//        env.getCheckpointConfig().setCheckpointTimeout(60 * 1000L);
//        env.getCheckpointConfig().enableExternalizedCheckpoints(
//                CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION
//        );
//        env.setRestartStrategy(
//                RestartStrategies.failureRateRestart(
//                        3, Time.days(1L), Time.minutes(1L)
//                )
//        );
//        env.setStateBackend(new HashMapStateBackend());
//        env.getCheckpointConfig().setCheckpointStorage(
//                "hdfs://hadoop102:8020/ck"
//        );
//        System.setProperty("HADOOP_USER_NAME", "atguigu");

        // TODO 3. 从 Kafka dwd_traffic_page_log 主题中读取页面浏览日志数据
        String topic = "dwd_traffic_page_log";
        String groupId = "dws_traffic_source_keyword_page_view_window";
        tableEnv.executeSql("create table page_log(" +
                "`common` map<string, string>," +
                //TODO 对应的是接收jsonObject
                "`page` map<string, string>," +
                "`ts` bigint," +
                //TODO 设置水位线，由于程序里面是毫秒数，然后FROM_UNIXTIME函数里面传入的是秒数，所以要除以1000
                "row_time AS TO_TIMESTAMP(FROM_UNIXTIME(ts/1000, 'yyyy-MM-dd HH:mm:ss'))," +
                //TODO 对于上面的字段设置水位线,延迟3秒
                "WATERMARK FOR row_time AS row_time - INTERVAL '3' SECOND" +
                ")" + KafkaUtil.getKafkaDDL(topic, groupId));

        // TODO 4. 从表中过滤搜索行为
        Table searchTable = tableEnv.sqlQuery("select " +
                "page['item'] full_word," +
                "row_time " +
                "from page_log " +
                "where page['item'] is not null " +
                //TODO 过来查找数据的页面
                "and page['last_page_id'] = 'search' " +
                "and page['item_type'] = 'keyword'");
        tableEnv.createTemporaryView("search_table", searchTable);

        // TODO 5. 使用自定义的UDTF函数对搜索的内容进行分词
        Table splitTable = tableEnv.sqlQuery("select " +
                "keyword," +
                "row_time " +
                "from search_table," +
                //TODO 把search_table表中的full_word使用自定义函数处理，由于ik_analyze是udtf所以侧写下表示出来lateral table
                "lateral table(ik_analyze(full_word))" +
                "as t(keyword)");
        tableEnv.createTemporaryView("split_table", splitTable);

        // TODO 6. 分组、开窗、聚合计算
        Table KeywordBeanSearch = tableEnv.sqlQuery("select " +
                "DATE_FORMAT(TUMBLE_START(row_time, INTERVAL '10' SECOND),'yyyy-MM-dd HH:mm:ss') stt," +
                "DATE_FORMAT(TUMBLE_END(row_time, INTERVAL '10' SECOND),'yyyy-MM-dd HH:mm:ss') edt,'" +
                GmallConstant.KEYWORD_SEARCH + "' source," +
                "keyword," +
                "count(*) keyword_count," +
                //TODO 这里的作用就是使用版本的左右在clickhourse里面起到幂等性的作用
                "UNIX_TIMESTAMP()*1000 ts " +
                "from split_table " +
                //TODO 这里使用TUMBLE对于row_time水位线进行开窗聚合,窗口的大小是10秒,根据分词以后的keyword进行分组聚合
                "GROUP BY TUMBLE(row_time, INTERVAL '10' SECOND),keyword");

        // TODO 7. 将动态表转换为流
        DataStream<KeywordBean> keywordBeanDS = tableEnv.toAppendStream(KeywordBeanSearch, KeywordBean.class);
        keywordBeanDS.print();

        // TODO 8. 将流中的数据写到ClickHouse中
//        SinkFunction<KeywordBean> jdbcSink = ClickHouseUtil.<KeywordBean>getJdbcSink(
//                "insert into dws_traffic_source_keyword_page_view_window values(?,?,?,?,?,?)");
//        keywordBeanDS.addSink(jdbcSink);

        env.execute();
    }
}

输出的结果

3> KeywordBean(stt=2020-06-14 10:54:00, edt=2020-06-14 10:54:10, source=SEARCH, keyword=小米, keyword_count=13, ts=1659063253000)
4> KeywordBean(stt=2020-06-14 10:54:00, edt=2020-06-14 10:54:10, source=SEARCH, keyword=电视, keyword_count=2, ts=1659063253000)
4> KeywordBean(stt=2020-06-14 10:54:00, edt=2020-06-14 10:54:10, source=SEARCH, keyword=盒子, keyword_count=7, ts=1659063253000)
4> KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=盒子, keyword_count=6, ts=1659063263000)
3> KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=小米, keyword_count=16, ts=1659063263000)
4> KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=图书, keyword_count=2, ts=1659063263000)
3> KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=手机, keyword_count=5, ts=1659063263000)
4> KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=电视, keyword_count=10, ts=1659063263000)
1> KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=口红, keyword_count=6, ts=1659063263000)
2> KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=苹果, keyword_count=5, ts=1659063263000)
1> KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=ps5, keyword_count=7, ts=1659063263000)
2> KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=iphone11, keyword_count=5, ts=1659063263000)

结果流保存到clickhouse

加入依赖

<dependency>
            <groupId>ru.yandex.clickhouse</groupId>
            <artifactId>clickhouse-jdbc</artifactId>
            <version>0.1.54</version>
        </dependency>

启动clickhouse

sudo systemctl start clickhouse-server

clickhouse-client -m

创建表

drop table if exists dws_traffic_source_keyword_page_view_window;
create table if not exists dws_traffic_source_keyword_page_view_window
(
    stt           DateTime,
    edt           DateTime,
    source        String,
    keyword       String,
    keyword_count UInt64,
    ts            UInt64
)engine = ReplacingMergeTree(ts)
partition by toYYYYMMDD(stt)
//order by用来去重复，保证数据的幂等性
order by (stt, edt, source, keyword);

去重复的例子数据

KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=苹果, keyword_count=5, ts=1659063263000)
KeywordBean(stt=2020-06-14 10:54:10, edt=2020-06-14 10:54:20, source=SEARCH, keyword=ps5, keyword_count=7, ts=1659063263000)

工具类

public class ClickHouseUtil {
    // ClickHouse 驱动
    public static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver";

    // ClickHouse 连接 URL,gmall_rebuild是数据库create database gmall_rebuild;
    public static final String CLICKHOUSE_URL = "jdbc:clickhouse://master:8123/gmall_rebuild";

    public static <T> SinkFunction<T> getJdbcSink(String sql) {
        return JdbcSink.<T>sink(
                sql,
                new JdbcStatementBuilder<T>() {
                    @Override
                    public void accept(PreparedStatement preparedStatement, T obj) throws SQLException {
                        Field[] declaredFields = obj.getClass().getDeclaredFields();
                        int skipNum = 0;
                        for (int i = 0; i < declaredFields.length; i++) {
                            Field declaredField = declaredFields[i];
                            //使用这个自定义注解的作用是如果有些字段是 不想保存的时候可以标记它
                            TransientSink transientSink = declaredField.getAnnotation(TransientSink.class);
                            if (transientSink != null) {
                                skipNum++;
                                continue;
                            }
                            declaredField.setAccessible(true);
                            try {
                                Object value = declaredField.get(obj);
                                preparedStatement.setObject(i + 1 - skipNum, value);
                            } catch (IllegalAccessException e) {
                                System.out.println("ClickHouse 数据插入 SQL 占位符传参异常 ~");
                                e.printStackTrace();
                            }
                        }
                    }
                },
                JdbcExecutionOptions.builder()
                        .withBatchIntervalMs(5000L)
                        .withBatchSize(5)
                        .build(),
                new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
                        .withDriverName(CLICKHOUSE_DRIVER)
                        .withUrl(CLICKHOUSE_URL)
                        .build()
        );
    }
}

注解

(作用就是把bean里面不是clickhouse里面的字段去掉不不处理)

@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
public @interface TransientSink {
}

最终代码

public class DwsTrafficSourceKeywordPageViewWindow {
    public static void main(String[] args) throws Exception {

        // TODO 1. 基本环境准备
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);
        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
        // 注册自定义函数
        tableEnv.createTemporarySystemFunction("ik_analyze", KeywordUDTF.class);

        // TODO 2. 检查点设置
//        env.enableCheckpointing(3000L, CheckpointingMode.EXACTLY_ONCE);
//        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(3000L);
//        env.getCheckpointConfig().setCheckpointTimeout(60 * 1000L);
//        env.getCheckpointConfig().enableExternalizedCheckpoints(
//                CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION
//        );
//        env.setRestartStrategy(
//                RestartStrategies.failureRateRestart(
//                        3, Time.days(1L), Time.minutes(1L)
//                )
//        );
//        env.setStateBackend(new HashMapStateBackend());
//        env.getCheckpointConfig().setCheckpointStorage(
//                "hdfs://hadoop102:8020/ck"
//        );
//        System.setProperty("HADOOP_USER_NAME", "atguigu");

        // TODO 3. 从 Kafka dwd_traffic_page_log 主题中读取页面浏览日志数据
        String topic = "dwd_traffic_page_log";
        String groupId = "dws_traffic_source_keyword_page_view_window";
        tableEnv.executeSql("create table page_log(" +
                "`common` map<string, string>," +
                //TODO 对应的是接收jsonObject
                "`page` map<string, string>," +
                "`ts` bigint," +
                //TODO 设置水位线，由于程序里面是毫秒数，然后FROM_UNIXTIME函数里面传入的是秒数，所以要除以1000
                "row_time AS TO_TIMESTAMP(FROM_UNIXTIME(ts/1000, 'yyyy-MM-dd HH:mm:ss'))," +
                //TODO 对于上面的字段设置水位线,延迟3秒
                "WATERMARK FOR row_time AS row_time - INTERVAL '3' SECOND" +
                ")" + KafkaUtil.getKafkaDDL(topic, groupId));

        // TODO 4. 从表中过滤搜索行为
        Table searchTable = tableEnv.sqlQuery("select " +
                "page['item'] full_word," +
                "row_time " +
                "from page_log " +
                "where page['item'] is not null " +
                //TODO 过来查找数据的页面
                "and page['last_page_id'] = 'search' " +
                "and page['item_type'] = 'keyword'");
        tableEnv.createTemporaryView("search_table", searchTable);

        // TODO 5. 使用自定义的UDTF函数对搜索的内容进行分词
        Table splitTable = tableEnv.sqlQuery("select " +
                "keyword," +
                "row_time " +
                "from search_table," +
                //TODO 把search_table表中的full_word使用自定义函数处理，由于ik_analyze是udtf所以侧写下表示出来lateral table
                "lateral table(ik_analyze(full_word))" +
                "as t(keyword)");
        tableEnv.createTemporaryView("split_table", splitTable);

        // TODO 6. 分组、开窗、聚合计算
        Table KeywordBeanSearch = tableEnv.sqlQuery("select " +
                "DATE_FORMAT(TUMBLE_START(row_time, INTERVAL '10' SECOND),'yyyy-MM-dd HH:mm:ss') stt," +
                "DATE_FORMAT(TUMBLE_END(row_time, INTERVAL '10' SECOND),'yyyy-MM-dd HH:mm:ss') edt,'" +
                GmallConstant.KEYWORD_SEARCH + "' source," +
                "keyword," +
                "count(*) keyword_count," +
                //TODO 这里的作用就是使用版本的左右在clickhourse里面起到幂等性的作用
                "UNIX_TIMESTAMP()*1000 ts " +
                "from split_table " +
                //TODO 这里使用TUMBLE对于row_time水位线进行开窗聚合,窗口的大小是10秒,根据分词以后的keyword进行分组聚合
                "GROUP BY TUMBLE(row_time, INTERVAL '10' SECOND),keyword");

        // TODO 7. 将动态表转换为流
        DataStream<KeywordBean> keywordBeanDS = tableEnv.toAppendStream(KeywordBeanSearch, KeywordBean.class);

        // TODO 8. 将流中的数据写到ClickHouse中
        SinkFunction<KeywordBean> jdbcSink = ClickHouseUtil.<KeywordBean>getJdbcSink(
                "insert into dws_traffic_source_keyword_page_view_window values(?,?,?,?,?,?)");
        keywordBeanDS.addSink(jdbcSink);

        env.execute();
    }
}

得到clickhouse的结果

spark存入clickhouse自动建表_servlet

附录(项目所涉及的所有maven依赖)

<build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>8</source>
                    <target>8</target>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>3.0.0</version>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

    <properties>
        <flink.version>1.13.0</flink.version>
        <java.version>1.8</java.version>
        <scala.binary.version>2.12</scala.binary.version>
        <slf4j.version>1.7.30</slf4j.version>
        <scala.version>2.12</scala.version>
    </properties>
    <dependencies>
        <!-- 引入 Flink 相关依赖-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-cep_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- 引入日志管理相关依赖-->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>${slf4j.version}</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>${slf4j.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-to-slf4j</artifactId>
            <version>2.14.0</version>
        </dependency>

        <dependency>
            <groupId>org.apache.bahir</groupId>
            <artifactId>flink-connector-redis_2.11</artifactId>
            <version>1.0</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch7_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>


        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.47</version>
        </dependency>


        <!--        这里的依赖是一个 Java 的“桥接器”（bridge），主要就是负责 Table API 和下层 DataStream-->
        <!--        API 的连接支持，按照不同的语言分为 Java 版和 Scala 版。-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!--        如果我们希望在本地的集成开发环境（IDE）里运行 Table API 和 SQL，还需要引入以下-->
        <!--        依赖-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>


        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-cep_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>


        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>RELEASE</version>
            <scope>compile</scope>
        </dependency>


        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>${flink.version}</version>
        </dependency>


        <dependency>
            <groupId>com.janeluo</groupId>
            <artifactId>ikanalyzer</artifactId>
            <version>2012_u6</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.41</version>
        </dependency>

        <dependency>
            <groupId>ru.yandex.clickhouse</groupId>
            <artifactId>clickhouse-jdbc</artifactId>
            <version>0.1.54</version>
        </dependency>

    </dependencies>

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。

上一篇：dnf安装mysql后

下一篇：Python打开windows系统下的挂载盘下的文件

提问和评论都可以，用心的回复会被更多人看到评论

发布评论

相关文章

官方博客	全部文章	热门标签	班级博客
了解我们	网站地图	意见反馈

鸿蒙开发者社区	51CTO学堂
51CTO	软考资讯

spark存入clickhouse自动建表

spark存入clickhouse自动建表

简介

前置知识

ik分词

实战

原始数据

依赖

工具类

自定义UTTF函数

分词工具类

将kafka里面的数据装换成表

常量类

表装换成对象流的bean

应用实现模板

输出的结果

结果流保存到clickhouse

加入依赖

启动clickhouse

创建表

工具类

注解

最终代码

得到clickhouse的结果

附录(项目所涉及的所有maven依赖)

51CTO博客