flume读取HDFS数据到kafka

转载

mob64ca14116c53 2024-11-02 13:53:42

文章标签 flume读取HDFS数据到kafka linux flume mysql sql 文章分类 架构后端开发

文章目录

Flume下载
创建数据库表
MysqlSink代码
Flume配置
Flume启动
使用DBCP连接池
dbcp遇到的问题
测试
遇到的问题

Flume下载

这里使用Flume1.8.0，从官方下载tar包
下载地址：
http://archive.apache.org/dist/flume/1.8.0/apache-flume-1.8.0-bin.tar.gz 下载后上传至服务器或虚拟机中，解压，将解压后文件夹简化命名

[root@worker opt]# tar -zxvf apache-flume-1.8.0-bin.tar.gz

[root@worker opt]# ll
total 645452
-rw-r--r-- 1 root root  58688757 Jul 15 15:58 apache-flume-1.8.0-bin.tar.gz
drwxr-xr-x 7 root root       187 Jul 15 15:58 flume-1.8.0

增加环境变量

[root@worker opt]# vim /etc/profile

export FLUME_HOME=/opt/flume-1.8.0
export PATH=$PATH:$FLUME_HOME/bin

使环境变量生效

[root@worker opt]# source /etc/profile

创建数据库表

# 建库
create database flume01;

# 建表
CREATE TABLE IF NOT EXISTS `table01`(
   `id` INT UNSIGNED AUTO_INCREMENT,
   `name` VARCHAR(100) NOT NULL,
   `age` INT NOT NULL,
   `insert_date` DATE,
   PRIMARY KEY ( `id` )
)ENGINE=InnoDB DEFAULT CHARSET=utf8;

# 测试插入一条数据
insert into table01 (name,age,insert_date) values('zhangsan',20,'2020-01-02 18:09:09');

MysqlSink代码

pom中属性和依赖

<properties>
        <maven.compiler.target>1.8</maven.compiler.target>
        <maven.compiler.source>1.8</maven.compiler.source>
        <version.flume>1.8.0</version.flume>
    </properties>


    <dependencies>
        <dependency>
            <groupId>org.apache.flume</groupId>
            <artifactId>flume-ng-core</artifactId>
            <version>${version.flume}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flume</groupId>
            <artifactId>flume-ng-configuration</artifactId>
            <version>${version.flume}</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.15</version>
        </dependency>
    </dependencies>

数据实体类

package entity;

import java.util.Date;

public class FlumeEntity {
    private String name;
    private int age;
    private Date date;

    public void setName(String name) {
         = name;
    }

    public void setAge(int age) {
        this.age = age;
    }

    public void setDate(Date date) {
        this.date = date;
    }

    public String getName() {
        return name;
    }

    public int getAge() {
        return age;
    }

    public Date getDate() {
        return date;
    }
}

MysqlSink主体类

package sink;

import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import entity.FlumeEntity;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;

public class MysqlSink extends AbstractSink implements Configurable {

    private Logger logger = LoggerFactory.getLogger(MysqlSink.class);
    private String hostname;
    private String port;
    private String databaseName;
    private String tableName;
    private String user;
    private String password;
    private PreparedStatement preparedStatement;
    private Connection conn;
    private int batchSize;
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

    public MysqlSink() {
        ("MysqlSink start...");
    }


    @Override
    public void configure(Context context) {
        hostname = context.getString("hostname");
        Preconditions.checkNotNull(hostname, "hostname must be set!!");
        port = context.getString("port");
        Preconditions.checkNotNull(port, "port must be set!!");
        databaseName = context.getString("databaseName");
        Preconditions.checkNotNull(databaseName, "databaseName must be set!!");
        tableName = context.getString("tableName");
        Preconditions.checkNotNull(tableName, "tableName must be set!!");
        user = context.getString("user");
        Preconditions.checkNotNull(user, "user must be set!!");
        password = context.getString("password");
        Preconditions.checkNotNull(password, "password must be set!!");
        batchSize = context.getInteger("batchSize", 100);
        Preconditions.checkNotNull(batchSize > 0, "batchSize must be a positive number!!");
    }

    @Override
    public void start() {
        super.start();
        try {
            //调用Class.forName()方法加载驱动程序
            Class.forName("com.mysql.jdbc.Driver");
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }

        String url = "jdbc:mysql://" + hostname + ":" + port + "/" + databaseName;
        //调用DriverManager对象的getConnection()方法，获得一个Connection对象

        try {
            conn = DriverManager.getConnection(url, user, password);
            conn.setAutoCommit(false);
            //创建一个Statement对象
            preparedStatement = conn.prepareStatement("insert into " + tableName +
                    " (name,age,insert_date) values (?,?,?)");

        } catch (SQLException e) {
            e.printStackTrace();
            System.exit(1);
        }

    }

    @Override
    public void stop() {
        super.stop();
        if (preparedStatement != null) {
            try {
                preparedStatement.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }

        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }



    @Override
    public Status process() throws EventDeliveryException {
        Status result = Status.READY;
        Channel channel = getChannel();
        Transaction transaction = channel.getTransaction();
        Event event;
        String content;

        List<FlumeEntity> lists = new ArrayList<>();
        transaction.begin();

        try {
            for (int i = 0; i < batchSize; i++) {
                event = channel.take();
                if (event != null) {//对事件进行处理
                    //event 的 body 为   "exec tail$i , abel"
                    content = new String(event.getBody());
                    FlumeEntity entity = new FlumeEntity();
                    if (content.contains(",")) {
                        String[] lines = content.split(",");
                        if (lines.length != 3) {
                            logger.warn("record missing : " + content);
                            continue;
                        }
                        //存储 event 的 content
                        entity.setName(lines[0]);
                        entity.setAge(Integer.parseInt(lines[1]));
                        entity.setDate(sdf.parse(lines[2]));
                    }else{
                        logger.warn("record missing : " + content);
                    }
                    lists.add(entity);
                } else {
                    result = Status.BACKOFF;
                    break;
                }
            }

            if (lists.size() > 0) {
                preparedStatement.clearBatch();
                for (FlumeEntity entity : lists) {
                    preparedStatement.setString(1, entity.getName());
                    preparedStatement.setInt(2, entity.getAge());
                    preparedStatement.setDate(3,new java.sql.Date(entity.getDate().getTime()));
                    preparedStatement.addBatch();
                }
                preparedStatement.executeBatch();

                conn.commit();
            }
            transaction.commit();
        } catch (Exception e) {
            try {
                transaction.rollback();
            } catch (Exception e2) {
                logger.error("Exception in rollback. Rollback might not have been" +
                        "successful.", e2);
            }
            logger.error("Failed to commit transaction." +
                    "Transaction rolled back.", e);
            Throwables.propagate(e);
        } finally {
            transaction.close();
        }
        return result;

    }
}

将程序打成jar包，我这里直接使用maven clean 和maven package打的包，之后将该jar包放置到FLUME_HOME/lib目录下，同时也需要将mysql-connector-java-8.0.15.jar放置该目录下。

Flume配置

#a1表示代理名称
a1.sources=s1
a1.sinks=mysqlSink
a1.channels=c1
#配置sources  监控目录是否有文件数据生成
a1.sources.s1.type=spooldir
a1.sources.s1.spoolDir=/opt/cr1000/spooldir
# 匹配文件名包含15min且以.dat结尾的文件名
# a1.sources.s1.includePattern = ^(.)*15min(.)*\\.dat$
a1.sources.s1.ignorePattern = ^(.)*\\.tmp$
a1.sources.s1.channels = c1
 
#配置sinks 将检测到的数据sink到mysql上
a1.sinks.mysqlSink.type=sink.MysqlSink
a1.sinks.mysqlSink.hostname=192.168.2.103
a1.sinks.mysqlSink.port=3306
a1.sinks.mysqlSink.databaseName=flume01
a1.sinks.mysqlSink.tableName=table01
a1.sinks.mysqlSink.user=root
a1.sinks.mysqlSink.password=123456
a1.sinks.mysqlSink.channel=c1

 
#通道是以内存方式存储
#配置channels
a1.channels.c1.type=memory
a1.channels.c1.capacity=10000
a1.channels.c1.transactionCapacity=1000
a1.channels.c1.checkpointDir=/opt/cr1000/checkpoint
a1.channels.c1.dataDirs=/opt/cr1000/data

Flume启动

进入到FLUME_HOME目录下

[root@worker flume-1.8.0]# ./bin/flume-ng agent -c ./conf -f ./conf/mysqlsink.conf -n a1 -Dflume.root.logger=INFO,console

使用DBCP连接池

使用传统的DBCP连接，需要的jar包在flume目录/lib下已提供
POM文件中增加依赖，版本与flume下的jar包保持一致

<dependency>
            <groupId>commons-dbcp</groupId>
            <artifactId>commons-dbcp</artifactId>
            <version>1.4</version>
        </dependency>
        <dependency>
            <groupId>commons-pool</groupId>
            <artifactId>commons-pool</artifactId>
            <version>1.5.5</version>
        </dependency>

编写JdbcUtils类

package util;

import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbcp.BasicDataSourceFactory;

import javax.sql.DataSource;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;

public class JdbcUtils {

    private JdbcUtils() {}

    private static DataSource ds;

    static {
        BasicDataSource bds = new BasicDataSource();
        bds.setDriverClassName("com.mysql.jdbc.Driver");
        bds.setUrl("jdbc:mysql://192.168.2.103/flume01");
        bds.setUsername("root");
        bds.setPassword("123456");

        bds.setInitialSize(5);
        bds.setMaxActive(10);
        //bds.setMinIdle(3);
        ds = bds;
    }

    /*static {
        try {
            InputStream in = JdbcUtils.class.getClassLoader().getResourceAsStream(
                    "config.properties");//通过类加载的方式获得获得配置文件的流
            Properties prop = new Properties();
            prop.load(in);//通过流将配置项加载进来。
            //通过工厂类读取
            ds = BasicDataSourceFactory.createDataSource(prop);
        } catch (Exception e) {
            throw new ExceptionInInitializerError();
        }
    }*/

    public static DataSource getDataSource() {
        return ds;
    }

    public static Connection getConnection() throws SQLException {
        return ds.getConnection();
    }

    public static void release(ResultSet rs, Statement stmt, Connection conn){
        if(rs!=null) {
            try{
                rs.close();
            }catch(SQLException e){
                e.printStackTrace();
            }
            rs = null;
        }
        if(stmt!=null){
            try{
                stmt.close();
            }catch(SQLException e){
                e.printStackTrace();
            }
            stmt = null;
        }
        if(conn!= null) {
            try{
                conn.close();
            }catch(SQLException e){
                e.printStackTrace();
            }
            conn = null;
        }
    }
}

MysqlSink类中更改参考

@Override
    public void start() {
        super.start();
        /*try {
            //调用Class.forName()方法加载驱动程序
            Class.forName("com.mysql.jdbc.Driver");
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }

        String url = "jdbc:mysql://" + hostname + ":" + port + "/" + databaseName;
       */
        try {
        //调用DriverManager对象的getConnection()方法，获得一个Connection对象
            //conn = DriverManager.getConnection(url, user, password);
            conn = JdbcUtils.getConnection();

使用配置文件dbcp.properties，根据需要自己配置即可

#连接设置
driverClassName=com.mysql.jdbc.Driver
url=jdbc:mysql://192.168.2.103/flume01
username=root
password=123456
#<!-- 初始化连接 -->
initialSize=5
#最大连接数量
maxActive=10
#<!-- 最大空闲连接 -->
# maxIdle=20
#<!-- 最小空闲连接 -->
#minIdle=5
#<!-- 超时等待时间以毫秒为单位 6000毫秒/1000等于60秒 -->
#maxWait=60000
#<!-- 连接编码设置 -->
#connectionProperties=useUnicode=true;characterEncoding=gbk
#指定由连接池所创建的连接的自动提交（auto-commit）状态。
#defaultAutoCommit=true

配置文件上传放置到flume安装目录/conf下，注释掉上边代码中的写死的那个静态代码块就行

static {
        try {
            # 使用了绝对路径
            String path = "/opt/flume-1.8.0/conf/dbcp.properties";
            InputStream in= new BufferedInputStream(new FileInputStream(path));
            Properties prop = new Properties();
            prop.load(in);//通过流将配置项加载进来。
            //通过工厂类读取
            ds = BasicDataSourceFactory.createDataSource(prop);
        } catch (Exception e) {
            throw new ExceptionInInitializerError();
        }
    }

dbcp遇到的问题

遇到类似这样的错误，具体原因是某个连接长时间不活动了，mysql就把这个连接关闭了，但是连接池不知道，仍然拿这个连接去操作，结果自然出错。

The last packet successfully received from the server was 904,023 milliseconds ago.  The last packet sent successfully to the server was 904,023 milliseconds ago.
	at com.mysql.cj.jdbc.exceptions.SQLError.createCommunicationsException(SQLError.java:174)
	at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:64)
	at com.mysql.cj.jdbc.ConnectionImpl.setAutoCommit(ConnectionImpl.java:2074)
	at org.apache.commons.dbcp.DelegatingConnection.setAutoCommit(DelegatingConnection.java:371)
	at org.apache.commons.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.setAutoCommit(PoolingDataSource.java:328)
	at sink.SensorMysqlSinkForLinux.getSensorIdByJDBC(SensorMysqlSinkForLinux.java:316)
	at sink.SensorMysqlSinkForLinux.process(SensorMysqlSinkForLinux.java:158)
	at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:67)
	at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:145)
	at java.lang.Thread.run(Thread.java:745)
Caused by: com.mysql.cj.exceptions.CJCommunicationsException: Communications link failure

查看mysql服务的等待超时时间，默认是8小时，解决办法之一是修改这个参数，最大有效时间是24天（超过了无效）

show variables like '%timeout';
wait_timeout	28800

修改参数

[mysqld]
.....
wait_timeout=31536000
interactive_timeout=31536000
.....

第2种解决办法是修改dbcp配置文件，增加以下参数

#SQL查询,用来验证从连接池取出的连接
validationQuery=SELECT 1
#指明连接是否被空闲连接回收器(如果有)进行检验，如果检测失败，则连接将被从池中去除
testWhileIdle=true
#在空闲连接回收器线程运行期间休眠的时间值,以毫秒为单位，一般比minEvictableIdleTimeMillis小
timeBetweenEvictionRunsMillis=300000
#在每次空闲连接回收器线程(如果有)运行时检查的连接数量，最好和maxActive一致
numTestsPerEvictionRun=10
#连接池中连接，在时间段内一直空闲，被逐出连接池的时间(1000*60*60)，以毫秒为单位
minEvictableIdleTimeMillis=3600000

测试

随便新建一个文件，我这里保存成test_data1.data

zhangsan2,22,2020-04-04 11:11:11
zhangsan3,23,2020-04-04 11:11:12
zhangsan4,24,2020-04-04 11:11:13
zhangsan5,25,2020-04-04 11:11:14

然后将该文件上传到/opt/cr1000/spooldir目录下，观察日志，有程序中的日志MysqlSink start...

2020-07-15 20:27:26,475 (conf-file-poller-0) [INFO - sink.MysqlSink.<init>(MysqlSink.java:35)] MysqlSink start...
2020-07-15 20:27:26,476 (conf-file-poller-0) [INFO - org.apache.flume.node.AbstractConfigurationProvider.getConfiguration(AbstractConfigurationProvider.java:116)] Channel c1 connected to [s1, mysqlSink]
2020-07-15 20:27:26,481 (conf-file-poller-0) [INFO - org.apache.flume.node.Application.startAllComponents(Application.java:137)] Starting new configuration:{ sourceRunners:{s1=EventDrivenSourceRunner: { source:Spool Directory source s1: { spoolDir: /opt/cr1000/spooldir } }} sinkRunners:{mysqlSink=SinkRunner: { policy:org.apache.flume.sink.DefaultSinkProcessor@36ffdfe1 counterGroup:{ name:null counters:{} } }} channels:{c1=org.apache.flume.channel.MemoryChannel{name: c1}} }
2020-07-15 20:27:26,498 (conf-file-poller-0) [INFO - org.apache.flume.node.Application.startAllComponents(Application.java:144)] Starting Channel c1
2020-07-15 20:27:26,512 (conf-file-poller-0) [INFO - org.apache.flume.node.Application.startAllComponents(Application.java:159)] Waiting for channel: c1 to start. Sleeping for 500 ms
2020-07-15 20:27:26,588 (lifecycleSupervisor-1-0) [INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.register(MonitoredCounterGroup.java:119)] Monitored counter group for type: CHANNEL, name: c1: Successfully registered new MBean.
2020-07-15 20:27:26,588 (lifecycleSupervisor-1-0) [INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.start(MonitoredCounterGroup.java:95)] Component type: CHANNEL, name: c1 started
2020-07-15 20:27:27,013 (conf-file-poller-0) [INFO - org.apache.flume.node.Application.startAllComponents(Application.java:171)] Starting Sink mysqlSink
2020-07-15 20:27:27,017 (conf-file-poller-0) [INFO - org.apache.flume.node.Application.startAllComponents(Application.java:182)] Starting Source s1
2020-07-15 20:27:27,019 (lifecycleSupervisor-1-4) [INFO - org.apache.flume.source.SpoolDirectorySource.start(SpoolDirectorySource.java:83)] SpoolDirectorySource source starting with directory: /opt/cr1000/spooldir
2020-07-15 20:27:27,141 (lifecycleSupervisor-1-4) [INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.register(MonitoredCounterGroup.java:119)] Monitored counter group for type: SOURCE, name: s1: Successfully registered new MBean.
2020-07-15 20:27:27,141 (lifecycleSupervisor-1-4) [INFO - org.apache.flume.instrumentation.MonitoredCounterGroup.start(MonitoredCounterGroup.java:95)] Component type: SOURCE, name: s1 started
Loading class `com.mysql.jdbc.Driver'. This is deprecated. The new driver class is `com.mysql.cj.jdbc.Driver'. The driver is automatically registered via the SPI and manual loading of the driver class is generally unnecessary.
2020-07-15 20:29:16,244 (pool-3-thread-1) [INFO - org.apache.flume.client.avro.ReliableSpoolingFileEventReader.readEvents(ReliableSpoolingFileEventReader.java:324)] Last read took us just up to a file boundary. Rolling to the next file, if there is one.
2020-07-15 20:29:16,244 (pool-3-thread-1) [INFO - org.apache.flume.client.avro.ReliableSpoolingFileEventReader.rollCurrentFile(ReliableSpoolingFileEventReader.java:433)] Preparing to move file /opt/cr1000/spooldir/test_data1.data to /opt/cr1000/spooldir/test_data1.data.COMPLETED

数据库查看：

mysql> use flume01;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
mysql> select * from table01;
+----+-----------+-----+-------------+
| id | name      | age | insert_date |
+----+-----------+-----+-------------+
|  1 | zhangesan |  20 | 2020-01-02  |
|  2 | zhangsan2 |  22 | 2020-04-03  |
|  3 | zhangsan3 |  23 | 2020-04-03  |
|  4 | zhangsan4 |  24 | 2020-04-03  |
|  5 | zhangsan5 |  25 | 2020-04-03  |
+----+-----------+-----+-------------+
5 rows in set (0.00 sec)

遇到的问题

1、

2020-07-15 20:00:26,860 (conf-file-poller-0) [WARN - org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateSinks(FlumeConfiguration.java:697)] Could not configure sink  mysqlSink due to: No channel configured for sink: mysqlSink
org.apache.flume.conf.ConfigurationException: No channel configured for sink: mysqlSink
	at org.apache.flume.conf.sink.SinkConfiguration.configure(SinkConfiguration.java:52)
	at org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateSinks(FlumeConfiguration.java:680)
	at org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.isValid(FlumeConfiguration.java:347)
	at org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.access$000(FlumeConfiguration.java:212)
	at org.apache.flume.conf.FlumeConfiguration.validateConfiguration(FlumeConfiguration.java:126)
	at org.apache.flume.conf.FlumeConfiguration.<init>(FlumeConfiguration.java:108)
	at org.apache.flume.node.PropertiesFileConfigurationProvider.getFlumeConfiguration(PropertiesFileConfigurationProvider.java:194)
	at org.apache.flume.node.AbstractConfigurationProvider.getConfiguration(AbstractConfigurationProvider.java:93)
	at org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run(PollingPropertiesFileConfigurationProvider.java:141)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

解决办法：a1.sinks.s1.channels应该为channel

flume读取HDFS数据到kafka_flume

2、

2020-07-15 20:03:36,706 (conf-file-poller-0) [ERROR - org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run(PollingPropertiesFileConfigurationProvider.java:143)] Failed to load configuration data. Exception follows.
org.apache.flume.FlumeException: Unable to load sink type: com.wim.mysqlsink.sink.MysqlSink, class: com.wim.mysqlsink.sink.MysqlSink
	at org.apache.flume.sink.DefaultSinkFactory.getClass(DefaultSinkFactory.java:70)
	at org.apache.flume.sink.DefaultSinkFactory.create(DefaultSinkFactory.java:43)
	at org.apache.flume.node.AbstractConfigurationProvider.loadSinks(AbstractConfigurationProvider.java:408)
	at org.apache.flume.node.AbstractConfigurationProvider.getConfiguration(AbstractConfigurationProvider.java:102)
	at org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run(PollingPropertiesFileConfigurationProvider.java:141)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

解决办法：
在参数a1.sinks.mysqlSink.type=sink.MysqlSink设置中，自定义的类只需要写上包名.类名就行了。

3、

java.sql.SQLException: null,  message from server: "Host 'worker.learn.bigdata' is not allowed to connect to this MySQL server"
	at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:129)
	at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:97)
	at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:122)
	at com.mysql.cj.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:835)
	at com.mysql.cj.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:455)
	at com.mysql.cj.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:240)
	at com.mysql.cj.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:199)
	at java.sql.DriverManager.getConnection(DriverManager.java:664)
	at java.sql.DriverManager.getConnection(DriverManager.java:247)
	at sink.MysqlSink.start(MysqlSink.java:71)
	at org.apache.flume.sink.DefaultSinkProcessor.start(DefaultSinkProcessor.java:45)
	at org.apache.flume.SinkRunner.start(SinkRunner.java:79)
	at org.apache.flume.lifecycle.LifecycleSupervisor$MonitorRunnable.run(LifecycleSupervisor.java:249)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

解决办法：

mysql> use mysql;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
mysql> select host from user where user='root';
+-----------+
| host      |
+-----------+
| localhost |
+-----------+
1 row in set (0.01 sec)

mysql> update user set host = '%' where user ='root';
Query OK, 1 row affected (0.01 sec)
Rows matched: 1  Changed: 1  Warnings: 0

mysql> select host from user where user='root';
+------+
| host |
+------+
| %    |
+------+
1 row in set (0.00 sec)

mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。