send data to Flume client-sdk flume使用之httpSource

转载

mb5fd86ddc9c8d5 2017-11-11 01:27:00 ©著作权

文章标签 json apache java hadoop ide 文章分类 其他其它

send data to Flume client-sdk flume使用之httpSource
https://blog.51cto.com/u_15057858/4626784

package com.mycom;


import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.flume.Channel;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.Transaction;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class MyFlumeSink extends AbstractSink implements Configurable {
    private static final Logger logger = LoggerFactory.getLogger(MyFlumeSink.class);
    private static final String PROP_KEY_ROOTPATH = "fileName";
    private String fileName;

    @Override
    public void configure(Context context) {
        // TODO Auto-generated method stub
        fileName = context.getString(PROP_KEY_ROOTPATH);
    }

    @Override
    public Status process() throws EventDeliveryException {
        // TODO Auto-generated method stub
        Channel ch = getChannel();
        Transaction txn = ch.getTransaction();
        Event event = null;
        txn.begin();
        while (true) {
            event = ch.take();
            if (event != null) {
                break;
            }
        }
        try {

            logger.debug("Get event.");

            String body = new String(event.getBody());
            System.out.println("event.getBody()-----" + body);

            String res = body + ":" + System.currentTimeMillis() + "\r\n";
            File file = new File(fileName);
            FileOutputStream fos = null;
            try {
                fos = new FileOutputStream(file, true);
            } catch (FileNotFoundException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            try {
                fos.write(res.getBytes());
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            try {
                fos.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            txn.commit();
            return Status.READY;
        } catch (Throwable th) {
            txn.rollback();

            if (th instanceof Error) {
                throw (Error) th;
            } else {
                throw new EventDeliveryException(th);
            }
        } finally {
            txn.close();
        }

    }

}

[root@d1 apache-flume-1.8.0-bin]# cat conf/http_test.conf 
a1.sources=r1
a1.sinks=k1
a1.channels=c1

a1.sources.r1.type=http
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=50000
a1.sources.r1.channels=c1

a1.sinks.k1.type=logger
a1.sinks.k1.channel=c1

a1.sinks.k1.type = com.mycom.MyFlumeSink
a1.sinks.k1.fileName=/home/mysinks.txt


a1.channels.c1.type=memory
#a1.channels.c1.capacity=1000
#a1.channels.c1.transactionCapacity=100
a1.channels.c1.capacity=8
a1.channels.c1.transactionCapacity=4
[root@d1 apache-flume-1.8.0-bin]#

[root@d1 home]# cat ~/.bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

. ~/.bashrc

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

JAVA_HOME=/usr/local/jdk;export JAVA_HOME;

HADOOP_PREFIX=/home/hadoop-2.9.1;export HADOOP_PREFIX;HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop;export HADOOP_CONF_DIR;HADOOP_HOME=/home/hadoop-2.9.1;export HADOOP_HOME;HADOOP_PREFIX=/home/hadoop-2.9.1;export HADOOP_PREFIX;HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop;export HADOOP_CONF_DIR;HADOOP_HOME=/home/hadoop-2.9.1;export HADOOP_HOME;HADOOP_YARN_HOME=$HADOOP_PREFIX;export HADOOP_YARN_HOME;

[root@d1 home]#

bin/flume-ng agent -c /home/apache-flume-1.8.0-bin/conf/ -f /home/apache-flume-1.8.0-bin/conf/http_test.conf -n a1 -Dflume.root.logger=INFO,console -C /home/MyBgJavaLan/target/MyAid-1.0.0-jar-with-dependencies.jar

[INFO]

[INFO] --- maven-jar-plugin:2.3.2:jar (default-jar) @ MyAid ---

[INFO] Building jar: /home/MyBgJavaLan/target/MyAid-1.0.0.jar

[INFO]

[INFO] --- maven-assembly-plugin:2.4:single (make-assembly) @ MyAid ---

[INFO] artifact net.minidev:json-smart: checking for updates from central

[INFO] artifact net.minidev:json-smart: checking for updates from dynamodb-local-oregon

[INFO] artifact net.minidev:json-smart: checking for updates from apache.snapshots.https

[INFO] artifact net.minidev:json-smart: checking for updates from repository.jboss.org

[INFO] Building jar: /home/MyBgJavaLan/target/MyAid-1.0.0-jar-with-dependencies.jar

[INFO] ------------------------------------------------------------------------

[INFO] BUILD SUCCESS

[INFO] ------------------------------------------------------------------------

[INFO] Total time: 58.121s

[INFO] Finished at: Tue Aug 07 17:33:01 CST 2018

[INFO] Final Memory: 39M/749M

[INFO] ------------------------------------------------------------------------

[root@d1 MyBgJavaLan]# mvn clean;mvn compile;mvn package;

存在的问题：

[DEBUG] 2018-08-07 17:39:59,103 method:org.apache.flume.SinkRunner.stop(SinkRunner.java:104)