package com.mycom;


import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.flume.Channel;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.Transaction;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class MyFlumeSink extends AbstractSink implements Configurable {
private static final Logger logger = LoggerFactory.getLogger(MyFlumeSink.class);
private static final String PROP_KEY_ROOTPATH = "fileName";
private String fileName;

@Override
public void configure(Context context) {
// TODO Auto-generated method stub
fileName = context.getString(PROP_KEY_ROOTPATH);
}

@Override
public Status process() throws EventDeliveryException {
// TODO Auto-generated method stub
Channel ch = getChannel();
Transaction txn = ch.getTransaction();
Event event = null;
txn.begin();
while (true) {
event = ch.take();
if (event != null) {
break;
}
}
try {

logger.debug("Get event.");

String body = new String(event.getBody());
System.out.println("event.getBody()-----" + body);

String res = body + ":" + System.currentTimeMillis() + "\r\n";
File file = new File(fileName);
FileOutputStream fos = null;
try {
fos = new FileOutputStream(file, true);
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
fos.write(res.getBytes());
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
fos.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
txn.commit();
return Status.READY;
} catch (Throwable th) {
txn.rollback();

if (th instanceof Error) {
throw (Error) th;
} else {
throw new EventDeliveryException(th);
}
} finally {
txn.close();
}

}

}


  

 

 



[root@d1 apache-flume-1.8.0-bin]# cat conf/http_test.conf 
a1.sources=r1
a1.sinks=k1
a1.channels=c1

a1.sources.r1.type=http
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=50000
a1.sources.r1.channels=c1

a1.sinks.k1.type=logger
a1.sinks.k1.channel=c1

a1.sinks.k1.type = com.mycom.MyFlumeSink
a1.sinks.k1.fileName=/home/mysinks.txt


a1.channels.c1.type=memory
#a1.channels.c1.capacity=1000
#a1.channels.c1.transactionCapacity=100
a1.channels.c1.capacity=8
a1.channels.c1.transactionCapacity=4
[root@d1 apache-flume-1.8.0-bin]#


  

[root@d1 home]# cat ~/.bash_profile

# .bash_profile

# Get the aliases and functions

if [ -f ~/.bashrc ]; then

. ~/.bashrc

fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH

JAVA_HOME=/usr/local/jdk;export JAVA_HOME;

HADOOP_PREFIX=/home/hadoop-2.9.1;export HADOOP_PREFIX;HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop;export HADOOP_CONF_DIR;HADOOP_HOME=/home/hadoop-2.9.1;export HADOOP_HOME;HADOOP_PREFIX=/home/hadoop-2.9.1;export HADOOP_PREFIX;HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop;export HADOOP_CONF_DIR;HADOOP_HOME=/home/hadoop-2.9.1;export HADOOP_HOME;HADOOP_YARN_HOME=$HADOOP_PREFIX;export HADOOP_YARN_HOME;

[root@d1 home]#

 

 

 


 

bin/flume-ng agent -c /home/apache-flume-1.8.0-bin/conf/  -f  /home/apache-flume-1.8.0-bin/conf/http_test.conf -n a1 -Dflume.root.logger=INFO,console -C /home/MyBgJavaLan/target/MyAid-1.0.0-jar-with-dependencies.jar

 

 

[INFO]

[INFO] --- maven-jar-plugin:2.3.2:jar (default-jar) @ MyAid ---

[INFO] Building jar: /home/MyBgJavaLan/target/MyAid-1.0.0.jar

[INFO]

[INFO] --- maven-assembly-plugin:2.4:single (make-assembly) @ MyAid ---

[INFO] artifact net.minidev:json-smart: checking for updates from central

[INFO] artifact net.minidev:json-smart: checking for updates from dynamodb-local-oregon

[INFO] artifact net.minidev:json-smart: checking for updates from apache.snapshots.https

[INFO] artifact net.minidev:json-smart: checking for updates from repository.jboss.org

[INFO] Building jar: /home/MyBgJavaLan/target/MyAid-1.0.0-jar-with-dependencies.jar

[INFO] ------------------------------------------------------------------------

[INFO] BUILD SUCCESS

[INFO] ------------------------------------------------------------------------

[INFO] Total time: 58.121s

[INFO] Finished at: Tue Aug 07 17:33:01 CST 2018

[INFO] Final Memory: 39M/749M

[INFO] ------------------------------------------------------------------------

[root@d1 MyBgJavaLan]# mvn clean;mvn compile;mvn package;

 

 

存在的问题:

[DEBUG] 2018-08-07 17:39:59,103 method:org.apache.flume.SinkRunner.stop(SinkRunner.java:104)

Waiting for runner thread to exit

[DEBUG] 2018-08-07 17:39:59,604 method:org.apache.flume.SinkRunner.stop(SinkRunner.java:104)

Waiting for runner thread to exit

[DEBUG] 2018-08-07 17:40:00,104 method:org.apache.flume.SinkRunner.stop(SinkRunner.java:104)

Waiting for runner thread to exit

[DEBUG] 2018-08-07 17:40:00,604 method:org.apache.flume.SinkRunner.stop(SinkRunner.java:104)

Waiting for runner thread to exit

[DEBUG] 2018-08-07 17:40:01,105 method:org.apache.flume.SinkRunner.stop(SinkRunner.java:104)

Waiting for runner thread to exit

[DEBUG] 2018-08-07 17:40:01,605 method:org.apache.flume.SinkRunner.stop(SinkRunner.java:104)

Waiting for runner thread to exit

 

 



package com.mycom;


import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.flume.Channel;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.Transaction;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.gson.JsonArray;
import com.google.gson.JsonIOException;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonSyntaxException;

public class MyFlumeSink extends AbstractSink implements Configurable {
private static final Logger logger = LoggerFactory.getLogger(MyFlumeSink.class);
private static final String PROP_KEY_ROOTPATH = "fileName";
private String fileName;

@Override
public void configure(Context context) {
// TODO Auto-generated method stub
fileName = context.getString(PROP_KEY_ROOTPATH);
}

@Override
public Status process() throws EventDeliveryException {
// TODO Auto-generated method stub
Channel ch = getChannel();
Transaction txn = ch.getTransaction();
Event event = null;
txn.begin();
while (true) {
event = ch.take();
if (event != null) {
break;
}
}
try {

logger.debug("Get event.");


// request.getParameter("username")
// JSON json = JSONObject.parseObject(event.getBody());
//JSONObject jsonObject1 = JSONObject.parseObject(JSON_OBJ_STR); //因为JSONObject继承了JSON,所以这样也是可以的

// System.out.println(jsonObject.getString("studentName")+":"+jsonObject.getInteger("studentAge"));
// String body = jsonObject.getString("body");
//
//old

//JAVA解析JSON数据 - monsterLin

// String body = new String(event.getBody());
String body = event.getBody().toString();

JsonParser parse = new JsonParser(); //创建json解析器
JsonObject json = (JsonObject) parse.parse(body); //创建jsonObject对象
String bodyReal = json.get("body").getAsString();
System.out.println("event.getBody()-----" + bodyReal);
String res = bodyReal + ":" + System.currentTimeMillis() + "\r\n";
File file = new File(fileName);
FileOutputStream fos = null;
try {
fos = new FileOutputStream(file, true);
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
fos.write(res.getBytes());
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
fos.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
txn.commit();
return Status.READY;
} catch (Throwable th) {
txn.rollback();

if (th instanceof Error) {
throw (Error) th;
} else {
throw new EventDeliveryException(th);
}
} finally {
txn.close();
}

}

}


 

 



package com.mycom;

import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.api.RpcClient;
import org.apache.flume.api.RpcClientFactory;
import org.apache.flume.event.EventBuilder;

import java.nio.charset.Charset;
import java.text.SimpleDateFormat;

//http://flume.apache.org/FlumeDeveloperGuide.html

public class MyAppFlume {
public static void main(String[] args) {
MyRpcClientFacade client = new MyRpcClientFacade();
// Initialize client with the remote Flume agent's host and port
// client.init("hadoop1", 41414);
// client.init("192.168.3.101", 41414);
client.init("0.0.0.0", 41414);

// Send 10 events to the remote Flume agent. That agent should be configured to listen with an AvroSource.
String sampleData = "Hello Flume!";

SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd--HH-mm-ss");
long cTm = System.currentTimeMillis();
String df = sdf.format(cTm);
System.out.println(df);

String str_ = "";
for (int i = 0; i < 14; i++) {
str_ = str_ + i + df + "加油!!" + sampleData;
}

System.out.println(str_);

// for (int i = 0; i < 20; i++) {
// String str = i + "------------" + str_ + df + "加油!!" + sampleData;
// System.out.println(str);
// client.sendDataToFlume(str);
// }

String str = "对StringBuilder抛出ArrayIndexOutOfBoundsException的探究 - .csdn.net/liu_005/article/details/73699604" + "------------" + str_ + df + "加油!!" + sampleData;
System.out.println(str);
client.sendDataToFlume(str);

client.cleanUp();
}
}

class MyRpcClientFacade {
private RpcClient client;
private String hostname;
private int port;

public void init(String hostname, int port) {
// Setup the RPC connection
this.hostname = hostname;
this.port = port;
this.client = RpcClientFactory.getDefaultInstance(hostname, port);
// Use the following method to create a thrift client(instead of the above line);
// this.client=RpcClientFactory.getThriftInstance(hostname,port);

}

public void sendDataToFlume(String data) {
// Create a Flume Event object that encapsulate the sample data
Event event = EventBuilder.withBody(data, Charset.forName("UTF-8"));
System.out.println("--->");
System.out.println(data);
System.out.println(event);
System.out.println("<--->");


/*
* int getBatchSize();

void append(Event var1) throws EventDeliveryException;

void appendBatch(List<Event> var1) throws EventDeliveryException;

boolean isActive();

void close() throws FlumeException;
*
* */
// Send the event
try {
this.client.append(event);
System.out.println("<----------client.append(event)--------------->");

} catch (EventDeliveryException e) {
e.printStackTrace();

System.out.println(e);

// clean up and recreate the client
client.close();
client = null;
client = RpcClientFactory.getDefaultInstance(hostname, port);
}
}

public void cleanUp() {
// Close the RPC connection
client.close();
}
}


  

【动手】

 

 分布式日志Va

版本V a

待废除php,引入go网关 

protected function data2unifiedlog($data)
{
/*
[{
"headers" : {
"timestamp" : "434324343",
"host" : "random_host.example.com"
},
"body" : "random_body"
},
{
"headers" : {
"namenode" : "namenode.example.com",
"datanode" : "random_datanode.example.com"
},
"body" : "really_random_body"
}]
*/
$ch = curl_init();
$now = date('y-m-d h:i:s', time()) . gethostname();
$post_data_json = '[{
"headers" : {
"timestamp" : "434324343",
"host" : "random_host.example.com"
},
"body" : \'str86677' . $now . '\'}]';
$s = '{"hostname":' . gethostname() . ',"timestamp":"' . time() . '"';
foreach ($data as $k => $v) {
$s = $s . ',"' . $k . '":"' . $v . '"';
}
$s = $s . '}';
$post_data_json = '[{
"headers" : {
"timestamp" : "434324343",
"host" : "random_host.example.com"
},
"body" : \'' . $s . '\'}]';
$curlopt_url = 'http://101.201.41.72:50000';
curl_setopt($ch, CURLOPT_URL, $curlopt_url);
curl_setopt($ch, CURLOPT_HEADER, TRUE);
curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type:application/json'));
curl_setopt($ch, CURLOPT_POSTFIELDS, $post_data_json);
curl_exec($ch);
curl_close($ch);
}


str8667718-11-02 04:23:28a

str8667718-11-02 04:23:28b

str8667718-11-02 04:23:28a

str8667718-11-02 04:23:28c

str8667718-11-02 04:23:28b

str8667718-11-02 04:23:29a

str8667718-11-02 04:23:29a

str8667718-11-02 04:23:29c

str8667718-11-02 04:23:29b

str8667718-11-02 04:23:29c

str8667718-11-02 04:23:29a

str8667718-11-02 04:23:29c

str8667718-11-02 04:23:29c

str8667718-11-02 04:23:29a

str8667718-11-02 04:23:29b

str8667718-11-02 04:23:29c

str8667718-11-02 04:23:30b

str8667718-11-02 04:23:30b

str8667718-11-02 04:23:30c

str8667718-11-02 04:23:30a

str8667718-11-02 04:23:30c

str8667718-11-02 04:23:30a

主机名
a1.sources=r1
a1.sinks=k1
a1.channels=c1
a1.sources.r1.type=http
a1.sources.r1.bind=0.0.0.0
a1.sources.r1.port=50000
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1
#a1.sinks.k1.type = com.product.FlumeApp
a1.sinks.k1.type = file_roll
a1.sinks.k1.sink.directory = /data/UnifiedLog/log
a1.sinks.k1.batchSize=1
#a1.sinks.k1.pathManager=%y%m%d%H%M%S
a1.sinks.k1.pathManager.extension=log
a1.sinks.k1.pathManager.prefix=webTrack
a1.sinks.k1.rollInterval=0
a1.sinks.k1.sink.serializer = text
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
#a1.channels.c1.capacity=8
#a1.channels.c1.transactionCapacity=4
启动
export FlumeHome=/data/UnifiedLog/flume; $FlumeHome/bin/flume-ng agent -c $FlumeHome/conf/  -f  $FlumeHome/conf/httpSourceApp.conf -n a1 -Dflume.root.logger=INFO,console  -Xms10240m  -Xmx10240m ;
进程


root 23619 1.4 7.9 16559800 2608576 pts/3 Sl+ 15:31 1:05 /usr/java/jdk1.8.0_101/bin/java -Xmx20m -Dflume.root.logger=INFO,console -Xms10240m -Xmx10240m -cp /data/UnifiedLog/flume/conf:/data/UnifiedLog/flume/lib/*:/lib/* -Djava.library.path= org.apache.flume.node.Application -f /data/UnifiedLog/flume/conf/httpSourceApp.conf -n a1

 

  端口

tcp 0 0 0.0.0.0:50000 0.0.0.0:* LISTEN 23619/java

 

  日志目录

8.0K -rw-r--r-- 1 root root 5.3K Nov 2 16:32 1541143920191-102

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-103

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-104

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-105

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-106

8.0K -rw-r--r-- 1 root root 5.3K Nov 2 16:32 1541143920191-107

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-108

8.0K -rw-r--r-- 1 root root 5.3K Nov 2 16:32 1541143920191-109

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-110

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-111

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-112

8.0K -rw-r--r-- 1 root root 5.3K Nov 2 16:32 1541143920191-113

8.0K -rw-r--r-- 1 root root 5.3K Nov 2 16:32 1541143920191-114

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-115

8.0K -rw-r--r-- 1 root root 5.3K Nov 2 16:32 1541143920191-116

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-117

8.0K -rw-r--r-- 1 root root 8.0K Nov 2 16:32 1541143920191-118

8.0K -rw-r--r-- 1 root root 5.3K Nov 2 16:32 1541143920191-119

4.0K -rw-r--r-- 1 root root 2.7K Nov 2 16:32 1541143920191-120

 【动手】

 

 https://github.com/apache/flume/blob/c5168c902634e8ea1f25ec578ed0b7055b246d68/flume-ng-core/src/test/java/org/apache/flume/source/http/TestHTTPSource.java


@Test

 

public void testSimpleUTF16() throws IOException, InterruptedException {

 

 

 

StringEntity input = new StringEntity("[{\"headers\":{\"a\": \"b\"},\"body\": \"random_body\"},"

 

+ "{\"headers\":{\"e\": \"f\"},\"body\": \"random_body2\"}]", "UTF-16");

 

input.setContentType("application/json; charset=utf-16");

 

postRequest.setEntity(input);

 

 

 

HttpResponse response = httpClient.execute(postRequest);

 

 

 

Assert.assertEquals(HttpServletResponse.SC_OK,

 

response.getStatusLine().getStatusCode());

 

Transaction tx = httpChannel.getTransaction();

 

tx.begin();

 

Event e = httpChannel.take();

 

Assert.assertNotNull(e);

 

Assert.assertEquals("b", e.getHeaders().get("a"));

 

Assert.assertEquals("random_body", new String(e.getBody(), "UTF-16"));

 

 

 

e = httpChannel.take();

 

Assert.assertNotNull(e);

 

Assert.assertEquals("f", e.getHeaders().get("e"));

 

Assert.assertEquals("random_body2", new String(e.getBody(), "UTF-16"));

 

tx.commit();

 

tx.close();

 

}

 

 

 

@Test

 

public void testInvalid() throws Exception {

 

StringEntity input = new StringEntity("[{\"a\": \"b\",[\"d\":\"e\"],\"body\": \"random_body\"},"

 

+ "{\"e\": \"f\",\"body\": \"random_body2\"}]");

 

input.setContentType("application/json");

 

postRequest.setEntity(input);

 

HttpResponse response = httpClient.execute(postRequest);

 

 

 

Assert.assertEquals(HttpServletResponse.SC_BAD_REQUEST,

 

response.getStatusLine().getStatusCode());

 

SourceCounter sc = (SourceCounter) Whitebox.getInternalState(httpSource, "sourceCounter");

 

Assert.assertEquals(1, sc.getEventReadFail());

 

 

 

}

 

github  查测试用例

 

headers  application/json

body [{"headers" : {"a":"b", "c":"d"},"body": "random_body"}, {"headers" : {"e": "f"},"body": "random_body2"}]