Flume安装与配置与测试

1. 上传 apache-flume-1.6.0-bin.tar.gz 到 node1/opt/apps 目录下
2. 解压并重命名

[root@node1 apps]# tar -zxvf apache-flume-1.6.0-bin.tar.gz -C /opt/ 
[root@node1 apps]# cd /opt 
[root@node1 opt]# mv apache-flume-1.6.0-bin/ flume

3. 删除 docs 目录

[root@node1 opt]# ls 
apps data flume hadoop-2.6.5 hbase-0.98 
[root@node1 opt]# cd flume/ 
[root@node1 flume]# ls 
bin CHANGELOG conf DEVNOTES docs lib LICENSE NOTICE README RELEASE-NOTES tools 
[root@node1 flume]# rm -rf docs/

4. 修改配置

[root@node1 conf]# mv flume-env.sh.template flume-env.sh 
[root@node1 conf]# vim flume-env.sh export 
JAVA_HOME=/usr/java/default

5. 配置 flume 环境变量，并让配置文件生效

[root@node1 ~]# vim /etc/profile 
export FLUME_HOME=/opt/flume 
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HBA SE_HOME/bin:$FLUME_HOME/bin 
[root@node1 ~]# source /etc/profile

6. 验证安装是否成功

[root@node1 ~]# flume-ng version
Flume 1.6.0
Source code repository: https://git-wip-us.apache.org/repos/asf/flume.git
Revision: 2561a23240a71ba20bf288c7c2cda88f443c2080
Compiled by hshreedharan on Mon May 11 11:15:44 PDT 2015
From source with checksum b29e416802ce9ece3269d34233baf43f

7. 在这里，我们给出一个示例配置文件，描述一个单节点 Flume 部署。 此配置允许用户生成 事件并随后将其记录到控制台。

# example.conf: A single-node Flume configuration 
# Name the components on this agent 
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source 
a1.sources.r1.type = netcat
a1.sources.r1.bind = node1
a1.sources.r1.port = 44444
# Describe the sink 
a1.sinks.k1.type = logger
# Use a channel which buffers events in memory 
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel 
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

8. 查看帮助

[root@node1 conf]# flume-ng help 
Usage: /opt/flume/bin/flume-ng <command> [options]... 
-Dproperty=value sets a Java system property value 指定 java 的系统属性值 
-Xproperty=value sets a Java -X option agent options: 
--name,-n <name> the name of this agent (required)指定代理 的名称
--conf-file,-f <file> specify a config file (required if -z missing)如果不指定-z 选项，则必须指定配置文件 
--zkConnString,-z <str> specify the ZooKeeper connection to use

9. 启动

[root@node1 ~]#flume-ng agent -n a1 --conf-file option.properties -Dflume.root.logger=INFO,console

10. 打开一个连接 node2 的 xshell 终端，安装 telnet

[root@node2 ~]# yum install telnet -y 
[root@node2 ~]# telnet node1 44444

[root@node2 ~]# telnet node1 4444
Trying 192.168.134.101...
telnet: connect to address 192.168.134.101: Connection refused
[root@node2 ~]# telnet node1 44444
Trying 192.168.134.101...
Connected to node1.
Escape character is '^]'.
A
OK
B
OK
HELLO YQQ
OK

11. 查看 node1 的变化

21/11/19 18:54:44 INFO source.NetcatSource: Created serverSocket:sun.nio.ch.ServerSocketChannelImpl[/192.168.134.101:44444]
21/11/19 19:07:06 INFO sink.LoggerSink: Event: { headers:{} body: 41 0D                                           A. }
21/11/19 19:07:17 INFO sink.LoggerSink: Event: { headers:{} body: 42 0D                                           B. }
21/11/19 19:07:32 INFO sink.LoggerSink: Event: { headers:{} body: 48 45 4C 4C 4F 20 59 51 51 0D                   HELLO YQQ. }