一 简介
在上一篇中介绍了elk安装和收集日志，这一篇我们在中间加个数据库，让filebeat直接把日志发送给redis,然后logstash在去redis里面取出来
大致流程:filebeat---redis---logstash---es--kibana
二 redis安装
下载 https://redis.io/download
三 logstash配置
用redis，配置文件需要更改一下
[root@elk logstash]# cat conf.d/logstash.conf
input {
redis {
host => "172.17.199.231"
port => 6379
key => "filebeat"
db => "0"
data_type => "list"
}
}
filter{
grok {
match => ["message", "%{SYSLOGBASE} %{GREEDYDATA:message}"]
overwrite => ["message"]
}
}
output {
if [app] == "www" {
if [type] == "tiantian-system-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-system-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-system-service-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-system-service-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-assets-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-assets-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-collection-admin-node6" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-collection-admin-node6-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-collection-job-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-collection-job-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-credit-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-credit-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-jinbaodai-api-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-jinbaodai-api-node4-%{+YYYY.MM.dd}"
}
}
}
stdout { codec=> rubydebug }
}四 filebeat设置
https://www.elastic.co/guide/en/beats/filebeat/1.3/redis-output.html #filebeat设置redis
[root@node1 ~]# grep -v "^ *#" /etc/filebeat/filebeat.yml |grep -v "^$"
filebeat.inputs:
- type: log
paths:
- /datalog/service/jz-asset/node1/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: jz-asset-node1
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output:
redis:
hosts: ["172.17.199.231:6379"] #发送给redis
save_topology: true
index: "filebeat"
db: 0
db_topology: 1
timeout: 5
reconnect_interval: 1
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~只有这两个地方需要更改一下，其余地方不用设置
