elasticsearch 日志抓取使用

[root@server5 elk]# rpm -ivh logstash-2.3.3-1.noarch.rpm [root@server5 logstash]# /opt/logstash/bin/logstash -e 'input {stdin { } } output { stdout { codec => rubydebug } }' Settings: Default pipeline workers: 1 Pipeline main started hello #随便写 { "message" => "hello", "@version" => "1", "@timestamp" => "2018-12-08T03:58:53.761Z", "host" => "server5" } redhat { "message" => "redhat", "@version" => "1", "@timestamp" => "2018-12-08T03:59:05.366Z", "host" => "server5" } ^CSIGINT received. Shutting down the agent. {:level=>:warn} stopping pipeline {:id=>"main"}

Pipeline main has been shutdown [root@server5 logstash]# /opt/logstash/bin/logstash -e 'input {stdin { } } output { stdout { codec => rubydebug } elasticsearch { hosts => ["172.25.135.5"] index => "logstash-%{+YYYY.MM.dd}"} }' [root@server5 logstash]# cd /etc/logstash/conf.d/ [root@server5 conf.d]# ls [root@server5 conf.d]# vim es.conf

[root@server5 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf [root@server5 conf.d]# cat /tmp/testfile custom format: asdjkl [root@server5 conf.d]# chmod 644 /var/log/messages [root@server5 conf.d]# logger "hello" #没有日志文件写一些，有可略过 [root@server5 conf.d]# logger "hello" [root@server5 conf.d]# logger "hello" [root@server5 conf.d]# logger "hello"

[root@server5 conf.d]# vim es.conf input { file { path => "/var/log/messages" start_position => "beginning" } }

output {

stdout {

codec => rubydebug

}

    elasticsearch {
            hosts => ["172.25.135.5"]
            index => "message-%{+YYYY.MM.dd}"
    }

file {

path => "/tmp/testfile"

codec => line { format => "custom format: %{message}"}

}

} [root@server5 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf [root@server5 conf.d]# cd [root@server5 ~]# l. . .bashrc .pki .. .cache .sincedb_452905a167cf4509fd08acb964fdb20c .bash_history .cshrc .ssh .bash_logout .mysql_history .tcshrc .bash_profile .oracle_jre_usage .viminfo [root@server5 ~]# cat .sincedb_452905a167cf4509fd08acb964fdb20c 267849 0 64768 517 [root@server5 ~]# ls -i /var/log/messages 267849 /var/log/messages #两者id一样 [root@server5 conf.d]# vim /etc/rsyslog.conf .* @@172.25.135.5:514 #末尾添加，server7，8同样添加，同样重启以下服务 [root@server5 conf.d]# /etc/init.d/elasticsearch restart [root@server5 conf.d]# /etc/init.d/rsyslog restart
[root@server5 conf.d]# vim es.conf input {

file {

path => "/var/log/messages"

start_position => "beginning"

}

   syslog {
          port  => 514
    }

}

output { stdout { codec => rubydebug } elasticsearch { hosts => ["172.25.135.5"] index => "message-%{+YYYY.MM.dd}" }

file {

path => "/tmp/testfile"

codec => line { format => "custom format: %{message}"}

}

}

[root@server5 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf #为了方便看： 可以给server7，8加点日志 [root@server7 ~]# logger server2 [root@server7 ~]# logger server2 [root@server7 ~]# logger server2 [root@server8 vm]# logger server3 [root@server8 vm]# logger server3 [root@server8 vm]# logger server3 刷新可以看出： [root@server5 conf.d]# cd [root@server5 ~]# l. . .bash_profile .mysql_history .ssh .. .bashrc .oracle_jre_usage .tcshrc .bash_history .cache .pki .viminfo .bash_logout .cshrc .sincedb_452905a167cf4509fd08acb964fdb20c [root@server5 ~]# rm -fr .sincedb_452905a167cf4509fd08acb964fdb20c [root@server5 ~]# cd - /etc/logstash/conf.d [root@server5 conf.d]# ls es.conf [root@server5 conf.d]# vim es.conf input { file { path => "/var/log/elasticsearch/my-es.log" start_position => "beginning" }

syslog {

port => 514

}

} #filter {

multiline {

type => "eslog"

pattern => "^["

negate => true

what => "previous"

}

#} output {

stdout {

codec => rubydebug

}

    elasticsearch {
            hosts => ["172.25.135.5"]
            index => "es-%{+YYYY.MM.dd}"
    }

file {

path => "/tmp/testfile"

codec => line { format => "custom format: %{message}"}

}

}

[root@server5 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf #新建es log 刷新浏览器可以看出：