ELK安装和配置笔记

logstash安装和配置

配置片段

IPv4地址转整数

ruby {
        id => "ndr_ruby_1"
        code => "
            ipstr = event.get('clientip')
            if ipstr =~ /\./ 
                iparray =  ipstr.split('.')
                ip_i = iparray[0].to_i * 16777216 + iparray[1].to_i * 65536 + iparray[2].to_i*256 + iparray[3].to_i
            else
                ip_i = 0
            end
            event.set('clientip_i',ip_i)
        "
    } #ruby

日期转换问题

索引分片和日期转换及Kibana显示日期问题。参看下面链接

https://www.cnblogs.com/zzb666/p/14148898.html

@timestamp使用日志生成时间，但会转换成UTC标准时间，即北京时间-8小时。生成一个日志索引日期字段index_date，保证日志按日志生成时间进入索引。kibana显示时，@timestamp会自动转换成北京时间显示。

input{　　...   }

filter{
　　date {
　　　　match => ["log_time", "yyyy-MM-dd HH:mm:ss"]
　　　　target => "@timestamp"
　　}
   ruby{
       code => "event.set('index_date', (event.get('@timestamp').time.localtime + 8*60*60).strftime('%Y.%m.%d'))"
   }
}
output {
     elasticsearch {
         hosts => [...]         
         index => "log-%{index_date}"
         #index => "log-%{+YYYY.MM.dd}"
     }
}

filter配置

将日志生成时间设置到@timestamp中

filter {
    if  [logstash-input] == "waf"{
        grok  {
            id => "waf-grok-1"
            match => ["message","tag:%{DATA:tag} site_id:%{NUMBER:site_id}  protect_id:%{NUMBER:protect_id}  dst_ip:%{IP:dst_ip}  dst_port:%{NUMBER:dst_port}  src_ip:%{IP:src_ip}  src_port:%{NUMBER:src_port}  method:%{WORD:method}  domain:%{GREEDYDATA:domain}  uri:%{GREEDYDATA:uri}  alertlevel:%{DATA:alertlevel}  event_type:%{DATA:event_type}  stat_time:%{DATA:stat_time}  policy_id:%{DATA:policy_id}  rule_id:%{DATA:rule_id}  action:%{WORD:action}  block:%{WORD:block}  block_info:%{DATA:block_info}  http:%{GREEDYDATA:http}"]
            remove_field => ["message","block_info"]
        }
        date {
             match => [ "stat_time", "yyyy-MM-dd HH:mm:ss" ]
             target => "@timestamp"
        }
        ruby {
             code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60);
                      event.set('@timestamp',event.get('timestamp'))"
             remove_field => ["timestamp","stat_time"]
        }
    }
}

jdbc+sqlite配置

下载jdbc_sqlite驱动地址：https://github.com/xerial/sqlite-jdbc

然后拷贝到自己配置的路径。

filter {

     jdbc_static {
      loaders => [ 
        {
          id => "remote-whitelist_test"
          query => "select ip, status,descr from whitelist_test order by ip"
          local_table => "whitelist_test"
        }
      ]
      local_db_objects => [
        {
          name => "whitelist_test"
          index_columns => ["ip"]
          columns => [
            ["ip", "varchar(50)"],
            ["status","int"],
            ["descr", "varchar(255)"]
          ]
        }
      ]
      local_lookups => [ 
        {
          id => "local-whitelist_test"
          query => "SELECT ip,status,descr FROM whitelist_test"
          target => "whitelist_test"
        }
      ]
      # using add_field here to add & rename values to the event root
      add_field => { whiteip => "%{[whitelist_test][0][ip]}" } 
      add_field => { whiteip_desc => "%{[whitelist_test][0][descr]}" }
      remove_field => ["whitelist"]
      staging_directory => "/tmp/logstash/jdbc_static/import_data"
      loader_schedule => "12 */2 * * *" # run loaders every 2 hours
      #jdbc_user => "logstash"
      #jdbc_password => "example"
      jdbc_driver_class => "org.sqlite.JDBC"
      jdbc_driver_library => "/usr/share/logstash/vendor/jar/jdbc/sqlite-jdbc-3.41.2.1.jar"
      jdbc_connection_string => "jdbc:sqlite:/home/test/testparams.db"
    }
 
}

sqlite创建表

sqlite3 /home/test/testparams.db

#表
create table whitelist_test(
    IP char(50)  primary key not null,
    status int no null,
    descr text,
    begin_time TEXT not null,
    end_time TEXT
);

insert into whitelist_test(ip,status,descr,begin_time,end_time)
values('192.168.255.1',0,'测试IP','2023-05-08 12:00:00',null);

内网IP的经纬度配置

这个配置主要是解决将内网IP在地图上显示的办法。将内网IP段及对应的地点名称和经纬度存放在IPdist.txt文件中，通过logstash的ruby过滤器，根据访问IP取出来，然后使用kibana的地图功能在地图上显示出来。

filter { 
 if [logstash-input] == "weblog"{
    grok  { 
        id => "weblog-grok-1"
        match => { "message" => "%{IP:clientip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{DATA:request} HTTP/%{DATA:httpversion}\" %{NUMBER:status:int} %{GREEDYDATA:bytes}" }
    }
    if ("" in [clientip]) {
        ruby {
            id => "weblog-ruby-1"
            code => "
                require 'ipaddr'
                mess = event.to_hash
                my_ipaddr = mess['clientip']
              
                File.open('/opt/logstash/IPdist.txt', 'r') do |file|
                    file.each_line do |line|
                        items = line.split(',')
                        low = IPAddr.new(items[0]).to_i    #IP段的起始地址，转成整数
                        high = IPAddr.new(items[1]).to_i   #IP段的结束地址，转成整数
                        myip = IPAddr.new(my_ipaddr).to_i  #需要判断的IP，转成整数
                        if (low..high) === myip
                            event.set('location_tag',items[2])    #取地址名称
                            #将经纬度生成浮点数数组存放到字段src_location里。
                            event.set('src_location',Array[ items[4].to_f, items[3].to_f ] )  
                            break
                        end
                    end
                end            
            "
         }
    }
 }
}

IPdist.txt文件示意:

10.0.0.0,10.255.255.255,A地,35.5306,120.3250,
172.16.0.0,172.31.255.255,B地,36.5516,121.2470,
192.168.0.0,192.168.255.255,C地,37.5517,122.1455,
....

注意，要定义索引模板，将src_location字段映射为地理坐标。

地址的经纬度提取方法： 在百度地图开放平台-->工具支持-->坐标拾取器,可以提取地址的经纬度。

logstash安装

1. java版本要求

6.7.y 以前版本，推荐使用 java8

6.7.y - 7.17.y ，推荐使用java8或java11

8.x以后，推荐使用oracle/OpenJava 11或17

2. 下载安装包

https://www.elastic.co/cn/downloads/logstash  或 https://www.elastic.co/cn/downloads/past-releases#logstash 下载历史版本

3. 直接解压安装包即可

将安装包拷贝到安装目录下，

tar zxvf logstash-7.17.16-linux-x86_64.tar.gz
cd  ./logstash-7.17.16/bin/
./logstash -e 'input{stdin{}} output{stdout{codec=>rubydebug}}'

4. 配置

修改 logstash-7.17.16-linux-x86_64/config目录下的 startup.optins

LS_HOME=/data/logstash-7.17.16
LS_SETTINGS_DIR=/data/logstash-7.17.16/config
LS_USER=root
LS_GROUP=root

配置pipelines

   修改logstash-7.17.16-linux-x86_64/config目录下的pipelines.yml,添加一个管道。logstash启动时，会读取这个管道配置文件，然后会自动检测并加载指定目录下的所有配置文件。

   配置此文件后，启动logstash时，就不需要用-f参数专门指定配置文件了。

- pipeline.id: waf
  path.config: "/data/logstash-7.17.16/config/conf.d/waf_*.conf"

kibana安装配置

创建用户

#groupadd elk
#useradd -g elk -s /bin/bash -md /home/kibana kibana

解压并修改own

# tar -zxvf kibana-7.17.16-linux-x86_64.tar.gz
# chown -Rf kibana:elk /opt/kibana-7.17.16-linux-x86_64

修改配置文件

vi ./config/kibana.yml  #取消注释
server.port:5601
server.host: 0.0.0.0
elasticsearch.hosts:["http://localhost:9200"]

配置syetemctl启动脚本

#vi /usr/lib/systemd/system/kibana.service
[Unit]
Description=kibana
After=network.target

[Service]
Type=simple
User=kibana
ExecStart=/opt/kibana/bin/kibana
PrivateTmp=true

[Install]
WantedBy=multi-user.target

#systemctl daemon-reload
#systemctl start kibana && systemctl enable kibana

filebeat安装配置

1 下载,安装

wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.4.3-linux-x86_64.tar.gz

mv filebeat-8.4.3-linux-x86_64.tar.gz   /opt
cd /opt
tar -zxvf filebeat-8.4.3-linux-x86_64.tar.gz
cd filebeat-8.4.3-linux-x86_64

2 配置

修改filebeat.yml

# ========================= Filebeat inputs ======================
filebeat.inputs:

# Each - is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.

# filestream is an input for collecting log messages from files.
- type: filestream

  # Unique ID among all inputs, an ID is required.
  id: my-filestream-id

  # Change to true to enable this input configuration.
  enabled: true      #设为true,启用这个input配置

  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /var/log/*.log      #日志路径
    #- c:\programdata\elasticsearch\logs\*

  # Exclude lines. A list of regular expressions to match. It drops the lines that are
  # matching any regular expression from the list.
  # Line filtering happens after the parsers pipeline. If you would like to filter lines
  # before parsers, use include_message parser.
  #exclude_lines: ['^DBG']

  # Include lines. A list of regular expressions to match. It exports the lines that are
  # matching any regular expression from the list.
  # Line filtering happens after the parsers pipeline. If you would like to filter lines
  # before parsers, use include_message parser.
  #include_lines: ['^ERR', '^WARN']

  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
  # are matching any regular expression from the list. By default, no files are dropped.
  #prospector.scanner.exclude_files: ['.gz$']

  # Optional additional fields. These fields can be freely picked
  # to add additional information to the crawled log files for filtering
  #fields:
  #  level: debug
  #  review: 1
 processors:
    #- add_host_metadata:
    #    when.not.contains.tags: forwarded
    #- add_cloud_metadata: ~
    #- add_docker_metadata: ~
    #- add_kubernetes_metadata: ~
    - drop_fields:   #剔除不需要的字段，减少数据量
        fields: ["log","input","ecs","agent","tags"]
        ignore_missing: false

  # =================== Outputs ===============================
  # ---------------------------- Elasticsearch Output ----------------------------
  #本配置传给logstash,所以本部分全部注释掉
  ...
  # ------------------------------ Logstash Output -------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["192.168.0.100:5044"]

  # Optional SSL. By default is off.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"

3 启动

#启动
nohup ./filebeat -e -c filebeat.yml >/dev/null 2>&1 &

#指定配置文件启动
./filebeat run -e -c filebeat.yml -d "publish"

4 清除标记

 Filebeat 会将文件读取位置记录 /opt/filebeat-8.4.3-linux-x86/data/registry 文件夹中，想重新从文件开始读取需要删除 registry 文件夹，然后重启Filebeat 。