ELSA(全称:Enterprise Log Search and Archive)是一款基于syslog-ng(新一代日志收集器,但目前多数Linux发现版都不带此工具)、MySQL的开源级企业日志归档查询工具,由于它和Sphinx的完美搭配,支持全文索引可以像搜索Web一样轻松地搜索上亿个日志中的任意字符串(前提是你的服务器配置足够高)。单节点ELSA日志采集系统的工作原理图如下所示:

上面这张架构图可以看出ELSA从架构上分为三层:

日志接收器,由syslog-ng完成负责接收来自本地、网络以及导入的日志文件 日志存储索引,存储由MySQL数据库完成,索引由sphinx完成。 Web前端 。 ELSA利用syslog-ng的pattern-db解析器进行有效的日志规范化,并利用Sphinx全文索引进行日志搜索。系统内部API将查询结果汇总后,发送给客户端,整个系统是异步执行,可以跑多个查询。接收器syslog-ng在接收日志时并没有进行归一化处理(类比OSSIM-Agent插件),所以对日志的正则表达式计算量不大,可以在syslog-ng中保持高效的日志接收率,系统大部分有Perl脚本组成,MySQL每秒可插入100K行数据。Sphinx在索引中为新插入的行建立索引,每个2小时会重新建立一次永久索引。整个系统最大效率发挥时每秒钟可以处理100K条日志。

如果你具备ELK实战经验的话,可以把ELSA理解为简版的ELK系统,结构简单,速度快。安装(感兴趣的朋友可以在基于Debian(包括Ubuntu)的OS上测试,在ELSA Google Code主页上获取安装tar包)比较简单就不介绍了,下面直接切入正题。

1.采集Windows服务器日志

我们可以采用Eventlog-to-Syslog工具将Windows平台的日志发送到ELSA服务器 方法: 将evtsys.exe和evtsys.dll复制到系统目录下输入下面命令 evtsys.exe -i -h ELSA服务器的IP 志将使用syslog协议发送到您的ELSA服务器,在该服务器中,日志将被解析为“ WINDOWS”类

2.采集Linux系统及相关服务的日志

Linux/Unix系统都有rsyslog 或 Syslogd进程,在其配置文件中加入下面的配置即可

. @ELSA服务器IP

3.配置文件

ELSA的主要配置文件是/etc/elsa_node.conf

{ # 本地数据库连接信息 "database" : { "db": "syslog", "data_db": "syslog_data", "dsn" : "dbi:mysql:database=syslog", "username" : "elsa", "password" : "biglog" }, // 系统协调锁的目录 "lockfile_dir": "/opt/elsa/node/tmp/locks",

    "num_indexes": 200,

//如果要归档日志,请保留此项 "archive": { # Uncomment to establish a retention period in days for archive logs #"days": 90, "percentage": 33, "table_size": 10000000 }, //日志大小限制+索引大小。设置为磁盘总空间的95-90%。 "log_size_limit" : 8000000000, "sphinx" : {

            "indexer": "/usr/bin/indexer",
            
            "allowed_temp_percent" : 40,
          
            "allowed_mem_percent": 25
            "host" : "127.0.0.1",
            "port" : 9312,
    "mysql_port" : 9306,
            
            "config_file" : "/etc/sphinxsearch/sphinx.conf",
           
            "index_path" : "/nsm/elsa/data/sphinx",
            
            "index_interval" : 60,
            
            "perm_index_size" : 10000000,
            # Where the optional stopwords file is
            "stopwords": {
                    "file": "/etc/sphinxsearch/sphinx_stopwords.txt",
                    "top_n": 0,
                    "interval": 0,
                    "whitelist": []
            },
            
            "pid_file": "/var/run/sphinxsearch/searchd.pid"
    },
 
    "logdir" : "/nsm/elsa/data/elsa/log",
"mysql_dir": "/nsm/elsa/data/elsa/mysql",
   
    "num_log_readers" : 1,
   #调试跟踪级别
    "debug_level" : "TRACE",
    
    "buffer_dir" : "/nsm/elsa/data/elsa/tmp/buffers/",
   
    "log_parse_errors": 1,
   
    "stats" : {
            "retention_days": 365
    },
 
    "min_expected_hosts": 2

} ELSA的Web配置文件 /etc/elsa_web.conf

{ #定义API密钥 "apikeys": { "elsa": "b7292980d34c99e2581d36681831667b" }, "version": { "Author": "mcholste", "Date": "2014-07-17 15:12:58 -0700 (Thu, 17 Jul 2014)", "Rev": "1205", "Sphinx": "Sphinx 2.1.9" }, "peers": { "127.0.0.1": { "url": "http://127.0.0.1:3154/", "username": "elsa", "apikey": "b7292980d34c99e2581d36681831667b" } }, "admin_email_address": "root@localhost", "connectors": { }, "dashboards": { }, "datasources": { }, "transforms": { "whois": { "known_subnets": { "10.0.0.0": { "end": "10.255.255.255", "org": "MyOrg" }, "192.168.0.0": { "end": "192.168.255.255", "org": "MyOrg" }, "172.16.0.0": { "end": "172.31.255.255", "org": "MyOrg" } }, "known_orgs": { "MyOrg": { "name": "MyOrg", "org": "MyOrg", "descr": "MyOrg", "cc": "US", "country": "United States", "city": "Anytown", "state": "Somestate" } } }, "parse": { "tld": [ { "field": "domain", "pattern": "\.([a-zA-Z]+)$", "extractions": [ "tld" ] }, { "field": "site", "pattern": "\.([a-zA-Z]+)$", "extractions": [ "tld" ] }, { "field": "uri", "pattern": "\.([a-zA-Z]+)(:|/|$)", "extractions": [ "tld" ] } ], "url": [ { "field": "uri", "pattern": "(?:(?<proto>[a-zA-Z]+)://)?(?:(?<username>[^/]+):(?<password>[^/]+)@)?(?<domain>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|[^/]+\.(?<tld>[a-zA-Z]+))(?::(?<port>\d+))?(?<resource>/[^?])?(?:\?(?<query_string>.))?$", "extractions": [ "proto", "username", "password", "domain", "tld", "port", "resource", "query_string" ] } ], "mimetype": [ { "field": "msg", "pattern": "["'\(\\s\|;:["'\)\]\s\|;:]", "extractions": [ "mime", "type", "subtype" ] } ] } }, "plugins": { "SNORT": "Info::Snort", "WINDOWS": "Info::Windows", "URL": "Info::Url", "BRO_NOTICE": "Info::Bro" }, "info": { "snort": { "url_templates": [ "http://doc.emergingthreats.net/bin/view/Main/%d" ] }, "url": { "url_templates": [ "http://whois.domaintools.com/%s" ] }, "windows": { "url_templates": [ "http://www.ultimatewindowssecurity.com/securitylog/encyclopedia/event.aspx?eventid=%d" ] } }, "max_concurrent_archive_queries": 4, "schedule_interval": 60, "node_info_cache_timeout": 60, "email": { "display_address": "noreply-elsa@example.com", "base_url": "http://elsa/", "subject": "ELSA Alert" }, "link_key": "secret", "yui": { "local": "inc" }, "data_db": { "db": "syslog", "username": "elsa", "password": "biglog" }, "meta_db": { "dsn": "dbi:mysql:database=elsa_web", "username": "elsa", "password": "biglog" }, "auth": { "method": "security_onion" }, "admin_groups": [ "system", "admin" ], "auth_db": { "dsn": "dbi:mysql:database=securityonion_db", "username": "root", "password": "", "auth_statement": "SELECT PASSWORD(password) FROM user_info WHERE username=?", "email_statement": "SELECT email FROM user_info WHERE username=?" }, "peer_id_multiplier": 1000000000000, "query_timeout": 55, "pcap_url": "/capme", "logdir": "/nsm/elsa/data/elsa/log", "buffer_dir": "/nsm/elsa/data/elsa/tmp/buffers", "debug_level": "TRACE", "default_start_time_offset": 2, "livetail": { "poll_interval": 5, "time_limit": 3600 } }

4.典型应用场景(截图)

着重对ELSA软件的几个重点功能进行展示。

1.连接数 Top N

2.动态仪表盘展示

动态展示单位时间内处理日志的数量、查询量、采集主机的地址以及日志类型等参数。

3.查询日志详细信息

我们在Field Summary(字段摘要)中发现这些日志有15个字段(主机IP、进程名称、源地址、源端口、目的地址、目的端口、协议类型、输入字节数量、服务类型、持续时间、输出字节、输入数据包数量、输出数据包数量、国家代码等),每个字段后面是出现的次数,各个字段之间通过“|”符号分割。

4.查询ossec日志信息

5.侦测到针对MySQL 3306端口扫描报警日志信息

6.端口扫描报警日志信息

7.Ping报警日志信息

有关日志分析的相关话题大家可以阅读畅销书《Unix/Linux网络日志分析与流量监控》。