ELK收集中断,定位问题到redis发现redis内存不足,所以迁移到新的redis上
elasticsearch 删除索引语句:
curl -XDELETE http://localhost:9200/logstash-nginx.access-2016.05.31
错误1:Failed to send event to Redis
Failed to send event to Redis {:event=>#<LogStash::Event:0x4062cce3 @metadata_accessors=#<LogStash::Util::Accessors:0x7cbdd35c @store={"path"=>"/app/local/log/nginx/ckl_access.log"}, @lut={"[path]"=>[{"path"=>"/app/local/log/nginx/ckl_access.log"}, "path"]}>, @cancelled=false, @data={"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, @metadata={"path"=>"/app/local/log/nginx/ckl_access.log"}, @accessors=#<LogStash::Util::Accessors:0x64f18bac @store={"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, @lut={"path"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "path"], "host"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "host"], "type"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "type"], "host_name"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "host_name"], "[type]"=>[{"message"=>"116.209.58.13 | 03/Aug/2016:16:01:01 +0800 | POST /api/v1 HTTP/1.1 | 200 | 69 | {\\x22os\\x22:\\x221\\x22,\\x22v\\x22:\\x222.0.2\\x22,\\x22m\\x22:\\x22user.isFollow\\x22,\\x22ver\\x22:\\x224\\x22,\\x22channel\\x22:\\x22OT_bdhn\\x22,\\x22p\\x22:{\\x22roomId\\x22:\\x221743331\\x22}} | 97 | - | Mozilla/5.0 (Linux; Android 4.4.4; C630Lw Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 QMTV/2.0.2 CHANNEL/OT_bdhn | - | 61.136.167.55 | 0.016 | 0.022 | -", "@version"=>"1", "@timestamp"=>"2016-08-03T08:52:11.442Z", "path"=>"/app/local/log/nginx/ckl_access.log", "host"=>"0.0.0.0", "type"=>"nginx.access", "host_name"=>"ckl_access_front-web15"}, "type"]}>>, :identity=>"default", :exception=>#<Redis::CommandError: OOM command not allowed when used memory > 'maxmemory'.>, :backtrace=>["/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis/client.rb:121:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis.rb:1070:in `rpush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis.rb:58:in `synchronize'", "/opt/logstash/vendor/jruby/lib/ruby/1.9/monitor.rb:211:in `mon_synchronize'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis.rb:58:in `synchronize'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/redis-3.3.0/lib/redis.rb:1069:in `rpush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-redis-2.0.5/lib/logstash/outputs/redis.rb:246:in `send_to_redis'", "org/jruby/RubyProc.java:281:in `call'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-codec-json-2.1.4/lib/logstash/codecs/json.rb:42:in `encode'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-redis-2.0.5/lib/logstash/outputs/redis.rb:152:in `receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/outputs/base.rb:83:in `multi_receive'", "org/jruby/RubyArray.java:1613:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/outputs/base.rb:83:in `multi_receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/output_delegator.rb:130:in `worker_multi_receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/output_delegator.rb:114:in `multi_receive'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:301:in `output_batch'", "org/jruby/RubyHash.java:1342:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:301:in `output_batch'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:232:in `worker_loop'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in `start_workers'"], :level=>:warn}
发现如下:
commanderror oom command not allowed when used memory
定位为redis问题
修改redis 配置文件,max-memeory选项调整为当前内存的90即可
错误2:elasticsearch` is obsolete and is no longer available
{:timestamp=>"2016-08-03T17:21:26.451000+0800", :message=>"Pipeline aborted due to error", :exception=>#<LogStash::ConfigurationError: The setting `host` in plugin `elasticsearch` is obsolete and is no longer available. Please use the 'hosts' setting instead. You can specify multiple entries separated by comma in 'host:port' format. If you have any questions about this, you are invited to visit https://discuss.elastic.co/c/logstash and ask.>, :backtrace=>["/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/config/mixin.rb:87:in `config_init'", "org/jruby/RubyHash.java:1342:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/config/mixin.rb:71:in `config_init'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/outputs/base.rb:63:in `initialize'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/output_delegator.rb:74:in `register'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/pipeline.rb:181:in `start_workers'", "org/jruby/RubyArray.java:1613:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/pipeline.rb:181:in `start_workers'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/pipeline.rb:136:in `run'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.2-java/lib/logstash/agent.rb:465:in `start_pipeline'"], :level=>:error}
原配置文件:
elasticsearch { hosts => ["10.11.11.12"] protocol => "http" index => "logstash-%{type}-%{+YYYY.MM.dd}" document_type => "%{type}" workers => 5 flush_size => 3840 idle_flush_time => 10 template_overwrite => true }
解决:
新的配置文件:
elasticsearch { hosts => "10.11.11.12" index => "logstash-%{type}-%{+YYYY.MM.dd}" document_type => "%{type}" workers => 5 flush_size => 3840 idle_flush_time => 10 template_overwrite => true }
错误3: Permission denied
Errno::EACCES: Permission denied - /tmp/logstash-log/log-2016.08.03.log initialize at org/jruby/RubyFile.java:370 new at org/jruby/RubyIO.java:853 open at /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-file-2.2.5/lib/logstash/outputs/file.rb:264 write_event at /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-file-2.2.5/lib/logstash/outputs/file.rb:162
logstash 日志目录修改:
chown -R logstash.logstash logstash-log/
错误4,内存耗尽,redis的lists太长,没有清理
写个简单清理脚本,半个小时执行一次:
#!/bin/bash CUR_TIME=`date +%Y-%m-%d' '%H:%M:%S` ALL_COUNT=`/usr/local/redis/bin/redis-cli LLEN logstash:redis:nginx.access` HALF_COUNT=`expr $ALL_COUNT / 2` echo $ALL_COUNT $HALF_COUNT echo "=================" echo $CUR_TIME /usr/local/redis/bin/redis-cli LTRIM logstash:redis:nginx.access 0 $HALF_COUNT
添加定时任务:
*/30 * * * * /bin/bash /opt/script/clean_redis.sh >> /tmp/ckl.txt 2>&1