实验环境
cat /etc/redhat-release && uname -a
CentOS Linux release 7.8.2003 (Core)
Linux centos7 3.10.0-1127.el7.x86_64
实验软件
jdk-8u152-linux-x64.tar.gz
elasticsearch-6.1.0.rpm
kibana-6.1.0-x86_64.rpm
logstash-6.1.0.rpm
软件安装
ntpdate && hwclock -w
echo SELINUX=disabled > /etc/sysconfig/selinux
systemctl stop firewalld && systemctl disable firewalld
yum clean all && yum makecache fast -y
yum install -y lrzsz wget nginx
tar zxvf /root/jdk-8u152-linux-x64.tar.gz && mv /root/jdk1.8.0_152 /usr/local/java
cp -pv /etc/profile /etc/profile.bak && ln -s /usr/local/java/bin/java /usr/bin/java
cat >> /etc/profile << EOF
> export JAVA_HOME=/usr/local/java
> export PATH=$PATH:$JAVA_HOME/bin
EOF && source /etc/profile && java -version
java version "1.8.0_152"
安装elasticsearch 服务端
rpm -ivh elasticsearch-6.1.0.rpm
mkdir -pv /var/lib/elasticsearch/{data,log} 创建es数据/日志目录
chown -R elasticsearch:elasticsearch /var/lib/elasticsearch/ 目录授权
cp -pv /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.bak
cat /etc/elasticsearch/elasticsearch.yml
path.data: /data data存放路径/var/lib/elasticsearch/data/
path.logs: /log logs日志路径/var/lib/elasticsearch/log
bootstrap.memory_lock: false 内存不使用交换分区
bootstrap.memory_lock 为true时9200不会被监听
network.host: 0.0.0.0 允许所有ip连接elasticsearch
http.port: 9200 监听端口9200
systemctl daemon-reload
systemctl start elasticsearch && systemctl enable elasticsearch
curl http://localhost:9200
{
"name" : "E8fRgJF",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "sGIkZFjJTGmtLLAHWEcV2Q",
"version" : {
"number" : "6.1.0",
"build_hash" : "c0c1ba0",
"build_date" : "2017-12-12T12:32:54.550Z",
"build_snapshot" : false,
"lucene_version" : "7.1.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}
安装logstash 客户端
rpm -ivh logstash-6.1.0.rpm
mkdir -pv /var/lib/logstash/{data,log}
chown -R logstash:logstash /var/lib/logstash/ 授权目录
cp -pv /etc/logstash/logstash.yml /etc/logstash/logstash.yml.bak
cat /etc/logstash/logstash.yml
path.data: /var/lib/logstash/data/ 数据存储目录/var/lib/logstash/data/
path.config: /etc/logstash/conf.d 管道配置文件路径为/etc/logstash/conf.d
path.logs: /log/ 日志存储路径/var/lib/logstash/log
touch /etc/logstash/conf.d/system.conf
cat /etc/logstash/conf.d/system.conf
input {
file {
path => "/var/log/messages"
type => "system-log"
start_position => "beginning"
}
}
output {
elasticsearch {
hosts => "192.168.1.15:9200"
index => "system_log-%{+YYYY.MM.dd}"
}
}
chmod 777 /var/log/messages 重要操作
systemctl enable logstash && systemctl start logstash
ln -s /usr/share/logstash/bin/logstash /bin/
logstash -e 'input { stdin { } } output { stdout {} }'
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
The stdin plugin is now waiting for input:
abc 输入
2021-01-23T07:52:41.260Z centos7 abc
logstash -e 'input { stdin { } } output { elasticsearch { hosts => ["192.168.1.15:9200"] } stdout { codec => rubydebug }}'
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
The stdin plugin is now waiting for input:
test 输入
{
"host" => "centos7",
"@timestamp" => 2021-01-23T07:54:24.294Z,
"@version" => "1",
"message" => "test"
}
安装Kibana
rpm -ivh kibana-6.1.0-x86_64.rpm
cp -pv /etc/kibana/kibana.yml /etc/kibana/kibana.yml.bak
cat /etc/kibana/kibana.yml
server.port: 5601 kibana5601端口
server.host: "0.0.0.0" 所有ip访问5601端口
elasticsearch.url: "http://localhost:9200" 监听elasticsearch_ip
i18n.defaultLocale: "zh-CN" 添加识别中文
systemctl enable kibana && systemctl restart kibana
curl -X GET http://127.0.0.1:9200 elasticsearch 查看集群统计信息
curl -X GET 'http://localhost:9200/_cluster/stats?pretty' 查看集群状态
curl -X GET 'localhost:9200/_cat/health?v'
curl -X PUT HTTP://localhost:9200/test_index?pretty 创建索引:test_index
curl -X GET HTTP://localhost:9200/_cat/indices?v 查看所有索引
curl -X DELETE 'localhost:9200/test_index?pretty' 删除索引：test_index
netstat -tuplna | grep LISTEN
tcp 0 0 127.0.0.1:58642 127.0.0.1:9200 ESTABLISHED 5020/node
tcp6 0 0 :::9200 :::* LISTEN 3083/java
tcp6 0 0 :::9300 :::* LISTEN 3083/java
tcp 0 0 0.0.0.0:5601 0.0.0.0:* LISTEN 5020/node
tcp6 0 0 127.0.0.1:9600 :::* LISTEN 8801/java
http://serverip:5601/app/kibana#/home?_g=()
创建索引
复制索引
docker部署efk
cat /proc/cpuinfo | grep "physical id" | uniq | wc -l 软件运行最低处理器
2
free -h | grep Mem
Mem: 3.7G 软件运行最低内存
echo "vm.max_map_count=262144" >> /etc/sysctl.conf
sysctl -p | grep vm
vm.max_map_count = 262144
yum install -y yum-utils docker-ce-18.06.2.ce nginx
systemctl daemon-reload
systemctl start docker nginx && systemctl enable docker
docker --version
Docker version 18.06.2-ce, build 6d37f41
配置elasticsearch服务端
mkdir -pv /home/es
docker run -itd -p 9200:9200 -p 9300:9300
--restart always --user root:root
-e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms512m -Xmx512m"
-v /home/es:/home --net=host
--name=elasticsearch docker.elastic.co/elasticsearch/elasticsearch:7.1.0
curl http://localhost:9200
{
"name" : "centos7",
"cluster_name" : "docker-cluster",
"cluster_uuid" : "KGScsWugRISFGKh4vAP8MA",
"version" : {
"number" : "7.1.0",
"build_flavor" : "default",
"build_type" : "docker",
"build_hash" : "606a173",
"build_date" : "2019-05-16T00:43:15.323135Z",
"build_snapshot" : false,
"lucene_version" : "8.0.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
http://serverip:9200/
{
"name" : "centos7",
"cluster_name" : "docker-cluster",
"cluster_uuid" : "KGScsWugRISFGKh4vAP8MA",
"version" : {
"number" : "7.1.0",
"build_flavor" : "default",
"build_type" : "docker",
"build_hash" : "606a173",
"build_date" : "2019-05-16T00:43:15.323135Z",
"build_snapshot" : false,
"lucene_version" : "8.0.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
docker logs -f elasticsearch
docker exec -it elasticsearch /bin/bash 进入es容器
配置kibana服务端
docker run -itd -p 5601:5601
--restart always --user root:root --net=host
--name=kibana docker.elastic.co/kibana/kibana:7.1.0
cat /root/kibana.yml
# Default Kibana configuration for docker target
: kibana
server.host: "0.0.0.0"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN" 添加中文模块
docker cp /root/kibana.yml kibana:/usr/share/kibana/config/kibana.yml
docker restart kibana && docker logs -f kibana
docker logs kibana | grep 5601 查看kibana容器日志
log [05:14:00.170] [info][listening] Server running at http://0:5601
docker exec -it kibana /bin/bash 进入kiban容器
配置filebeat服务端
docker run -itd
--restart always --user root:root -v /var/log/nginx:/mnt/log
-e setup.kibana.host=192.168.10.18:5601 -e output.elasticsearch.hosts=["192.168.10.18:9200"]
--net=host --name=filebeat docker.elastic.co/beats/filebeat:7.1.0
docker logs -f filebeat 查看filebeat容器日志
docker exec -it filebeat /bin/bash -c 'ls /mnt/log'
access.log error.log
netstat -tuplna | grep LISTEN
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 799/nginx: master
tcp 0 0 0.0.0.0:9100 0.0.0.0:* LISTEN 43631/grunt
tcp6 0 0 :::9200 :::* LISTEN 43292/java
tcp6 0 0 :::9300 :::* LISTEN 43292/java
tcp 0 0 0.0.0.0:5601 0.0.0.0:* LISTEN 44262/node
http://serverip:5601/
