Linux部署Elasticsearch

1 下载

[root@localhost ~]# cd /home/elk

1.1 ELK7.8.1

[root@localhost elk]# wget https://artifacts.elastic.co/downloads/logstash/logstash-7.8.1.tar.gz
[root@localhost elk]# wget https://artifacts.elastic.co/downloads/kibana/kibana-7.8.1-linux-x86_64.tar.gz
[root@localhost elk]# wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.8.1-linux-x86_64.tar.gz

1.2 ELK 7.6.2

[root@localhost elk]# wget https://artifacts.elastic.co/downloads/logstash/logstash-7.6.2.tar.gz
[root@localhost elk]# wget https://artifacts.elastic.co/downloads/kibana/kibana-7.6.2-linux-x86_64.tar.gz
[root@localhost elk]# wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.6.2-linux-x86_64.tar.gz

2 Elasticsearch安装

2.1 解压

[root@localhost elk]# tar -zxvf elasticsearch-7.8.1-linux-x86_64.tar.gz
[root@localhost elk]# cd elasticsearch-7.8.1/

2.2 修改配置

2.2.1 jvm.options调整内存大小

[root@localhost elasticsearch-7.8.1]# vi config/jvm.options

#-Xms4g
#-Xmx4g
-Xms2m
-Xmx2m

2.2.2 elasticsearch.yml

[root@localhost elasticsearch-7.8.1]# vi config/elasticsearch.yml

#cluster.name: my-application
cluster.name: es-cloud

#node.name: node-1
node.name: node-1

#network.host: 192.168.0.1
network.host: 0.0.0.0

#cluster.initial_master_nodes: ["node-1", "node-2"]
cluster.initial_master_nodes: ["node-1"]	#集群需要至少一个节点

##末尾添加
indices.memory.index_buffer_size: 40%
thread_pool.write.size: 3
thread_pool.write.queue_size: 1000

#允许跨域访问
http.cors.enabled: true
http.cors.allow-origin: "*"

2.3 创建用户

#创建es用户，adduser会在/home目录下会自动创建同名文件夹
[root@localhost elasticsearch-7.8.1]# adduser es
#修改es用户密码，接着输入两次密码123456
[root@localhost elasticsearch-7.8.1]# passwd es
#给新用户文件夹权限
[root@localhost home]# chown -R es /home/elk

2.4 启动

#切换成新建的es用户
[root@localhost elk]# su es
[es@localhost elk]$ cd elasticsearch-7.8.1/
[es@localhost elasticsearch-7.8.1]$ ./bin/elasticsearch

2.4.1 排错

ERROR: [2] bootstrap checks failed
[1]: max file descriptors [4096] for elasticsearch process is too low, increase to at least [65535]
[2]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]

#修改系统内核参数
#切换到root用户
[es@localhost elasticsearch-7.8.1]$ su
#修改用户资源限制limits.conf
[root@localhost elasticsearch-7.8.1]# vi /etc/security/limits.conf 
#es : 表示用户，* 表示所有用户
#soft xxx : 代表警告的设定，可以超过这个设定值，但是超过后会有警告。
#hard xxx : 代表严格的设定，不允许超过这个设定的值
#nofile : 是每个进程可以打开的文件数的限制
#nproc  : 是操作系统级别对每个用户创建的进程数的限制
es soft nofile 65536  #es用户可打开的文件描述符的最大数(超过会警告)
es hard nofile 65536  #es用户可打开的文件描述符的最大数(超过会报错)
es soft nproc  4096   #es用户可用的最大进程数量(超过会警告)
es hard nproc  4096   #es用户可用的最大进程数量(超过会报错)
#修改进程可以拥有的VMA(虚拟内存区域)的数量
[root@localhost elasticsearch-7.8.1]# vi /etc/sysctl.conf 
vm.max_map_count=262144
#使修改立即生效
[root@localhost elasticsearch-7.8.1]# sysctl -p

2.4.2 后台启动

#切换成es用户来启动
[root@localhost elasticsearch-7.8.1]# su es
[es@localhost elasticsearch-7.8.1]$ nohup ./bin/elasticsearch &
#查看控制台
[root@localhost elasticsearch-7.8.1]# tail -f nohup.out

#关闭
[root@localhost elasticsearch-7.8.1]# kill -9 2382（进程号）

2.5 访问

浏览器请求http://192.168.56.13:9200

{
  "name" : "node-1",
  "cluster_name" : "my-application",
  "cluster_uuid" : "WKr8kYjGQgqNsSjokqU7dg",
  "version" : {
    "number" : "7.8.1",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "b5ca9c58fb664ca8bf9e4057fc229b3396bf3a89",
    "build_date" : "2020-07-21T16:40:44.668009Z",
    "build_snapshot" : false,
    "lucene_version" : "8.5.1",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

3 Elasticsearch集群部署

3.1 解压并拷贝3份es

elasticsearch-a elasticsearch-b elasticsearch-c

3.2 修改elasticsearch.yml

3.2.1 修改elasticsearch-a

# 集群名称
cluster.name: my-application
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
# 节点名称
node.name: node-1
# 是否有资格主节点
node.master: true
# 是否存储数据
node.data: true
# 最大集群节点数，因为3个节点，所有配置3
node.max_local_storage_nodes: 3
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
# 数据存储路径
path.data: /home/elk/elasticsearch-a/data
#
# Path to log files:
# 日志存储路径
path.logs: /home/elk/elasticsearch-a/logs
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
# 网关地址
network.host: 0.0.0.0
#
# Set a custom port for HTTP:
# 端口
http.port: 9200
# 内部节点之间通讯端口
transport.tcp.port: 9300
#
# For more information, consult the network module documentation.
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when this node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
# es7.x之后新增配置，写入候选主节点的设备地址，在开启服务后可以被选为主节点
discovery.seed_hosts: ["127.0.0.1:9300", "127.0.0.1:9400", "127.0.0.1:9500"]
#
# Bootstrap the cluster using an initial set of master-eligible nodes:
# es7.x之后新增配置，初始化一个新的集群时需要此配置来选举master
cluster.initial_master_nodes: ["node-1", "node-2", "node-3"]
#
# For more information, consult the discovery and cluster formation module documentation.
#
# ---------------------------------- Gateway -----------------------------------
#
# Block initial recovery after a full cluster restart until N nodes are started:
#
#gateway.recover_after_nodes: 3
#
# For more information, consult the gateway module documentation.
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true

#允许跨域访问
http.cors.enabled: true
http.cors.allow-origin: "*"

3.2.2 修改elasticsearch-b

只修改了，2个端口、数据和日志存储路径

# 集群名称
cluster.name: my-application
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
# 节点名称
node.name: node-2
# 是否有资格主节点
node.master: true
# 是否存储数据
node.data: true
# 最大集群节点数，因为3个节点，所有配置3
node.max_local_storage_nodes: 3
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
# 数据存储路径
path.data: /home/elk/elasticsearch-b/data
#
# Path to log files:
# 日志存储路径
path.logs: /home/elk/elasticsearch-b/logs
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
# 网关地址
network.host: 0.0.0.0
#
# Set a custom port for HTTP:
# 端口
http.port: 9201
# 内部节点之间通讯端口
transport.tcp.port: 9400
#
# For more information, consult the network module documentation.
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when this node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
# es7.x之后新增配置，写入候选主节点的设备地址，在开启服务后可以被选为主节点
discovery.seed_hosts: ["127.0.0.1:9300", "127.0.0.1:9400", "127.0.0.1:9500"]
#
# Bootstrap the cluster using an initial set of master-eligible nodes:
# es7.x之后新增配置，初始化一个新的集群时需要此配置来选举master
cluster.initial_master_nodes: ["node-1", "node-2", "node-3"]
#
# For more information, consult the discovery and cluster formation module documentation.
#
# ---------------------------------- Gateway -----------------------------------
#
# Block initial recovery after a full cluster restart until N nodes are started:
#
#gateway.recover_after_nodes: 3
#
# For more information, consult the gateway module documentation.
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true

#允许跨域访问
http.cors.enabled: true
http.cors.allow-origin: "*"

3.2.3 修改elasticsearch-c

只修改了，2个端口、数据和日志存储路径

# 集群名称
cluster.name: my-application
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
# 节点名称
node.name: node-3
# 是否有资格主节点
node.master: true
# 是否存储数据
node.data: true
# 最大集群节点数，因为3个节点，所有配置3
node.max_local_storage_nodes: 3
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
# 数据存储路径
path.data: /home/elk/elasticsearch-c/data
#
# Path to log files:
# 日志存储路径
path.logs: /home/elk/elasticsearch-c/logs
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
# 网关地址
network.host: 0.0.0.0
#
# Set a custom port for HTTP:
# 端口
http.port: 9202
# 内部节点之间通讯端口
transport.tcp.port: 9500
#
# For more information, consult the network module documentation.
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when this node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
# es7.x之后新增配置，写入候选主节点的设备地址，在开启服务后可以被选为主节点
discovery.seed_hosts: ["127.0.0.1:9300", "127.0.0.1:9400", "127.0.0.1:9500"]
#
# Bootstrap the cluster using an initial set of master-eligible nodes:
# es7.x之后新增配置，初始化一个新的集群时需要此配置来选举master
cluster.initial_master_nodes: ["node-1", "node-2", "node-3"]
#
# For more information, consult the discovery and cluster formation module documentation.
#
# ---------------------------------- Gateway -----------------------------------
#
# Block initial recovery after a full cluster restart until N nodes are started:
#
#gateway.recover_after_nodes: 3
#
# For more information, consult the gateway module documentation.
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true

#允许跨域访问
http.cors.enabled: true
http.cors.allow-origin: "*"

3.3 获取集群的健康状态

http://192.168.56.13:9200/_cat/health?v

epoch      timestamp cluster        status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1624453943 13:12:23  my-application green           3         3     12   6    0    0        0             0                  -                100.0%

ES集群概念

Cluster集群

一个ElasticSearch集群由一个或多个节点(Node)组成，每个集群都有一个共同的集群名称作为标识。

Node节点

一个ElasticSearch实例即一个Node，一台机器可以有多个实例，正常使用下每个实例应该会部署在不同机器上。ElasticSearch的配置文件中可以通过node.master、node.data来设置节点类型。

　　node.master：表示节点是否具有称为主节点的资格

　　　　true代表的是有资格竞选主节点

　　　　false代表的是没有资格竞选主节点

　　node.data：表示节点是否存储数据
　　

Node节点组合

主节点+数据节点(master+data)

节点即有成为主节点的资格，又存储数据

node.master: true
node.data: true

数据节点(data)

节点没有成为主节点的资格，不参与选举，只会存储数据

node.master: false
node.data: true

客户端节点(client)

不会成为主节点，也不会存储数据，主要是针对海量请求的时候，可以进行负载均衡

node.master: false
node.data: false

索引分片

每个索引有一个或多个分片，每个分片存储不同的数据。分片可分为主分片(primary shard)和复制分片(replica shard)，复制分片是主分片的拷贝。默认每个主分片有一个复制分片，一个索引的复制分片的数量可以动态地调整，复制分片匆匆不与它的主分片在同一个节点上。