ETCD入门指南

原创

运维开发故事 2021-12-29 14:14:17 ©著作权

©著作权归作者所有：来自51CTO博客作者运维开发故事的原创作品，请联系作者获取转载授权，否则将追究法律责任

前言

之前对etcd不是很了解，于是下定决心学习一下。随手把过程记录了一下，希望对大家有帮助。

环境说明如下：

序号	IP	主机名	说明
1	10.211.55.50	etcd1	etcd version为3.4.16，虚拟机，系统为CentOS 7.7，使用自签证书
2	10.211.55.51	etcd2
3	10.211.55.52	etcd3

一、集群搭建

我这里将etcd部署在虚拟机上，采取静态配置，即在etcd各节点在配置中指定。

所有节点下载etcd并解压、移动至/usr/local/bin/，并给予对应权限

# 下载地址：https://github.com/etcd-io/etcd/releases
wget https://github.com/coreos/etcd/releases/download/v3.4.16/etcd-v3.4.16-linux-amd64.tar.gz
tar -zxvf etcd-v3.4.16-linux-amd64.tar.gz
cd etcd*
mv etcdctl etcd /usr/local/bin
chmod +x /usr/local/bin/etcd*

准备证书文件

# 证书文件 ca-config.json etcd-ca-csr.json etcd-csr.json
cat ca-config.json
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}

cat etcd-ca-csr-json
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shenzhen",
      "L": "Shenzhen",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ]
}


cat etcd-csr.json
{
    "CN": "etcd",
    "hosts": [
      "127.0.0.1",
      "10.211.55.50",
      "10.211.55.51",
      "10.211.55.52"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "etcd",
            "OU": "Etcd Security"
        }
    ]
}

生成etcd-ca证书

cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
# 查看证书文件
[root@etcd1 opt]# ls -al
total 36
drwxr-xr-x.  2 root root  186 May 13 21:53 .
dr-xr-xr-x. 17 root root  244 Oct 21  2020 ..
-rw-------   1 root root  294 Nov 27  2019 ca-config.json
-rw-r--r--   1 root root 1005 May 13 21:53 etcd-ca.csr
-rw-------   1 root root  212 Nov 27  2019 etcd-ca-csr.json
-rw-------   1 root root 1675 May 13 21:53 etcd-ca-key.pem
-rw-r--r--   1 root root 1371 May 13 21:53 etcd-ca.pem
-rw-------   1 root root  374 May 13 21:48 etcd-csr.json

生成etcd证书

cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
# 查看证书
ls -al
total 36
drwxr-xr-x.  2 root root  186 May 13 21:53 .
dr-xr-xr-x. 17 root root  244 Oct 21  2020 ..
-rw-------   1 root root  294 Nov 27  2019 ca-config.json
-rw-r--r--   1 root root 1005 May 13 21:53 etcd-ca.csr
-rw-------   1 root root  212 Nov 27  2019 etcd-ca-csr.json
-rw-------   1 root root 1675 May 13 21:53 etcd-ca-key.pem
-rw-r--r--   1 root root 1371 May 13 21:53 etcd-ca.pem
-rw-r--r--   1 root root 1078 May 13 21:53 etcd.csr
-rw-------   1 root root  374 May 13 21:48 etcd-csr.json
-rw-------   1 root root 1679 May 13 21:53 etcd-key.pem
-rw-r--r--   1 root root 1456 May 13 21:53 etcd.pem

mkdir -pv /etc/etcd/ssl && cp etcd*.pem /etc/etcd/ssl
# 复制证书目录到其他节点
scp -r /etc/etcd root@etcd2
scp -r /etc/etcd root@etcd3

配置成系统服务

cat etcd.service # 要根据每个节点进行修改IP和name
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
ExecStart=/usr/local/bin/etcd \
--name=etcd1 \
--cert-file=/etc/etcd/ssl/etcd.pem \
--key-file=/etc/etcd/ssl/etcd-key.pem \
--peer-cert-file=/etc/etcd/ssl/etcd.pem \
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem \
--initial-advertise-peer-urls=https://10.211.55.50:2380 \
--listen-peer-urls=https://10.211.55.50:2380 \
--listen-client-urls=https://10.211.55.50:2379,http://127.0.0.1:2379 \
--advertise-client-urls=https://10.211.55.50:2379 \
--initial-cluster-token=etcd-cluster-0 \
--initial-cluster=etcd1=https://10.211.55.50:2380,etcd2=https://10.211.55.51:2380,etcd3=https://10.211.55.52:2380 \
--initial-cluster-state=new \
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

启动etcd

# 三个节点启动etcd服务
systemctl daemon-reload && systemctl enable etcd && systemctl start etcd

8.检查集群状态

# 检查member
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.52:2379 member list -w table
+------------------+---------+-------+---------------------------+---------------------------+------------+
|        ID        | STATUS  | NAME  |        PEER ADDRS         |       CLIENT ADDRS        | IS LEARNER |
+------------------+---------+-------+---------------------------+---------------------------+------------+
|  ca2cb14b2acc776 | started | etcd3 | https://10.211.55.52:2380 | https://10.211.55.52:2379 |      false |
| 31f517c6aefb1a37 | started | etcd1 | https://10.211.55.50:2380 | https://10.211.55.50:2379 |      false |
| ad1ee26f89ef12f4 | started | etcd2 | https://10.211.55.51:2380 | https://10.211.55.51:2379 |      false |
+------------------+---------+-------+---------------------------+---------------------------+------------+

# 查看是否有LD
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 endpoint status -w table
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|         ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.211.55.50:2379 | 31f517c6aefb1a37 |  3.4.16 |   25 kB |      true |      false |       645 |         54 |                 54 |        |
| https://10.211.55.51:2379 | ad1ee26f89ef12f4 |  3.4.16 |   20 kB |     false |      false |       645 |         54 |                 54 |        |
| https://10.211.55.52:2379 |  ca2cb14b2acc776 |  3.4.16 |   20 kB |     false |      false |       645 |         54 |                 54 |        |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

# 尝试写入数据
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379  put foo4 bar4

# 读取数据
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 get  foo4
foo4
bar4

# 获取所有key
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 get --prefix  --keys-only ''
foo
foo2
foo3
foo4

# 只获取value
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 get  --print-value-only --prefix=true ''
bar
bar2
bar3
bar

# 获取指定key、rev的数据
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 get  foo4 -w json
{"header":{"cluster_id":9218922165880849559,"member_id":910513364439713654,"revision":10,"raft_term":645},"kvs":[{"key":"Zm9vNA==","create_revision":7,"mod_revision":10,"version":4,"value":"eGl5YW5neGl4aQ=="}],"count":1}

etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 get  foo4 -w json --rev=8
{"header":{"cluster_id":9218922165880849559,"member_id":3599809619259234871,"revision":10,"raft_term":645},"kvs":[{"key":"Zm9vNA==","create_revision":7,"mod_revision":8,"version":2,"value":"eGl5YW5neGl4aWE="}],"count":1}

二、成员变更

删除成员

因为我的资源有限，所以先模拟删除成员再添加成员。

# 先检查集群状态并获取成员ID
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 endpoint status -w table
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|         ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.211.55.50:2379 | 31f517c6aefb1a37 |  3.4.16 |   25 kB |      true |      false |       645 |         54 |                 54 |        |
| https://10.211.55.51:2379 | ad1ee26f89ef12f4 |  3.4.16 |   20 kB |     false |      false |       645 |         54 |                 54 |        |
| https://10.211.55.52:2379 |  ca2cb14b2acc776 |  3.4.16 |   20 kB |     false |      false |       645 |         54 |                 54 |        |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

# 通过ID删除成员，这里以删除10.211.55.52为例
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379 member remove ca2cb14b2acc776
Member  ca2cb14b2acc776 removed from cluster 7ff030ddad4d8497

# 检查集群状态，只有两个节点了
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 endpoint status -w table
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|         ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.211.55.50:2379 | 31f517c6aefb1a37 |  3.4.16 |   25 kB |      true |      false |       645 |         58 |                 58 |        |
| https://10.211.55.51:2379 | ad1ee26f89ef12f4 |  3.4.16 |   20 kB |     false |      false |       645 |         58 |                 58 |        |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

# 查看10.211.55.52 etcd状态，发现etcd
systemctl status etcd
● etcd.service - Etcd Server
   Loaded: loaded (/etc/systemd/system/etcd.service; enabled; vendor preset: disabled)
   Active: inactive (dead) since Sun 2021-07-04 11:39:42 CST; 1min 31s ago
   ......
   
# 尝试写入数据
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379  put foo4 bar8
OK

# 尝试获取写入的数据
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379  get  foo4 
foo4
bar8

添加成员

添加成员主要分为两步，一是通过etcdctl添加成员，二是根据第一步输出的配置修改etcd配置并启动etcd

# 先通过etcdctl添加成员
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379  member add etcd3 --peer-urls="https://10.211.55.52:2380"
Member 89088d035064124d added to cluster 7ff030ddad4d8497

ETCD_NAME="etcd3"
ETCD_INITIAL_CLUSTER="etcd1=https://10.211.55.50:2380,etcd3=https://10.211.55.52:2380,etcd2=https://10.211.55.51:2380"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.211.55.52:2380"
ETCD_INITIAL_CLUSTER_STATE="existing"

# 更改etcd配置并启动etcd，注意要保证etcd数据目录为空
systemctl cat etcd
# /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
ExecStart=/usr/local/bin/etcd \
--name=etcd3 \
--cert-file=/etc/etcd/ssl/etcd.pem \
--key-file=/etc/etcd/ssl/etcd-key.pem \
--peer-cert-file=/etc/etcd/ssl/etcd.pem \
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem \
--initial-advertise-peer-urls=https://10.211.55.52:2380 \
--listen-peer-urls=https://10.211.55.52:2380 \
--listen-client-urls=https://10.211.55.52:2379,http://127.0.0.1:2379 \
--advertise-client-urls=https://10.211.55.52:2379 \
--initial-cluster-token=etcd-cluster-0 \
--initial-cluster=etcd1=https://10.211.55.50:2380,etcd2=https://10.211.55.51:2380,etcd3=https://10.211.55.52:2380 \
--initial-cluster-state=existing \
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

# 启动etcd
systemctl start etcd && systemctl status etcd

# 检查集群状态
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 endpoint status -w table
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|         ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.211.55.50:2379 | 31f517c6aefb1a37 |  3.4.16 |   25 kB |      true |      false |       645 |         62 |                 62 |        |
| https://10.211.55.51:2379 | ad1ee26f89ef12f4 |  3.4.16 |   20 kB |     false |      false |       645 |         62 |                 62 |        |
| https://10.211.55.52:2379 | 89088d035064124d |  3.4.16 |   20 kB |     false |      false |       645 |         62 |                 62 |        |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

三、数据备份

数据备份我这里直接使用snapshot人工备份好了,仅用于测试而已。

# 先写入一个数据，方便后面做测试
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379  put xiyangxixi  boys
OK

etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379  get xiyangxixi
xiyangxixi
boys

# 备份数据
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379 snapshot save snapshot-xiyangxixi.db
{"level":"info","ts":1625373129.2212617,"caller":"snapshot/v3_snapshot.go:119","msg":"created temporary db file","path":"snapshot-xiyangxixi.db.part"}
{"level":"info","ts":"2021-07-04T12:32:09.228+0800","caller":"clientv3/maintenance.go:200","msg":"opened snapshot stream; downloading"}
{"level":"info","ts":1625373129.2287908,"caller":"snapshot/v3_snapshot.go:127","msg":"fetching snapshot","endpoint":"https://10.211.55.50:2379"}
{"level":"info","ts":"2021-07-04T12:32:09.231+0800","caller":"clientv3/maintenance.go:208","msg":"completed snapshot read; closing"}
{"level":"info","ts":1625373129.2318149,"caller":"snapshot/v3_snapshot.go:142","msg":"fetched snapshot","endpoint":"https://10.211.55.50:2379","size":"25 kB","took":0.010441635}
{"level":"info","ts":1625373129.2318769,"caller":"snapshot/v3_snapshot.go:152","msg":"saved","path":"snapshot-xiyangxixi.db"}
Snapshot saved at snapshot-xiyangxixi.db

四、集群数据恢复

需要使用同一个备份文件。这里以上面的snapshot-xiyangxixi.db进行模拟

# 注意：
# 1、最好指定一下data-dir我这里没指定，结果数据都到etcd1.etcd、etcd2.etcd、etcd3.etcd中去了，这样子还得修改etcd.service文件
# 2、恢复数据前，之前的数据目录记得备份
# 3、如果是单节点故障，可以考虑删除该节点再添加该节点
# etcd1上执行
etcdctl snapshot restore /root/snapshot-xiyangxixi.db  --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem  --name etcd1 --initial-advertise-peer-urls=https://10.211.55.50:2380 --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd1=https://10.211.55.50:2380,etcd2=https://10.211.55.51:2380,etcd3=https://10.211.55.52:2380
{"level":"info","ts":1625373977.7445557,"caller":"snapshot/v3_snapshot.go:296","msg":"restoring snapshot","path":"/root/snapshot-xiyangxixi.db","wal-dir":"etcd1.etcd/member/wal","data-dir":"etcd1.etcd","snap-dir":"etcd1.etcd/member/snap"}
{"level":"info","ts":1625373977.7477813,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"a914ec3525c16b4","local-member-id":"0","added-peer-id":"517f2ff3c4b9581b","added-peer-peer-urls":["https://10.211.55.52:2380"]}
{"level":"info","ts":1625373977.7478256,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"a914ec3525c16b4","local-member-id":"0","added-peer-id":"6856401bd8388bec","added-peer-peer-urls":["https://10.211.55.50:2380"]}
{"level":"info","ts":1625373977.7478426,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"a914ec3525c16b4","local-member-id":"0","added-peer-id":"6ea7d2ec70c556d5","added-peer-peer-urls":["https://10.211.55.51:2380"]}
{"level":"info","ts":1625373977.7509031,"caller":"snapshot/v3_snapshot.go:309","msg":"restored snapshot","path":"/root/snapshot-xiyangxixi.db","wal-dir":"etcd1.etcd/member/wal","data-dir":"etcd1.etcd","snap-dir":"etcd1.etcd/member/snap"}

# etcd2上执行
etcdctl snapshot restore /root/snapshot-xiyangxixi.db  --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem  --name etcd2 --initial-advertise-peer-urls=https://10.211.55.51:2380 --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd1=https://10.211.55.50:2380,etcd2=https://10.211.55.51:2380,etcd3=https://10.211.55.52:2380
{"level":"info","ts":1625374104.1673598,"caller":"snapshot/v3_snapshot.go:296","msg":"restoring snapshot","path":"/root/snapshot-xiyangxixi.db","wal-dir":"etcd2.etcd/member/wal","data-dir":"etcd2.etcd","snap-dir":"etcd2.etcd/member/snap"}
{"level":"info","ts":1625374104.1709266,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"a914ec3525c16b4","local-member-id":"0","added-peer-id":"517f2ff3c4b9581b","added-peer-peer-urls":["https://10.211.55.52:2380"]}
{"level":"info","ts":1625374104.1709714,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"a914ec3525c16b4","local-member-id":"0","added-peer-id":"6856401bd8388bec","added-peer-peer-urls":["https://10.211.55.50:2380"]}
{"level":"info","ts":1625374104.1709878,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"a914ec3525c16b4","local-member-id":"0","added-peer-id":"6ea7d2ec70c556d5","added-peer-peer-urls":["https://10.211.55.51:2380"]}
{"level":"info","ts":1625374104.1741195,"caller":"snapshot/v3_snapshot.go:309","msg":"restored snapshot","path":"/root/snapshot-xiyangxixi.db","wal-dir":"etcd2.etcd/member/wal","data-dir":"etcd2.etcd","snap-dir":"etcd2.etcd/member/snap"}

# etcd3上执行
etcdctl snapshot restore /root/snapshot-xiyangxixi.db  --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem  --name etcd3 --initial-advertise-peer-urls=https://10.211.55.52:2380 --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd1=https://10.211.55.50:2380,etcd2=https://10.211.55.51:2380,etcd3=https://10.211.55.52:2380
{"level":"info","ts":1625374158.713205,"caller":"snapshot/v3_snapshot.go:296","msg":"restoring snapshot","path":"/root/snapshot-xiyangxixi.db","wal-dir":"etcd3.etcd/member/wal","data-dir":"etcd3.etcd","snap-dir":"etcd3.etcd/member/snap"}
{"level":"info","ts":1625374158.7164843,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"a914ec3525c16b4","local-member-id":"0","added-peer-id":"517f2ff3c4b9581b","added-peer-peer-urls":["https://10.211.55.52:2380"]}
{"level":"info","ts":1625374158.7165308,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"a914ec3525c16b4","local-member-id":"0","added-peer-id":"6856401bd8388bec","added-peer-peer-urls":["https://10.211.55.50:2380"]}
{"level":"info","ts":1625374158.7165465,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"a914ec3525c16b4","local-member-id":"0","added-peer-id":"6ea7d2ec70c556d5","added-peer-peer-urls":["https://10.211.55.51:2380"]}
{"level":"info","ts":1625374158.7190611,"caller":"snapshot/v3_snapshot.go:309","msg":"restored snapshot","path":"/root/snapshot-xiyangxixi.db","wal-dir":"etcd3.etcd/member/wal","data-dir":"etcd3.etcd","snap-dir":"etcd3.etcd/member/snap"}

# 启动etcd
systemctl daemon-reload && systemctl start etcd

# 查看etcd集群状态
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379 endpoint status -w table
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|         ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://10.211.55.50:2379 | 6856401bd8388bec |  3.4.16 |   25 kB |     false |      false |         4 |          8 |                  8 |        |
| https://10.211.55.51:2379 | 6ea7d2ec70c556d5 |  3.4.16 |   25 kB |      true |      false |         4 |          8 |                  8 |        |
| https://10.211.55.52:2379 | 517f2ff3c4b9581b |  3.4.16 |   25 kB |     false |      false |         4 |          8 |                  8 |        |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

# 获取key为xiyangxixi的数据
etcdctl --cacert /etc/etcd/ssl/etcd-ca.pem --cert /etc/etcd/ssl/etcd.pem  --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://10.211.55.50:2379,https://10.211.55.51:2379,https://10.211.55.52:2379  get xiyangxixi
xiyangxixi
boys