1. 部署galera集群
利用四台主机cicd、node1、node2、node3来搭建galera集群。
1> 上传压缩包至cicd,解压得到ansible配置文件
[root@cicd ~]# ls
anaconda-ks.cfg ansible-galera.tar.gz
[root@cicd ~]# tar zxf ansible-galera.tar.gz
[root@cicd ~]# ls
anaconda-ks.cfg ansible-galera ansible-galera.tar.gz
[root@cicd ~]# cd ansible-galera/
[root@cicd ansible-galera]# ls
bin delete.yml hosts install.yml roles
2> 安装ansible
[root@cicd ~]# mount /dev/cdrom /mnt
mount: /dev/sr0 is write-protected, mounting read-only
[root@cicd ~]# yum install ansible -y
3> 清空ansible目录文件,将解压得到的文件复制到ansible目录下
[root@cicd ~]# cd /etc/ansible/
[root@cicd ansible]# ls
ansible.cfg hosts roles
[root@cicd ansible]# rm -rf *
[root@cicd ansible]# ls
[root@cicd ansible]# mv /root/ansible-galera/* .
[root@cicd ansible]# ls
bin delete.yml hosts install.yml roles
4> 集群进行域名解析,使每个节点都能静态解析。
[root@cicd ansible]# vim /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.16.95 cicd
192.168.16.96 node1
192.168.16.98 node2
192.168.16.99 node3
[root@cicd ansible]# scp /etc/hosts 192.168.16.96:/etc/hosts
[root@cicd ansible]# scp /etc/hosts 192.168.16.98:/etc/hosts
[root@cicd ansible]# scp /etc/hosts 192.168.16.99:/etc/hosts
5> 在部署节点设置无秘钥登录
[root@cicd ansible]# ssh-keygen
[root@cicd ansible]# ssh-copy-id node1
[root@cicd ansible]# ssh-copy-id node2
[root@cicd ansible]# ssh-copy-id node3
[root@cicd ansible]# ssh-copy-id cicd
6> bin目录用来存放配置文件以及镜像、安装脚本,因此要将配置文件、镜像和安装版本放在该目录下。
[root@cicd ansible]# ls
bin delete.yml hosts install.yml roles
[root@cicd ansible]# cd bin
[root@cicd bin]# ls
VERSION.md
[root@cicd bin]# cd
[root@cicd ~]# ls
anaconda-ks.cfg ansible-galera ansible-galera.tar.gz galera.tar.gz
[root@cicd ~]# tar zxf galera.tar.gz
[root@cicd ~]# ls
anaconda-ks.cfg ansible-galera.tar.gz galera.tar.gz
ansible-galera config.tar.gz mariadb-galera.ta
[root@cicd ~]# mv config.tar.gz mariadb-galera.tar /etc/ansible/bin/
[root@cicd ~]# cd /etc/ansible/bin/
[root@cicd bin]# ls
config.tar.gz mariadb-galera.tar VERSION.md
7> 在四台机器上部署docker
[root@cicd ~]# wget http://download2.yunwei.edu/shell/docker.tar.gz
[root@cicd ~]# tar zxf docker.tar.gz
[root@cicd ~]# cd docker
[root@cicd docker]# bash docker.sh
四台均部署。
8> 编辑配置文件hosts
[root@cicd ~]# cd /etc/ansible
[root@cicd ansible]# ls
bin delete.yml hosts install.yml roles
[root@cicd ansible]# vim hosts
[all] #集群节点
192.168.16.96
192.168.16.98
192.168.16.99
[master] #主节点
192.168.16.96
[slave1] #从节点
192.168.16.98
[slave2] #从节点
192.168.16.99
[all:vars] #变量
#定义集群ip
master="192.168.16.96"
slave1="192.168.16.98"
slave2="192.168.16.99"
#部署目录,即 ansible 工作目录
base_dir="/etc/ansible"
#默认软件安装目录
opt_dir="/opt"
9> 执行yml文件,部署ansible集群,得到docker-galera集群
[root@cicd ~]# cd /etc/ansible/
[root@cicd ansible]# ls
bin delete.yml hosts install.yml roles
[root@cicd ansible]# ansible-playbook install.yml
10> 查看容器即数据库结果,验证是否安装成功。
[root@node1 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
10cda5ecde16 panubo/mariadb-galera "/galera-entrypoint.…" 5 minutes ago Up 5 minutes galera1
[root@node2 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
bc1dca8dea63 panubo/mariadb-galera "/galera-entrypoint.…" 6 minutes ago Up 34 seconds galera2
[root@node3 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
fd542150b5cc panubo/mariadb-galera "/galera-entrypoint.…" 7 minutes ago Up 22 seconds galera3
[root@node1 ~]# docker exec -it 10cda5ecde16 /bin/bash
root@node1:/# mysql -uroot -p123
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 3 |
+--------------------+-------+
[root@node2 opt]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
9abff0435fa2 panubo/mariadb-galera "/galera-entrypoint.…" 2 hours ago Up 2 hours galera2
[root@node2 opt]# docker exec -it 9abff0435fa2 /bin/bash
root@node2:/# mysql -uroot -p123
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 4
Server version: 10.1.32-MariaDB-1~jessie mariadb.org binary distribution
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 3 |
+--------------------+-------+
[root@node3 opt]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
[root@node3 opt]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
26b3b9dee6ea panubo/mariadb-galera "/galera-entrypoint.…" 2 hours ago Up 2 hours galera3
[root@node3 opt]# docker exec -it 26b3b9dee6ea /bin/bash
root@node3:/# mysql -uroot -p123;
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 4
Server version: 10.1.32-MariaDB-1~jessie mariadb.org binary distribution
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 3 |
+--------------------+-------+
1 row in set (0.01 sec)
11> 各文件内容
[root@node ansible]# cat install.yml
- hosts:
- all
roles:
- ready
- hosts:
- master
roles:
- master
- hosts:
- slave1
roles:
- slave1
- hosts:
- slave2
roles:
- slave2
对应的main.yml文件在目录roles下面
[root@node ansible]# cd roles
[root@node roles]# ls
master ready slave1 slave2
[root@node ansible]# cd roles
[root@node roles]# ls
master ready slave1 slave2
[root@node roles]# cd master/
[root@node master]# ls
tasks
[root@node master]# cd tasks/
[root@node tasks]# ls
main.yml
[root@node tasks]# cat main.yml
#部署wsrep进程mariadb-master
- name: 启动wsrep进程
shell: "cd {{ opt_dir }}/mariadb/ && sh master.sh"
[root@node tasks]# cd /etc/ansible/roles/ready/
[root@node ready]# ls
tasks templates
[root@node ready]# cd tasks/
[root@node tasks]# ls
main.yml
[root@node tasks]# cat main.yml
#准备工作目录:
- name: prepare some dirs
file: name={{ item }} state=directory mode=0755
with_items:
- "{{ opt_dir }}/mariadb"
## ---------下载 配置文件及镜像 -----------
- name: download configfile and iamge
copy: src={{ base_dir }}/bin/{{ item }} dest={{ opt_dir }}/mariadb/{{ item }} mode=0755
with_items:
- mariadb-galera.tar
- config.tar.gz
#解压配置文件
- name: 解压config
shell: "tar zxf {{ opt_dir }}/mariadb/config.tar.gz -C {{ opt_dir }}/mariadb/"
#导入mariadb-galera镜像
- name: 导入mariadb-galera
shell: "cd {{ opt_dir }}/mariadb/ && docker load -i mariadb-galera.tar"
## ---------分发 脚本文件 -----------
- name: download configfile and iamge
template: src={{ item }} dest={{ opt_dir }}/mariadb/{{ item }} mode=0755
with_items:
- master.sh
- slave1.sh
- slave2.sh
[root@node roles]# ls
master ready slave1 slave2
[root@node roles]# cd ready/
[root@node ready]# ls
tasks templates
[root@node ready]# cd templates/
[root@node templates]# ls
master.sh slave1.sh slave2.sh
[root@node templates]# cat master.sh
docker run -d --net host --name galera1 \
-e WSREP_NODE_ADDRESS={{ master }} \
-e WSREP_CLUSTER_ADDRESS=gcomm://{{ master }}:4567,{{ slave1 }}:4567,{{ slave2 }}:4567 \
-e MYSQL_ROOT_PASSWORD=123 \
-p 3306:3306 \
-p 4567:4567/udp \
-p 4567-4568:4567-4568 \
-p 4444:4444 \
-v /opt/mariadb/config/mysql:/etc/mysql \
-v /opt/mariadb/config/data:/var/lib/mysql:Z \
--restart=always \
panubo/mariadb-galera mysqld --wsrep-new-cluster
[root@node templates]# cat slave1.sh
docker run -d --net host --name galera2 \
-e WSREP_NODE_ADDRESS={{ slave1 }} \
-e WSREP_CLUSTER_ADDRESS=gcomm://{{ master }}:4567,{{ slave1 }}:4567,{{ slave2 }}:4567 \
-p 3306:3306 \
-p 4567:4567/udp \
-p 4567-4568:4567-4568 \
-p 4444:4444 \
-v /opt/mariadb/config/mysql:/etc/mysql \
-v /opt/mariadb/config/data:/var/lib/mysql:Z \
--restart=always \
panubo/mariadb-galera mysqld
[root@node templates]# cat slave2.sh
docker run -d --net host --name galera3 \
-e WSREP_NODE_ADDRESS={{ slave2 }} \
-e WSREP_CLUSTER_ADDRESS=gcomm://{{ master }}:4567,{{ slave1 }}:4567,{{ slave2 }}:4567 \
-p 3306:3306 \
-p 4567:4567/udp \
-p 4567-4568:4567-4568 \
-p 4444:4444 \
-v /opt/mariadb/config/mysql:/etc/mysql \
-v /opt/mariadb/config/data:/var/lib/mysql:Z \
--restart=always \
panubo/mariadb-galera mysqld
[root@node roles]# cd slave1
[root@node slave1]# ls
tasks
[root@node slave1]# cd tasks/
[root@node tasks]# ls
main.yml
[root@node tasks]# cat main.yml
#部署wsrep进程mariadb-master
- name: 启动salve进程
shell: "cd {{ opt_dir }}/mariadb/ && sh slave1.sh"
[root@node tasks]# cat main.yml
#部署wsrep进程mariadb-master
- name: 启动salve进程
shell: "cd {{ opt_dir }}/mariadb/ && sh slave2.sh"
[root@node tasks]#
11> 删除部署的docker-galance
删除该集群用/etc/ansible/delete.yum文件
文件内容
[root@node ansible]# cat delete.yml
- hosts:
- master
tasks:
- name: stop mariadb-galera-docker
shell: "docker stop galera1 && docker rm galera1"
- hosts:
- slave1
tasks:
- name: stop mariadb-galera-docker
shell: "docker stop galera2 && docker rm galera2"
- hosts:
- slave2
tasks:
- name: stop mariadb-galera-docker
shell: "docker stop galera3 && docker rm galera3"
- hosts:
- all
tasks:
#清理文件及镜像
- name: 清理目录和文件
file: name={{ opt_dir }}/mariadb state=absent
- name: rm image
shell: "docker rmi 75feb1db734a"
执行删除
[root@node ansible]# ansible-playbook delete.yml
2. 集群故障恢复
1> 集群中某slave节点down机,重启该节点后重启docker容器即可;
[root@node ~]# mysql -h 192.168.16.96 -uroot -p123 #-h:指定节点数据库登录
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 4
Server version: 10.1.32-MariaDB-1~jessie mariadb.org binary distribution
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 3 |
+--------------------+-------+
模拟节点node3 down机
[root@node3 ~]# powerof
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 2 |
+--------------------+-------+
再将节点重新启动,查看数据库
Last login: Wed Jul 17 04:57:57 2019 from 192.168.16.95
[root@node3 ~]#
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 3 |
+--------------------+-------+
2> 集群中master节点down机,因为master节点已经切换重启该master节点后master-docker容器不能正常启动,需要将原先master-docker容器删除,再以非wrster方式启动一个普通容器;
[root@node1 ~]# shutdown -h now
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 2 |
+--------------------+-------+
重启node1
Last login: Wed Jul 17 04:57:41 2019 from 192.168.16.95
[root@node1 ~]#
[root@node1 ~]# docker ps -a #由于被抢占了master,一直处于重启状态
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
e919dd79cb44 panubo/mariadb-galera "/galera-entrypoint.…" 34 minutes ago Restarting (1) 11 seconds ago galera1
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 2 |
+--------------------+-------+
将该容器删除,然后再以普通容器的方式启动
[root@node1 ~]# docker stop e919dd79cb44
e919dd79cb44
[root@node1 ~]# docker rm e919dd79cb44
e919dd79cb44
修改从节点启动脚本,以普通容器形式启动
[root@node1 opt]# cd
[root@node1 ~]# cd /opt/mariadb/
[root@node1 mariadb]# ls
config config.tar.gz mariadb-galera.tar master.sh slave1.sh slave2.sh
[root@node1 mariadb]# vim slave1.sh
docker run -d --net host --name galera1 \
-e WSREP_NODE_ADDRESS=192.168.16.96 \
-e WSREP_CLUSTER_ADDRESS=gcomm://192.168.16.96:4567,192.168.16.98:4567,192.168.16.99:4567 \
-p 3306:3306 \
-p 4567:4567/udp \
-p 4567-4568:4567-4568 \
-p 4444:4444 \
-v /opt/mariadb/config/mysql:/etc/mysql \
-v /opt/mariadb/config/data:/var/lib/mysql:Z \
--restart=always \
panubo/mariadb-galera mysqld
[root@node1 mariadb]# docker run -d --net host --name galera1 \
> -e WSREP_NODE_ADDRESS=192.168.16.96 \
> -e WSREP_CLUSTER_ADDRESS=gcomm://192.168.16.96:4567,192.168.16.98:4567,192.168.16.99:4567 \
> -p 3306:3306 \
> -p 4567:4567/udp \
> -p 4567-4568:4567-4568 \
> -p 4444:4444 \
> -v /opt/mariadb/config/mysql:/etc/mysql \
> -v /opt/mariadb/config/data:/var/lib/mysql:Z \
> --restart=always \
> panubo/mariadb-galera mysqld
WARNING: Published ports are discarded when using host network mode
354383d0d45ee204e1cf5b9f5f0196216ea68a8df3b61a5618eb0cb1abe90e7b #重启成功
"slave1.sh" 11L, 383C written
[root@node1 mariadb]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
354383d0d45e panubo/mariadb-galera "/galera-entrypoint.…" 10 minutes ago Up 10 minutes galera1
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 3 |
+--------------------+-------+
3> 集群全部意外down机(如断电),强制找一台节点作为master节点,将/opt/mariadb/data目录中的文件grastate.dat文件的safe_to_bootstrap:0改为1,之后在该节点启动wsrap进程的docker。
模拟三个节点down
[root@node ~]# ansible all -m shell -a 'shutdown -h now'
192.168.16.96 | UNREACHABLE! => {
"changed": false,
"msg": "Failed to connect to the host via ssh: Shared connection to 192.168.16.96 closed.",
"unreachable": true
}
192.168.16.99 | UNREACHABLE! => {
"changed": false,
"msg": "Failed to connect to the host via ssh: Shared connection to 192.168.16.99 closed.",
"unreachable": true
}
192.168.16.98 | UNREACHABLE! => {
"changed": false,
"msg": "Failed to connect to the host via ssh: Shared connection to 192.168.16.98 closed.",
"unreachable": true
}
再将三个节点开机
Last login: Wed Jul 17 05:53:07 2019 from 192.168.16.95
[root@node1 ~]#
Last login: Wed Jul 17 05:52:33 2019 from 192.168.16.95
[root@node2 ~]#
Last login: Wed Jul 17 05:53:06 2019 from 192.168.16.95
[root@node3 ~]#
无法登陆数据库
[root@node ~]# mysql -h 192.168.16.99 -uroot -p123
ERROR 2003 (HY000): Can't connect to MySQL server on '192.168.16.99' (111)
[root@node ~]# mysql -h 192.168.16.98 -uroot -p123
ERROR 2003 (HY000): Can't connect to MySQL server on '192.168.16.99' (111)
[root@node ~]# mysql -h 192.168.16.96-uroot -p123
ERROR 2003 (HY000): Can't connect to MySQL server on '192.168.16.99' (111)
但是各节点容器是启动的
[root@node ~]# ansible all -m shell -a 'docker ps'
192.168.16.96 | CHANGED | rc=0 >>
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
354383d0d45e panubo/mariadb-galera "/galera-entrypoint.…" 18 minutes ago Up 20 seconds galera1
192.168.16.99 | CHANGED | rc=0 >>
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
57f1d19388c2 panubo/mariadb-galera "/galera-entrypoint.…" About an hour ago Up 21 seconds galera3
192.168.16.98 | CHANGED | rc=0 >>
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
e96ee6196980 panubo/mariadb-galera "/galera-entrypoint.…" About an hour ago Up 16 seconds galera2
修改文件grastate.dat
[root@node1 ~]# cd /opt/mariadb/config/data/
[root@node1 data]# ls
aria_log.00000001 grastate.dat ib_logfile1 performance_schema
aria_log_control ibdata1 multi-master.info
galera.cache ib_logfile0 mysql
[root@node1 data]# vim grastate.dat
# GALERA saved state
version: 2.1
uuid: 8c85e11d-f45d-11e8-8ed7-dbc6d595b08d
seqno: -1
safe_to_bootstrap: 1
删除galera1容器,由于做了映射。并不会删除数据库
[root@node1 data]# docker stop 354383d0d45e
354383d0d45e
[root@node1 data]# docker rm 354383d0d45e
354383d0d45e
执行master.sh脚本,人为启动mode1为master
[root@node1 ~]# cd /opt/mariadb/
[root@node1 mariadb]# ls
config config.tar.gz mariadb-galera.tar master.sh slave1.sh slave2.sh
[root@node1 mariadb]# bash master.sh
WARNING: Published ports are discarded when using host network mode
2c9900b470a429484e1e959d9d4b4ab4bbfbd0238abaf67214c82cdf7388f404
[root@node1 mariadb]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
2c9900b470a4 panubo/mariadb-galera "/galera-entrypoint.…" 23 seconds ago Up 23 seconds galera1
查看数据库,其他两节点自动加入
[root@node ~]# mysql -h 192.168.16.96 -uroot -p123
MariaDB [(none)]> SHOW STATUS LIKE 'wsrep_cluster_size';
+--------------------+-------+
| Variable_name | Value |
+--------------------+-------+
| wsrep_cluster_size | 3 |
+--------------------+-------+