存储高可用 -- keepalived+DRBD+NFS高可用共享存储

keepalived+DRBD+NFS高可用共享存储

• ​​一、BRBD 环境部署​​
• ​​二、配置 NFS 共享存储​​
• ​​三、Keepalived 实现集群高可用​​
• ​​四、测试​​

• ​​1、模拟 NFS 服务故障​​
• ​​2、测试 keepalived 故障​​

一、BRBD 环境部署

环境准备
配置 hosts解析

---node01
[root@node02 ~]# vim  /etc/hosts +
192.168.5.11 node01
192.168.5.12 node02

---node02
[root@node02 ~]# vim  /etc/hosts +
192.168.5.11 node01
192.168.5.12 node02

配置 免密钥互信

---node01
[root@node01 ~]# ssh-keygen -f ~/.ssh/id_rsa -P '' -q
[root@node01 ~]# ssh-copy-id node01
[root@node01 ~]# ssh-copy-id node02

---node02
[root@node02 ~]# ssh-keygen -f ~/.ssh/id_rsa -P '' -q
[root@node02 ~]# ssh-copy-id node01
[root@node02 ~]# ssh-copy-id node02

配置 DRBD yum 源

---node01
[root@node01 ~]# vim /etc/yum.repos.d/drbd.repo 
[drbd]
name=drbd
baseurl=https://mirrors.tuna.tsinghua.edu.cn/elrepo/elrepo/el7/x86_64/
gpgcheck=0

---node02
[root@node02 ~]# vim /etc/yum.repos.d/drbd.repo 
[drbd]
name=drbd
baseurl=https://mirrors.tuna.tsinghua.edu.cn/elrepo/elrepo/el7/x86_64/
gpgcheck=0

安装 DRBD

[root@node01 ~]# yum install -y drbd84-utils kmod-drbd84
[root@node01 ~]# yum install -y drbd84-utils kmod-drbd84

加载内核

---node01
[root@node01 ~]# lsmod | grep drbd
[root@node01 ~]# modprobe drbd
[root@node01 ~]# lsmod | grep drbd
drbd                  397041  0 
libcrc32c              12644  2 xfs,drbd

---node02
[root@node02 ~]# lsmod | grep drbd
[root@node02 ~]# modprobe drbd
[root@node02 ~]# lsmod | grep drbd
drbd                  397041  0 
libcrc32c              12644  2 xfs,drbd

修改 DRBD 配置文件

[root@node01 ~]# vim /etc/drbd.d/global_common.conf 
# DRBD is the result of over a decade of development by LINBIT.
# In case you need professional services for DRBD or have
# feature requests visit http://www.linbit.com

global {
  usage-count no;

  # Decide what kind of udev symlinks you want for "implicit" volumes
  # (those without explicit volume <vnr> {} block, implied vnr=0):
  # /dev/drbd/by-resource/<resource>/<vnr>   (explicit volumes)
  # /dev/drbd/by-resource/<resource>         (default for implict)
  udev-always-use-vnr; # treat implicit the same as explicit volumes

  # minor-count dialog-refresh disable-ip-verification
  # cmd-timeout-short 5; cmd-timeout-medium 121; cmd-timeout-long 600;
}

common {
        protocol C;
  handlers {
    # These are EXAMPLE handlers only.
    # They may have severe implications,
    # like hard resetting the node under certain circumstances.
    # Be careful when choosing your poison.

    pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
    pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
    local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
    # fence-peer "/usr/lib/drbd/crm-fence-peer.sh";
    # split-brain "/usr/lib/drbd/notify-split-brain.sh root";
    # out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root";
    # before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
    # after-resync-target /usr/lib/drbd/unsnapshot-resync-target-lvm.sh;
    # quorum-lost "/usr/lib/drbd/notify-quorum-lost.sh root";
  }

  startup {
    # wfc-timeout degr-wfc-timeout outdated-wfc-timeout wait-after-sb
  }

  options {
    # cpu-mask on-no-data-accessible

    # RECOMMENDED for three or more storage nodes with DRBD 9:
    # quorum majority;
    # on-no-quorum suspend-io | io-error;
  }

  disk {
                on-io-error detach;
    # size on-io-error fencing disk-barrier disk-flushes
    # disk-drain md-flushes resync-rate resync-after al-extents
                # c-plan-ahead c-delay-target c-fill-target c-max-rate
                # c-min-rate disk-timeout
  }

  net {
                cram-hmac-alg "sha1";
    shared-secret "nfs-HA";
    allow-two-primaries;
  }
          syncer { rate 1000M; }
}

[root@node01 ~]# vim /etc/drbd.d/nfs.res 
resource nfs {
  meta-disk internal;
  device /dev/drbd1;
  disk /dev/nfs/share; 
  
  on node01 {
  address 192.168.5.11:7789;
  }

  on node02 {
  address 192.168.5.12:7789;
  }
}


[root@node01 ~]# scp /etc/drbd.d/* node02:/etc/drbd.d/

LVM 格式化

---node01
[root@node01 ~]# vgcreate nfs /dev/sdb
[root@node01 ~]# lvcreate --size 2G --name share nfs

---node02
[root@node01 ~]# vgcreate nfs /dev/sdb
[root@node01 ~]# lvcreate --size 2G --name share nfs

创建用户

---node01
[root@node01 ~]# useradd -M -s /sbin/nologin haclient
[root@node01 ~]# chgrp haclient /lib/drbd/drbdsetup-84
[root@node01 ~]# chmod o-x /lib/drbd/drbdsetup-84
[root@node01 ~]# chmod u+s /lib/drbd/drbdsetup-84
[root@node01 ~]# chgrp haclient /usr/sbin/drbdmeta
[root@node01 ~]# chmod o-x /usr/sbin/drbdmeta
[root@node01 ~]# chmod u+s /usr/sbin/drbdmeta

---node02
[root@node02 ~]# useradd -M -s /sbin/nologin haclient
[root@node02 ~]# chgrp haclient /lib/drbd/drbdsetup-84
[root@node02 ~]# chmod o-x /lib/drbd/drbdsetup-84
[root@node02 ~]# chmod u+s /lib/drbd/drbdsetup-84
[root@node02 ~]# chgrp haclient /usr/sbin/drbdmeta
[root@node02 ~]# chmod o-x /usr/sbin/drbdmeta
[root@node02 ~]# chmod u+s /usr/sbin/drbdmeta

激活资源

[root@node01 ~]# drbdadm create-md data
[root@node02 ~]# drbdadm create-md data

查看 DRBD 状态

node01

node02

node01 强制升级

drbdadm -- --force primary nfs

再次查看 DRBD 状态

node01

node02

格式化、挂载 DRBD

[root@node01 ~]# mkdir data
[root@node01 ~]# mkfs.xfs /dev/drbd1
[root@node01 ~]# mount /dev/drbd1 data

查看挂载

DRBD 挂载 网络数据同步情况

[root@node01 ~]# echo "123" > data/a.txt
[root@node01 ~]# cat data/a.txt
123

二、配置 NFS 共享存储

---node01
[root@node01 ~]# yum -y install rpcbind nfs-utils
[root@node01 ~]# vim /etc/exports
[root@node01 ~]# systemctl start rpcbind
[root@node01 ~]# systemctl start nfs

---node02
[root@node02 ~]# yum -y install rpcbind nfs-utils
[root@node02 ~]# systemctl start rpcbind
[root@node02 ~]# systemctl start nfs

三、Keepalived 实现集群高可用

安装 keepalived

[root@node01 ~]# yum install keepalived -y
[root@node02 ~]# yum install keepalived -y

修改keepalived 配置文件

[root@node01 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
  notification_email {
  root@localhost
  }
   
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id DRBD_HA_MASTER
}
   
vrrp_script chk_nfs {
    script "/etc/keepalived/check_nfs.sh"
    interval 5
  }
  vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
      auth_type PASS
      auth_pass 1111
    }
    track_script {
      chk_nfs
    }
  notify_stop /etc/keepalived/notify_stop.sh       
  notify_master /etc/keepalived/notify_master.sh   
  virtual_ipaddress {
    192.168.5.100
  }
}

提供 Keepalived + DRBD 集群脚本

---node01
[root@node01 ~]# mkdir /etc/keepalived/logs
[root@node01 ~]# vim /etc/keepalived/check_nfs.sh
/sbin/service nfs status &>/dev/null
if [ $? -ne 0 ];then
   /sbin/service nfs restart
   /sbin/service nfs status &>/dev/null
   if [ $? -ne 0 ];then
      umount /dev/drbd1
      drbdadm secondary nfs
      /sbin/service keepalived stop
   fi
fi

[root@node01 ~]# vim /etc/keepalived/notify_master.sh
#!/bin/bash
  
time=`date "+%F  %T"`
echo -e "$time    ------notify_master------\n" >> /etc/keepalived/logs/notify_master.log
/sbin/drbdadm primary nfs &>> /etc/keepalived/logs/notify_master.log
/bin/mount /dev/drbd1 data &>> /etc/keepalived/logs/notify_master.log
/sbin/service nfs restart &>> /etc/keepalived/logs/notify_master.log
echo -e "\n" >> /etc/keepalived/logs/notify_master.log

[root@node01 ~]# vim /etc/keepalived/notify_stop.sh 
#!/bin/bash
time=`date "+%F  %H:%M:%S"`
echo -e "$time  ------notify_stop------\n" >> /etc/keepalived/logs/notify_stop.log
/sbin/service nfs stop &>> /etc/keepalived/logs/notify_stop.log
/bin/umount data &>> /etc/keepalived/logs/notify_stop.log
/sbin/drbdadm secondary nfs &>> /etc/keepalived/logs/notify_stop.log
echo -e "\n" >> /etc/keepalived/logs/notify_stop.log

[root@node01 ~]# chmod +x /etc/keepalived/*.sh
[root@node01 ~]# ll /etc/keepalived/*.sh
-rwxr-xr-x 1 root root 248 Mar  9 14:06 /etc/keepalived/check_nfs.sh
-rwxr-xr-x 1 root root 389 Mar  9 14:07 /etc/keepalived/notify_master.sh
-rwxr-xr-x 1 root root 367 Mar  9 14:09 /etc/keepalived/notify_stop.sh

[root@node01 ~]# scp /etc/keepalived/* node02:/etc/keepalived/

---node02
[root@node02 ~]# mkdir /etc/keepalived/logs

启动 keepalived 服务

[root@node01 ~]# systemctl start keepalived.service
[root@node02 ~]# systemctl start keepalived.service

查看 VIP

node01

node02

四、测试

1、模拟 NFS 服务故障

node01

node02

实现 NFS 故障 恢复！

2、测试 keepalived 故障

MASTER keepalived 故障

node02

MASTER keepalived 故障恢复

node02

实现 Keepalived VIP 漂移！