ansible管理机:192.168.8.35 名称:kick

tomcat主机: 192.168.8.234,192.168.8.235   

VIP  : 192.168.8.100

系统版本: Centos7.2


wKioL1lRzV2jTqQfAAAoKl49FBQ419.png-wh_50


##### 实验软件下载地址

配置文件:  http://pan.baidu.com/s/1eSou1Bs

NFS:           http://pan.baidu.com/s/1kVieBgN

DRBD:        http://pan.baidu.com/s/1nvkIxtR



######### 下载drbd 软件及模块

wget http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm

wget http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpm


######### 将下载软件复制到各节点中去,也可以直接下载

[root@kick ~]# ansible store -m copy -a "src=drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpm dest=/tmp/"

[root@kick ~]# ansible store -m copy -a "src=kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm dest=/tmp/"


######### 三步可以合一起直接yum -y install http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm

[root@kick ~]# ansible store -m shell -a "yum -y install /tmp/drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpm"

[root@kick ~]# ansible store -m shell -a "yum -y reinstall /tmp/kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm"


######### 加载drbd模块

[root@kick ~]# ansible store -m shell -a "modprobe drbd"


######### 查看drbd模块是否已加载上

[root@kick ~]# ansible store -m shell -a "lsmod | grep drbd"

192.168.8.235 | SUCCESS | rc=0 >>

drbd                  392583  0 

libcrc32c              12644  2 xfs,drbd


192.168.8.234 | SUCCESS | rc=0 >>

drbd                  392583  0 

libcrc32c              12644  2 xfs,drbd


# 新加一个磁盘分区不需要格式化


######### 配置drbd全局配置及资源

[root@kick ~]# ansible store -m copy -a "src=global_common.conf dest=/etc/drbd.d/"

[root@kick ~]# ansible store -m copy -a "src=mydrbd.res dest=/etc/drbd.d/"


[root@kick ~]# vim global_common.conf

global {

usage-count no;

}


common {

handlers {


pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";

pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";

local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";

}


startup {

}


options {

}


disk {

on-io-error detach;

}


net {

cram-hmac-alg "sha1";

shared-secret "xiong123";

}

syncer {

rate 1024M;

}

}

################# 资源叫mydrbd,主机名称一定要在本地的/etc/hosts中增加主机名称解析否则会报错

[root@kick ~]# vim mydrbd.res

resource mydrbd{

  device /dev/drbd0;

  disk /dev/sdb1;

  meta-disk internal;


on store1 {

  address 192.168.8.234:7789;

}


on store2 {

  address 192.168.8.235:7789;

}

}


# 更改hosts主机名称

[root@kick ~]# cat /etc/hosts

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4

::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.8.234 store1

192.168.8.235 store2


[root@kick ~]# ansible store -m copy -a "src=/etc/hosts dest=/etc/hosts"


######### 配置mydrbd资源并启动服务,查看资源信息两个没有主的时候都是seconary

[root@kick ~]# ansible store -m shell -a "drbdadm create-md mydrbd"

[root@kick ~]# ansible store -m shell -a "systemctl start drbd"

[root@kick ~]# ansible store -m shell -a "drbd-overview"

# 随便登陆一台设置成主节点

[root@store1 ~]# drbdadm primary --force mydrbd

# 测试查看




######### 安装nfs软件,也可以直接安装nfs-utils-1.3.0-0.33.el7.x86_64.rpm 这个版本,最后有错误记录

[root@kick ~]# ansible store -m yum -a "name=nfs-utils state=installed"



######### 配置keepalived,直接使用ansible安装即可

################# 主库 ##################################

! Configuration File for keepalived


global_defs {

   notification_email {

        xiong@localhost

   }

   notification_email_from root@localhost

   smtp_server 127.0.0.1

   smtp_connect_timeout 30

   router_id LVS_DEVEL

}

vrrp_script check_run {

        script "/etc/keepalived/notify_check.sh"

        interval 10

        weight 10

}


vrrp_instance nfs {

    state MASTER

    interface eno16780032

    virtual_router_id 70

    priority 100

    advert_int 1

    nopreempt                   # 不让它强制抢占资源

    authentication {

        auth_type PASS

        auth_pass PPVk7CXXZ5U

    }


    track_script {

        check_run weight 20                            # 检查节点信息状态,关闭keepalived,之后它会调取notify_stop 

    }


    notify_stop /etc/keepalived/notify_stop.sh         # 当节点stop后停止所有服务

    notify_master /etc/keepalived/notify_master.sh     # 当节点成为master时开启所有服务


    virtual_ipaddress {

        192.168.8.100

    }


}

################# 备库 ##################################

! Configuration File for keepalived


global_defs {

   notification_email {

        xiong@localhost

   }

   notification_email_from root@localhost

   smtp_server 127.0.0.1

   smtp_connect_timeout 30

   router_id LVS_DEVEL

}

vrrp_script check_run {

        script "/etc/keepalived/notify_check.sh"

        interval 10

        weight 10

}


vrrp_instance nfs {

    state BACKUP

    interface eno16780032

    virtual_router_id 70

    priority 95

    advert_int 1

    nopreempt                     # 不让它强制抢占资源

    authentication {

        auth_type PASS

        auth_pass PPVk7CXXZ5U

    }


    track_script {

        check_run weight 20

    }


    notify_stop /etc/keepalived/notify_stop.sh

    notify_master /etc/keepalived/notify_master.sh


    virtual_ipaddress {

        192.168.8.100

    }


}

#############################################################


######### 配置NFS

[root@kick ~]# cat /etc/exports

/data   192.168.8.0/24(rw,async,all_squash,anonuid=2021,anongid=2020)


######### 将配置文件复制到store两个drbd服务器中

[root@kick data]# ansible store -m copy -a "src=exports dest=/etc/"


######### 创建mysql用户mysql组 后续使用

[root@kick /]# ansible store -m shell -a "groupadd -g 2020 mysql"

[root@kick /]# ansible store -m shell -a "useradd -g 2020 -u 2021 mysql -s /sbin/nologin"


[root@kick /]# ansible store -m shell -a "chown mysql.mysql /data/ -R"


######### 复制脚本文件到各节点中

[root@kick ~]# ansible store -m copy -a "src=notify_master.sh dest=/etc/keepalived/"

[root@kick ~]# ansible store -m copy -a "src=notify_stop.sh dest=/etc/keepalived/"

[root@kick ~]# ansible store -m copy -a "src=notify_check.sh dest=/etc/keepalived/"

[root@kick ~]# ansible store -m copy -a "src=keepalived_store_drbd.conf dest=/etc/keepalived/keepalived.conf"



######################### 脚本内容 #####################################################

[root@kick ~]# cat notify_master.sh      # 当keepalived为主节点时重启所有服务并设置drbd为主

#!/bin/bash

#

systemctl  start rpcbind &>/dev/null

systemctl  start nfs &>/dev/null

systemctl  start drbd &>/dev/null

/usr/sbin/drbdadm primary mydrbd &>/tmp/node_change.txt

mount /dev/drbd0 /data &>/tmp/node_change.txt



[root@kick ~]# cat notify_stop.sh # 当检测节点为back时直接停停止各项服务

#!/bin/bash

#

umount /data

systemctl  stop rpcbind &>/dev/null

systemctl  stop nfs &>/dev/null

drbdadm secondary mydrbd &>/tmp/node_change.txt

echo -e "\n" &> /tmp/node_change.txt

systemctl stop keepalived &>/dev/null


[root@kick ~]# cat notify_check.sh     # 检查服务的正常性 当NFS服务不正常时直接停止keepalived然后再调取notify_stop脚本 

#!/bin/sh


###检查nfs可用性:进程和是否能够挂载   # 备用脚本###### check使用下面那个

systemctl  status nfs &>/dev/null

if [ $? -ne 0 ];then

    ###如果服务状态不正常,先尝试重启服务

    systemctl  restart restart

    systemctl  status nfs &>/dev/null

    if [ $? -ne 0 ];then

        ###若重启nfs服务后,仍不正常

        ###卸载drbd设备

        umount /dev/drbd0

        ###将drbd主降级为备

        drbdadm secondary r0

        #关闭keepalived

systemctl  stop keepalived

    fi

fi


################################ 检查三项服务正常,当其中的任何一个不正常都直接干掉 ##################

#/bin/bash

#

systemctl  status rpcbind &>/dev/null

VAR1=$?

systemctl  status nfs &>/dev/null

VAR2=$?

systemctl  status drbd &>/dev/null

VAR3=$?


if [[ $VAR1 -ne 0 || $VAR2 -ne 0 || $VAR3 -ne 0 ]];then

        systemctl  stop rpcbind &>/dev/null

        systemctl  stop nfs &>/dev/null

        umount /data

        drbdadm secondary mydrbd &>/tmp/node_change.txt

        echo -e "\n" >> /tmp/node_change.txt

        systemctl stop keepalived

        systemctl restart drbd

fi



##########################################################################################


############## 启动服务 ###########

[root@kick data]# ansible store -m shell -a "systemctl start keepalived"

[root@kick data]# ansible store -m shell -a "systemctl start rpcbind"

[root@kick data]# ansible store -m shell -a "systemctl start nfs"

############### 配合日志查看 ############

[root@kick data]# ansible store -m shell -a "systemctl status keepalived"

[root@kick data]# ansible store -m shell -a "systemctl status nfs"

[root@kick data]# ansible store -m shell -a "cat /var/log/message"



##########################################################################################




# 检查VIP地址以及NFS是否挂载

[root@kick ~]# showmount -e 192.168.8.100

Export list for 192.168.8.100:

/data 192.168.8.0/24




################################## NFS 出现的一些错误 #################################

[root@kick ~]# mount -t nfs 192.168.8.234:/data /data

mount.nfs: access denied by server while mounting 192.168.8.234:/data


#在/etc/exports中增加insecure,http://blog.chinaunix.net/uid-20554957-id-3444786.html

/data   192.168.8.0(rw,insecure,no_root_squash,async)



[root@store1 data]# systemctl restart nfs



###### 一直报这个错 ###### 排查确认是 centos7 关于nfs 1.3.0-0.21.el7_2 版本的bug

Jun 27 09:29:52 store1 nfsdcltrack[12174]: sqlite_insert_client: insert statement prepare failed: table clients has 2 columns but 3 values were supplied


# 将版本直接升级至nfs-utils-1.3.0-0.33.el7.x86_64

# 需要的软件包 libtirpc需要大于0.2.4-0.7,libtirpc-devel,nfs-utils-1.3.0-0.33.el7

[root@kick ~]# ansible store -m copy -a "src=libtirpc-0.2.4-0.8.el7.x86_64.rpm dest=/tmp"

[root@kick ~]# ansible store -m copy -a "src=libtirpc-devel-0.2.4-0.8.el7.x86_64.rpm dest=/tmp"

[root@kick ~]# ansible store -m copy -a "src=nfs-utils-1.3.0-0.33.el7.x86_64.rpm dest=/tmp"



[root@kick ~]# ansible store -m shell -a "rpm -Uvh /tmp/libtirpc-0.2.4-0.8.el7.x86_64.rpm"

[root@kick ~]# ansible store -m shell -a "rpm -Uvh /tmp/libtirpc-devel-0.2.4-0.8.el7.x86_64.rpm"

[root@kick ~]# ansible store -m shell -a "rpm -Uvh /tmp/nfs-utils-1.3.0-0.33.el7.x86_64.rpm"