环境：
- 202.106.0.17: as Client
- 202.106.0.27: as router
- 192.168.205.37: as lvs1
- 192.168.205.47: as lvs2
- 192.168.205.57: as websrv1
- 192.168.205.67: as websrv2
- 192.168.205.77: as websrv3
- 192.168.205.87: as websrv4 注：所有操作系统默认停用firewalld,iptable为空,关闭selinux
版本：
- OS: centos 7 1810 with mini install
- keepalived
- httpd
目地：
使用keepalived 监控四台web服务器，两台为一组，各使用一个虚拟IP地址，lvs1为vip1的主vip2的从，lvs2为vip2的主，vip1的从，互相主备, 并使用lvs做为负载均衡，并能自动的发现服务down机并自动移除故障主机，从而实现无故障调度。
配置router
- 开启路由转发功能
[root@router data]#ech 1 > /proc/sys/net/ipv4/ip_forward
- 增加一个10.1.1.1在router的eth1上, 由于我要到lvs1和lvs2服务器时必须有路由，由于是两个主机的虚拟Ip是很有可能飘移，所以当其中一台down时无法确定在哪台主机上，为了安全不加路由，直接用IP直连。
[root@router data]#ip a a 10.1.1.1/24 dev eth0:1
先配置四台real server
- 在app1的两台服务器192.168.205.57/67上运行如下脚本
[root@websrv1 data]#cat lvs_dr_rs.sh
#!/bin/bash
vip=10.1.1.100
gateway=192.168.205.27
mask='24'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "this is `hostname`" > /var/www/html/index.html
case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
ip a a $vip/$mask dev $dev #broadcast $vip up
ip route add default via $gateway dev eth0
#route add -host $vip dev $dev
echo "The RS Server is Ready!"
;;
stop)
systemctl restart network
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo "The RS Server is Canceled!"
;;
*)
echo "Usage: $(basename $0) start|stop"
exit 1
;;
esac
- 在app2的两台服务器192.168.205.77/87上运行如下脚本
[root@websrv3 data]#cat lvs_dr_rs.sh
#!/bin/bash
vip=10.1.1.200
gateway=192.168.205.27
mask='24'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "this is `hostname`" > /var/www/html/index.html
case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
ip address add $vip/$mask dev $dev #broadcast $vip up
ip route add default via $gateway dev eth0
#route add -host $vip dev $dev
echo "The RS Server is Ready!"
;;
stop)
systemctl restart network
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo "The RS Server is Canceled!"
;;
*)
echo "Usage: $(basename $0) start|stop"
exit 1
;;
esac
在两台lvs上做同样的动作（除特别说明）
- 安装httpd服务，将sorry server放在本地的两台主机中
yum install httpd
echo server under maintenance > /var/www/html/index.html
- 一定要加默认路由到router,如果加正确可以工作，但sorry server不会正常
ip route add default via 192.168.205.27 dev eth0
- 为了访问方便最好是将两个lvs服务器之间做ssh key验证，用下面的方法就不用在47上再做一次了，真接实现互认证
ssh-keygen
ssh-copy-id 127.0.0.1
scp -r /root/.ssh 192.168.205.47:/root
- 最好将host文件加入两台主机名称解析
vi /etc/hosts
192.168.205.37 websrv1
192.168.205.47 websrv2
scp /etc/hosts 192.168.205.47:/etc
- 安装keepalive, 为了能看清如何加的lvs策略，我们把iplvadm也装上
yum install keepalived ipvsadm
- 在lvs1上修改配置文件
[root@lsv1 ~]#vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalive@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS1
vrrp_mcast_group4 224.0.0.100
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 37
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass centos
}
virtual_ipaddress {
10.1.1.100/24 dev eth0 label eth0:0
}
}
vrrp_instance VI_2 {
state BACKUP
interface eth0
virtual_router_id 47
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass centos
}
virtual_ipaddress {
10.1.1.200/24 dev eth0 label eth0:1
}
}
virtual_server 10.1.1.100 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 192.168.205.57 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 192.168.205.67 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
}
virtual_server 10.1.1.200 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 192.168.205.77 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 192.168.205.87 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
}
11. 为了方便将lsv1的keepalive.conf复制到lvs2上，并进行修改
[root@lvs2 ~]#vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalive@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS2
vrrp_mcast_group4 224.0.0.100
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 37
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass centos
}
virtual_ipaddress {
10.1.1.100/24 dev eth0 label eth0:0
}
}
vrrp_instance VI_2 {
state MASTER
interface eth0
virtual_router_id 47
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass centos
}
virtual_ipaddress {
10.1.1.200/24 dev eth0 label eth0:1
}
}
virtual_server 10.1.1.100 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 192.168.205.57 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 192.168.205.67 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
}
virtual_server 10.1.1.200 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 192.168.205.77 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 192.168.205.87 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
}
- 起动keepalived服务
systemctl start keepalived
- 在lvs1看到状态为RR调度
[root@lsv1 ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.1.1.100:80 rr
-> 192.168.205.57:80 Route 1 0 0
-> 192.168.205.67:80 Route 1 0 0
TCP 10.1.1.200:80 rr
-> 192.168.205.77:80 Route 1 0 0
-> 192.168.205.87:80 Route 1 0 0
- 在lvs1中看到只一10.1.1.100IP， 在Lvs2中会看到只有10.1.1.200IP
[root@lsv1 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:56:e1:ea brd ff:ff:ff:ff:ff:ff
inet 192.168.205.37/24 brd 192.168.205.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet 10.1.1.100/24 scope global eth0:0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe56:e1ea/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@lvs2 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:37:f9:93 brd ff:ff:ff:ff:ff:ff
inet 192.168.205.47/24 brd 192.168.205.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet 10.1.1.200/24 scope global eth0:1
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe37:f993/64 scope link noprefixroute
valid_lft forever preferred_lft forever
测试
- 在client上运行一个循环进行测试
[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
this is websrv1
this is websrv2
this is websrv1
this is websrv2
this is websrv1
this is websrv2
[root@client ~]#while : ;do curl 10.1.1.200; sleep 0.5; done
this is websrv4
this is websrv3
this is websrv4
this is websrv3
this is websrv4
- 停掉web1，再测试，发现只会调度到web2
[root@websrv1 data]#systemctl stop httpd
[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
this is websrv2
this is websrv2
this is websrv2
this is websrv2
this is websrv2
- 停掉web2，再测试，发现sorry server带替工作，并在lvs1上可以看到127.0.0.1加载
[root@websrv2 ~]#systemctl stop httpd
[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
server under maintenance
server under maintenance
server under maintenance
server under maitnenance
[root@lsv1 ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.1.1.100:80 rr
-> 127.0.0.1:80 Route 1 0 4
TCP 10.1.1.200:80 rr
-> 192.168.205.77:80 Route 1 0 0
-> 192.168.205.87:80 Route 1 0 0
- 恢复两个websrv1 和websrv2,并停掉lvs2, 发现没有影响，但可以看到两个vip全部回到lvs1上
[root@websrv1 ~]#systemctl start httpd
[root@websrv2 ~]#systemctl start httpd
[root@lvs2 ~]#systemctl stop keepalived
[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
this is websrv2
this is websrv1
this is websrv2
this is websrv1
[root@client ~]#while : ;do curl 10.1.1.200; sleep 0.5; done
this is websrv3
this is websrv4
this is websrv3
this is websrv4
[root@lsv1 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:56:e1:ea brd ff:ff:ff:ff:ff:ff
inet 192.168.205.37/24 brd 192.168.205.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet 10.1.1.100/24 scope global eth0:0
valid_lft forever preferred_lft forever
inet 10.1.1.200/24 scope global secondary eth0:1
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe56:e1ea/64 scope link noprefixroute
valid_lft forever preferred_lft forever
- 将lvs2恢复状态，因为有抢占功能又回到原来的主和备份上
[root@client ~]#while : ;do curl 10.1.1.100; sleep 0.5; done
this is websrv2
this is websrv1
this is websrv2
[root@client ~]#while : ;do curl 10.1.1.200; sleep 0.5; done
this is websrv4
this is websrv3
this is websrv4
[root@lsv1 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:56:e1:ea brd ff:ff:ff:ff:ff:ff
inet 192.168.205.37/24 brd 192.168.205.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet 10.1.1.100/24 scope global eth0:0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe56:e1ea/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@lvs2 ~]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:37:f9:93 brd ff:ff:ff:ff:ff:ff
inet 192.168.205.47/24 brd 192.168.205.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet 10.1.1.200/24 scope global eth0:1
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe37:f993/64 scope link noprefixroute
valid_lft forever preferred_lft forever
