nginx+keepalived双主高可用负载均衡

原创

linuxzkq 2015-08-25 23:15:32 博主文章分类：架构 ©著作权

文章标签 Nginx+keepalived双主高可 文章分类 Nginx 服务器

©著作权归作者所有：来自51CTO博客作者linuxzkq的原创作品，请联系作者获取转载授权，否则将追究法律责任

实验环境及软件版本：
CentOS版本： 6.6(2.6.32.-504.el6.x86_64)
nginx版本： nginx-1.6.3
keepalived版本：keepalived-1.2.7

主LB1：LB-110-05

主LB2：LB-111-06

一、安装准备及依赖（用SecureCRT的交互窗口同时对两台LB操作，只贴出LB1的操作过程在此）

[root@LB-110-05 ~]# mkdir tools

[root@LB-110-05 ~]# mkdir /application

[root@LB-110-05 ~]# yum -y install pcre pcre-devel openssl openssl-devel gcc gcc-c++ make automake popt-devel

[root@LB-110-05 ~]# cd tools

[root@LB-110-05 tools]# tar xf nginx-1.6.3.tar.gz

二、Nginx+keepalived安装实战（用SecureCRT的交互窗口同时对两台LB操作，只贴出LB1的操作过程在此）

1. 安装nginx

[root@LB-110-05 tools]# cd nginx-1.6.3

[root@LB-110-05 nginx-1.6.3]# useradd nginx -s /sbin/nologin -M

[root@LB-110-05 nginx-1.6.3]# ./configure --prefix=/application/nginx-1.6.3 --with-http_stub_status_module --with-http_ssl_module --user=nginx --group=nginx

[root@LB-110-05 nginx-1.6.3]# echo $? #检查上一步操作是否正确，正确返回结果为0，反之为1
0

[root@LB-110-05 nginx-1.6.3]# make && make install

[root@LB-110-05 nginx-1.6.3]# ln -s /application/nginx-1.6.3 /application/nginx

[root@LB-110-05 nginx-1.6.3]# /application/nginx/sbin/nginx -t #检查nginx的语法是否正确和测试是否成功
nginx: the configuration file /application/nginx-1.6.3/conf/nginx.conf syntax is ok
nginx: configuration file /application/nginx-1.6.3/conf/nginx.conf test is successful

[root@LB-110-05 nginx-1.6.3]# /application/nginx/sbin/nginx #启动nginx服务

[root@LB-110-05 nginx-1.6.3]# netstat -tunlp|grep 80 #检查nginx服务是否启动成功

tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 4329/nginx

2. 安装keepalived

[root@LB-110-05 nginx-1.6.3]# cd..

[root@LB-110-05 tools]# tar xf keepalived-1.2.7.tar.gz

[root@LB-110-05 tools]# cd keepalived-1.2.7

[root@LB-110-05 keepalived-1.2.7]# ./configure

[root@LB-110-05 keepalived-1.2.7]# make && make install

[root@LB-110-05 keepalived-1.2.7]# cp /usr/local/etc/rc.d/init.d/keepalived /etc/rc.d/init.d/

[root@LB-110-05 keepalived-1.2.7]# cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/

[root@LB-110-05 keepalived-1.2.7]# mkdir /etc/keepalived

[root@LB-110-05 keepalived-1.2.7]# cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/

[root@LB-110-05 keepalived-1.2.7]# cp /usr/local/sbin/keepalived /usr/sbin/

3. 加入开机启动

[root@LB-110-05 ~]# cat >>/etc/rc.local<<EOF
> /usr/local/nginx/sbin/nginx
> /etc/init.d/keepalived start
> EOF

[root@LB-110-05 ~]# cat /etc/rc.local
#!/bin/sh
#
# This script will be executed *after* all the other init scripts.
# You can put your own initialization stuff in here if you don't
# want to do the full Sys V style init stuff.

touch /var/lock/subsys/local
/application/nginx/sbin/nginx
/etc/init.d/keepalived start

三、配置nginx+keepalived

1. 配置nginx

[root@LB-110-05 ~]# cd /application/nginx/conf

[root@LB-110-05 conf]# cp nginx.conf nginx.conf.bak

[root@LB-110-05 conf]# vi nginx.conf

user nginx nginx;
worker_processes 1;
events {
worker_connections 1024;
}

http {
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;

upstream myserver{
ip_hash; #用ip哈希算法保持会话
server 10.0.0.7:80 max_fails=3 fail_timeout=20s;
server 10.0.0.8:80 max_fails=3 fail_timeout=20s;
}
server {
listen 80;
server_name 192.168.0.110;
location / {
index index.php index.htm index.html;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_next_upstream http_500 http_502 http_503 error timeout invalid_header;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_pass http://myserver;
}
}
}

保存退出，把LB1的nginx.conf配置文件用scp推送到LB2的/application/nginx/conf目录下就行。

[root@LB-110-05 conf]# /application/nginx/sbin/nginx -t
nginx: the configuration file /application/nginx-1.6.3/conf/nginx.conf syntax is ok
nginx: configuration file /application/nginx-1.6.3/conf/nginx.conf test is successful

[root@LB-110-05 conf]# /application/nginx/sbin/nginx -s reload #平滑重启nginx,不影响服务使用，提高用户体验

[root@LB-110-05 conf]# ps -ef|grep nginx|grep -v grep #检查nginx服务是否启动成功
root 4329 1 0 17:08 ? 00:00:00 nginx: master process /application/nginx/sbin/nginx
nginx 6330 4329 0 18:01 ? 00:00:00 nginx: worker process

2. 配置keepalived

2.1 LB1的keepalived配置

[root@LB-110-05 ~]# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

[root@LB-110-05 ~]# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id Nginx_DEVEL
}

vrrp_instance Nginx_HA1 {
state MASTER
interface eth1
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.0.110/24 dev eth1
}
}
vrrp_instance Nginx_HA2 {
state BACKUP
interface eth1
virtual_router_id 52
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.0.111/24 dev eth1
}
}

2.2 LB2的keepalived配置

[root@LB-111-06 ~]# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived

vrrp_instance Nginx_HA1 {
state BACKUP
interface eth2
virtual_router_id 51
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.0.110/24 dev eth2
}
}
vrrp_instance Nginx_HA2 {
state MASTER
interface eth2
virtual_router_id 52
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.0.111/24 dev eth2
}
}

3. 启动keepalived

[root@LB-110-05~]# /etc/init.d/keepalived start #先启动主LB1

[root@LB-111-06 ~]# /etc/init.d/keepalived start #随后再启动主LB2

四、查看服务启动成功后的结果

[root@LB-110-05 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:5c:2d:57 brd ff:ff:ff:ff:ff:ff
inet 192.168.0.110/24 brd 192.168.0.255 scope global eth0
inet6 fe80::20c:29ff:fe5c:2d57/64 scope link
valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:5c:2d:61 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.5/24 brd 10.0.0.255 scope global eth1
inet 192.168.0.110/24 scope global eth1
inet6 fe80::20c:29ff:fe5c:2d61/64 scope link
valid_lft forever preferred_lft forever

[root@LB-111-06 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:50:8e:3a brd ff:ff:ff:ff:ff:ff
inet 192.168.0.111/24 brd 192.168.0.255 scope global eth1
inet6 fe80::20c:29ff:fe50:8e3a/64 scope link
valid_lft forever preferred_lft forever
3: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:50:8e:44 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.6/24 brd 10.0.0.255 scope global eth2
inet 192.168.0.111/24 scope global eth2
inet6 fe80::20c:29ff:fe50:8e44/64 scope link
valid_lft forever preferred_lft forever

五、测试高可用

[root@LB-111-06 ~]# /etc/init.d/keepalived stop
Stopping keepalived: [ OK ]
[root@LB-111-06 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:50:8e:3a brd ff:ff:ff:ff:ff:ff
inet 192.168.0.111/24 brd 192.168.0.255 scope global eth1
inet6 fe80::20c:29ff:fe50:8e3a/64 scope link
valid_lft forever preferred_lft forever
3: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:50:8e:44 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.6/24 brd 10.0.0.255 scope global eth2
inet6 fe80::20c:29ff:fe50:8e44/64 scope link
valid_lft forever preferred_lft forever

[root@LB-110-05 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:5c:2d:57 brd ff:ff:ff:ff:ff:ff
inet 192.168.0.110/24 brd 192.168.0.255 scope global eth0
inet6 fe80::20c:29ff:fe5c:2d57/64 scope link
valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:5c:2d:61 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.5/24 brd 10.0.0.255 scope global eth1
inet 192.168.0.110/24 scope global eth1
inet 192.168.0.111/24 scope global secondary eth1
inet6 fe80::20c:29ff:fe5c:2d61/64 scope link
valid_lft forever preferred_lft forever