Nginx+Keepalived部署
翻译
- 安装Nginx（http://nginx.org/）
yum install -y gcc gcc+ gcc-c++ openssl openssl-devel
groupadd www
useradd -g www www
mkdir -p /data/logs
chown
cd /usr/local/src
tar zxvf pcre-7.9.tar.gz
cd pcre-7.9
./configure
make && make install
cd ..
tar xzvf nginx-0.8.15.tar.gz
cd nginx-0.8.15
./configure --user=www --group=www --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module
make && make install
cd ..
- 配置nginx
vim /usr/local/nginx/conf/nginx.conf
user www www;
worker_processor 8;
pid /user/local/nginx/logs/nginx.pid;
worker_rlimit_nofile 51200;
events
{
use epoll;
worker_connections 51200;
}
http{
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 8m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
upstream backend {
ip_hash;
server 192.168.1.106:80; #web server 1
server 192.168.1.107:80; #web server 2
}
server {
listen 80;
server_name www.lpaitun.com;
location / {
root /var/www/html;
index index.php index.htm index.html;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://backend;
}
location /nginx {
access_log off;
auth_basic "NginxStatus";
#auth_basic_user_file /usr/local/nginx/htpasswd;
}
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
access_log /data/logs/access.log access;
}
}
- 运行Nginx
分别在两台Nginx负载均衡主机上执行/usr/local/nginx/sbin/nginx 启动Nginx进程。
使用 lsof -i:80 查看。 - 使用Keepalived实现两台Nginx主机的冗余互备
- 安装Keepalived（https://www.keepalived.org/）
tar xzvf keepalived-1.1.15.tar.gz
cd keepalived-1.1.15
./configure --prefix=/usr/local/keepalived
make
make install
cp /usr/local/keepalived/sbin/keepalived /usr/sbin/
cp /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/
cp
mkdir /etc/keepalived && cd
配置内容
! Configuration File for keepalived
global_defs {
notification_email {
vivi@163.com
}
notification_email_from keepalived@163.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
mcast_src_ip 192.168.1.103 #发送组播包的原地址。默认使用绑定网卡的地址。Nginx Master
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.1.108 # VIP
}
}
! Configuration File for keepalived
global_defs {
notification_email {
bee@163.com
}
notification_email_from keepalived@163.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
mcast_src_ip 192.168.1.104 #发送组播包的原地址。默认使用绑定网卡的地址。Nginx backup
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.1.108
}
}
service keepalived start
tail
通过命令 ip a 可以看到vrrp的虚接口sit0
- VRRP的重要知识点
~ 通信组播地址 224.0.0.18，且ttl=1。
~ VRID=0,1,2,…,255
~ 虚拟MAC 00-00-5E-00-01-[VRID]
~ 只有Master才会周期地向224.0.0.18发送advertisement通告报文。若backup在连续3个时间间隔内收不到advertisement报文，或者收到priority为0的advertisement则启动新一轮的选举。
~ priority=0,1,2,…,255 数值大者优先。0表示放弃Master角色。
~ 如果接口地址与VIP相同，则接口地址所在设备为VIP的owner。owner自动获得最高优先级255。
~ 当priority相同时，接口IP大的设备夺得Master角色。
~ preempt 用于配置抢占策略。指示高优先级的设备如何抢占Master角色。
~ 认证：“明文认证” 和 “IP头认证”。VRID及密钥要相同。
明文认证
防止配置错误。
IP头认证
防止攻击
- 监控Nginx进程
思路：
后台监控Nginx进程。如果Nginx进程消失，则尝试重启Nginx进程，如果还是不行，就停止本机的Keepalived服务，让另一台Nginx主机接手。
#!/bin/bash
while :
do
nginxpid=`ps -C nginx --no-header | wc -l`
if [ $nginxpid -eq 0 ]; then
/usr/local/nginx/sbin/nginx
sleep 5 #让出CPU让别人干点啥
if [ $nginxpid -eq 0 ]; then
/etc/init.d/keepalived stop
fi
fi
sleep 5
done
然后交给后台运行（no hang up）
nohup /bin/bash /root/nginx_pid.sh &
- Nginx作为负载均衡器在实际中遇到的问题
5.1 如何让Nginx支持HTTPS
在Nginx负载均衡器上开启SSL功能，监听TCP 443端口。将证书放在Nginx
负载均衡器上（而不是后边的Web服务器）。
nginx的配置文件如下：
server{
listen 443;
server_name www.bee.com;
ssl on;
ssl_certificate /usr/local/nginx/keys/www.bee.com.crt;
ssl_certificate_keys /usr/local/nginx/keys/www.bee.com.key;
ssl_protocols SSLv3 TLSv1;
ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:-LOW:-SSLv2:-EXP;
}5.2 如何让后端的Apache服务器获取用户的真实IP
一般情况下运行在后端的Apache服务器获得的客户端IP是Nginx负载均衡服务器的IP，或者是127.0.0.1。Apache服务器日志上显示的也都是内网IP。可以借助Nginx的日志来得到真实用户的IP。更通用的做法是修改Nginx Proxy的参数。
在Nginx配置文件中加入如下配置：
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
还需结合Apache的第三方模块。
http://stderr.net/apache/rpaf http://stderr.net/apache/rpaf/download
安装模块
tar xzvf mod_rpaf-0.6.tar.gz
cd
完成后会在http.conf的LoadModule区域多出：
LoadModule mod_rpaf-2.0.so_module modules/mod_rpaf-2.0.so
需修改为
LoadModule rpaf_module modules/mod_rpaf-2.0.so
RPAFenable On
RPAFsethostname On
# 192的地址是Nginx均衡器IP
5.3 正确区分Nginx的分发请求
客户端浏览器向Nginx发出http请求，Nginx是在做分发，而非正则跳转，跳转由后端的Web服务器负责。前端Nginx既是负载均衡器，又做反向代理。Nginx默认正则优先。
upstream mysrv {
ip_hash;
server 192.168.110.62;
server 192.168.110.63;
}
upstream myjpg {
server 192.168.110.3:88;
}
server {
listen 80;
server_name www.bee.1.com;
proxy_redirect off; # http 302
location ~ ^/StockInfo {
proxy_pass http://myjpg;
}
}
