系统环境：
SUSE Linux Enterprise Server 10 SP1 (x86_64)
注：所有软件包都放置在/data/software目录下
nginx_tcp_proxy_module：
https://github.com/yaoweibin/nginx_tcp_proxy_module
nginx-hmux-module：
https://github.com/wangbin579/nginx-hmux-module
ngx_cache_purge：
http://labs.frickle.com/files/
#---------------------------------------------------------------------------------------------------------------------------------------------
1、内存管理库
# tar -zxvf libunwind-1.0.1.tar.gz
# cd libunwind-1.0.1
# LAGS=-fPIC ./configure --prefix=/usr/local
# make LAGS=-fPIC
# make LAGS=-fPIC install
# tar -zxvf gperftools-2.0.tar.gz
# cd gperftools-2.0
# ./configure --prefix=/usr/local
# make && make install
#---------------------------------------------------------------------------------------------------------------------------------------------
2、正则库
# tar -xvzf pcre-8.32.tar.gz
# cd pcre-8.32
# LAGS=-fPIC ./configure --prefix=/usr/local
# make LAGS=-fPIC
# make LAGS=-fPIC install
#---------------------------------------------------------------------------------------------------------------------------------------------
3、OpenSSL库
# tar xvzf openssl-1.0.1g.tar.gz
# cd openssl-1.0.1g
# ./config shared --prefix=/usr/local
# make && make install
#---------------------------------------------------------------------------------------------------------------------------------------------
4、IP地理位置定位组件
# tar xvzf GeoIP-latest.tar.gz
# cd GeoIP-1.5.0
# ./configure --prefix=/usr/local
# make && make install
#---------------------------------------------------------------------------------------------------------------------------------------------
5、相关目录创建
# mkdir -p /data/nginx_temp/{nginx_client,nginx_proxy,nginx_fastcgi,nginx_temp,nginx_cache}
# mkdir -p /data/logs/{nginx,web} /data/web/{data,conf}
#---------------------------------------------------------------------------------------------------------------------------------------------
6、Tengine编译安装
# tar xvzf nginx-hmux-module-1.3.tar.gz
# tar xvzf nginx_tcp_proxy_module-0.4.5.tar.gz
# tar xvzf tengine-1.5.2.tar.gz
# cd tengine-1.5.2
# patch -p1 < ../nginx_tcp_proxy_module-0.4.5/tcp.patch
# ./configure --prefix=/usr/local/nginx \
--lock-path=/var/lock/nginx.lock \
--pid-path=/var/run/nginx.pid \
--error-log-path=/data/logs/nginx/error.log \
--http-log-path=/data/logs/nginx/access.log \
--user=nobody \
--group=nogroup \
--with-pcre=../pcre-8.32 \
--with-pcre-opt=-fPIC \
--with-openssl=../openssl-1.0.1g \
--with-openssl-opt=-fPIC \
--with-backtrace_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--with-http_realip_module \
--with-http_concat_module=shared \
--with-http_sysguard_module=shared \
--with-http_limit_conn_module=shared \
--with-http_limit_req_module=shared \
--with-http_split_clients_module=shared \
--with-http_footer_filter_module=shared \
--with-http_geoip_module=shared \
--with-http_sub_module=shared \
--with-http_access_module=shared \
--with-http_upstream_ip_hash_module=shared \
--with-http_upstream_least_conn_module=shared \
--with-http_referer_module=shared \
--with-http_rewrite_module=shared \
--with-http_memcached_module=shared \
--with-http_upstream_session_sticky_module=shared \
--with-http_addition_module=shared \
--with-http_xslt_module=shared \
--with-http_p_w_picpath_filter_module=shared \
--with-http_user_agent_module=shared \
--with-http_empty_gif_module=shared \
--with-http_browser_module=shared \
--with-google_perftools_module \
--with-http_map_module=shared \
--with-http_userid_filter_module=shared \
--with-http_charset_filter_module=shared \
--with-http_trim_filter_module=shared \
--with-http_lua_module=shared \
--without-http_fastcgi_module \
--without-http_uwsgi_module \
--without-http_scgi_module \
--without-select_module \
--without-poll_module \
--add-module=../nginx-hmux-module-1.3 \
--add-module=../nginx_tcp_proxy_module-0.4.5 \
--with-ld-opt='-ltcmalloc_minimal' \
--http-client-body-temp-path=/data/nginx_temp/nginx_client \
--http-proxy-temp-path=/data/nginx_temp/nginx_proxy \
--http-fastcgi-temp-path=/data/nginx_temp/nginx_fastcgi
# make && make install
#---------------------------------------------------------------------------------------------------------------------------------------------
7、Tengine缓存刷新模块
# cd /data/software
# tar xvzf ngx_cache_purge-2.0.tar.gz
# ./dso_tool --add-module=/data/software/ngx_cache_purge-2.0
#---------------------------------------------------------------------------------------------------------------------------------------------
8、Tengine配置
# rm -f /usr/local/nginx/html/*.html
# rm -f /usr/local/nginx/conf/*.default
# mkdir /usr/local/nginx/conf/SET
# vim /usr/local/nginx/conf/nginx.conf
user nobody nogroup;
worker_processes auto;
worker_cpu_affinity auto;
error_log /data/logs/nginx/error.log crit;
pid /var/run/nginx.pid;
google_perftools_profiles /var/tmp/tcmalloc;
worker_rlimit_nofile 65535;
dso {
load ngx_http_rewrite_module.so;
load ngx_http_access_module.so;
load ngx_http_concat_module.so;
load ngx_http_limit_conn_module.so;
load ngx_http_limit_req_module.so;
load ngx_http_sysguard_module.so;
load ngx_http_upstream_session_sticky_module.so;
load ngx_http_cache_purge_module.so;
load ngx_http_trim_filter_module.so;
}
events {
use epoll;
worker_connections 10240;
}
http {
server_tokens off;
server_tag off;
autoindex off;
access_log off;
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 10m;
client_body_buffer_size 256k;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
proxy_connect_timeout 600;
proxy_read_timeout 600;
proxy_send_timeout 600;
proxy_buffer_size 128k;
proxy_buffers 4 128k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
proxy_headers_hash_max_size 1024;
proxy_headers_hash_bucket_size 128;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_temp_path /data/nginx_temp/nginx_temp;
proxy_cache_path /data/nginx_temp/nginx_cache levels=1:2 keys_zone=cache_one:2048m inactive=30m max_size=60g;
# backend web server address pool
include SET/*.conf;
log_format access '$remote_addr - $remote_user [$time_local] "$request"'
'$status $body_bytes_sent "$http_referer"'
'"$http_user_agent" $http_x_forwarded_for';
# system resource overload protect
server {
sysguard on;
sysguard_load load=10.5 action=/loadlimit;
sysguard_mem swapratio=20% action=/swaplimit;
sysguard_mem free=100M action=/freelimit;
location /loadlimit {
return 503;
}
location /swaplimit {
return 503;
}
location /freelimit {
return 503;
}
}
# refuse request server by ipaddr
server {
server_name _;
return 404;
}
# web page cache and proxy setting
include /data/web/conf/*.conf;
}# vim /usr/local/nginx/conf/SET/NORTH1.conf
upstream NORTH1_SERVER_PROXY {
consistent_hash $request_uri;
server 192.168.1.101:80 weight=1;
server 192.168.1.102:80 weight=1;
server 192.168.1.103:80 weight=1;
server 192.168.1.104:80 weight=1;
session_sticky;
check interval=3000 rise=2 fall=5 timeout=1000 type=http;
check_http_send "GET / HTTP/1.0\r\n\r\n";
check_http_expect_alive http_2xx http_3xx;
}# mkdir -p /data/logs/web/test.qq.com
# vim /data/web/conf/test.qq.com.conf
server {
listen 80;
server_name test.qq.com;
index index.html index.htm index.php;
root /data/nginx_temp/nginx_cache;
access_log on;
trim on;
trim_jscss on;
location / {
proxy_next_upstream http_500 http_502 http_503 http_504 error timeout invalid_header;
proxy_pass http://NORTH1_SERVER_PROXY;
#存在静态首页时，才需添加此规则
if (-d $request_filename) {
rewrite ^/(.*)$ http://$host/index.html break;
}
}
location ~ .*\. (php)?$ {
proxy_next_upstream http_500 http_502 http_503 http_504 error timeout invalid_header;
proxy_pass http://NORTH1_SERVER_PROXY;
}
location ~ /purge(/.*) {
allow 127.0.0.1;
allow 192.168.1.0/24;
deny all;
proxy_cache_purge cache_one $host$1$is_args$args;
}
location ~ .*\.(htm|html|js|css|gif|jpg|jpeg|png|bmp|ico|swf|flv)$ {
proxy_next_upstream http_500 http_502 http_503 http_504 error timeout invalid_header;
proxy_cache cache_one;
proxy_cache_valid 200 304 15m;
proxy_cache_valid 301 302 10m;
proxy_cache_valid any 1m;
proxy_cache_key $host$uri$is_args$args;
add_header Ten-webcache '$upstream_cache_status from $host';
proxy_pass http://NORTH1_SERVER_PROXY;
expires 30m;
}
location ~ /\.ht {
deny all;
}
access_log /data/logs/web/test.qq.com/access.log access;
}#---------------------------------------------------------------------------------------------------------------------------------------------
9、Tengine启动脚本
# vim /etc/init.d/nginx
#!/bin/sh
#
# nginx - this script start and stop the nginx daemon
#
# chkconfig: 2345 55 25
# description: Startup script for nginx
# processname: nginx
# config: /usr/local/nginx/conf/nginx.conf
# pidfile: /var/run/nginx.pid
#
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
DAEMON=/usr/local/nginx/sbin/nginx
CONFIGFILE=/usr/local/nginx/conf/nginx.conf
PIDFILE=/var/run/nginx.pid
SCRIPTNAME=/etc/init.d/nginx
LOCKFILE=/var/lock/nginx.lock
set -e
[ -x "$DAEMON" ] || exit 0
start() {
echo "Startting Nginx......"
[ -x $DAEMON ] || exit 5
[ -f $CONFIGFILE ] || exit 6
$DAEMON -c $CONFIGFILE || echo -n "Nginx already running!"
[ $? -eq 0 ] && touch $LOCKFILE
}
stop() {
echo "Stopping Nginx......"
MPID=`ps aux | grep nginx | awk '/master/{print $2}'`
if [ "${MPID}X" != "X" ]; then
kill -QUIT $MPID
[ $? -eq 0 ] && rm -f $LOCKFILE
else
echo "Nginx server is not running!"
fi
}
reload() {
echo "Reloading Nginx......"
MPID=`ps aux | grep nginx | awk '/master/{print $2}'`
if [ "${MPID}X" != "X" ]; then
kill -HUP $MPID
else
echo "Nginx can't reload!"
fi
}
case "$1" in
start)
start
;;
stop)
stop
;;
reload)
reload
;;
restart)
stop
sleep 1
start
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|reload|restart}"
exit 3
;;
esac
exit 0# chmod +x /etc/init.d/nginx
# chkconfig --add nginx
# service nginx start
#---------------------------------------------------------------------------------------------------------------------------------------------
10、Tengine健康检测
# mkdir -p /data/web/data/mycheckweb.act.qq.com
# echo "OK" > /data/web/data/mycheckweb.act.qq.com/index.html
# echo "你的内网IP mycheckweb.act.qq.com" >> /etc/hosts
# touch /var/lock/check_web.lock
#vim /data/web/conf/checkweb_for_nginx.conf
server {
listen 80;
server_name mycheckweb.act.qq.com;
access_log off;
location / {
root /data/web/data/mycheckweb.act.qq.com;
index index.html;
}
location ~ health_status {
check_status;
allow 127.0.0.1;
allow 192.168.1.0/24;
deny all;
}
}# vim /usr/local/nginx/sbin/check_web.sh
#!/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin
retval=`ping -c 3 mycheckweb.act.qq.com | awk '/received/ {print $4}'`
[[ ${retval} -eq 0 ]] && exit 1
retval=`curl -I -s "http://mycheckweb.act.qq.com" | grep "200 OK"`
if [[ "${retval}x" = "x" ]]; then
[[ -e /usr/local/nginx ]] && /sbin/service nginx restart >/dev/null 2>&1
fi#chmod +x /usr/local/nginx/sbin/check_web.sh
# crontab -e
*/5 * * * * (flock --timeout=0 /var/lock/check_web.lock /usr/local/nginx/sbin/check_web.sh >/dev/null 2>&1)
#---------------------------------------------------------------------------------------------------------------------------------------------
11、Tengine访问日志切割与清理
# vim /usr/local/nginx/sbin/cut_nginx_log.sh
#!/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin
## the nginx access logs base path
WEBLOG_PATH="/data/logs/web"
retval=`ps aux | grep ngin[x] | wc -l`
if [ ${retval} -eq 0 ]; then
echo "The daemon process for nginx has no found."
exit 1
fi
## avoid errors for USR1 signal, and modify 750 privilege
chown -R nobody:nogroup /data/logs/{nginx,web}
chmod -R 750 /data/logs/{nginx,web}
## cut nginx access logs
for LOGFILE in `find ${WEBLOG_PATH} -type f -name access.log`
do
LOGPATH=`dirname ${LOGFILE}`
mv ${LOGPATH}/access.log ${LOGPATH}/access_$(date -d "yesterday" +"%Y-%m-%d").log
done
kill -USR1 `ps aux | grep nginx | awk '/master/{print $2}'`
## and then modify original privileges
chown -R nobody:nogroup /data/logs/{nginx,web}
chmod -R 640 /data/logs/{nginx,web}
## clear 10 days ago's nginx access logs
LOGFILE=access_$(date -d "10 days ago" +"%Y-%m-%d").log
find ${WEBLOG_PATH} -type f -name ${LOGFILE} -exec rm -f {} \;# crontab -e
00 00 * * * /bin/bash /usr/local/nginx/sbin/cut_nginx_log.sh >/dev/null 2>&1
#---------------------------------------------------------------------------------------------------------------------------------------------
12、系统优化
##网络参数设置
# vim /etc/sysctl.conf
net.ipv4.tcp_syncookies = 1 net.ipv4.tcp_tw_reuse = 1 net.ipv4.tcp_tw_recycle = 1 net.ipv4.tcp_fin_timeout = 30 net.ipv4.tcp_keepalive_time = 1200 net.ipv4.ip_local_port_range = 1024 65000 net.ipv4.tcp_max_syn_backlog = 8192 net.ipv4.tcp_max_tw_buckets = 80000 net.core.somaxconn = 32768 net.ipv4.tcp_keepalive_probes = 5 net.ipv4.tcp_keepalive_intvl = 20 net.core.wmem_default = 8388608 net.core.rmem_default = 8388608 net.core.rmem_max = 16777216 net.core.wmem_max = 16777216 net.ipv4.tcp_rmem = 4096 87380 16777216 net.ipv4.tcp_wmem = 4096 65536 16777216 net.core.netdev_max_backlog = 32768 net.ipv4.tcp_timestamps = 0 net.ipv4.tcp_synack_retries = 2 net.ipv4.tcp_syn_retries = 2 net.ipv4.tcp_retries2 = 5 net.ipv4.tcp_mem = 41943040 73400320 94371840 net.ipv4.tcp_max_orphans = 3276800 fs.file-max = 1300000
# sysctl -p
## 文件描述符设置
# echo "ulimit -SHn 65535" >> /etc/profile
# source /etc/profile
#---------------------------------------------------------------------------------------------------------------------------------------------
13、测试
本地HOSTS绑定访问
http://mycheckweb.act.qq.com/health_status
