一、yum安装consul
#安装yum-utils
yum install -y yum-utils
#配置consul的下载仓库
yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
#必须上面步骤,不然会找不到仓库
yum -y install consul
#查看版本
consul -v
二、启动consul(单节点)
#创建consul目录
mkdir /home/consul-data
#后台启动
nohup consul agent -server -data-dir=/home/consul-data/ -node=node1 -bind=0.0.0.0 -bootstrap-expect=1 -client=0.0.0.0 -ui > /home/consul-data/consul.log 2>&1 &
#关闭防火墙或开放端口8500
systemctl stop firewalld
- –server 指定server agent
- –data-dir l数据存储路径
- –bootstrap-expect 期望的server节点数目,consul一直等到指定sever数目才会引导整个集群
- –bind:该地址用来在集群内部的通讯,集群内的所有节点到地址都必须是可达的,默认是0.0.0.0
- –node:节点在集群中的名称,在一个集群中必须是唯一的,默认是该节点的主机名
- –ui: web的管理ui,查看服务和节点,可以通过访问:8500端口要访问UI
- –client:提供HTTP、DNS、RPC等服务,默认是127.0.0.1,不对外提供服务,如果需要则改成0.0.0.0
单节点扩容(需要扩容时再做)
#扩容192.168.100.30
nohup consul agent -bind=0.0.0.0 -client=0.0.0.0 -data-dir=/home/consul-data/ -node=node2 -join=192.168.100.30 > /home/consul-data/consul.log 2>&1 &
三、错误处理
1、Multiple private IPv4 addresses found. Please configure one with 'bind' and/or 'advertise'.
出现这个问题的原因是,有多张网卡,所以就有了多个ip4的地址,解决方案,就是绑定ip地址到局域网卡,
加上 -bind 192.168.xxx.xxx 参数即可
2. data_dir cannot be empty
consul agent -server运行的时候需要指定 data_dir格式如下
consul agent -server -bind 192.168.100.9 -client 0.0.0.0 -ui -data-dir=/home/consul-data/
3.consul ui界面只能在本机访问,不能在其他电脑访问的解决办法,启动方法如下
consul agent -dev -client 0.0.0.0 -ui
四、访问consul
或使用命令访问
curl localhost:8500/v1/catalog/nodes
返回内容
[{"ID":"3f9f219a-f15e-82a9-0a23-889f324b1fe9","Node":"node1","Address":"192.168.100.7","Datacenter":"dc1","TaggedAddresses":{"lan":"192.168.100.7","lan_ipv4":"192.168.100.7","wan":"192.168.100.7","wan_ipv4":"192.168.100.7"},"Meta":{"consul-network-segment":""},"CreateIndex":12,"ModifyIndex":17}]
四、关闭consul
#关闭命令
consul leave
#后台启动使用这个
ps -ef | grep consul
kill -9 PID(Consul进程ID)
五、使用PostMan 注册Http服务
//地址 使用put请求
http://192.168.100.7:8500/v1/catalog/register
//发送json格式数据
//参数1
{"Datacenter": "dc1",
"Node":"skus1",
"Address":"192.168.100.37",
"Service": {
"Id" :"skuprice-skus1",
"Service": "skuprice",
"tags": ["prod"],
"Port": 80
}}
有几个注册几个
六、发现Http服务
http://192.168.100.7:8500/v1/catalog/service/skuprice
[{"ID":"","Node":"skus1","Address":"192.168.100.37","Datacenter":"dc1","TaggedAddresses":null,"NodeMeta":null,"ServiceKind":"","ServiceID":"skuprice-skus1","ServiceName":"skuprice","ServiceTags":["prod"],"ServiceAddress":"","ServiceWeights":{"Passing":1,"Warning":1},"ServiceMeta":{},"ServicePort":80,"ServiceSocketPath":"","ServiceEnableTagOverride":false,"ServiceProxy":{"Mode":"","MeshGateway":{},"Expose":{}},"ServiceConnect":{},"CreateIndex":96,"ModifyIndex":310},{"ID":"","Node":"skus2","Address":"192.168.100.39","Datacenter":"dc1","TaggedAddresses":null,"NodeMeta":null,"ServiceKind":"","ServiceID":"skuprice-skus2","ServiceName":"skuprice","ServiceTags":["prod"],"ServiceAddress":"","ServiceWeights":{"Passing":1,"Warning":1},"ServiceMeta":{},"ServicePort":80,"ServiceSocketPath":"","ServiceEnableTagOverride":false,"ServiceProxy":{"Mode":"","MeshGateway":{},"Expose":{}},"ServiceConnect":{},"CreateIndex":101,"ModifyIndex":308}]
七、删除http服务
http://
192.168.100.7:8500/v1/agent/service/deregister/{ID}
八、nginx安装-使用编译安装需要把upsync模块编译进去
wget -c https://github.com/weibocom/nginx-upsync-module/archive/refs/heads/master.zip
unzip nginx-upsync-module-master.zip
wget -c https://github.com/weibocom/nginx-upsync-module/archive/refs/tags/v2.1.3.tar.gz
tar -zxvf v2.1.3.tar.gz
mv nginx-upsync-module-2.1.3 nginx-upsync-module
#编译需要增加upsync模块
--add-module=/root/nginx-upsync-module
nginx -V 查看已安装模块
upsync参考文档:
https://github.com/weibocom/nginx-upsync-module
九、实现nginx动态负载均衡配置
user www www;
worker_processes auto;
error_log /www/wwwlogs/nginx_error.log crit;
pid /www/server/nginx/logs/nginx.pid;
worker_rlimit_nofile 51200;
stream {
log_format tcp_format '$time_local|$remote_addr|$protocol|$status|$bytes_sent|$bytes_received|$session_time|$upstream_addr|$upstream_bytes_sent|$upstream_bytes_received|$upstream_connect_time';
access_log /www/wwwlogs/tcp-access.log tcp_format;
error_log /www/wwwlogs/tcp-error.log;
include /www/server/panel/vhost/nginx/tcp/*.conf;
}
events
{
use epoll;
worker_connections 51200;
multi_accept on;
}
http
{
include mime.types;
#include luawaf.conf;
include proxy.conf;
default_type application/octet-stream;
server_names_hash_bucket_size 512;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 50m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 256k;
fastcgi_intercept_errors on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.1;
gzip_comp_level 2;
gzip_types text/plain application/javascript application/x-javascript text/javascript text/css application/xml;
gzip_vary on;
gzip_proxied expired no-cache no-store private auth;
gzip_disable "MSIE [1-6]\.";
limit_conn_zone $binary_remote_addr zone=perip:10m;
limit_conn_zone $server_name zone=perserver:10m;
server_tokens off;
access_log off;
upstream api.sku.xxx.cn {
ip_hash;
server 192.168.100.3:80;
#server 192.168.100.27:80;
#server 192.168.100.29:80;
upsync 192.168.100.7:8500/v1/kv/upstreams/skuprice upsync_timeout=6m upsync_interval=500ms upsync_type=consul strong_dependency=off;
#动态的拉去ConsulServer 相关负载均衡信息持久化到硬盘上
upsync_dump_path /www/server/nginx/conf/vhost/servers_skuprice.conf;
include /www/server/nginx/conf/vhost/servers_skuprice.conf;
}
server
{
listen 80;
server_name api.sku.xxx.cn;
location / {
proxy_pass http://api.sku.xxx.cn;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# 显示具体负载的机器的ip,X-Route-Ip随便命名
add_header X-Route-Ip $upstream_addr;
add_header X-Route-Status $upstream_status;
# 文件不能下载 设置缓存
proxy_buffering on;
proxy_buffer_size 512k;
proxy_buffers 2 512k;
proxy_busy_buffers_size 512k;
proxy_temp_path /www/server/nginx/proxy_temp 1 2;
proxy_max_temp_file_size 100M;
proxy_temp_file_write_size 512k;
# 文件不能下载结束
#nginx跟后端服务器连接超时时间(代理连接超时)默认60s
proxy_connect_timeout 1800;
#后端服务器数据回传时间(代理发送超时)默认值60s
proxy_read_timeout 1800;
#连接成功后,后端服务器响应时间(代理接收超时)默认值60s
proxy_send_timeout 1800;
}
}
include /www/server/panel/vhost/nginx/*.conf;
}
改动只有两处
#实现动态负载均衡
upstream api.sku.xxx.cn {
ip_hash;
server 192.168.100.3:80;
#server 192.168.100.27:80;
#server 192.168.100.29:80;
upsync 192.168.100.7:8500/v1/kv/upstreams/skuprice upsync_timeout=6m upsync_interval=500ms upsync_type=consul strong_dependency=off;
upsync_dump_path /www/server/nginx/conf/vhost/servers_skuprice.conf;
#include /www/server/nginx/conf/vhost/servers_skuprice.conf;
}
server
{
listen 80;
server_name api.sku.xxx.cn;
location / {
proxy_pass http://api.sku.xxx.cn;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# 显示具体负载的机器的ip,X-Route-Ip随便命名
add_header X-Route-Ip $upstream_addr;
add_header X-Route-Status $upstream_status;
# 文件不能下载 设置缓存
proxy_buffering on;
proxy_buffer_size 512k;
proxy_buffers 2 512k;
proxy_busy_buffers_size 512k;
proxy_temp_path /www/server/nginx/proxy_temp 1 2;
proxy_max_temp_file_size 100M;
proxy_temp_file_write_size 512k;
# 文件不能下载结束
}
}
重启生效
十、负载均衡测试
# 使用curl 请求,一定要是put请求
curl -X PUT http://192.168.100.7:8500/v1/kv/upstreams/skuprice/192.168.100.27:80
curl -X PUT http://192.168.100.7:8500/v1/kv/upstreams/skuprice/192.168.100.29:80
curl -X PUT http://192.168.100.7:8500/v1/kv/upstreams/skuprice/192.168.100.3:80
#或者使用postman调用consul提供的api来添加key,value
http://192.168.254.134:8500/v1/kv/upstreams/springbootserver/192.168.0.116:8081
# 甚至可以图形化界面进行操作,手动添加key value,手动添加时注意如果是创建文件夹需要在最后加一个正斜杠 :/
十一、在consul图形中添加key/value,设置权重
{"weight":1, "max_fails":2, "fail_timeout":10, "down":0}
在linux下实现Nginx + consul + upsync 完成动态负载均衡已完成
遇到问题:
1.session问题
后来发现上传和下载在不同的机子上,轮询会出现异常不存在情况,后来配置了nginx哈希解决
2.图片上传问题
负载均衡后上传图片只存放于服务器A或B上,用户打开后会发现图片偶尔会没有显示出来现象
解决办法:上传图片目录设置为共享目录
搭建linux下nfs文件共享服务器
这里我选择了一台新机192.168.100.24做为图片共享存储
搭建步骤
1、安装nfs所需软件包
yum install rpcbind nfs-utils
2、创建测试文件并赋权限
mkdir /home/storage
chown www.www -R /home/storage
chmod -R 777 /home/storage
cd /home/storage
#创建一个文件测试,我这里只直把原来图片都上传到这个目录下了
3、修改配置文件
vim /etc/exports
/home/data 192.168.100.0/24(rw,sync,no_root_squash)
/home/storage 192.168.100.0/24(rw,sync,no_root_squash)
#共享文件路径 允许共享网段(共享文件可执行权限)
共享文件可执行权限有:
ro 只读访问
rw 读写访问
sync 所有数据在请求时写入共享
hide 在NFS共享目录中不共享其子目录
no_hide 共享NFS目录的子目录
all_squash 共享文件的UID和GID映射匿名用户anonymous,适合公用目录。
no_all_squash 保留共享文件的UID和GID(默认)
root_squash root用户的所有请求映射成如anonymous用户一样的权限(默认)
no_root_squas root用户具有根目录的完全管理访问权限
4、开启服务
systemctl start rpcbind nfs
#设置开机自启
echo "192.168.100.24:/home/storage /nfs nfs4 defaults 0 0" >> /etc/fstab
echo "192.168.100.24:/home/data /nfs2 nfs4 defaults 0 0" >> /etc/fstab
echo "192.168.100.24:/home/xiaochengxu /nfs3 nfs4 defaults 0 0" >> /etc/fstab
mount -av
#注意 上面的/nfs 是根目录下文件夹 目录是没有的 需要创建挂载点 每个名称不能一样
5、查看共享信息
#在100.3、100.27、100.29负载均衡机子安装showmount
yum -y install showmount
showmount -e 192.168.100.24 (此处ip地址为搭建服务器主机地址,在需要引用的机子上运行查看)
6、挂载并访问nfs服务器
#挂载共享目录
mount 192.168.100.24:/home/storage/ /www/wwwroot/api.sku.xxx.cn/public/storage/
mount 192.168.100.24:/home/xiaochengxu/ /www/wwwroot/api.sku.xxx.cn/public/xiaochengxu/
#查看
df
报错问题
chown: changing ownership of ‘20230315/2023031509155343618.jpg’: Operation not permitted
检查nfs设置权限
在nfs上添加no_root_squash权限,完成后进行重启
systemctl restart rpcbind
systemctl restart nfs
iptables开端口
扩展:umount使用
#!/bin/bash
cd /www/wwwroot/
if [ ! -d "/www/wwwroot/skuprice" ]; then
rm -rf skuprice
fi
git clone git@192.168.100.201:gitlab-instance-428d607212/skuprice.git
#cp -a api.sku.xxx.cn/public/storage ./skuprice/public/
umount /www/wwwroot/api.sku.xxx.cn/public/storage/
rm -rf api.sku.xxx.cn
if [ ! -d "/www/wwwroot/api.sku.xxx.cn" ]; then
mv ./skuprice ./api.sku.xxx.cn
else
cp -a ./skuprice/* ./api.sku.xxx.cn/
rm -rf skuprice
fi
cd /www/wwwroot/api.sku.xxx.cn
php init prod
php think migrate:run
chown -R www.www /www/wwwroot/api.sku.xxx.cn
cd /www/wwwroot/api.sku.xxx.cn
php think export fresh
nohup php think export excel >/dev/null 2>&1 &
rm -rf /www/wwwroot/api.sku.xxx.cn/public/storage/*
mount 192.168.100.24:/home/storage/ /www/wwwroot/api.sku.xxx.cn/public/storage/
echo '发布完成'
小程序shell部署程序
#!/bin/bash
cd /www/wwwroot/
if [ ! -d "/www/wwwroot/xiaochengxu" ]; then
rm -rf xiaochengxu
fi
git clone git@192.168.100.201:gitlab-instance-428d60722/xiaochengxu.git
umount /www/wwwroot/www.xxx.com.cn/upload/
rm -rf www.xxx.com.cn
if [ ! -d "/www/wwwroot/www.xxx.com.cn" ]; then
mv ./xiaochengxu ./www.xxx.com.cn
else
cp -a ./xiaochengxu/* ./www.xxx.com.cn/
rm -rf xiaochengxu
fi
cd /www/wwwroot/www.xxx.com.cn
php init prod
cd /www/wwwroot/www.xxx.com.cn
rm -rf /www/wwwroot/www.xxx.com.cn/upload/*
mount 192.168.100.11:/home/storage/ /www/wwwroot/www.xxx.com.cn/upload/
chmod 777 -R /www/wwwroot/www.xxx.com.cn/upload/
chown www.www -R /www/wwwroot/www.xxx.com.cn
/www/server/panel/pyenv/bin/supervisorctl -c /etc/supervisor/supervisord.conf restart all
echo '发布完成'
原始nginx配置
user www www;
worker_processes auto;
error_log /www/wwwlogs/nginx_error.log crit;
pid /www/server/nginx/logs/nginx.pid;
worker_rlimit_nofile 51200;
stream {
log_format tcp_format '$time_local|$remote_addr|$protocol|$status|$bytes_sent|$bytes_received|$session_time|$upstream_addr|$upstream_bytes_sent|$upstream_bytes_received|$upstream_connect_time';
access_log /www/wwwlogs/tcp-access.log tcp_format;
error_log /www/wwwlogs/tcp-error.log;
include /www/server/panel/vhost/nginx/tcp/*.conf;
}
events
{
use epoll;
worker_connections 51200;
multi_accept on;
}
http
{
include mime.types;
#include luawaf.conf;
include proxy.conf;
lua_package_path "/www/server/nginx/lib/lua/?.lua;;";
default_type application/octet-stream;
server_names_hash_bucket_size 512;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 50m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 256k;
fastcgi_intercept_errors on;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.1;
gzip_comp_level 2;
gzip_types text/plain application/javascript application/x-javascript text/javascript text/css application/xml;
gzip_vary on;
gzip_proxied expired no-cache no-store private auth;
gzip_disable "MSIE [1-6]\.";
limit_conn_zone $binary_remote_addr zone=perip:10m;
limit_conn_zone $server_name zone=perserver:10m;
server_tokens off;
access_log off;
server
{
listen 888;
server_name phpmyadmin;
index index.html index.htm index.php;
root /www/server/phpmyadmin;
#error_page 404 /404.html;
include enable-php.conf;
location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)$
{
expires 30d;
}
location ~ .*\.(js|css)?$
{
expires 12h;
}
location ~ /\.
{
deny all;
}
access_log /www/wwwlogs/access.log;
}
include /www/server/panel/vhost/nginx/*.conf;
}