k8s维护长连接负载均衡 k8s实现负载均衡

转载

doscommand 2024-04-01 22:15:04

文章标签 k8s维护长连接负载均衡 nginx vim 配置文件 文章分类 运维

前言：

接上一篇多节点部署（2）部署负载均衡

部署环境

负载均衡
Nginx1:192.168.13.128/24
Nginx2:192.168.13.129/24
Master节点
master1:192.168.13.131/24 kube-apiserver kube-controller-manager kube-scheduler etcd
master2:192.168.13.130/24 kube-apiserver kube-controller-manager kube-scheduler etcd
Node节点
node1:192.168.13.132/24 kubelet kube-proxy docker flannel etcd
node2:192.168.13.133/24 kubelet kube-proxy docker flannel etcd

负载均衡部署

1，在两台Nginx服务器上安装nginx服务

[root@nginx01 ~]# rz -E    ##上传nginx脚本和keepalive配置文件
[root@nginx01 ~]# ls      
keepalived.conf   nginx.sh 
[root@nginx01 ~]# systemctl stop firewalld.service   ##关闭防火墙
[root@nginx01 ~]# setenforce 0

vim nginx.sh   ##nginx脚本
cat > /etc/yum.repos.d/nginx.repo << EOF
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/
gpgcheck=0
EOF


stream {

	 log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
		access_log  /var/log/nginx/k8s-access.log  main;

		upstream k8s-apiserver {
				server 10.0.0.3:6443;
				server 10.0.0.8:6443;
		}
		server {
								listen 6443;
								proxy_pass k8s-apiserver;
		}
		}

[root@nginx01 ~]# vim /etc/yum.repos.d/nginx.repo   ##配置nginx的yum源
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/
gpgcheck=0
[root@nginx01 ~]# yum list   ##更新yum
[root@nginx01 ~]# yum install -y nginx  ##下载Nginx

2，修改两台nginx服务器的配置文件

[root@nginx01 ~]# vim /etc/nginx/nginx.conf
events {
		worker_connections  1024;
}     ##在此处下面添加四层转发配置

stream {

	 log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent'; 
		access_log  /var/log/nginx/k8s-access.log  main;

		upstream k8s-apiserver {
				server 192.168.13.131:6443;   ##master01地址
				server 192.168.13.130:6443;   ##master02地址
		}   
		server {
								listen 6443;
								proxy_pass k8s-apiserver;
		}           
		}
[root@nginx01 ~]# systemctl start nginx  ##开启nginx服务
##可以修改/usr/share/nginx/html/index.html主页区分主master从backup
##浏览器查看两个nginx网站

k8s维护长连接负载均衡 k8s实现负载均衡_配置文件

k8s维护长连接负载均衡 k8s实现负载均衡_k8s维护长连接负载均衡_02

3，在两台nginx上部署keepalived服务，并修改配置文件

[root@nginx01 ~]# yum install -y keepalived  ##安装keepalived服务
[root@nginx01 ~]# cp keepalived.conf /etc/keepalived/keepalived.conf  ##复制配置文件
[root@nginx01 ~]# vim /etc/keepalived/keepalived.conf  ##主master的配置文件修改
! Configuration File for keepalived 

global_defs { 
	 # 接收邮件地址 
	 notification_email { 
		 acassen@firewall.loc 
		 failover@firewall.loc 
		 sysadmin@firewall.loc 
	 } 
	 # 邮件发送地址 
	 notification_email_from Alexandre.Cassen@firewall.loc  
	 smtp_server 127.0.0.1 
	 smtp_connect_timeout 30 
	 router_id NGINX_MASTER 
} 

vrrp_script check_nginx {
		script "/etc/nginx/check_nginx.sh"     ##nginx检查脚本，需要自己去编辑的
}

vrrp_instance VI_1 { 
		state MASTER    ##主服务
		interface ens33
		virtual_router_id 51 ## VRRP 路由 ID实例，每个实例是唯一的 
		priority 100    ## 优先级，备服务器设置 90 
		advert_int 1    ## 指定VRRP 心跳包通告间隔时间，默认1秒 
		authentication { 
				auth_type PASS      ##验证不需要修改，主从一致
				auth_pass 1111 
		}  
		virtual_ipaddress { 
				192.168.13.100/24     ##虚拟ip地址
		} 
		track_script {
				check_nginx
		} 
}

[root@nginx02 ~]# vim /etc/keepalived/keepalived.conf  ##备backup的配置文件修改
! Configuration File for keepalived 

global_defs { 
	 # 接收邮件地址 
	 notification_email { 
		 acassen@firewall.loc 
		 failover@firewall.loc 
		 sysadmin@firewall.loc 
	 } 
	 # 邮件发送地址 
	 notification_email_from Alexandre.Cassen@firewall.loc  
	 smtp_server 127.0.0.1 
	 smtp_connect_timeout 30 
	 router_id NGINX_MASTER 
} 

vrrp_script check_nginx {
		script "/etc/nginx/check_nginx.sh"     ##nginx检查脚本
}

vrrp_instance VI_1 { 
		state BACKUP 
		interface ens33
		virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
		priority 90    # 优先级，备服务器设置 90 
		advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
		authentication { 
				auth_type PASS      
				auth_pass 1111 
		}  
		virtual_ipaddress { 
				192.168.13.100/24 
		} 
		track_script {
				check_nginx
		} 
}
[root@nginx01 ~]#  vim /etc/nginx/check_nginx.sh   ##编辑nginx检查脚本
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
		systemctl stop keepalived
fi
[root@nginx01 ~]# chmod +x /etc/nginx/check_nginx.sh   ##给执行权限
[root@nginx01 ~]# systemctl start keepalived.service   ##开启服务
[root@nginx01 ~]# ip a   ##查看地址信息
		inet 192.168.13.100/24 scope global secondary ens33   ##漂移地址在master上

4，验证地址漂移

##在nginx01中关闭nginx服务，此时keepalived服务也关闭了（check_nginx.sh）
[root@nginx01 ~]# pkill nginx
##在nginx02中查看漂移地址
[root@nginx02 ~]# ip a  ##此时13.100在nginx02上
##在nginx01上恢复nginx和keepalived服务，查看漂移地址
[root@nginx01 ~]# systemctl start nginx
[root@nginx01 ~]# systemctl start keepalived.service 
[root@nginx01 ~]# ip a   ##此时漂移地址又到了nginx01上
##用浏览器访问虚拟ip

k8s维护长连接负载均衡 k8s实现负载均衡_配置文件_03

5，在所有的node节点服务器上统一配置VIP

[root@node01 ~]# vim /opt/kubernetes/cfg/bootstrap.kubeconfig 
server: https://192.168.13.100:6443
[root@node01 ~]# vim /opt/kubernetes/cfg/kubelet.kubeconfig
server: https://192.168.13.100:6443
[root@node01 ~]# vim /opt/kubernetes/cfg/kube-proxy.kubeconfig
server: https://192.168.13.100:6443
[root@node01 ~]# cd /opt/kubernetes/cfg/  ##切换到配置文件目录
[root@node01 cfg]# grep 100 *   ##查看修改的情况
bootstrap.kubeconfig:    server: https://192.168.13.100:6443
kubelet.kubeconfig:    server: https://192.168.13.100:6443
kube-proxy.kubeconfig:    server: https://192.168.13.100:6443
[root@node01 cfg]# systemctl restart kubelet.service   ##重启两个服务
[root@node01 cfg]# systemctl restart kube-proxy.service
##在nginx01上查看访问日志
[root@nginx01 ~]# tail /var/log/nginx/k8s-access.log
192.168.13.132 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0
192.168.13.132 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0
192.168.13.132 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0
192.168.13.133 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0
192.168.13.133 k8s-apiserver - [10/Feb/2020:13:17:11 +0800] 502 0

6，在matser01上创建pod

[root@master01 ~]# kubectl get pods   ##查看pod
No resources found.
[root@master01 ~]# kubectl run nginx --image=nginx   ##创建pod
[root@master01 ~]# kubectl get pods    ##查看pod状态是正在创建的状态
NAME                    READY   STATUS              RESTARTS   AGE
nginx-dbddb74b8-brjlj   0/1     ContainerCreating   0          86s
[root@master01 ~]# kubectl get pods   ##此时pod是已经运行的状态
NAME                    READY   STATUS    RESTARTS   AGE
nginx-dbddb74b8-brjlj   1/1     Running   0          87s
[root@master01 ~]# kubectl logs nginx-dbddb74b8-brjlj  ##此时日志文件不能查看
[root@master01 ~]# kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=system:anonymous
##提权后日志文件就可以查看了
[root@master01 ~]# kubectl get pods -o wide  ##查看pod网络，此时pod容器分配到node01上
NAME                    READY   STATUS    RESTARTS   AGE     IP            NODE             NOMINATED NODE
nginx-dbddb74b8-brjlj   1/1     Running   0          5m18s   172.17.45.2   192.168.13.132   <none>

7，在对应的网段node01上访问

[root@node01 cfg]# curl 172.17.45.2  ##此时就可以访问nginx信息
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
##用node01虚拟机的浏览器访问

k8s维护长连接负载均衡 k8s实现负载均衡_vim_04

8，此时在master01上查看日志文件

[root@master01 ~]# kubectl logs nginx-dbddb74b8-brjlj                             
172.17.45.1 - - [10/Feb/2020:05:29:23 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"

负载均衡部署完成，未完待续…

8，此时在master01上查看日志文件

[root@master01 ~]# kubectl logs nginx-dbddb74b8-brjlj                             
172.17.45.1 - - [10/Feb/2020:05:29:23 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"