一、kubemark介绍
背景介绍
kubemark 是 K8s 官方给出的性能测试工具，能够不受任何资源限制，模拟出一个大规模 K8s 集群。其主要架构如图所示:需要一个外部 K8s 集群（external cluster） 以及一个机器节点运行 kubemark master，即另外一个 K8s 集群，但是只有一个 master 节点。我们需要在 external cluster 中部署运行 hollow pod，这些 pod 会主动向 kubemark 集群注册，并成为 kubemark 集群中的 hollow node(虚拟节点)。然后我们就可以在 kubemark 集群中进行 e2e 测试。虽然与真实集群的稍微有点误差，不过可以代表真实集群的数据
二、kubemark构建
- 编译kubemark可执行文件
下载与环境一直的k8s源码包文件，安装go环境
#########查看目前环境为1.18.6版本
[root@node1 kubemark]# kubectl get node
NAME STATUS ROLES AGE VERSION
node1 Ready master 18h v1.18.6
node2 Ready master 18h v1.18.6
node3 Ready master 18h v1.18.6
########下载源码包以及安装go环境
[root@node1 kubemark]# ll | grep kubernetes
drwxr-xr-x 21 root root 4096 Mar 31 14:13 kubernetes-1.18.6
-rw-r--r-- 1 root root 454873856 Apr 12 16:29 kubernetes-1.18.6.tar.gz
########go版本
[root@node1 kubemark]# go version
go version go1.16.14 linux/amd64
[root@node1 kubemark]#
#######编译kubemark
cd /root/test/kubemark/kubernetes-1.18.6
make kubemark GOGCFLAGS="-N -l"
编译好的文件在_output/bin/目录下：
[root@node1 kubernetes-1.18.6]# cd _output/bin/
[root@node1 bin]# ll
total 282868
-rwxr-xr-x 1 root root 6217728 Mar 31 14:13 conversion-gen
-rwxr-xr-x 1 root root 5971968 Apr 12 17:08 deepcopy-gen
-rwxr-xr-x 1 root root 5955584 Mar 31 14:13 defaulter-gen
-rwxr-xr-x 1 root root 113429456 Apr 6 10:59 e2e.test
-rwxr-xr-x 1 root root 3554775 Apr 12 17:08 go2make
-rwxr-xr-x 1 root root 1966080 Mar 31 14:14 go-bindata
-rwxr-xr-x 1 root root 41275392 Mar 31 15:45 kubectl
-rwxr-xr-x 1 root root 101385080 Apr 12 17:08 kubemark
-rwxr-xr-x 1 root root 9895936 Mar 31 14:13 openapi-gen
########编译好的工具下载解压即用
链接：https://pan.baidu.com/s/1iWTaZRz4nUZJ6W7mzC5mjA
提取码：db37
--来自百度网盘超级会员V6的分享
########################- 构建kubemark镜像
将生成的 kubemark 二进制文件从 _output/bin 复制到 cluster/images/kubemark 目录下。
[root@node1 kubernetes-1.18.6]# cp _output/bin/kubemark cluster/images/kubemark/
[root@node1 kubemark]# ll
total 99028
-rw-r--r-- 1 root root 791 Jul 16 2020 BUILD
-rw-r--r-- 1 root root 877 Apr 12 17:35 Dockerfile
-rwxr-xr-x 1 root root 101385080 Apr 12 17:08 kubemark
-rw-r--r-- 1 root root 1268 Jul 16 2020 Makefile
-rw-r--r-- 1 root root 147 Jul 16 2020 OWNERS
####需要替换Dockerfile中使用的镜像，默认镜像不可用。
FROM centos:7
COPY kubemark /kubemark
#######运行命令进行编译。或者使用下面云盘链接中编译好的镜像。
IMAGE_TAG=v1.14.8 make build ###不指定tag默认为latest。
####修改tag上传至registry，方便其他节点使用。
链接：https://pan.baidu.com/s/1WI0V88sU4TTRtHfA9EmZjw
提取码：3zdw
--来自百度网盘超级会员V6的分享三、启动kubemark集群
- 配置基础环境
在执行节点执行以下命令创建ns、cm、secret
kubectl create ns kubemark
kubectl create configmap node-configmap -n kubemark --from-literal=content.type="test-cluster"
kubectl create secret generic kubeconfig --type=Opaque --namespace=kubemark --from-file=kubelet.kubeconfig=/root/.kube/config --from-file=kubeproxy.kubeconfig=/root/.kube/config- 修改maxPods值
修改配置文件vim /var/lib/kubelet/config.yaml
maxPods: 1000000
然后重启kubelet
systemctl restart kubelet- 启动hollow node节点
1、设置物理node标签
kubectl label node node1 name=hollow-node
kubectl label node node2 name=hollow-node
kubectl label node node3 name=hollow-node
###########或者设置为不可调度
kubectl taint node node1 key=value:NoSchedule
kubectl taint node node2 key=value:NoSchedule
kubectl taint node node3 key=value:NoSchedule
2、执行hollow-node.yaml文件创建虚拟节点
kubectl apply -f hollow-node.yaml
###yaml文件内容如下：
apiVersion: apps/v1
kind: Deployment
metadata:
name: hollow-node
namespace: kubemark
labels:
name: hollow-node
spec:
replicas: 1000 ###启动的虚拟节点的数量
selector:
matchLabels:
name: hollow-node
template:
metadata:
labels:
name: hollow-node
spec:
nodeSelector:
name: hollow-node
initContainers:
- name: init-inotify-limit
image: busybox
imagePullPolicy: IfNotPresent
command: ['sysctl', '-w', 'fs.inotify.max_user_instances=524288']
securityContext:
privileged: true
volumes:
- name: kubeconfig-volume
secret:
secretName: kubeconfig
containers:
- name: hollow-kubelet
image: 172.18.30.4:5000/kubemark:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 4194
- containerPort: 10250
- containerPort: 10255
env:
- name: CONTENT_TYPE
valueFrom:
configMapKeyRef:
name: node-configmap
key: content.type
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
command:
- /bin/sh
- -c
- /kubemark --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --alsologtostderr --v=2
volumeMounts:
- name: kubeconfig-volume
mountPath: /kubeconfig
readOnly: true
securityContext:
privileged: true
- name: hollow-proxy
image: 172.18.30.4:5000/kubemark:latest
imagePullPolicy: IfNotPresent
env:
- name: CONTENT_TYPE
valueFrom:
configMapKeyRef:
name: node-configmap
key: content.type
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
command:
- /bin/sh
- -c
- /kubemark --morph=proxy --name=$(NODE_NAME) --use-real-proxier=false --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --alsologtostderr --v=2
volumeMounts:
- name: kubeconfig-volume
mountPath: /kubeconfig
readOnly: true
tolerations:
- key: key
value: value
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution: # 硬策略
nodeSelectorTerms:
- matchExpressions:
- key: name
operator: In
values:
- hollow-node
#################################
#################################
#################################
如果集群是通过域名访问的，需要在hollow-node.yaml文件中添加如下配置：
spec:
hostAliases:
- ip: "10.233.0.1" ###如果是高可用，则填写集群的vip地址
hostnames:
- "vip.sanyi.com" ###集群域名
nodeSelector:
name: hollow-node
##############################
##############################
##############################
c查看已经注册好的hollow node节点，如下：
[root@node1 ~]# kubectl get pod -n kubemark -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
hollow-node-7cdbb7d69-6jqnv 2/2 Running 0 71s 172.25.135.58 node3 <none> <none>
hollow-node-7cdbb7d69-prhx5 2/2 Running 0 71s 172.25.104.44 node2 <none> <none>
hollow-node-7cdbb7d69-szn5m 2/2 Running 0 71s 172.25.104.43 node2 <none> <none>
[root@node1 ~]#
[root@node1 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
hollow-node-7cdbb7d69-6jqnv Ready <none> 66s v1.18.6
hollow-node-7cdbb7d69-prhx5 Ready <none> 66s v1.18.6
hollow-node-7cdbb7d69-szn5m Ready <none> 66s v1.18.6
node1 Ready master 19h v1.18.6
node2 Ready master 19h v1.18.6
node3 Ready master 19h v1.18.6
每个hollow node节点也会启动相应的k8s组件，如下：
[root@node1 ~]# kubectl get pod -A -o wide | grep hollow
kube-system calico-node-724sq 0/1 Init:0/3 0 94s 192.168.192.168 hollow-node-7cdbb7d69-6jqnv <none> <none>
kube-system calico-node-bzq7p 0/1 Init:0/3 0 94s 192.168.192.168 hollow-node-7cdbb7d69-szn5m <none> <none>
kube-system calico-node-d7255 0/1 Init:0/3 0 94s 192.168.192.168 hollow-node-7cdbb7d69-prhx5 <none> <none>
kube-system kube-proxy-bqph2 1/1 Running 0 94s 192.168.192.168 hollow-node-7cdbb7d69-6jqnv <none> <none>
kube-system kube-proxy-gtl5z 1/1 Running 0 94s 192.168.192.168 hollow-node-7cdbb7d69-prhx5 <none> <none>
kube-system kube-proxy-qvkcb 1/1 Running 0 94s 192.168.192.168 hollow-node-7cdbb7d69-szn5m <none> <none>
kubemark hollow-node-7cdbb7d69-6jqnv 2/2 Running 0 103s 172.25.135.58 node3 <none> <none>
kubemark hollow-node-7cdbb7d69-prhx5 2/2 Running 0 103s 172.25.104.44 node2 <none> <none>
kubemark hollow-node-7cdbb7d69-szn5m 2/2 Running 0 103s 172.25.104.43 node2 <none> <none>
kubesphere-logging-system fluent-bit-8lvg4 1/1 Running 0 94s 192.168.192.168 hollow-node-7cdbb7d69-6jqnv <none> <none>
kubesphere-logging-system fluent-bit-lvdw9 1/1 Running 0 94s 192.168.192.168 hollow-node-7cdbb7d69-prhx5 <none> <none>
kubesphere-logging-system fluent-bit-scgtq 1/1 Running 0 94s 192.168.192.168 hollow-node-7cdbb7d69-szn5m <none> <none>
kubesphere-monitoring-system node-exporter-8b9ts 2/2 Running 0 94s 192.168.192.168 hollow-node-7cdbb7d69-6jqnv <none> <none>
kubesphere-monitoring-system node-exporter-pmbst 2/2 Running 0 94s 192.168.192.168 hollow-node-7cdbb7d69-szn5m <none> <none>
kubesphere-monitoring-system node-exporter-tzjrg 2/2 Running 0 94s 192.168.192.168 hollow-node-7cdbb7d69-prhx5 <none> <none>
velero restic-7b8tm 1/1 Running 0 84s 192.168.192.168 hollow-node-7cdbb7d69-szn5m <none> <none>
velero restic-8rfvc 1/1 Running 0 84s 192.168.192.168 hollow-node-7cdbb7d69-6jqnv <none> <none>
velero restic-cm654 1/1 Running 0 84s 192.168.192.168 hollow-node-7cdbb7d69-prhx5 <none> <none>
[root@node1 ~]#- 在hollow节点启动pod测试
1、为hollow node节点添加标签，如下：
[root@node1 kubemark]# kubectl label node hollow-node-7cdbb7d69-6jqnv app=nginx
node/hollow-node-7cdbb7d69-6jqnv labeled
[root@node1 kubemark]# kubectl label node hollow-node-7cdbb7d69-prhx5 app=nginx
node/hollow-node-7cdbb7d69-prhx5 labeled
[root@node1 kubemark]# kubectl label node hollow-node-7cdbb7d69-szn5m app=nginx
node/hollow-node-7cdbb7d69-szn5m labeled
2、执行yaml脚本，在hollow节点启动pod，如下：
kubectl apply -f deploy-pod.yaml
###yaml问价内容如下：
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deployment
labels:
app: nginx
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
nodeSelector: ####匹配hollow node节点标签，如果物理node设置了NoSchedule则不需要添加
app: nginx
containers:
- name: nginx-deploy
image: nginx:latest ###不会真的去下载镜像
imagePullPolicy: IfNotPresent
3、查看启动好的pod节点信息，如下：
[root@node1 ~]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
deployment-5bd6b744d4-5fnwr 1/1 Running 0 9s 192.168.192.168 hollow-node-7cdbb7d69-szn5m <none> <none>
deployment-5bd6b744d4-cx6wv 1/1 Running 0 9s 192.168.192.168 hollow-node-7cdbb7d69-6jqnv <none> <none>
deployment-5bd6b744d4-fh29s 1/1 Running 0 9s 192.168.192.168 hollow-node-7cdbb7d69-prhx5 <none> <none>四、perf-test压测工具使用
- clusterloader工具构建
下载perf-test源码包，与目前k8s环境版本一致，如下:
[root@node1 perf-test]# ll
total 46512
drwxr-xr-x 14 root root 4096 Apr 18 11:15 perf-tests-release-1.18
-rw-r--r-- 1 root root 47622183 Apr 19 14:18 perf-tests-release-1.18.zip
####工具编译
[root@node1 perf-test]# cd perf-tests-release-1.18/clusterloader2/
[root@node1 clusterloader2]# go build -o clusterloader './cmd/'
###编译完成后查看已经存在clusterloader文件，如下：
[root@node1 clusterloader2]# ll
total 37964
drwxr-xr-x 2 root root 43 Apr 18 11:14 api
-rw-r--r-- 1 root root 2631 Apr 18 11:14 clean-up-old-snapshots.sh
-rwxrwxrwx 1 root root 38557259 Apr 18 11:14 clusterloader ########
drwxr-xr-x 2 root root 30 Apr 18 11:14 cmd
drwxr-xr-x 2 root root 97 Apr 18 11:15 docs
drwxr-xr-x 2 root root 40 Apr 18 11:15 drivers
-rw-r--r-- 1 root root 49 Apr 18 11:14 go.mod
-rw-r--r-- 1 root root 2080 Apr 25 15:41 junit.xml
-rw-r--r-- 1 root root 126 Apr 18 11:14 OWNERS
-rw-r--r-- 1 root root 315 Apr 18 11:14 perf.sh
drwxr-xr-x 16 root root 223 Apr 18 11:16 pkg
-rw-r--r-- 1 root root 6444 Apr 18 11:14 README.md
drwxr-xr-x 2 root root 4096 Apr 18 11:15 reports
-rw-r--r-- 1 root root 1485 Apr 18 11:14 run-e2e.sh
-rw-r--r-- 1 root root 239 Apr 19 15:18 sh
drwxr-xr-x 11 root root 166 Apr 18 11:23 testing
-rw-r--r-- 1 root root 271115 Apr 18 11:14 test.log
drwxr-xr-x 11 root root 215 Apr 18 11:26 vendor
#####编译好的工具，解压即用
链接：https://pan.baidu.com/s/1Ukv1g6SsNkgHCDZ0lxP-7Q
提取码：xsm0
--来自百度网盘超级会员V6的分享- 参数修改
clusterloader2的测试配置文件在testing目录下。可以参考修改配置
按修改后的测试配置文件，指定参数变量，执行clusterloader测试
[root@node1 clusterloader2]# cd testing/density/
You have mail in /var/spool/mail/root
[root@node1 density]# ll
total 32
drwxr-xr-x 2 root root 27 Apr 19 14:22 2000_nodes
drwxr-xr-x 2 root root 27 Apr 18 11:18 5000_nodes
drwxr-xr-x 2 root root 40 Apr 18 11:18 600_nodes
-rw-r--r-- 1 root root 9626 Apr 19 14:21 config.yaml ####运行的文件
-rw-r--r-- 1 root root 968 Apr 19 14:25 deployment.yaml
-rw-r--r-- 1 root root 8583 Apr 18 11:16 high-density-config.yaml
drwxr-xr-x 5 root root 93 Apr 19 14:43 legacy
drwxr-xr-x 5 root root 78 Apr 18 11:23 scheduler
-rw-r--r-- 1 root root 543 Apr 18 11:16 scheduler-suite.yaml
###此文件中需要修改的参数如下：
1、Nodes，属于配置文件上下文参数，如果不指定，测试工具会抓取实际环境中的可用的节点数，进行设置
2、NODES_PER_NAMESPACE， 每个ns下的nodes数。这里需注意: NODES > NODES_PER_NAMESPACE
3、PODS_PER_NODE，每个节点下的pod数
4、MIN_LATENCY_PODS这个数值会跟 PODS_PER_NODE比较 选取最大的，作为LATENCY测试的参数。因为LATENCY测试一般使用较多pod 数，即$MIN_LATENCY_PODS
5、测试中会有测试使用的资源参数，这里需要对实际情况进行config.yaml调整。
• LATENCY_POD_CPU
• LATENCY_POD_MEMORY
• 其它自定义资源数量，可以在config.yaml文件中添加配置
此次运行pod数量如下：
• NODES_PER_NAMESPACE 2
• PODS_PER_NODE 40
• MIN_LATENCY_PODS 40
####还需要修改次目录下的deployment.yaml文件，如下：
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{.Name}}
labels:
group: {{.Group}}
spec:
replicas: {{.Replicas}}
selector:
matchLabels:
name: {{.Name}}
template:
metadata:
labels:
name: {{.Name}}
group: {{.Group}}
spec:
nodeSelector: ###添加此参数，让pod启动在hollow节点上。
app: nginx
containers:
- image: 172.18.30.5:5000/pause:3.2
imagePullPolicy: IfNotPresent
name: {{.Name}}
ports:
resources:
requests:
cpu: {{.CpuRequest}}
memory: {{.MemoryRequest}}- 工具运行
cd /root/perf-tests-release-1.18/clusterloader2
####运行如下命令，提换实际ip地址
[root@node1 clusterloader2]# ./clusterloader --kubeconfig=/root/.kube/config --mastername=192.168.1.117 --masterip=192.168.1.117 --master-internal-ip=192.168.1.117 --testconfig=/root/go/perf-tests-release-1.18/clusterloader2/testing/density/config.yaml --report-dir=/tmp --alsologtostderr 2>&1 | tee /tmp/clusterload.log
###此工具会先启动三个名为exec-pod的pod，之后通过配置的depolyment启动pod。
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
