1. 前期准备
[root@sea ~]# yum -y update
# 说明: 如果没有关闭swap ，在k8s cluster 初始化或 worknode join cluster时都会失败；关闭 swap后在重启OS时，有可以会导致OS启动失败，具体原因待查；
[root@sea ~]# grep swapoff /etc/rc.local
swapoff -a
[root@sea ~]# swapoff -a
[root@sea ~]# grep swap /etc/fstab
#/dev/mapper/centos-swap swap swap defaults 0 0
# 时钟同步
[root@sea ~]# yum -y install ntp ntpdate
[root@sea ~]# ntpdate cn.pool.ntp.org
# 关闭防火墙和 selinux
[root@sea ~]# systemctl stop firewalld && systemctl disable firewalld
[root@sea ~]# getenforce
Disabled
# 本地解析
[root@sea ~]# tail -1 /etc/hosts
10.168.25.62 sea
# 调整系统参数
[root@sea ~]# cat <<EOF >> /etc/sysctl.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
vm.swappiness=0
EOF
[root@sea ~]# sysctl -p
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
# 确保iptables的参数都为1
[root@sea ~]# cat /proc/sys/net/bridge/bridge-nf-call-iptables
1
[root@sea ~]# cat /proc/sys/net/bridge/bridge-nf-call-ip6tables
1
# 关闭iptables转发，使用ipvs转发数据。
[root@sea ~]# cat > /etc/sysconfig/modules/ipvs.modules <<EOF
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
[root@sea ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod |grep -e -i ip_vs -e nf_conntrack_ipv4
[root@sea ~]# reboot
2. 安装 Kubernetes v1.16.0 和 docker-ce v.19.03.12
# 导入 k8s 安装包所需要的证书
[root@sea ~]# wget https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
[root@sea ~]# wget https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
[root@sea ~]# rpm --import rpm-package-key.gpg
[root@sea ~]# rpm --import yum-key.gpg
# 安装必要的一些系统工具
[root@sea ~]# yum install -y yum-utils
# 安装软件源信息
[root@sea ~]# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
[root@sea ~]# sed -i 's+download.docker.com+mirrors.aliyun.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo
[root@sea ~]# cat /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
# 更新并安装 kubernetes、docker-ce
[root@sea ~]# yum makecache fast
[root@sea ~]# yum install kubelet-1.16.0 kubeadm-1.16.0 kubectl-1.16.0 docker-ce-19.03.12 docker-ce-cli-19.03.12 containerd.io ipvsadm ipset
[root@sea ~]# systemctl enable docker && systemctl start docker
[root@sea ~]# systemctl enable kubelet
# 验证 docker-ce 的版本
[root@sea ~]# docker version
Client: Docker Engine - Community
Version: 19.03.12
API version: 1.40
Go version: go1.13.10
Git commit: 48a66213fe
Built: Mon Jun 22 15:46:54 2020
OS/Arch: linux/amd64
Experimental: false
Server: Docker Engine - Community
Engine:
Version: 19.03.12
API version: 1.40 (minimum version 1.12)
Go version: go1.13.10
Git commit: 48a66213fe
Built: Mon Jun 22 15:45:28 2020
OS/Arch: linux/amd64
Experimental: false
containerd:
Version: 1.5.11
GitCommit: 3df54a852345ae127d1fa3092b95168e4a88e2f8
docker-init:
Version: 0.18.0
GitCommit: fec3683
# 验证 kubectl 的版本
[root@sea ~]# kubectl version
Client Version: version.Info{Major:"1", Minor:"16", GitVersion:"v1.16.0", GitCommit:"2bd9643cee5b3b3a5ecbd3af49d09018f0773c77", GitTreeState:"clean", BuildDate:"2019-09-18T14:36:53Z", GoVersion:"go1.12.9", Compiler:"gc", Platform:"linux/amd64"}
The connection to the server localhost:8080 was refused - did you specify the right host or port?
# 忽略swap
[root@sea ~]# cat /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--fail-swap-on=false"
3. 修改 docker 配置
# 设置 docker runtime 、cgroup 及其他配置
[root@sea ~]# cat /etc/docker/daemon.json
{
"registry-mirrors": ["https://hub-mirror.c.163.com", "https://docker.mirrors.ustc.edu.cn"],
"storage-driver": "overlay2",
"exec-opts": ["native.cgroupdriver=systemd"],
"default-shm-size": "1G",
"default-ulimits": {
"memlock": {"name": "memlock", "soft": -1, "hard": -1},
"stack": {"name": "stack", "soft": 67108864, "hard": 67108864},
"nofile": {"name": "nofile", "soft": 65536, "hard": 65536}
}
}
# 重启 docker
[root@sea ~]# systemctl daemon-reload && systemctl restart docker
[root@sea ~]# systemctl status docker
● docker.service - Docker Application Container Engine
Loaded: loaded (/usr/lib/systemd/system/docker.service; enabled; vendor preset: disabled)
Active: active (running) since 四 2022-05-05 21:18:45 CST; 5s ago
Docs: https://docs.docker.com
Main PID: 8270 (dockerd)
Tasks: 23
Memory: 50.8M
CGroup: /system.slice/docker.service
└─8270 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
4. 部署 Kubernetes集群
# 生成集群初始化文件，修改配置选项
[root@sea ~]# kubeadm config print init-defaults > kubeadm-init.yaml
# 修改部分配置 （有注解的地方都是需要修改的）
[root@sea ~]# cat kubeadm-init.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.*************
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 10.***.**.** # 当前主机、k8s master 的 IP
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: sea
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.16.0
networking:
dnsDomain: cluster.local
podSubnet: 10.226.0.0/16 # pod ip网段
serviceSubnet: 10.98.0.0/16 # service ip网段
scheduler: {}
--- # 增加以下三行，表示使用 ipvs mode
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: "ipvs"
# 下载 init.yaml 中所需 images
[root@sea ~]# kubeadm config images pull --config kubeadm-init.yaml
[root@sea ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
k8s.gcr.io/kube-apiserver v1.16.0 b305571ca60a 2 years ago 217MB
k8s.gcr.io/kube-controller-manager v1.16.0 06a629a7e51c 2 years ago 163MB
k8s.gcr.io/kube-proxy v1.16.0 c21b0c7400f9 2 years ago 86.1MB
k8s.gcr.io/kube-scheduler v1.16.0 301ddc62b80b 2 years ago 87.3MB
k8s.gcr.io/etcd 3.3.15-0 b2756210eeab 2 years ago 247MB
k8s.gcr.io/coredns 1.6.2 bf261d157914 2 years ago 44.1MB
k8s.gcr.io/pause 3.1 da86e6ba6ca1 4 years ago 742kB
# 无法下载国外镜像的话，可以改用 aliyun 的源先下载然后修改 tag 即可
# 先查看当前 kubeadm-init.yaml 中所需要的镜像及 tag
[root@sea ~]# kubeadm config images list --config kubeadm-init.yaml
k8s.gcr.io/kube-apiserver:v1.16.0
k8s.gcr.io/kube-controller-manager:v1.16.0
k8s.gcr.io/kube-scheduler:v1.16.0
k8s.gcr.io/kube-proxy:v1.16.0
k8s.gcr.io/pause:3.1
k8s.gcr.io/etcd:3.3.15-0
k8s.gcr.io/coredns:1.6.2
[root@sea ~]# images=(kube-apiserver:v1.16.0 kube-controller-manager:v1.16.0 kube-scheduler:v1.16.0 kube-proxy:v1.16.0 pause:3.1 etcd:3.3.15-0 coredns:1.6.2)
[root@sea ~]# for imageName in ${images[@]};do
> docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName
> docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName k8s.gcr.io/$imageName
> docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName
> done
# 初始化 Kubernetes 集群
[root@sea ~]# kubeadm init --config kubeadm-init.yaml
......
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 10.***.**.**:6443 --token abcdef.***************** \
--discovery-token-ca-cert-hash sha256:************************************
[root@sea ~]# mkdir -p $HOME/.kube
[root@sea ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@sea ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config
5. 去除 master 节点的污点，让 pod 可以在 master 节点运行
# 查看 master 节点的污点
[root@sea ~]# kubectl describe nodes sea |grep -i taints
Taints: node-role-kubernetes.io/master:NoSchedule
# 删除 master 节点的污点
[root@sea ~]# kubectl taint node k8s-master node-role.kubernetes.io/master:NoSchedule-
# 再次查看 master 节点已经没有污点了
[root@sea ~]# kubectl describe nodes sea |grep -i taints
Taints: <none>
6. 部署网络插件 flannel
说明：我这里由于不能下载 flannel 国外镜像源，所以找了一个之前用过的 kube-flannel.yaml 文件，如果可以下载国外镜像源，推荐使用以下命令安装 flannel
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
[root@sea ~]# cat kube-flannel.yml
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp.flannel.unprivileged
annotations:
seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
privileged: false
volumes:
- configMap
- secret
- emptyDir
- hostPath
allowedHostPaths:
- pathPrefix: "/etc/cni/net.d"
- pathPrefix: "/etc/kube-flannel"
- pathPrefix: "/run/flannel"
readOnlyRootFilesystem: false
# Users and groups
runAsUser:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
fsGroup:
rule: RunAsAny
# Privilege Escalation
allowPrivilegeEscalation: false
defaultAllowPrivilegeEscalation: false
# Capabilities
allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
defaultAddCapabilities: []
requiredDropCapabilities: []
# Host namespaces
hostPID: false
hostIPC: false
hostNetwork: true
hostPorts:
- min: 0
max: 65535
# SELinux
seLinux:
# SELinux is unused in CaaSP
rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
rules:
- apiGroups: ['extensions']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
tier: node
app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
"Network": "10.244.0.0/16",
"Backend": {
"Type": "vxlan"
}
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel-ds
namespace: kube-system
labels:
tier: node
app: flannel
spec:
selector:
matchLabels:
app: flannel
template:
metadata:
labels:
tier: node
app: flannel
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
hostNetwork: true
priorityClassName: system-node-critical
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: flannel
initContainers:
- name: install-cni
image: quay.io/coreos/flannel:v0.14.0
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: quay.io/coreos/flannel:v0.14.0
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: run
mountPath: /run/flannel
- name: flannel-cfg
mountPath: /etc/kube-flannel/
volumes:
- name: run
hostPath:
path: /run/flannel
- name: cni
hostPath:
path: /etc/cni/net.d
- name: flannel-cfg
configMap:
name: kube-flannel-cfg
[root@sea ~]# kubectl create -f kube-flannel.yml
7. 验证安装
# 验证 flannel 及其他 Kubernetes 组件
[root@sea ~]# kubectl get all -n kube-system
NAME READY STATUS RESTARTS AGE
pod/coredns-5644d7b6d9-96csd 1/1 Running 0 38m
pod/coredns-5644d7b6d9-t52z6 1/1 Running 0 38m
pod/etcd-sea 1/1 Running 0 37m
pod/kube-apiserver-sea 1/1 Running 0 37m
pod/kube-controller-manager-sea 1/1 Running 0 37m
pod/kube-flannel-ds-zmprb 1/1 Running 0 16m
pod/kube-proxy-hh6nl 1/1 Running 0 38m
pod/kube-scheduler-sea 1/1 Running 0 37m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kube-dns ClusterIP 10.98.0.10 <none> 53/UDP,53/TCP,9153/TCP 38m
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
daemonset.apps/kube-flannel-ds 1 1 1 1 1 <none> 22m
daemonset.apps/kube-proxy 1 1 1 1 1 beta.kubernetes.io/os=linux 38m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/coredns 2/2 2 2 38m
NAME DESIRED CURRENT READY AGE
replicaset.apps/coredns-5644d7b6d9 2 2 2 38m
# 验证 Kubernetes master 的状态
[root@sea ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
sea Ready master 39m v1.16.0
# 验证 Kubernetes 核心组件的状态
[root@sea ~]# kubectl get cs -o yaml
apiVersion: v1
items:
- apiVersion: v1
conditions:
- message: ok
status: "True"
type: Healthy
kind: ComponentStatus
metadata:
creationTimestamp: null
name: controller-manager
selfLink: /api/v1/componentstatuses/controller-manager
- apiVersion: v1
conditions:
- message: ok
status: "True"
type: Healthy
kind: ComponentStatus
metadata:
creationTimestamp: null
name: scheduler
selfLink: /api/v1/componentstatuses/scheduler
- apiVersion: v1
conditions:
- message: '{"health":"true"}'
status: "True"
type: Healthy
kind: ComponentStatus
metadata:
creationTimestamp: null
name: etcd-0
selfLink: /api/v1/componentstatuses/etcd-0
kind: List
metadata:
resourceVersion: ""
selfLink: ""
# 将 k8s 和 docker-ce 的源重命名，避免 yum update 时更新版本
[root@sea ~]# mv /etc/yum.repos.d/docker-ce.repo{,.bak}
[root@sea ~]# mv /etc/yum.repos.d/kubernetes.repo{,.bak}
[root@sea ~]# ls /etc/yum.repos.d/
CentOS-Base.repo CentOS-Debuginfo.repo CentOS-Media.repo CentOS-Vault.repo docker-ce.repo.bak kubernetes.repo.bak
CentOS-CR.repo CentOS-fasttrack.repo CentOS-Sources.repo CentOS-x86_64-kernel.repo elrepo.repo
8.上述资源都正常可重启 master 机器，并再次查看集群中的资源是否正常
[root@sea ~]# reboot
[root@sea ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
sea Ready master 49m v1.16.0
[root@sea ~]# kubectl get all -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system pod/coredns-5644d7b6d9-96csd 1/1 Running 1 49m
kube-system pod/coredns-5644d7b6d9-t52z6 1/1 Running 1 49m
kube-system pod/etcd-sea 1/1 Running 1 48m
kube-system pod/kube-apiserver-sea 1/1 Running 1 48m
kube-system pod/kube-controller-manager-sea 1/1 Running 1 48m
kube-system pod/kube-flannel-ds-zmprb 1/1 Running 1 27m
kube-system pod/kube-proxy-hh6nl 1/1 Running 1 49m
kube-system pod/kube-scheduler-sea 1/1 Running 1 48m
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default service/kubernetes ClusterIP 10.98.0.1 <none> 443/TCP 49m
kube-system service/kube-dns ClusterIP 10.98.0.10 <none> 53/UDP,53/TCP,9153/TCP 49m
NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
kube-system daemonset.apps/kube-flannel-ds 1 1 1 1 1 <none> 32m
kube-system daemonset.apps/kube-proxy 1 1 1 1 1 beta.kubernetes.io/os=linux 49m
NAMESPACE NAME READY UP-TO-DATE AVAILABLE AGE
kube-system deployment.apps/coredns 2/2 2 2 49m
NAMESPACE NAME DESIRED CURRENT READY AGE
kube-system replicaset.apps/coredns-5644d7b6d9 2 2 2 49m
