001 - 10分钟完成一套k8s部署 - 附脚本

服务器规划

整体规划

序号	服务器ip	角色	主机名	VM规格
1	192.168.100.7	Master,ceph	vm-01	2C/4G/100G/50G
2	192.168.100.8	worknode,ceph	vm-02	2C/4G/100G/50G
3	192.168.100.9	worknode,ceph	vm-03	2C/4G/100G/50G

VM规格信息

vi /home/virtmachines/vm-xml/vm-01.xml

qemu-img info /home/virtmachines/data-disk/vm-01.qcow2

virsh list --all

K8s环境准备

【vm-01】设置hosts与免密登录

cat >> /etc/hosts << 'EOF'

192.168.100.7 vm-01

192.168.100.8 vm-02

192.168.100.9 vm-03

192.168.100.10 vm-04

EOF

ssh-keygen

for i in {1..4}; do ssh-copy-id vm-0$i ; done

for i in {2..3}; do scp /etc/hosts vm-0$i:/etc/hosts ; done

【vm-01】配置yum源

cat <<EOF | sudo tee /etc/yum.repos.d/k8s.repo

[kubernetes]

name=Kubernetes

baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64

enabled=1

gpgcheck=0

repo_gpgcheck=0

gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg

https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg

EOF

for i in {2..3}; do scp /etc/yum.repos.d/k8s.repo vm-0$i:/etc/yum.repos.d/k8s.repo ; done

【vm-01】安装基本软件与配置

yum install -y ntpdate git net-tools telnet wget

mkdir -p /home/k8s/k8s-packages/ && cd /home/k8s/k8s-packages/

wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

wget -O /etc/yum.repos.d/docker-ce.repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum clean all && yum makecache

yum install --downloadonly --downloaddir=/home/k8s/k8s-packages/ kubelet-1.28.2-0 kubeadm-1.28.2-0 kubectl-1.28.2-0 docker-ce docker-ce-cli-y

cd /home/k8s/k8s-packages/ && rpm -ivh *.rpm --force

systemctl enable docker && systemctl start docker && systemctl status docker

【vm-01】拉取镜像并修改tag

for i in kube-apiserver:v1.28.7 kube-controller-manager:v1.28.7 kube-scheduler:v1.28.7 kube-proxy:v1.28.7 pause:3.9 pause:3.6 etcd:3.5.9-0 coredns:v1.10.1 ; do docker pull registry.aliyuncs.com/google_containers/$i ; done

docker images

for i in `docker images | grep registry.aliyuncs.com | awk -F ' ' '{print $1":"$2}'` ; do docker tag $i `echo $i | sed "s|registry.aliyuncs.com/google_containers|registry.k8s.io|"` ; done

docker tag registry.k8s.io/coredns:v1.10.1 registry.k8s.io/coredns/coredns:v1.10.1

docker images | grep registry.aliyuncs.com | awk -F ' ' '{print $1":"$2}' | xargs docker rmi

【vm-01】镜像save到本地

mkdir -p /home/k8s/images/coredns

for i in `docker images | grep registry.k8s.io | awk -F ' ' '{print $1":"$2}'` ; do docker save -o /home/k8s/images/${i:16:${#i}}.tar $i ; done

【vm-01】修改daemon.json

cat >> /etc/docker/daemon.json << 'EOF'

{

"registry-mirrors": ["https://registry.docker-cn.com"],

"exec-opts": ["native.cgroupdriver=systemd"],

"insecure-registries": ["192.168.100.7:5000"]

}

EOF

systemctl daemon-reload && systemctl restart docker

【vm-01】ctr导入镜像

cd /home/k8s/images && for i in ` ls | grep -E -v "kube-controllers:v3.26.4.tar|cni:v3.26.4.tar|node:v3.26.4.tar" ` ; do ctr -n=k8s.io image import $i ; done

docker save -o /home/k8s/images/coredns.coredns.tar registry.k8s.io/coredns/coredns:v1.10.1

ctr -n=k8s.io image list

【vm-01】启动镜像仓库

docker run -d -v /registry:/var/lib/registry --privileged=true -p 5000:5000 --restart=always --name registry registry:2

【vm-02,vm-03】安装rpm包，启动docker

for i in {2..3}; do ssh vm-0$i exec mkdir -p /home/k8s/k8s-packages/ ; done

for i in {2..3}; do scp -r /home/k8s/ vm-0$i:/home/; done

for i in {2..3}; do ssh vm-0$i exec rpm -ivh /home/k8s/k8s-packages/*.rpm --force ; done

for i in {2..3}; do scp /etc/docker/daemon.json vm-0$i:/etc/docker/daemon.json ; done

for i in {2..3}; do ssh vm-0$i "systemctl daemon-reload && systemctl restart docker"; done

【vm-01】push镜像到本地仓库

for i in `docker images | grep registry.k8s.io | awk -F ' ' '{print $1":"$2}'` ; do docker tag $i `echo $i | sed "s|registry.k8s.io|192.168.100.7:5000|"` ; done

for i in `docker images | grep 192.168.100.7:5000 | awk -F ' ' '{print $1":"$2}'` ; do docker push $i ; done

curl http://192.168.100.7:5000/v2/_catalog

【vm-01 ~ vm-04】关闭防火墙，disable selinux

for i in {1..4}; do ssh vm-0$i "systemctl disable firewalld.service && systemctl stop firewalld.service"; done

for i in {1..4}; do ssh vm-0$i 'swapoff -a && sed -i "s/\/dev\/mapper\/centos-swap/#\/dev\/mapper\/centos-swap/g" /etc/fstab '; done

for i in {1..4}; do ssh vm-0$i 'setenforce 0 && sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config '; done

【vm-01】更新k8s配置

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf

br_netfilter

EOF

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf

net.bridge.bridge-nf-call-ip6tables = 1

net.bridge.bridge-nf-call-iptables = 1

EOF

sudo sysctl --system

for i in {2..3}; do scp /etc/modules-load.d/k8s.conf vm-0$i:/etc/modules-load.d/k8s.conf ; done

for i in {2..3}; do scp /etc/sysctl.d/k8s.conf vm-0$i:/etc/sysctl.d/k8s.conf ; done

for i in {1..3}; do ssh vm-0$i "sudo sysctl --system && echo '-------------------------------------' && lsmod | grep br_netfilter"; done

【vm-01】更新containerd配置

for i in {1..3}; do ssh vm-0$i "mv /etc/containerd/config.toml /root/config.toml.bak ; containerd config default > /etc/containerd/config.toml"; done

for i in {1..3}; do ssh vm-0$i 'sed -i "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml'; done

for i in {1..3}; do ssh vm-0$i " systemctl enable --now containerd && systemctl restart containerd"; done

K8s安装

【vm-01】k8s初始化

## 指定版本初始化，否则可能会默认更高的版本而导致失败。

kubeadm init --kubernetes-version v1.28.7 --image-repository registry.aliyuncs.com/google_containers -v=2

报错：failed to do request: Head \"https://192.168.100.7:5000/v2/kube-apiserver/manifests/v1.28.7\": http: server gave HTTP response to HTTPS client"

, error: exit status 1

vi /usr/lib/systemd/system/docker.service

ExecStart增加配置：--insecure-registry 192.168.100.7:5000

注：需删除/etc/docker/daemon.json里的insecure-registry行，然后systemctl daemon-reload && systemctl restart docker。

##失败后用kubeadm reset 进行清理

##根据提示执行命令

mkdir -p $HOME/.kube && cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config

##vm-02,vm-03加入k8s集群，注：此时vm-02,vm-03的kubelet服务未启动（不影响）。

for i in {2..3}; do ssh vm-0$i 'systemctl status docker && systemctl status containerd && systemctl status kubelet' ; done && for i in {2..3}; do ssh vm-0$i 'systemctl start docker && systemctl start containerd && systemctl start kubelet' ; done

for i in {2..3}; do ssh vm-0$i 'kubeadm join 192.168.100.7:6443 --token 7axi7d.7p2mzgck4zzev1g9 --discovery-token-ca-cert-hash sha256:25ac49d78c83460c56e86e7973c6588519e5453a78fed9554404f6c15aa01a60-v=2'; done

以上命令中token来自kubeadm token list的TOKEN列

discovery-token-ca-cert-hash来自openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

执行完成后，vm-02和vm-03已加入集群。

【vm-01】安装calico

vi /home/k8s/calico.yaml 内容为网页中的值： https://github.com/projectcalico/calico/blob/release-v3.26/manifests/calico.yaml

确认calico/node,calico/cni,calico/kube-controllers的版本：

cat /home/k8s/calico.yaml | grep calico/node:

cat /home/k8s/calico.yaml | grep calico/cni:

cat /home/k8s/calico.yaml | grep calico/kube-controllers

根据版本拉取镜像：

for i in node:v3.26.4 cni:v3.26.4 kube-controllers:v3.26.4 ; do docker pull docker.io/calico/$i ; done

for i in `docker images | grep calico | awk -F ' ' '{print $1":"$2}'` ; do docker save -o /home/k8s/images/${i:7:${#i}}.tar $i ; done

cd /home/k8s/images && for i in ` ls *v3.26.4.tar ` ; do ctr -n=k8s.io image import $i ; done

kubectl apply -f /home/k8s/calico.yaml

此时vm-01为ready状态。

【vm-01】安装k9s

wget -O /home/k8s/k9s_Linux_amd64.tar.gz https://github.com/derailed/k9s/releases/download/v0.32.1/k9s_Linux_amd64.tar.gz && tar -zxvf /home/k8s/k9s_Linux_amd64.tar.gz -C /home/k8s/

for i in {1..3}; do scp /home/k8s/k9s vm-0$i:/usr/local/bin/ ; done

for i in {1..3}; do ssh vm-0$i "echo 'alias k9s=/usr/local/bin/k9s' >> ~/.bashrc " ; done

## 如果k9s不能运行则手动执行，非交互模式下for ** ssh可能行会无效

source ~/.bashrc

通过k9s查看pod状态，

【vm-02~vm-03】node节点配置

1. 检查cat /etc/containerd/config.toml | grep SystemdCgroup

SystemdCgroup = true

1. 重启服务for i in {1..3}; do ssh vm-0$i "systemctl daemon-reload && systemctl restart docker && systemctl restart containerd && systemctl restart kubelet" ; done
2. 设置开机启动

for i in {1..3}; do ssh vm-0$i "systemctl enable docker && systemctl enable containerd && systemctl enable kubelet" ; done

##vm-01在安装calico后变成ready状态。

for i in {2..3}; do ssh vm-0$i "mkdir -p $HOME/.kube" ;done

for i in {2..3}; do scp /etc/kubernetes/admin.conf vm-0$i:$HOME/.kube/config && ssh vm-0$i "mkdir -p $HOME/.kube && sudo chown $(id -u):$(id -g) $HOME/.kube/config" ; done

kubectl label node vm-02 kubernetes.io/role=worker && kubectl label node vm-03 kubernetes.io/role=worker

kubectl get nodes -o wide

kubectl get pods -n kube-system -o wide

kubectl describe node vm-02此时发现有报错：container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: cni plugin not initialized

将master（ready状态）的cni文件拷贝到work节点后即可。

for i in {2..3}; do scp /etc/cni/net.d/* vm-0$i:/etc/cni/net.d/ ; done

解决Pod失败的问题：

for i in {1..3} ; do ssh vm-0$i "sed -i 's/registry.k8s.io\/pause:3.6/registry.aliyuncs.com\/google_containers\/pause:3.9/g' /etc/containerd/config.toml" ; done

for i in {2..3}; do ssh vm-0$i "systemctl daemon-reload && systemctl restart docker && systemctl restart containerd && systemctl restart kubelet" ; done