================== start =


systemctl stop firewalld
systemctl disable firewalld
sed -ri 's/.*swap.*/#&/' /etc/fstab
swapoff -a


setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config



vi /etc/hosts
192.168.100.201 host201
192.168.100.202 host202
192.168.100.203 host203
192.168.100.204 host204
192.168.100.101 vip



vi /etc/modules-load.d/k8s.conf
overlay
br_netfilter
手动执行:
modprobe overlay
modprobe br_netfilter

或者

vi /etc/sysconfig/modules/ipvs.modules
modprobe br_netfilter
modprobe overlay
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs




vi /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
sysctl -p /etc/sysctl.d/k8s.conf



cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
EOF


yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum install ipvsadm ipset -y



yum install -y kubelet-1.24.1 kubeadm-1.24.1 kubectl-1.24.1 --disableexcludes=kubernetes
systemctl enable kubelet && systemctl start kubelet 或者systemctl enable --now kubelet


yum install -y docker-ce
systemctl start docker && systemctl enable docker
systemctl start containerd
systemctl status docker containerd kubelet




containerd config default > /etc/containerd/config.toml
sed -i "s#k8s.gcr.io/pause#registry.aliyuncs.com/google_containers/pause#g" /etc/containerd/config.toml
sed -i 's#SystemdCgroup = false#SystemdCgroup = true#g' /etc/containerd/config.toml
grep sandbox_image /etc/containerd/config.toml
grep SystemdCgroup /etc/containerd/config.toml
systemctl restart containerd


--- master 节点
kubeadm config images list # 查看镜像
k8s.gcr.io/kube-apiserver:v1.24.4
k8s.gcr.io/kube-controller-manager:v1.24.4
k8s.gcr.io/kube-scheduler:v1.24.4
k8s.gcr.io/kube-proxy:v1.24.4
k8s.gcr.io/pause:3.7
k8s.gcr.io/etcd:3.5.3-0
k8s.gcr.io/coredns/coredns:v1.8.6

替换为:
docker pull registry.aliyuncs.com/google_containers/kube-apiserver:v1.24.1
docker pull registry.aliyuncs.com/google_containers/kube-controller-manager:v1.24.1
docker pull registry.aliyuncs.com/google_containers/kube-scheduler:v1.24.1
docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.24.1
docker pull registry.aliyuncs.com/google_containers/pause:3.7
docker pull registry.aliyuncs.com/google_containers/etcd:3.5.3-0
docker pull registry.aliyuncs.com/google_containers/coredns:v1.8.6




kubeadm init \
--apiserver-advertise-address=192.168.100.201 \
--image-repository registry.aliyuncs.com/google_containers \
--control-plane-endpoint=vip \
--kubernetes-version v1.24.1 \
--service-cidr=10.1.0.0/16 \
--pod-network-cidr=10.244.0.0/16 \
--upload-certs

或者: kubeadm config print init-defaults > kubeadm.yaml



解释:--upload-certs 标志用来将在所有控制平面实例之间的共享证书上传到集群。如果正好相反,你更喜欢手动地通过控制平面节点或者使用自动化工具复制证书,请删除此标志
--certificate-key 使用指定的密钥解密从控制平面下载的证书。




k8s 节点状态是NotReady:
docker pull quay.io/coreos/flannel:v0.14.0
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

或者使用 https://docs.projectcalico.org/manifests/calico.yaml












master1 运行:
kubeadm token create --print-join-command 结果: kubeadm join vip:6443 --token uotivt.pxhxh9i39nfct6yc --discovery-token-ca-cert-hash sha256:4586f8ed7e5597cda40d764bf6ca7c5de91264bf3b773827565b00d3bf0aa1b6



kubeadm token create
kubeadm token list
kubeadm certs certificate-key # 要生成这样的密钥,可以使用以下命令
kubeadm init phase upload-certs --upload-certs # 重新上传证书并生成新的解密密钥,请在已加入集群节点的控制平面上使用以下命令




master2:
ssh root@host202 mkdir -p /etc/kubernetes/pki/etcd
scp /etc/kubernetes/admin.conf root@host202:/etc/kubernetes
scp /etc/kubernetes/pki/{ca.*,sa.*,front-proxy-ca.*} root@host202:/etc/kubernetes/pki
scp /etc/kubernetes/pki/etcd/ca.* root@host202:/etc/kubernetes/pki/etcd

kubeadm join vip:6443 --token k7rh8g.p4ppyy81htgqfgzf \
--discovery-token-ca-cert-hash sha256:7896b8119c53eb57016966cf01c6df03bf1f34de570ceecf57cac3f5785b2d6a \
--control-plane



master3:
ssh root@host204 mkdir -p /etc/kubernetes/pki/etcd
scp /etc/kubernetes/admin.conf root@host204:/etc/kubernetes
scp /etc/kubernetes/pki/{ca.*,sa.*,front-proxy-ca.*} root@host204:/etc/kubernetes/pki
scp /etc/kubernetes/pki/etcd/ca.* root@host204:/etc/kubernetes/pki/etcd

node:
kubeadm join vip:6443 --token k7rh8g.p4ppyy81htgqfgzf \
--discovery-token-ca-cert-hash sha256:7896b8119c53eb57016966cf01c6df03bf1f34de570ceecf57cac3f5785b2d6a


kubectl get nodes --show-labels
kubectl label nodes host203 node-role.kubernetes.io/node=
kubectl label nodes host203 node-role.kubernetes.io/worker=
kubectl describe nodes host203


说明:如果不想关闭交换分区,安装k8s的时候可以指定 vi /etc/sysconfig/kubelet添加:
KUBELET_EXTRA_ARGS="--fail-swap-on=false"
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd" 这里主要是和docker的驱动保持一致
KUBELET_CGROUP_ARGS="--cgroup-driver=systemd"
或者 vi /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS= --cgroup-driver=systemd --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock




kubectl edit cm kube-proxy -n=kube-system
修改mode: ipvs
或者
vi /etc/sysconfig/kubelet添加 KUBE_PROXY_MODE="ipvs"


kubernets 自v 1.24.0 后,就不再使用 docker.shim,替换采用 containerd 作为容器运行时端点。因此需要安装 containerd(在 docker 的基础下安装),上面安装 docker 的时候就自动安装了 containerd 了。这里的 docker 只是作为客户端而已。容器引擎还是 containerd。


kubeadm certs check-expiration 查看证书到期时间
kubeadm certs renew all 手动续订证书 该命令使用存储在/etc/kubernetes/pki中的CA(or front-proxy-CA)证书和密钥来更新证书
kubeadm alpha certs renew apiserver


问题:如出现coredns无法启动的情况,可能要检查/etc/kubernetes/manifests/kube-controller-manager.yaml文件
添加cidr参数

- --allocate-node-cidrs=true
- --cluster-cidr=10.244.0.0/16


问题:如果 kubectl get cs 发现集群不健康,更改以下两个文件
vim /etc/kubernetes/manifests/kube-scheduler.yaml
vim /etc/kubernetes/manifests/kube-controller-manager.yaml

## 测试kubernetes集群
kubectl create deployment nginx --image=nginx
kubectl expose deployment nginx --port=80 --type=NodePort
kubectl get pod,svc


# Node节点禁止调度(平滑维护)方式 cordon,drain,delete
cordon 停止调度
K8S再创建的pod资源,不会被调度到该节点。旧有的pod不会受到影响,仍正常对外提供服务。
禁止调度命令"kubectl cordon node_name"。
恢复调度命令"kubectl uncordon node_name"。

drain 驱逐节点
kubectl drain node_name --force --ignore-daemonsets --delete-local-data
恢复调度命令 kubectl uncordon node_name


kubectl delete node abc
# end ###########



# dashboard 安装

创建用户并授权
kubectl create serviceaccount dashboard-admin -n kubernetes-dashboard
kubectl create clusterrolebinding dashboard-admin-rb --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:dashboard-admin


kubectl create serviceaccount dashboard-admin -n kube-system
kubectl create clusterrolebinding abc --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin



nginx
wget -c http://nginx.org/download/nginx-1.16.1.tar.gz
./configure --with-stream --with-stream_ssl_module --with-http_stub_status_module

stream {
upstream k8s {
hash $remote_addr consistent;
server 192.168.254.18:6443 max_fails=3 fail_timeout=30s;
server 192.168.254.14:6443 max_fails=3 fail_timeout=30s;
server 192.168.254.19:6443 max_fails=3 fail_timeout=30s;
}
server{
listen 6443;
proxy_connect_timeout 1s;
proxy_timeout 10s;
proxy_pass k8s;
}
}



## dashboard

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.0/aio/deploy/recommended.yaml
修改:
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
type: NodePort
ports:
- port: 443
targetPort: 8443
nodePort: 30010




cat >ServiceAccount.yaml<<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
name: raoming
namespace: kubernetes-dashboard
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: dashboard-admin-binding
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
kind: ClusterRole
name: admin
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: raoming
namespace: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
name: dashboard-raoming-secret
namespace: kubernetes-dashboard
annotations:
kubernetes.io/service-account.name: "raoming"
EOF

kubectl apply -f ServiceAccount.yaml
kubectl -n kubernetes-dashboard create token rao
https://192.168.100.201:30010/













问题3:node 执行crictl命令报错解决方法:
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF








## metrics-server

wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
sed -i 's#k8s.gcr.io/metrics-server#registry.cn-hangzhou.aliyuncs.com/google_containers#g' components.yaml
kubectl apply -f components.yaml
kubectl get deploy,pod,svc -n kube-system -l k8s-app=metrics-server

vi /etc/kubernetes/manifests/kube-apiserver.yaml 文件command字段下增加
--enable-aggregator-routing=true
kubectl apply -f /etc/kubernetes/manifests/kube-apiserver.yaml
systemctl restart kubelet



常见命令:
kubectl get clusterrolebinding -n kubernetes-dashboard

kubectl get sa -n kubernetes-dashboard
admin-user 0 6h9m
default 0 6h30m
kubernetes-dashboard 0 6h30m
rao 0 12s
raoyuan 0 19m


kubectl describe ClusterRoleBinding rao-dashboard -n kubernetes-dashboard
Subjects:
Kind Name Namespace
---- ---- ---------
ServiceAccount rao kubernetes-dashboard


kubectl describe ClusterRoleBinding admin-user -n kubernetes-dashboard
Subjects:
Kind Name Namespace
---- ---- ---------
ServiceAccount raoyuan kubernetes-dashboard


kubectl describe secret rao-dashboard -n kubernetes-dashboard