k8s集群安装-1.30.4走底层docker

实验环境-初始化

系统：openeuler 24.03
cat /etc/hosts
192.168.0.35 kubeadmin-35
192.168.0.36 kubeadmin-36
192.168.0.37 kubeadmin-37
192.168.0.38 kubeadmin-38  
192.168.0.39 kubeadmin-39  #测试加入节点的机器

#初始化的时候ipvs转发先不开，跟老师保持一致
# 欧拉默认的在香港 得换掉
先更换源 不然速度很慢很慢
vim /etc/yum.repos.d/oepnEuler.repo
[OS]
name=OS
baseurl=http://mirrors.tuna.tsinghua.edu.cn/openeuler/openEuler-24.03-LTS/OS/$basearch/
enabled=1
gpgcheck=0

[everything]
name=everything
baseurl=http://mirrors.tuna.tsinghua.edu.cn/openeuler/openEuler-24.03-LTS/everything/$basearch/
enabled=1
gpgcheck=0

[EPOL]
name=EPOL
baseurl=http://mirrors.tuna.tsinghua.edu.cn/openeuler/openEuler-24.03-LTS/EPOL/main/$basearch/
enabled=1
gpgcheck=0

[debuginfo]
name=debuginfo
baseurl=http://mirrors.tuna.tsinghua.edu.cn/openeuler/openEuler-24.03-LTS/debuginfo/$basearch/
enabled=1
gpgcheck=0

[source]
name=source
baseurl=http://mirrors.tuna.tsinghua.edu.cn/openeuler/openEuler-24.03-LTS/source/
enabled=1
gpgcheck=0

[update]
name=update
baseurl=http://mirrors.tuna.tsinghua.edu.cn/openeuler/openEuler-24.03-LTS/update/$basearch/
enabled=1
gpgcheck=0

[update-source]
name=update-source
baseurl=http://mirrors.tuna.tsinghua.edu.cn/openeuler/openEuler-24.03-LTS/update/source/
enabled=1
gpgcheck=0



## 看看网络状态
ping www.baidu.com


vim  /etc/selinux/config
将SELINUX=enforcing改为SELINUX=disabled
setenforce 0 
genenforce 

# 关闭防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
systemctl stop iptables.service
systemctl disable iptables.service


#如有需求

iptables -P  INPUT  ACCEPT
iptables -P  FORWARD  ACCEPT
iptables -F
iptables -L  -n

更改初始密码

echo jichao | passwd root --stdin

设置网卡

vim /etc/sysconfig/network-scripts/ifcfg-ens

TYPE=Ethernet
PROXY_METHOD=none
NAME=ens160
DEVICE=ens160
IPADDR=192.168.0.35
PREFIX=24
GATEWAY=192.168.0.2
DNS1=223.5.5.5
DNS2=8.8.8.8
ONBOOT=yes

安装依赖包

yum install -y htop iftopconntrack ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git

yum install -y lrzsz git subversion numactl gpm unzip wget curl net-tools gpm gcc  gcc-c++  zip unzip  make

yum -y install wget jq psmisc vim net-tools nfs-utils telnet yum-utils device-mapper-persistent-data lvm2 git network-scripts tar curl -y

时间同步

https://developer.aliyun.com/mirror/NTP?spm=a2c6h.13651102.0.0.5a991b11xhPhjB

server ntp.aliyun.com iburst
stratumweight 0
driftfile /var/lib/chrony/drift
rtcsync
makestep 10 3
bindcmdaddress 127.0.0.1
bindcmdaddress ::1
keyfile /etc/chrony.keys
commandkey 1
generatecommandkey
logchange 0.5
logdir /var/log/chrony




chkconfig  chronyd  on
timedatectl  
systemctl  restart  chronyd    #重启校时服务
systemctl  enable  chronyd     #开机自启


ln  -sf  /usr/share/zoneinfo/Asia/Shanghai  /etc/localtime 
echo 'Asia/Shanghai' > /etc/timezone

进程最大使用权限

vim /etc/default/grub
numa=off
numactl --show

系统句柄数

ulimit -SHn 65535
cat >> /etc/security/limits.conf <<EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* seft memlock unlimited
* hard memlock unlimitedd
EOF

关闭swap

swapoff  -a
sed  -ri  's/.*swap.*/#&/'   /etc/fstab

免密登录

因为系统级别较高，要改为2048 RSA 不然会报错
./fgssh -user root -hosts "kubeadmin-35 kubeadmin-36 kubeadmin-37 kubeadmin-38 kubeadmin-39" -advanced -exverify -confirm

系统初始化

cat > /etc/sysctl.d/k8s_better.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
modprobe  br_netfilter
lsmod |grep  conntrack
modprobe  ip_conntrack
sysctl  -p  /etc/sysctl.d/k8s_better.conf

#ipvs转发
modprobe br_netfilter
cat > /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
#重启服务
systemctl restart systemd-modules-load.service
lsmod | grep -e ip_vs -e nf_conntrack


cat >> /etc/modules-load.d/ipvs.conf <<EOF 
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

systemctl restart systemd-modules-load.service
lsmod | grep -e ip_vs -e nf_conntrack

安装docker [ all nodes]

# 先配置源
vim /etc/yum.repos.d/docker-ce.repo
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/9/$basearch/stable
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
# 此处不是9的话 下载的都是 docker-ce el18了
yum makecache 


yum -y install docker-ce


把daemon.json导入进去

导入

systemctl enable docker
systemctl start docker

安装cri-docker[垫片][all]

https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.14/cri-dockerd-0.3.14-3.el8.x86_64.rpm

## 因为没有9  只有8

安装rpm包依赖[all]

scp 这个给其他的机器

#查看包需要什么依赖
rpm -ivh cri-dockerd-0.3.14-3.el8.x86_64.rpm.2 
yum -y install libcgroup

设置配置文件[all]

vim /usr/lib/systemd/system/cri-docker.service
----
修改第10行内容

ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 --container-runtime-endpoint fd://
systemctl start cri-docker
systemctl enable cri-docker

安装k8s

更换k8s源[all]

# 把里面的1.28改为1.30即可
https://developer.aliyun.com/mirror/kubernetes?spm=a2c6h.13651102.0.0.73281b11BEAEr6
###阿里云
cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/rpm/
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/rpm/repodata/repomd.xml.key
EOF
yum clean all
yum makecache

下载k8s[all]

yum list kubelet --showduplicates | sort -r | grep 1.30

yum -y install kubectl kubelet kubeadm

为了实现docker使用的cgroupdriver与kubelet使用的cgroup的一致性，建议修改如下文件内容
vim /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
设置kubelet为开机自启动即可，由于没有生成配置文件，集群初始化后自动启动
systemctl enable kubelet

查看需求镜像且下载[all]

准备k8s1.30.0 所需要的镜像
kubeadm config images list --kubernetes-version=v1.30.4

# 拉取镜像
kubeadm config images pull --image-repository registry.aliyuncs.com/google_containers --cri-socket=unix://var/run/cri-dockerd.sock

docker images

集群初始化

集群初始化
使用kubeadm init命令初始化
在kubeadmin-35上执行，报错请看k8s报错汇总
kubeadm init --kubernetes-version=v1.30.4 --pod-network-cidr=10.244.0.0/16 --apiserver-advertise-address=192.168.0.35 --image-repository registry.aliyuncs.com/google_containers --cri-socket=unix://var/run/cri-dockerd.sock
--apiserver-advertise-address 集群通告地址
--image-repository 由于默认拉取镜像地址k8s.gcr.io国内无法访问，这里指定阿里云镜像仓
--kubernetes-version K8s版本，与上面安装的一致
--service-cidr 集群内部虚拟网络，Pod统一访问入口
--pod-network-cidr Pod网络，，与下面部署的CNI网络组件yaml中保持一致

# 35
vim kubeinit-config.sh
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.0.35:6443 --token tor1uj.us7qfq0lst2gfq51 \
        --discovery-token-ca-cert-hash sha256:90b208a8a6d85ded20bf56b02fdebf89776813735f9c539b974a6e1bf4a3e2d6



# 36 37 38 
vim kubeinit-join.sh

kubeadm join 192.168.0.35:6443 --token tor1uj.us7qfq0lst2gfq51 \
        --discovery-token-ca-cert-hash sha256:90b208a8a6d85ded20bf56b02fdebf89776813735f9c539b974a6e1bf4a3e2d6 --cri-socket=unix://var/run/cri-dockerd.sock

部署网络插件

部署calico - yaml文件

wget --no-check-certificate https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/calico.yaml

- name: CALICO_IPV4POOL_CIDR
value: "10.244.0.0/16"
- name: IP_AUTODETECTION_METHOD
value: "interface=ens160"

docker pull calico/cni:v3.28.0
docker pull calico/node:v3.28.0
docker pull calico/kube-controllers:v3.28.0
kubectl apply -f calico.yaml
kubectl get nodes
kubectl get pods -n kube-system

kubectl apply -f recommended.yaml
kubectl apply -f dashboard-user.yaml

kubectl apply -f nginx-web.yaml
kubectl get svc
kubectl edit deploy  nginx-deployment

dashborad 创建token

kubectl get ns
kubectl create token admin-user -n kubernetes-dashboard
kubectl get svc -n kubernetes-dashboard



#
eyJhbGciOiJSUzI1NiIsImtpZCI6InFJWUI3Tm82eUNFY2VQak5RSUFQZ3VVOVBaX2c0dTI3X1hoQ0pvTjNDbGsifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNzI1NzI0MzMyLCJpYXQiOjE3MjU3MjA3MzIsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwianRpIjoiNDNiMWM3NGUtMjI5Ni00ZThmLTg5YzMtNWNkMGViZTNkMWZhIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJhZG1pbi11c2VyIiwidWlkIjoiODZiOGY1OTgtY2QwZi00ZDNhLTk1NjctNDdlNTQxODk2MjQ5In19LCJuYmYiOjE3MjU3MjA3MzIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDphZG1pbi11c2VyIn0.koHbvN3KEzJK3JsxdJqOSibvRuJuFasJsR6ufRCPUT_xmGwBlGyamOdMhibmIbaippPdKT6a_RIhhFdxygjYrJlm4i5jjObkVEvRZyJIS2er-qsJc9wGCIOhWU_OkvPLeEpWDIfyljYADpIRY8hq0axw1ZdyYjP3HINiK7Wa9-49LO36J_yDHPkYBLX2y2g8UwA8SA_qQ5N0WmdoYkRqYss04KB0aTzEWciBBUEu1gklhEpVsTPe3P_JHZbu-9UJwDpwMpD5bz3ie1D2I12sFgYwOlzTCI3z0mz6T7u9BKz4oKowBjhPFcj-bzmSB_hDiBqhl5Fm0-5umt92i67cmQ

登录界面

新节点如何加入？

# 把之前的步骤在重复一边 然后加入集群即可

安装Kuboard

wget https://addons.kuboard.cn/kuboard/kuboard-v3.yaml
# 获取官方的yaml文件
更改镜像策略

镜像我们通过阿里云自己下载的 

docker pull registry.cn-hangzhou.aliyuncs.com/gerencangku/jichaopersoncangku:eipworkkuboard
docker pull registry.cn-hangzhou.aliyuncs.com/gerencangku/jichaopersoncangku:eipworketcdhost

docker tag registry.cn-hangzhou.aliyuncs.com/gerencangku/jichaopersoncangku:eipworketcdhost eipwork/etcd-host:3.4.16-2^
docker tag registry.cn-hangzhou.aliyuncs.com/gerencangku/jichaopersoncangku:eipworkkuboard eipwork/kuboard:v3

标签

kubectl label nodes kubeadmin-35 k8s.kuboard.cn/role=etcd
kubectl label nodes kubeadmin-36 k8s.kuboard.cn/role=etcd
kubectl label nodes kubeadmin-37 k8s.kuboard.cn/role=etcd
kubectl label nodes kubeadmin-38 k8s.kuboard.cn/role=etcd
kubectl label nodes kubeadmin-39 k8s.kuboard.cn/role=etcd

kubectl label nodes kubeadmin-35 k8s.kuboard.cn/name=kuboard-etcd
kubectl label nodes kubeadmin-36 k8s.kuboard.cn/name=kuboard-etcd
kubectl label nodes kubeadmin-37 k8s.kuboard.cn/name=kuboard-etcd
kubectl label nodes kubeadmin-38 k8s.kuboard.cn/name=kuboard-etcd
kubectl label nodes kubeadmin-39 k8s.kuboard.cn/name=kuboard-etcd

kubectl apply -f kuboard.yaml

排查问题
kubectl describe pod    xxx   -n kuboard 
发现还是缺包 一共要三个包

docker  pull registry.cn-hangzhou.aliyuncs.com/gerencangku/jichaopersoncangku:agent3
docker tag registry.cn-hangzhou.aliyuncs.com/gerencangku/jichaopersoncangku:agent3 eipwork/kuboard-agent:v3

查看端口svc服务
kubectl get svc -n kuboard
192.168.0.35:30080
admin
Kuboard123