一、环境准备:
10.10.0.170 k8s-master-01
10.10.0.171 k8s-master-02
10.10.0.172 k8s-master-03
10.10.0.190 k8s-node-01
10.10.0.222 vip
二、初始化:
2.1 三台master(k8s-master-01、k8s-master-02、k8s-master-03)上执行如下脚本:
#!/bin/sh
#1 修改主机名, 并写入hosts文件中
ip=$(ifconfig |grep eth0 -A 1|grep -oP '(?<=inet )[\d\.]+(?=\s)')
echo ${ip}
if [ ${ip}x = '10.10.0.170'x ];then
echo "set hostname k8s-master-01"
hostnamectl set-hostname k8s-master-01
elif [ ${ip}x = '10.10.0.171'x ];then
echo "set hostname k8s-master-02"
hostnamectl set-hostname k8s-master-02
elif [ ${ip}x = '10.10.0.172'x ];then
echo "set hostname k8s-master-03"
hostnamectl set-hostname k8s-master-03
fi
echo "10.10.0.170 k8s-master-01" >> /etc/hosts
echo "10.10.0.171 k8s-master-02" >> /etc/hosts
echo "10.10.0.172 k8s-master-03" >> /etc/hosts
echo "10.10.0.190 k8s-node-01" >> /etc/hosts
#2 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
#3 关闭selinux
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
sed -i '/^SELINUX=/c SELINUX=disabled/' /etc/sysconfig/selinux
#4 关闭系统的swap
swapoff -a
sed -i 's/\(.*swap.*swap.*\)/#\1/' /etc/fstab
#5 配置sysctl
cat >/etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness=0
EOF
sysctl -p /etc/sysctl.d/k8s.conf > /dev/null
#6 修改本机时区及时间同步
rm -rf /etc/localtime
ln -s /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo "*/10 * * * * /usr/sbin/ntpdate -u time7.aliyun.com">> /var/spool/cron/root
#7 安装所需软已经docker ce
yum install epel-release tmux mysql lrzsz -y
yum remove docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-selinux \
docker-engine-selinux \
docker-engine -y
yum install -y yum-utils \
device-mapper-persistent-data \
lvm2
yum-config-manager \
--add-repo \
https://download.docker.com/linux/centos/docker-ce.repo
yum install -y docker-ce-18.06.1.ce -y
cat /etc/docker/daemon.json
{
"exec-opts": ["native.cgroupdriver=systemd"]
}
#8 安装kubelet kubeadm kubectl
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum install kubelet kubeadm kubectl -y
systemctl enable kubelet
systemctl enable docker
systemctl restart kubelet
systemctl restart docker
#9 keepalived安装
yum install keepalived -y
systemctl restart keepalived
systemctl enable keepalived
#10 重启服务器
reboot
(注:上述的2~8同时也需要在node节点机上执行。)
[root@k8s-master-01 ~]# cat /etc/keepalived/keepalived.conf :
[root@k8s-master-01 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
fzhlzfy@163.com
}
notification_email_from dba@dbserver.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id K8S-HA
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 150
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.10.0.222
}
}
[root@k8s-master-02 k8s-install]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
fzhlzfy@163.com
}
notification_email_from dba@dbserver.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id K8S-HA
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 51
priority 100
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.10.0.222
}
}
[root@k8s-master-03 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
fzhlzfy@163.com
}
notification_email_from dba@dbserver.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id K8S-HA
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 51
priority 90
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.10.0.222
}
}
keepalived是为了保证整个集群的高可用。
所有docker服务器修改docker运行参数(三台master):
vim /lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H=0.0.0.0:2375 -H unix:///var/run/docker.sock
systemctl daemon-reload && systemctl restart docker
三、etcd集群安装:
1、免秘钥登录:
k8s-master-01上执行:
ssh-keygen -t rsa(一路回车)
ssh-copy-id k8s-master-01
ssh-copy-id k8s-master-02
ssh-copy-id k8s-master-03
2、设置cfssl环境:
k8s-master-01上执行:
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
chmod +x cfssljson_linux-amd64
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
chmod +x cfssl-certinfo_linux-amd64
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
3、创建CA配置文件:
k8s-master-01上执行:
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "8760h"
},
"profiles": {
"kubernetes-Soulmate": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "8760h"
}
}
}
}
EOF
cat > ca-csr.json <<EOF
{
"CN": "kubernetes-Soulmate",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "shanghai",
"L": "shanghai",
"O": "k8s",
"OU": "System"
}
]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
cat > etcd-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"10.10.0.170",
"10.10.0.171",
"10.10.0.172"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "shanghai",
"L": "shanghai",
"O": "k8s",
"OU": "System"
}
]
}
EOF
cfssl gencert -ca=ca.pem \
-ca-key=ca-key.pem \
-config=ca-config.json \
-profile=kubernetes-Soulmate etcd-csr.json | cfssljson -bare etcd
[root@k8s-master-01 k8s-install]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem etcd.csr etcd-csr.json etcd-key.pem etcd.pem
4、cp证书:
k8s-master-01上执行:
[root@k8s-master-01 k8s-install]# mkdir /etc/etcd/ssl/
[root@k8s-master-01 k8s-install]# cp etcd.pem etcd-key.pem ca.pem /etc/etcd/ssl/
[root@k8s-master-01 k8s-install]# ssh -n k8s-master-02 "mkdir -p /etc/etcd/ssl && exit"
[root@k8s-master-01 k8s-install]# ssh -n k8s-master-03 "mkdir -p /etc/etcd/ssl && exit"
[root@k8s-master-01 k8s-install]# scp -r /etc/etcd/ssl/*.pem k8s-master-02:/etc/etcd/ssl/
ca.pem 100% 1387 1.4KB/s 00:00
etcd-key.pem 100% 1675 1.6KB/s 00:00
etcd.pem 100% 1452 1.4KB/s 00:00
[root@k8s-master-01 k8s-install]# scp -r /etc/etcd/ssl/*.pem k8s-master-03:/etc/etcd/ssl/
ca.pem 100% 1387 1.4KB/s 00:00
etcd-key.pem 100% 1675 1.6KB/s 00:00
etcd.pem 100% 1452 1.4KB/s 00:00
5、etcd安装:
三台master都执行:
yum install etcd -y
etcd.service配置文件:
[root@k8s-master-01 ~]# cat /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/bin/etcd \
--name k8s-master-01 \
--cert-file=/etc/etcd/ssl/etcd.pem \
--key-file=/etc/etcd/ssl/etcd-key.pem \
--peer-cert-file=/etc/etcd/ssl/etcd.pem \
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/etc/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \
--initial-advertise-peer-urls https://10.10.0.170:2380 \
--listen-peer-urls https://10.10.0.170:2380 \
--listen-client-urls https://10.10.0.170:2379,http://127.0.0.1:2379 \
--advertise-client-urls https://10.10.0.170:2379 \
--initial-cluster-token etcd-cluster-0 \
--initial-cluster k8s-master-01=https://10.10.0.170:2380,k8s-master-02=https://10.10.0.171:2380,k8s-master-03=https://10.10.0.172:2380 \
--initial-cluster-state new \
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
[root@k8s-master-02 ~]# cat /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/bin/etcd \
--name k8s-master-02 \
--cert-file=/etc/etcd/ssl/etcd.pem \
--key-file=/etc/etcd/ssl/etcd-key.pem \
--peer-cert-file=/etc/etcd/ssl/etcd.pem \
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/etc/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \
--initial-advertise-peer-urls https://10.10.0.171:2380 \
--listen-peer-urls https://10.10.0.171:2380 \
--listen-client-urls https://10.10.0.171:2379,http://127.0.0.1:2379 \
--advertise-client-urls https://10.10.0.171:2379 \
--initial-cluster-token etcd-cluster-0 \
--initial-cluster k8s-master-01=https://10.10.0.170:2380,k8s-master-02=https://10.10.0.171:2380,k8s-master-03=https://10.10.0.172:2380 \
--initial-cluster-state new \
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
[root@k8s-master-03 ~]# cat /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/bin/etcd \
--name k8s-master-03 \
--cert-file=/etc/etcd/ssl/etcd.pem \
--key-file=/etc/etcd/ssl/etcd-key.pem \
--peer-cert-file=/etc/etcd/ssl/etcd.pem \
--peer-key-file=/etc/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/etc/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \
--initial-advertise-peer-urls https://10.10.0.172:2380 \
--listen-peer-urls https://10.10.0.172:2380 \
--listen-client-urls https://10.10.0.172:2379,http://127.0.0.1:2379 \
--advertise-client-urls https://10.10.0.172:2379 \
--initial-cluster-token etcd-cluster-0 \
--initial-cluster k8s-master-01=https://10.10.0.170:2380,k8s-master-02=https://10.10.0.171:2380,k8s-master-03=https://10.10.0.172:2380 \
--initial-cluster-state new \
--data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
配置文件简单介绍,详细的解释可自己baidu、google:
--name
etcd集群中的节点名,这里可以随意,可区分且不重复就行
--listen-peer-urls
监听的用于节点之间通信的url,可监听多个,集群内部将通过这些url进行数据交互(如选举,数据同步等)
--initial-advertise-peer-urls
建议用于节点之间通信的url,节点间将以该值进行通信。
--listen-client-urls
监听的用于客户端通信的url,同样可以监听多个。
--advertise-client-urls
建议使用的客户端通信url,该值用于etcd代理或etcd成员与etcd节点通信。
--initial-cluster-token etcd-cluster-1
节点的token值,设置该值后集群将生成唯一id,并为每个节点也生成唯一id,当使用相同配置文件再启动一个集群时,只要该token值不一样,etcd集群就不会相互影响。
--initial-cluster
也就是集群中所有的initial-advertise-peer-urls 的合集
--initial-cluster-state new
新建集群的标志
三台master执行:
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd
验证etcd集群健康性:
三台都尝试:
[root@k8s-master-01 ~]# etcdctl --endpoints=https://10.10.0.170:2379,https://10.10.0.171:2379,https://10.10.0.172:2379 --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem cluster-health
member 1c25bde2973f71cf is healthy: got healthy result from https://10.10.0.172:2379
member 3222a6aebdf856ac is healthy: got healthy result from https://10.10.0.170:2379
member 5796b25a0b404b92 is healthy: got healthy result from https://10.10.0.171:2379
cluster is healthy
[root@k8s-master-02 ~]# etcdctl --endpoints=https://10.10.0.170:2379,https://10.10.0.171:2379,https://10.10.0.172:2379 --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem cluster-health
member 1c25bde2973f71cf is healthy: got healthy result from https://10.10.0.172:2379
member 3222a6aebdf856ac is healthy: got healthy result from https://10.10.0.170:2379
member 5796b25a0b404b92 is healthy: got healthy result from https://10.10.0.171:2379
cluster is healthy
[root@k8s-master-03 ~]# etcdctl --endpoints=https://10.10.0.170:2379,https://10.10.0.171:2379,https://10.10.0.172:2379 --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem cluster-health
member 1c25bde2973f71cf is healthy: got healthy result from https://10.10.0.172:2379
member 3222a6aebdf856ac is healthy: got healthy result from https://10.10.0.170:2379
member 5796b25a0b404b92 is healthy: got healthy result from https://10.10.0.171:2379
cluster is healthy
如上图所示,则表示集群健康。
四、kubeadm init初始化集群:
4.1~4.6在k8s-master-01上执行:
4.1 镜像准备:
[root@k8s-master-01 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
k8s.gcr.io/kube-proxy v1.12.1 61afff57f010 3 weeks ago 96.6MB
k8s.gcr.io/kube-controller-manager v1.12.1 aa2dd57c7329 3 weeks ago 164MB
k8s.gcr.io/kube-scheduler v1.12.1 d773ad20fd80 3 weeks ago 58.3MB
k8s.gcr.io/kube-apiserver v1.12.1 dcb029b5e3ad 3 weeks ago 194MB
k8s.gcr.io/coredns 1.2.2 367cdc8433a4 2 months ago 39.2MB
k8s.gcr.io/pause 3.1 da86e6ba6ca1 10 months ago 742kB
4.2 kubeadm-config.yaml文件:
[root@k8s-master-01 ~]# cat kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
kubernetesVersion: v1.12.1
apiServerCertSANs:
- 10.10.0.170
- 10.10.0.171
- 10.10.0.172
- k8s-master-01
- k8s-master-02
- k8s-master-03
- 10.10.0.222
api:
controlPlaneEndpoint: 10.10.0.222:8443
etcd:
external:
endpoints:
- https://10.10.0.170:2379
- https://10.10.0.171:2379
- https://10.10.0.172:2379
caFile: /etc/etcd/ssl/ca.pem
certFile: /etc/etcd/ssl/etcd.pem
keyFile: /etc/etcd/ssl/etcd-key.pem
networking:
# This CIDR is a Calico default. Substitute or remove for your CNI provider.
podSubnet: "10.244.0.0/16"
4.3 初始化:
[root@k8s-master-01 ~]# kubeadm init --config kubeadm-config.yaml
[init] using Kubernetes version: v1.12.1
[preflight] running pre-flight checks
[preflight/images] Pulling images required for setting up a Kubernetes cluster
[preflight/images] This might take a minute or two, depending on the speed of your internet connection
[preflight/images] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[preflight] Activating the kubelet service
[certificates] Generated ca certificate and key.
[certificates] Generated apiserver certificate and key.
[certificates] apiserver serving cert is signed for DNS names [k8s-master-01 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local k8s-master-01 k8s-master-02 k8s-master-03] and IPs [10.96.0.1 10.10.0.170 10.10.0.170 10.10.0.171 10.10.0.172 10.10.0.222]
[certificates] Generated apiserver-kubelet-client certificate and key.
[certificates] Generated front-proxy-ca certificate and key.
[certificates] Generated front-proxy-client certificate and key.
[certificates] valid certificates and keys now exist in "/etc/kubernetes/pki"
[certificates] Generated sa key and public key.
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/kubelet.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/controller-manager.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/scheduler.conf"
[controlplane] wrote Static Pod manifest for component kube-apiserver to "/etc/kubernetes/manifests/kube-apiserver.yaml"
[controlplane] wrote Static Pod manifest for component kube-controller-manager to "/etc/kubernetes/manifests/kube-controller-manager.yaml"
[controlplane] wrote Static Pod manifest for component kube-scheduler to "/etc/kubernetes/manifests/kube-scheduler.yaml"
[init] waiting for the kubelet to boot up the control plane as Static Pods from directory "/etc/kubernetes/manifests"
[init] this might take a minute or longer if the control plane images have to be pulled
[apiclient] All control plane components are healthy after 23.001756 seconds
[uploadconfig] storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.12" in namespace kube-system with the configuration for the kubelets in the cluster
[markmaster] Marking the node k8s-master-01 as master by adding the label "node-role.kubernetes.io/master=''"
[markmaster] Marking the node k8s-master-01 as master by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "k8s-master-01" as an annotation
[bootstraptoken] using token: 7igv4r.pfh4zf7h8eao43k7
[bootstraptoken] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstraptoken] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstraptoken] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstraptoken] creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
Your Kubernetes master has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of machines by running the following on each node
as root:
kubeadm join 10.10.0.170:6443 --token 7igv4r.pfh4zf7h8eao43k7 --discovery-token-ca-cert-hash sha256:8488d362ce896597e9d6f23c825b60447b6e1fdb494ce72d32843d02d2d4b200
4.4 环境配置:
[root@k8s-master-01 ~]# mkdir -p $HOME/.kube
[root@k8s-master-01 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@k8s-master-01 ~]# chown $(id -u):$(id -g) $HOME/.kube/config
4.5 检查集群状态:
[root@k8s-master-01 ~]# kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-0 Healthy {"health": "true"}
etcd-1 Healthy {"health": "true"}
etcd-2 Healthy {"health": "true"}
4.6 k8s证书cp:
[root@k8s-master-01 ~]# scp -r /etc/kubernetes/pki/ 10.10.0.171:/etc/kubernetes/
ca.key 100% 1679 1.6KB/s 00:00
ca.crt 100% 1025 1.0KB/s 00:00
apiserver.key 100% 1675 1.6KB/s 00:00
apiserver.crt 100% 1326 1.3KB/s 00:00
apiserver-kubelet-client.key 100% 1675 1.6KB/s 00:00
apiserver-kubelet-client.crt 100% 1099 1.1KB/s 00:00
front-proxy-ca.key 100% 1675 1.6KB/s 00:00
front-proxy-ca.crt 100% 1038 1.0KB/s 00:00
front-proxy-client.key 100% 1675 1.6KB/s 00:00
front-proxy-client.crt 100% 1058 1.0KB/s 00:00
sa.key 100% 1679 1.6KB/s 00:00
sa.pub 100% 451 0.4KB/s 00:00[root@k8s-master-01 ~]# scp -r /etc/kubernetes/pki/ 10.10.0.172:/etc/kubernetes/
ca.key 100% 1679 1.6KB/s 00:00
ca.crt 100% 1025 1.0KB/s 00:00
apiserver.key 100% 1675 1.6KB/s 00:00
apiserver.crt 100% 1326 1.3KB/s 00:00
apiserver-kubelet-client.key 100% 1675 1.6KB/s 00:00
apiserver-kubelet-client.crt 100% 1099 1.1KB/s 00:00
front-proxy-ca.key 100% 1675 1.6KB/s 00:00
front-proxy-ca.crt 100% 1038 1.0KB/s 00:00
front-proxy-client.key 100% 1675 1.6KB/s 00:00
front-proxy-client.crt 100% 1058 1.0KB/s 00:00
sa.key 100% 1679 1.6KB/s 00:00
sa.pub 100% 451 0.4KB/s 00:00
k8s-master-02(上诉4.1~4.5):
[root@k8s-master-02 ~]# kubeadm init --config kubeadm-config.yaml
[init] using Kubernetes version: v1.12.1
[preflight] running pre-flight checks
[preflight/images] Pulling images required for setting up a Kubernetes cluster
[preflight/images] This might take a minute or two, depending on the speed of your internet connection
[preflight/images] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[preflight] Activating the kubelet service
[certificates] Using the existing apiserver certificate and key.
[certificates] Using the existing apiserver-kubelet-client certificate and key.
[certificates] Using the existing front-proxy-client certificate and key.
[certificates] valid certificates and keys now exist in "/etc/kubernetes/pki"
[certificates] Using the existing sa key.
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/kubelet.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/controller-manager.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/scheduler.conf"
[controlplane] wrote Static Pod manifest for component kube-apiserver to "/etc/kubernetes/manifests/kube-apiserver.yaml"
[controlplane] wrote Static Pod manifest for component kube-controller-manager to "/etc/kubernetes/manifests/kube-controller-manager.yaml"
[controlplane] wrote Static Pod manifest for component kube-scheduler to "/etc/kubernetes/manifests/kube-scheduler.yaml"
[init] waiting for the kubelet to boot up the control plane as Static Pods from directory "/etc/kubernetes/manifests"
[init] this might take a minute or longer if the control plane images have to be pulled
[apiclient] All control plane components are healthy after 20.002010 seconds
[uploadconfig] storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.12" in namespace kube-system with the configuration for the kubelets in the cluster
[markmaster] Marking the node k8s-master-02 as master by adding the label "node-role.kubernetes.io/master=''"
[markmaster] Marking the node k8s-master-02 as master by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "k8s-master-02" as an annotation
[bootstraptoken] using token: z4q8gj.pyxlik9groyp6t3e
[bootstraptoken] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstraptoken] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstraptoken] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstraptoken] creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
Your Kubernetes master has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of machines by running the following on each node
as root:
kubeadm join 10.10.0.171:6443 --token z4q8gj.pyxlik9groyp6t3e --discovery-token-ca-cert-hash sha256:5149f28976005454d8b0da333648e66880aa9419bc0e639781ceab65c77034be
五、pod网络配置:
镜像如下:
k8s.gcr.io/coredns:1.2.2
quay.io/coreos/flannel:v0.10.0-amd64
5.1 配置前:
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 172.17.0.1 netmask 255.255.0.0 broadcast 172.17.255.255
ether 02:42:a1:0f:80:1e txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.10.0.170 netmask 255.255.255.0 broadcast 10.10.0.255
inet6 fe80::20c:29ff:fe22:d2ff prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:22:d2:ff txqueuelen 1000 (Ethernet)
RX packets 1444658 bytes 365717587 (348.7 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1339639 bytes 185797411 (177.1 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 0 (Local Loopback)
RX packets 480338 bytes 116529453 (111.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 480338 bytes 116529453 (111.1 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@k8s-master-01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
link/ether 00:0c:29:22:d2:ff brd ff:ff:ff:ff:ff:ff
inet 10.10.0.170/24 brd 10.10.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet 10.10.0.222/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe22:d2ff/64 scope link
valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN
link/ether 02:42:a1:0f:80:1e brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
5.2 安装flannel network:
[root@k8s-master-01 ~]# wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
[root@k8s-master-01 ~]# kubectl apply -f kube-flannel.yml
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.extensions/kube-flannel-ds-amd64 created
daemonset.extensions/kube-flannel-ds-arm64 created
daemonset.extensions/kube-flannel-ds-arm created
daemonset.extensions/kube-flannel-ds-ppc64le created
daemonset.extensions/kube-flannel-ds-s390x created
查看一下集群中的daemonset:
[root@k8s-master-01 ~]# kubectl get ds -l app=flannel -n kube-system
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
kube-flannel-ds-amd64 2 2 2 2 2 beta.kubernetes.io/arch=amd64 22m
kube-flannel-ds-arm 0 0 0 0 0 beta.kubernetes.io/arch=arm 22m
kube-flannel-ds-arm64 0 0 0 0 0 beta.kubernetes.io/arch=arm64 22m
kube-flannel-ds-ppc64le 0 0 0 0 0 beta.kubernetes.io/arch=ppc64le 22m
kube-flannel-ds-s390x 0 0 0 0 0 beta.kubernetes.io/arch=s390x 22m
查看pods:
[root@k8s-master-01 ~]# kubectl get pod --all-namespaces -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
kube-system coredns-576cbf47c7-nmphm 1/1 Running 0 23m 10.244.0.3 k8s-master-01 <none>
kube-system coredns-576cbf47c7-w5mhv 1/1 Running 0 23m 10.244.0.2 k8s-master-01 <none>
kube-system kube-apiserver-k8s-master-01 1/1 Running 0 178m 10.10.0.170 k8s-master-01 <none>
kube-system kube-apiserver-k8s-master-02 1/1 Running 0 11m 10.10.0.171 k8s-master-02 <none>
kube-system kube-controller-manager-k8s-master-01 1/1 Running 0 177m 10.10.0.170 k8s-master-01 <none>
kube-system kube-controller-manager-k8s-master-02 1/1 Running 0 11m 10.10.0.171 k8s-master-02 <none>
kube-system kube-flannel-ds-amd64-cl4kb 1/1 Running 1 24m 10.10.0.170 k8s-master-01 <none>
kube-system kube-flannel-ds-amd64-rghg4 1/1 Running 0 24m 10.10.0.171 k8s-master-02 <none>
kube-system kube-proxy-2vsqh 1/1 Running 0 150m 10.10.0.171 k8s-master-02 <none>
kube-system kube-proxy-wvtrz 1/1 Running 0 178m 10.10.0.170 k8s-master-01 <none>
kube-system kube-scheduler-k8s-master-01 1/1 Running 0 178m 10.10.0.170 k8s-master-01 <none>
kube-system kube-scheduler-k8s-master-02 1/1 Running 0 11m 10.10.0.171 k8s-master-02 <none>
查看此时的网络:
[root@k8s-master-01 ~]# ifconfig
cni0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 10.244.0.1 netmask 255.255.255.0 broadcast 0.0.0.0
inet6 fe80::74ef:2ff:fec2:6c85 prefixlen 64 scopeid 0x20<link>
ether 0a:58:0a:f4:00:01 txqueuelen 0 (Ethernet)
RX packets 5135 bytes 330511 (322.7 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 5136 bytes 1929848 (1.8 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 172.17.0.1 netmask 255.255.0.0 broadcast 172.17.255.255
ether 02:42:a1:0f:80:1e txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.10.0.170 netmask 255.255.255.0 broadcast 10.10.0.255
inet6 fe80::20c:29ff:fe22:d2ff prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:22:d2:ff txqueuelen 1000 (Ethernet)
RX packets 1727975 bytes 420636786 (401.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1613768 bytes 225024592 (214.6 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 10.244.0.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::98c0:baff:fed3:8de5 prefixlen 64 scopeid 0x20<link>
ether 9a:c0:ba:d3:8d:e5 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 10 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 0 (Local Loopback)
RX packets 590730 bytes 145157886 (138.4 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 590730 bytes 145157886 (138.4 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
veth5504c620: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet6 fe80::f499:6ff:fece:d24a prefixlen 64 scopeid 0x20<link>
ether f6:99:06:ce:d2:4a txqueuelen 0 (Ethernet)
RX packets 2564 bytes 200932 (196.2 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 2579 bytes 965054 (942.4 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
vetha0ab0abe: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet6 fe80::74ef:2ff:fec2:6c85 prefixlen 64 scopeid 0x20<link>
ether 76:ef:02:c2:6c:85 txqueuelen 0 (Ethernet)
RX packets 2571 bytes 201469 (196.7 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 2584 bytes 966816 (944.1 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@k8s-master-01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
link/ether 00:0c:29:22:d2:ff brd ff:ff:ff:ff:ff:ff
inet 10.10.0.170/24 brd 10.10.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet 10.10.0.222/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe22:d2ff/64 scope link
valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN
link/ether 02:42:a1:0f:80:1e brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
4: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN
link/ether 9a:c0:ba:d3:8d:e5 brd ff:ff:ff:ff:ff:ff
inet 10.244.0.0/32 scope global flannel.1
valid_lft forever preferred_lft forever
inet6 fe80::98c0:baff:fed3:8de5/64 scope link
valid_lft forever preferred_lft forever
5: cni0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP
link/ether 0a:58:0a:f4:00:01 brd ff:ff:ff:ff:ff:ff
inet 10.244.0.1/24 scope global cni0
valid_lft forever preferred_lft forever
inet6 fe80::74ef:2ff:fec2:6c85/64 scope link
valid_lft forever preferred_lft forever
6: vetha0ab0abe@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master cni0 state UP
link/ether 76:ef:02:c2:6c:85 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet6 fe80::74ef:2ff:fec2:6c85/64 scope link
valid_lft forever preferred_lft forever
7: veth5504c620@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master cni0 state UP
link/ether f6:99:06:ce:d2:4a brd ff:ff:ff:ff:ff:ff link-netnsid 1
inet6 fe80::f499:6ff:fece:d24a/64 scope link
valid_lft forever preferred_lft forever
六、把k8s-master-03加入集群(完全可以放在五中和k8s-master-02一起进行):
k8s-master-03上执行4.1~4.5:
[root@k8s-master-03 ~]# kubeadm init --config kubeadm-config.yaml
[init] using Kubernetes version: v1.12.1
[preflight] running pre-flight checks
[preflight/images] Pulling images required for setting up a Kubernetes cluster
[preflight/images] This might take a minute or two, depending on the speed of your internet connection
[preflight/images] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[preflight] Activating the kubelet service
[certificates] Using the existing apiserver certificate and key.
[certificates] Using the existing apiserver-kubelet-client certificate and key.
[certificates] Using the existing front-proxy-client certificate and key.
[certificates] valid certificates and keys now exist in "/etc/kubernetes/pki"
[certificates] Using the existing sa key.
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/admin.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/kubelet.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/controller-manager.conf"
[kubeconfig] Wrote KubeConfig file to disk: "/etc/kubernetes/scheduler.conf"
[controlplane] wrote Static Pod manifest for component kube-apiserver to "/etc/kubernetes/manifests/kube-apiserver.yaml"
[controlplane] wrote Static Pod manifest for component kube-controller-manager to "/etc/kubernetes/manifests/kube-controller-manager.yaml"
[controlplane] wrote Static Pod manifest for component kube-scheduler to "/etc/kubernetes/manifests/kube-scheduler.yaml"
[init] waiting for the kubelet to boot up the control plane as Static Pods from directory "/etc/kubernetes/manifests"
[init] this might take a minute or longer if the control plane images have to be pulled
[apiclient] All control plane components are healthy after 20.503277 seconds
[uploadconfig] storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.12" in namespace kube-system with the configuration for the kubelets in the cluster
[markmaster] Marking the node k8s-master-03 as master by adding the label "node-role.kubernetes.io/master=''"
[markmaster] Marking the node k8s-master-03 as master by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "k8s-master-03" as an annotation
[bootstraptoken] using token: ks930p.auijb1h0or3o87f9
[bootstraptoken] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstraptoken] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstraptoken] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstraptoken] creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
Your Kubernetes master has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of machines by running the following on each node
as root:
kubeadm join 10.10.0.172:6443 --token ks930p.auijb1h0or3o87f9 --discovery-token-ca-cert-hash sha256:8488d362ce896597e9d6f23c825b60447b6e1fdb494ce72d32843d02d2d4b200
[root@k8s-master-03 ~]# mkdir -p $HOME/.kube
[root@k8s-master-03 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@k8s-master-03 ~]# chown $(id -u):$(id -g) $HOME/.kube/config
注:上上图红色部分的来源:
[root@k8s-master-03 ~]# openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
8488d362ce896597e9d6f23c825b60447b6e1fdb494ce72d32843d02d2d4b200
(因此,即便token过期了,但是重新生成token后,token发生了变化,但ca证书sha256编码hash值却是不变的。)
六、检查所有pod(可在三台master上面分别执行):
[root@k8s-master-03 ~]# kubectl get po --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-576cbf47c7-nmphm 1/1 Running 0 22h
kube-system coredns-576cbf47c7-w5mhv 1/1 Running 0 22h
kube-system kube-apiserver-k8s-master-01 1/1 Running 0 25h
kube-system kube-apiserver-k8s-master-02 1/1 Running 0 22h
kube-system kube-apiserver-k8s-master-03 1/1 Running 0 11h
kube-system kube-controller-manager-k8s-master-01 1/1 Running 0 25h
kube-system kube-controller-manager-k8s-master-02 1/1 Running 0 22h
kube-system kube-controller-manager-k8s-master-03 1/1 Running 0 11h
kube-system kube-flannel-ds-amd64-cl4kb 1/1 Running 1 22h
kube-system kube-flannel-ds-amd64-prvvj 1/1 Running 0 11h
kube-system kube-flannel-ds-amd64-rghg4 1/1 Running 0 22h
kube-system kube-proxy-2vsqh 1/1 Running 0 24h
kube-system kube-proxy-mvf9h 1/1 Running 0 11h
kube-system kube-proxy-wvtrz 1/1 Running 0 25h
kube-system kube-scheduler-k8s-master-01 1/1 Running 0 25h
kube-system kube-scheduler-k8s-master-02 1/1 Running 0 22h
kube-system kube-scheduler-k8s-master-03 1/1 Running 0 11h
七、dashboard安装(我这里选择在k8s-master-03上安装dashboard):
7.1镜像准备:
k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.0
7.2获取yaml文件:
[root@k8s-master-03 ~]# wget https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml
对上述下载的文件kubernetes-dashboard.yaml做适当处理:
[root@k8s-master-03 ~]# cat kubernetes-dashboard.yaml
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------- Dashboard Secret ------------------- #
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kube-system
type: Opaque
---
# ------------------- Dashboard Service Account ------------------- #
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
---
# ------------------- Dashboard Role & Role Binding ------------------- #
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: kubernetes-dashboard-minimal
namespace: kube-system
rules:
# Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create"]
# Allow Dashboard to create 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["create"]
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"]
verbs: ["get", "update", "delete"]
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
# Allow Dashboard to get metrics from heapster.
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kubernetes-dashboard-minimal
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard-minimal
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kube-system
---
# ------------------- Dashboard Deployment ------------------- #
kind: Deployment
apiVersion: apps/v1beta2
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.0
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
# - --apiserver-host=http://my-address:port
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
---
# ------------------- Dashboard Service ------------------- #
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
type: NodePort
ports:
- port: 443
targetPort: 8443
nodePort: 30001
selector:
k8s-app: kubernetes-dashboard
(注:红色部分为添加部分:便于远程访问。)
7.3 create:
[root@k8s-master-03 ~]# kubectl apply -f kubernetes-dashboard.yaml
secret/kubernetes-dashboard-certs created
serviceaccount/kubernetes-dashboard created
role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
deployment.apps/kubernetes-dashboard created
service/kubernetes-dashboard created
[root@k8s-master-03 ~]# kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-576cbf47c7-nmphm 1/1 Running 0 23h
kube-system coredns-576cbf47c7-w5mhv 1/1 Running 0 23h
kube-system kube-apiserver-k8s-master-01 1/1 Running 0 25h
kube-system kube-apiserver-k8s-master-02 1/1 Running 0 22h
kube-system kube-apiserver-k8s-master-03 1/1 Running 0 12h
kube-system kube-controller-manager-k8s-master-01 1/1 Running 0 25h
kube-system kube-controller-manager-k8s-master-02 1/1 Running 0 22h
kube-system kube-controller-manager-k8s-master-03 1/1 Running 0 12h
kube-system kube-flannel-ds-amd64-cl4kb 1/1 Running 1 23h
kube-system kube-flannel-ds-amd64-prvvj 1/1 Running 0 12h
kube-system kube-flannel-ds-amd64-rghg4 1/1 Running 0 23h
kube-system kube-proxy-2vsqh 1/1 Running 0 25h
kube-system kube-proxy-mvf9h 1/1 Running 0 12h
kube-system kube-proxy-wvtrz 1/1 Running 0 25h
kube-system kube-scheduler-k8s-master-01 1/1 Running 0 25h
kube-system kube-scheduler-k8s-master-02 1/1 Running 0 22h
kube-system kube-scheduler-k8s-master-03 1/1 Running 0 12h
kube-system kubernetes-dashboard-77fd78f978-7rczc 1/1 Running 0 8m36s
(三台服务器都能看到该pod。)
7.4 创建登录令牌(k8s-master-03上执行):
[root@k8s-master-03 ~]# cat admin-user.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: admin
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin
namespace: kube-system
[root@k8s-master-03 ~]# kubectl create -f admin-user.yaml
serviceaccount/admin created
clusterrolebinding.rbac.authorization.k8s.io/admin created
[root@k8s-master-03 ~]# kubectl describe serviceaccount admin -n kube-system
Name: admin
Namespace: kube-system
Labels: k8s-app=kubernetes-dashboard
Annotations: <none>
Image pull secrets: <none>
Mountable secrets: admin-token-96xbr
Tokens: admin-token-96xbr
Events: <none>
[root@k8s-master-03 ~]# kubectl describe secret admin-token-96xbr -n kube-system
Name: admin-token-96xbr
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: admin
kubernetes.io/service-account.uid: 546a18f5-dddd-11e8-8392-000c29666ccc
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1025 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi10b2tlbi05NnhiciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJhZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjU0NmExOGY1LWRkZGQtMTFlOC04MzkyLTAwMGMyOTY2NmNjYyIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTphZG1pbiJ9.GInI4jFvfYMKGLoJ-5PhVm9d8MiJeXg97oJmgX3hMreAUAUdRGZz2VLSc0ig3msw_VBg8JYb2pPQjWpYCR2bwNXMrN-FDPq3Ym6wZMittLTmZCHcKwHKRWNnomKbQsJf6wE8dN6Dws-eSYA66NqI8PXiCKao3XnQVbKz9eFMcl7W4u0u4T_0T1I0xqEhlsPReGyTQ1RyHfdTphT32Wo7BELsAEN69xscHFaL7JQlgry_boHO3RnIr8S-7bSnJBCKOVJZ9NMu_2TyH_81lYQZASkQCh1H7BwJFXIETvG6zcxrTb8FSUtgtEc3OjIWPYFnlrdaPhSbvU54yHfTCWrUUw
访问https://10.10.0.222:30001输入上面得到的token,既可以得到下图:
八、dashboard插件heapster的安装:
8.1 镜像(最好是三个master都要有):
[root@k8s-master-01 ~]# docker images|grep heapster
k8s.gcr.io/heapster-amd64 v1.5.4 72d68eecf40c 3 months ago 75.3MB
k8s.gcr.io/heapster-influxdb-amd64 v1.3.3 577260d221db 14 months ago 12.5MB
k8s.gcr.io/heapster-grafana-amd64 v4.4.3 8cb3de219af7 14 months ago 152MB
8.2 获取yaml文件:
https://raw.githubusercontent.com/Lentil1016/kubeadm-ha/1.12.1/plugin/heapster.yaml
我对此文件做了修改,如下:
[root@k8s-master-03 ~]# cat heapster.yaml
apiVersion: v1
kind: Service
metadata:
name: monitoring-grafana
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "Grafana"
spec:
# On production clusters, consider setting up auth for grafana, and
# exposing Grafana either using a LoadBalancer or a public IP.
# type: LoadBalancer
type: NodePort
ports:
- port: 80
protocol: TCP
targetPort: ui
nodePort: 30005
selector:
k8s-app: influxGrafana
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: heapster
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: v1
kind: ConfigMap
metadata:
name: heapster-config
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: EnsureExists
data:
NannyConfiguration: |-
apiVersion: nannyconfig/v1alpha1
kind: NannyConfiguration
---
apiVersion: v1
kind: ConfigMap
metadata:
name: eventer-config
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: EnsureExists
data:
NannyConfiguration: |-
apiVersion: nannyconfig/v1alpha1
kind: NannyConfiguration
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: heapster-v1.5.4
namespace: kube-system
labels:
k8s-app: heapster
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
version: v1.5.4
spec:
replicas: 1
selector:
matchLabels:
k8s-app: heapster
version: v1.5.4
template:
metadata:
labels:
k8s-app: heapster
version: v1.5.4
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
spec:
priorityClassName: system-cluster-critical
containers:
- image: k8s.gcr.io/heapster-amd64:v1.5.4
name: heapster
livenessProbe:
httpGet:
path: /healthz
port: 8082
scheme: HTTP
initialDelaySeconds: 180
timeoutSeconds: 5
command:
- /heapster
- --source=kubernetes.summary_api:''
- --sink=influxdb:http://monitoring-influxdb:8086
- image: k8s.gcr.io/heapster-amd64:v1.5.4
name: eventer
command:
- /eventer
- --source=kubernetes:''
- --sink=influxdb:http://monitoring-influxdb:8086
volumes:
- name: heapster-config-volume
configMap:
name: heapster-config
- name: eventer-config-volume
configMap:
name: eventer-config
serviceAccountName: kubernetes-admin
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
- key: "CriticalAddonsOnly"
operator: "Exists"
---
kind: Service
apiVersion: v1
metadata:
name: heapster
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "Heapster"
spec:
type: NodePort
ports:
- port: 80
targetPort: 8082
nodePort: 30006
selector:
k8s-app: heapster
---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
name: monitoring-influxdb-grafana-v4
namespace: kube-system
labels:
k8s-app: influxGrafana
version: v4
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
replicas: 1
selector:
matchLabels:
k8s-app: influxGrafana
version: v4
template:
metadata:
labels:
k8s-app: influxGrafana
version: v4
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
spec:
priorityClassName: system-cluster-critical
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
- key: "CriticalAddonsOnly"
operator: "Exists"
containers:
- name: influxdb
image: k8s.gcr.io/heapster-influxdb-amd64:v1.3.3
resources:
limits:
cpu: 100m
memory: 500Mi
requests:
cpu: 100m
memory: 500Mi
ports:
- name: http
containerPort: 8083
- name: api
containerPort: 8086
volumeMounts:
- name: influxdb-persistent-storage
mountPath: /data
- name: grafana
image: k8s.gcr.io/heapster-grafana-amd64:v4.4.3
env:
resources:
# keep request = limit to keep this container in guaranteed class
limits:
cpu: 100m
memory: 100Mi
requests:
cpu: 100m
memory: 100Mi
env:
# This variable is required to setup templates in Grafana.
- name: INFLUXDB_SERVICE_URL
value: http://monitoring-influxdb:8086
# The following env variables are required to make Grafana accessible via
# the kubernetes api-server proxy. On production clusters, we recommend
# removing these env variables, setup auth for grafana, and expose the grafana
# service using a LoadBalancer or a public IP.
- name: GF_AUTH_BASIC_ENABLED
value: "false"
- name: GF_AUTH_ANONYMOUS_ENABLED
value: "true"
- name: GF_AUTH_ANONYMOUS_ORG_ROLE
value: Admin
- name: GF_SERVER_ROOT_URL
value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy/
ports:
- name: ui
containerPort: 3000
volumeMounts:
- name: grafana-persistent-storage
mountPath: /var
volumes:
- name: influxdb-persistent-storage
emptyDir: {}
- name: grafana-persistent-storage
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: monitoring-influxdb
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "InfluxDB"
spec:
type: NodePort
ports:
- name: http
port: 8083
targetPort: 8083
- name: api
port: 8086
targetPort: 8086
nodePort: 30007
selector:
k8s-app: influxGrafana
( 加了NodePort端口。)
[root@k8s-master-03 ~]# kubectl apply -f heapster.yaml
service/monitoring-grafana created
serviceaccount/heapster created
configmap/heapster-config created
configmap/eventer-config created
deployment.extensions/heapster-v1.5.4 created
service/heapster created
deployment.extensions/monitoring-influxdb-grafana-v4 created
service/monitoring-influxdb created
8.3 查看:
[root@k8s-master-03 ~]# kubectl get pods,svc --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system pod/coredns-576cbf47c7-nmphm 1/1 Running 0 24h
kube-system pod/coredns-576cbf47c7-w5mhv 1/1 Running 0 24h
kube-system pod/kube-apiserver-k8s-master-01 1/1 Running 0 27h
kube-system pod/kube-apiserver-k8s-master-02 1/1 Running 0 24h
kube-system pod/kube-apiserver-k8s-master-03 1/1 Running 0 13h
kube-system pod/kube-controller-manager-k8s-master-01 1/1 Running 0 27h
kube-system pod/kube-controller-manager-k8s-master-02 1/1 Running 0 24h
kube-system pod/kube-controller-manager-k8s-master-03 1/1 Running 0 13h
kube-system pod/kube-flannel-ds-amd64-cl4kb 1/1 Running 1 24h
kube-system pod/kube-flannel-ds-amd64-prvvj 1/1 Running 0 13h
kube-system pod/kube-flannel-ds-amd64-rghg4 1/1 Running 0 24h
kube-system pod/kube-proxy-2vsqh 1/1 Running 0 26h
kube-system pod/kube-proxy-mvf9h 1/1 Running 0 13h
kube-system pod/kube-proxy-wvtrz 1/1 Running 0 27h
kube-system pod/kube-scheduler-k8s-master-01 1/1 Running 0 27h
kube-system pod/kube-scheduler-k8s-master-02 1/1 Running 0 24h
kube-system pod/kube-scheduler-k8s-master-03 1/1 Running 0 13h
kube-system pod/kubernetes-dashboard-77fd78f978-7rczc 1/1 Running 0 108m
kube-system pod/monitoring-influxdb-grafana-v4-65cc9bb8c8-qmhb4 2/2 Running 0 9m56s
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 27h
kube-system service/heapster NodePort 10.101.21.123 <none> 80:30006/TCP 9m56s
kube-system service/kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP 27h
kube-system service/kubernetes-dashboard NodePort 10.108.219.183 <none> 443:30001/TCP 108m
kube-system service/monitoring-grafana NodePort 10.111.38.86 <none> 80:30005/TCP 9m56s
kube-system service/monitoring-influxdb NodePort 10.107.91.86 <none> 8083:30880/TCP,8086:30007/TCP 9m56s
九、k8s集群增加节点:
9.1 节点三组件:
kubelet、kube-proxy、docker、kubeadm
9.2 镜像:
k8s.gcr.io/kube-proxy:v1.12.1
k8s.gcr.io/pause:3.1
(kubelet、kubeadm、docker按照上面的方式安装即可,此处省。)
9.3 查看token列表(任何一个master节点均可):
[root@k8s-master-03 ~]# kubeadm token list
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
7igv4r.pfh4zf7h8eao43k7 <invalid> 2018-11-01T20:12:13+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
ks930p.auijb1h0or3o87f9 <invalid> 2018-11-02T09:41:31+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
q7tox4.5j53kpgdob45f49i <invalid> 2018-11-01T22:58:18+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
z4q8gj.pyxlik9groyp6t3e <invalid> 2018-11-01T20:40:28+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
似乎token都失效了,需要重新生成。
[root@k8s-master-03 ~]# kubeadm token create
I1102 18:24:59.302880 28667 version.go:93] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.txt": Get https://storage.googleapis.com/kubernetes-release/release/stable-1.txt: x509: certificate is valid for www.webhostingtest1.com, webhostingtest1.com, not storage.googleapis.com
I1102 18:24:59.302947 28667 version.go:94] falling back to the local client version: v1.12.2
txqfdo.1steqzihimchr82l
[root@k8s-master-03 ~]# openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
8488d362ce896597e9d6f23c825b60447b6e1fdb494ce72d32843d02d2d4b200
[root@k8s-master-03 ~]# kubeadm token list
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
7igv4r.pfh4zf7h8eao43k7 <invalid> 2018-11-01T20:12:13+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
ks930p.auijb1h0or3o87f9 <invalid> 2018-11-02T09:41:31+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
q7tox4.5j53kpgdob45f49i <invalid> 2018-11-01T22:58:18+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
txqfdo.1steqzihimchr82l 23h 2018-11-03T18:24:59+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
z4q8gj.pyxlik9groyp6t3e <invalid> 2018-11-01T20:40:28+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
[root@k8s-node-01 ~]# kubeadm join 10.10.0.172:6443 --token txqfdo.1steqzihimchr82l --discovery-token-ca-cert-hash sha256:8488d362ce896597e9d6f23c825b60447b6e1fdb494ce72d32843d02d2d4b200
[preflight] running pre-flight checks
[discovery] Trying to connect to API Server "10.10.0.172:6443"
[discovery] Created cluster-info discovery client, requesting info from "https://10.10.0.172:6443"
[discovery] Requesting info from "https://10.10.0.172:6443" again to validate TLS against the pinned public key
[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server "10.10.0.172:6443"
[discovery] Successfully established connection with API Server "10.10.0.172:6443"
[kubelet] Downloading configuration for the kubelet from the "kubelet-config-1.12" ConfigMap in the kube-system namespace
[kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[preflight] Activating the kubelet service
[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...
[patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "k8s-node-01" as an annotation
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
Run 'kubectl get nodes' on the master to see this node join the cluster.
可以使用--print-join-command来直接生成命令:
[root@offline-k8s-master ~]# kubeadm token create --print-join-command
kubeadm join 10.0.0.200:6443 --token jjbxr5.ee6c4kh6vof9zu1m --discovery-token-ca-cert-hash sha256:139438d7734c9edd08e1beb99dccabcd5c613b14f3a0f7abd07b097a746101ff
过几分钟在master上查看node:
[root@k8s-master-01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master-01 Ready master 46h v1.12.2
k8s-master-02 Ready master 45h v1.12.2
k8s-master-03 Ready master 32h v1.12.2
k8s-node-01 Ready <none> 79s v1.12.2
十、k8s命令支持tab快捷用法:
yum install -y bash-completion > /dev/null
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> /etc/profile
十一 、上面用到的k8s网络是flanne,如果选择calico网络,master上面需要的镜像如下:
[root@k8s-master-1 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
calico/node v3.6.1 b4d7c4247c3a 31 hours ago 73.1MB
calico/cni v3.6.1 c7d27197e298 31 hours ago 84.3MB
calico/kube-controllers v3.6.1 0bd1f99c7034 31 hours ago 50.9MB
k8s.gcr.io/kube-proxy v1.14.0 5cd54e388aba 3 days ago 82.1MB
k8s.gcr.io/kube-controller-manager v1.14.0 b95b1efa0436 3 days ago 158MB
k8s.gcr.io/kube-apiserver v1.14.0 ecf910f40d6e 3 days ago 210MB
k8s.gcr.io/kube-scheduler v1.14.0 00638a24688b 3 days ago 81.6MB
k8s.gcr.io/coredns 1.3.1 eb516548c180 2 months ago 40.3MB
k8s.gcr.io/etcd 3.3.10 2c4adeb21b4f 3 months ago 258MB
k8s.gcr.io/pause 3.1 da86e6ba6ca1 15 months ago 742kB
[root@k8s-master-1 ~]#
kubeadm init --kubernetes-version=1.14.0 --pod-network-cidr=20.10.0.0/16 --apiserver-advertise-address=10.20.26.21 --node-name=k8s-master-1
kubectl apply -f https://docs.projectcalico.org/v3.6/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
[root@k8s-master-1 ~]# kubectl get po -n kube-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-5cbcccc885-bwcwp 1/1 Running 0 72m
calico-node-64rjg 1/1 Running 0 32s
calico-node-blp9h 1/1 Running 0 72m
calico-node-xd4bq 1/1 Running 0 27m
coredns-fb8b8dccf-r8b8f 1/1 Running 0 88m
coredns-fb8b8dccf-v8jvx 1/1 Running 0 88m
etcd-k8s-master-1 1/1 Running 0 87m
kube-apiserver-k8s-master-1 1/1 Running 0 87m
kube-controller-manager-k8s-master-1 1/1 Running 0 87m
kube-proxy-9q7mz 1/1 Running 0 27m
kube-proxy-qnfvz 1/1 Running 0 88m
kube-proxy-xbstx 1/1 Running 0 31s
kube-scheduler-k8s-master-1 1/1 Running 0 87m
[root@k8s-master-1 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master-1 Ready master 88m v1.14.0
k8s-node-1 Ready <none> 27m v1.14.0
k8s-node-2 Ready <none> 34s v1.14.0
附件:
calico网络见下链接:
https://docs.projectcalico.org/v3.6/getting-started/kubernetes/