一、环境准备及安装

服务器环境准备

1.png

安装环境依赖

yum -y install epel-release yum-utils device-mapper-persistent-data lvm2 wget
#yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

准备所需 yum 源,在 master 上配置,然后执行下面命令复制到其他4个节点即可。

[root@localhost yum.repos.d]# for i in {103..106};do scp CentOS-Base-7.repo kubernetes.repo docker.repo root@192.168.10.$i:/etc/yum.repos.d/;done

CentOS yum源

cat > /etc/yum.repos.d/CentOS-Base-7.repo << EOF
[base]
name=CentOS-$releasever - Base - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/os/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
#released updates 
[updates]
name=CentOS-$releasever - Updates - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/updates/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras - mirrors.aliyun.com
baseurl=http://mirrors.aliyun.com/centos/$releasever/extras/$basearch/
gpgcheck=1
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/centosplus/$basearch/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
 
#contrib - packages by Centos Users
[contrib]
name=CentOS-$releasever - Contrib - mirrors.aliyun.com
failovermethod=priority
baseurl=http://mirrors.aliyun.com/centos/$releasever/contrib/$basearch/
gpgcheck=1
enabled=0
gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7
EOF

k8s aliyun源

cat > /etc/yum.repos.d/kubernetes.repo <<EOF 
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
    http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
k8s google源
cat > /etc/yum.repos.d/kubernetes.repo <<EOF 
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF

docker yum源

cat > /etc/yum.repos.d/docker-ce.repo << EOF
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/$basearch/stable
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-stable-debuginfo]
name=Docker CE Stable - Debuginfo $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-$basearch/stable
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-stable-source]
name=Docker CE Stable - Sources
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/stable
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-edge]
name=Docker CE Edge - $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/$basearch/edge
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-edge-debuginfo]
name=Docker CE Edge - Debuginfo $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-$basearch/edge
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-edge-source]
name=Docker CE Edge - Sources
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/edge
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-test]
name=Docker CE Test - $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/$basearch/test
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-test-debuginfo]
name=Docker CE Test - Debuginfo $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-$basearch/test
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-test-source]
name=Docker CE Test - Sources
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/test
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-nightly]
name=Docker CE Nightly - $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/$basearch/nightly
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-nightly-debuginfo]
name=Docker CE Nightly - Debuginfo $basearch
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-$basearch/nightly
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
[docker-ce-nightly-source]
name=Docker CE Nightly - Sources
baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/nightly
enabled=0
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
EOF

gcloud源

tee -a /etc/yum.repos.d/google-cloud-sdk.repo << EOM
[google-cloud-sdk]
name=Google Cloud SDK
baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOM

创建 yum 元数据

yum clean all
yum makecache fast

yum 安装 etcd、kubernetes

yum -y install etcd kubernetes google-cloud-sdk

最新版:

curl -fsSL https://get.docker.com/ | sh

注意:docker必须跟着kubernetes版本走


创建 kubernetes、etcd 密钥目录

mkdir -p /wdata/kubernetes/ssl

在 master 上建立免密登录私钥(执行 ssh-keygen ,一路回车)

[root@localhost ~]# ssh-keygen
[root@localhost ~]# for i in {103..106};do ssh-copy-id -i /root/.ssh/id_rsa.pub root@192.168.10.$i;done

二、启动

在master节点上执行

systemctl start etcd
systemctl start docker
systemctl start kube-apiserver
systemctl start kube-controller-manager
systemctl start kube-scheduler
systemctl start kubelet
systemctl start kube-proxy

在所有slave节点上执行

systemctl start etcd
systemctl start docker
systemctl start kubelet
systemctl start kube-proxy

三、开机启动

在master节点上执行

systemctl enable etcd
systemctl enable docker
systemctl enable kube-apiserver
systemctl enable kube-controller-manager
systemctl enable kube-scheduler
systemctl enable kubelet
systemctl enable kube-proxy

在所有slave节点上执行

systemctl enable etcd
systemctl enable docker
systemctl enable kubelet
systemctl enable kube-proxy

四、查询状态

在master节点上执行

systemctl status etcd
systemctl status docker
systemctl status kube-apiserver
systemctl status kube-controller-manager
systemctl status kube-scheduler
systemctl status kubelet
systemctl status kube-proxy

在所有slave节点上执行

systemctl status etcd
systemctl status docker
systemctl status kubelet
systemctl status kube-proxy

五、在所有节点编辑 /lib/systemd/system/docker.service ,在ExecStart=..上面加入

[root@localhost ~]# vim /lib/systemd/system/docker.service
ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT

重启docker

[root@localhost ~]# systemctl daemon-reload && systemctl restart docker && systemctl status docker

或者在master节点修改完之后分发到其他节点

for i in {103..106};do scp /lib/systemd/system/docker.service root@192.168.10.$i:/lib/systemd/system/;done

六、所有节点设置k8s启动参数

cat > /etc/sysctl.d/k8s.conf <<EOF 
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

[root@localhost ~]# sysctl -p /etc/sysctl.d/k8s.conf

七、在 master 需要安装CFSSL工具,这将会用来建立 TLS certificates。

[root@localhost ~]# export CFSSL_URL="https://pkg.cfssl.org/R1.2"
[root@localhost ~]# wget "${CFSSL_URL}/cfssl_linux-amd64" -O /usr/local/bin/cfssl
[root@localhost ~]# wget "${CFSSL_URL}/cfssljson_linux-amd64" -O /usr/local/bin/cfssljson
[root@localhost ~]# wget "${CFSSL_URL}/cfssl-certinfo_linux-amd64" -O /usr/local/bin/cfssl-certinfo
[root@localhost ~]# chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
[root@localhost ~]# export PATH=/usr/local/bin:$PATH

八、关闭selinux

sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config

九、修改系统 hosts 文件,加入下面 5 行

[root@localhost ~]# vim /etc/hosts
192.168.10.102    master
192.168.10.103    node1
192.168.10.104    node2
192.168.10.105    node3
192.168.10.106    node4

或者

cat >> /etc/hosts <<EOF
192.168.10.102    master
192.168.10.103    node1
192.168.10.104    node2
192.168.10.105    node3
192.168.10.106    node4
EOF

十、创建集群 CA 与 Certificates,在主节点master上进行操作

配置CA,进入/wdata/kubernetes/ssl目录,然后进入该目录下进行操作

[root@master ~]# cd /wdata/kubernetes/ssl
[root@master ssl]# cfssl print-defaults config > config.json
[root@master ssl]# cfssl print-defaults csr > csr.json# 根据config.json文件的格式创建如下的ca-config.json文件
# 过期时间设置成了 87600hcat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF

创建CA证书签名请求,即创建ca-csr.json,格式如下

cat >ca-csr.json<<EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ],
    "ca": {
       "expiry": "87600h"
    }
}
EOF

生成CA证书和私钥

cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem

创建Kubernetes证书签名请求文件 kubernetes-csr.json,注意记得替换相应ip

配置CA,进入 /wdata/kubernetes/ssl 目录进行操作

[root@master ssl]# cd /wdata/kubernetes/ssl
[root@master ssl]# cfssl print-defaults config > config.json
[root@master ssl]# cfssl print-defaults csr > csr.json

# 根据config.json文件的格式创建如下的ca-config.json文件 # 过期时间设置成了 87600h

cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF

创建CA证书签名请求,即创建ca-csr.json,格式如下

cat >ca-csr.json<<EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ],
    "ca": {
       "expiry": "87600h"
    }
}
EOF

创建 kubernetes-csr.json 文件

cat > kubernetes-csr.json <<EOF
{
    "CN": "kubernetes",
    "hosts": [
      "127.0.0.1",
      "192.168.10.102",
      "192.168.10.103",
      "192.168.10.104",
      "192.168.10.105",
      "192.168.10.106",
      "master",
      "node1",
      "node2",
      "node3",
      "node4"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

生成CA证书和私钥

cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem

生成kubernetes证书和私钥

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
ls kubernetes*
kubernetes.csr  kubernetes-csr.json  kubernetes-key.pem  kubernetes.pem

创建admin签名文件 admin-csr-json

cat >admin-csr.json <<EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

生成admin证书和密钥

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
ls admin*
admin.csr  admin-csr.json  admin-key.pem  admin.pem

创建kube-proxy 证书签名请求文件 kube-proxy-csr.json

cat > kube-proxy-csr.json<<EOF
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

生成 kube-proxy 客户端证书和密钥

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy
ls kube-proxy*
kube-proxy.csr  kube-proxy-csr.json  kube-proxy-key.pem  kube-proxy.pem

分发证书,将生成的证书和密钥文件(后缀名为.pem)拷贝到所有机器的/etc/kubernetes/ssl目录下备用

mkdir -p /etc/kubernetes/ssl
cp /wdata/kubernetes/ssl/*.pem /etc/kubernetes/ssl
cd /etc/kubernetes/ssl
for i in {103..106};do scp *.pem 192.168.10.$i:/etc/kubernetes/ssl/;done

创建 kubectl kuberconfig 文件

export KUBE_APISERVER="https://192.168.10.102:6443"

# 设置集群参数

kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=false --server=${KUBE_APISERVER}

# 设置客户端认证参数

kubectl config set-credentials admin --client-certificate=/etc/kubernetes/ssl/admin.pem --embed-certs=false --client-key=/etc/kubernetes/ssl/admin-key.pem

# 设置上下文参数

kubectl config set-context kubernetes --cluster=kubernetes --user=admin

# 设置默认上下文

kubectl config use-context kubernetes

创建 TLS Bootstrapping Token

export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
cat >  token.csv << EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
cp token.csv /etc/kubernetes/

创建 kubelet bootstrapping kubeconfig 文件

cd /etc/kubernetes
export KUBE_APISERVER="https://192.168.10.102:6443"

# 设置集群参数

kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=bootstrap.kubeconfig

# 设置客户端认证参数

kubectl config set-credentials kubelet-bootstrap --token=${BOOTSTRAP_TOKEN} --kubeconfig=bootstrap.kubeconfig

# 设置上下文参数

kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=bootstrap.kubeconfig

# 设置默认上下文

kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

创建 kube-proxy kubeconfig 文件

export KUBE_APISERVER="https://192.168.10.102:6443"

# 设置集群参数

kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=kube-proxy.kubeconfig

# 设置客户端认证参数

kubectl config set-credentials kube-proxy --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig

# 设置上下文参数

kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig

# 设置默认上下文

kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

分发 kubeconfig 文件,将两个 kubeconfig 文件分发到所有 node 节点服务器的 /etc/kubernetes/ 目录

for i in {103..106};do scp bootstrap.kubeconfig kube-proxy.kubeconfig root@192.168.10.$i:/etc/kubernetes/;done

到这里,创建证书以及密钥就告一段落了,相信有很多人都有所迷惑,因为刚才创建了好多密钥和证书,下面总结下证书的作用:

生成的 CA 证书和秘钥文件如下:

ca-key.pem

ca.pem

kubernetes-key.pem

kubernetes.pem

kube-proxy.pem

kube-proxy-key.pem

admin.pem

admin-key.pem

使用证书的组件如下:

etcd:使用 ca.pem、kubernetes-key.pem、kubernetes.pem;

kube-apiserver:使用 ca.pem、kubernetes-key.pem、kubernetes.pem;

kubelet:使用 ca.pem;

kube-proxy:使用 ca.pem、kube-proxy-key.pem、kube-proxy.pem;

kubectl:使用 ca.pem、admin-key.pem、admin.pem;

kube-controller-manager:使用 ca-key.pem、ca.pem

相信看完上面的总结就一目了然了,OK下面我们来进行etcd集群的安装。


十一、所有节点部署 etcd

Kuberntes 使用 etcd 来存储所有数据,下面我们来创建三节点etcd集群,也就是master、node1、node2前面我们已经创建了很多TLS证书,咱们这里就复用下kubernetes的证书,以下操作在所有节点执行。

由于上面已经 yum 安装了 etcd ,因此,以下1、2两部可忽略。

1、下载 etcd 源码文件

wget https://github.com/coreos/etcd/releases/download/v3.1.5/etcd-v3.1.5-linux-amd64.tar.gz
tar -xvf etcd-v3.1.5-linux-amd64.tar.gz
mv etcd-v3.1.5-linux-amd64/etcd* /usr/local/bin

2、更新 etcd.server 文件

[root@master ~]# vim /lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
User=etcd
# set GOMAXPROCS to number of processors
#ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\""
ExecStart=/usr/bin/etcd  \
    --name=${ETCD_NAME} \
    --data-dir=${ETCD_DATA_DIR} \
    --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS} \
    --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
    --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
    --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
    --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
    --initial-cluster-state=${ETCD_INITIAL_CLUSTER_STATE} \
    --initial-cluster=${ETCD_INITIAL_CLUSTER}
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

参数说明(下同):

name 节点名称

data-dir 指定节点的数据存储目录

listen-peer-urls 监听URL,用于与其他节点通讯

listen-client-urls 对外提供服务的地址:比如 http://ip:2379,http://127.0.0.1:2379 ,客户端会连接到这里和 etcd 交互

initial-advertise-peer-urls 该节点同伴监听地址,这个值会告诉集群中其他节点

initial-cluster-state 集群中所有节点的信息,格式为 node1=http://ip1:2380,node2=http://ip2:2380,… 。注意:这里的 node1 是节点的 --name 指定的名字;后面的 ip1:2380 是 --initial-advertise-peer-urls 指定的值

initial-cluster-state 新建集群的时候,这个值为 new ;假如已经存在的集群,这个值为 existing

initial-cluster-token 创建集群的 token,这个值每个集群保持唯一。这样的话,如果你要重新创建集群,即使配置和之前一样,也会再次生成新的集群和节点 uuid;否则会导致多个集群之间的冲突,造成未知的错误

advertise-client-urls 对外公告的该节点客户端监听地址,这个值会告诉集群中其他节点


注意:etcd 的数据目录为 /var/lib/etcd,需在启动服务前创建这个目录,否则启动服务的时候会报错“Failed at step CHDIR spawning /usr/bin/etcd: No such file or directory”;


3、环境变量配置文件/etc/etcd/etcd.conf,每个节点单独配置,共5个节点

#[Member]
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.10.102:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.10.102:2379"
ETCD_NAME="master"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.102:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.102:2379"
ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380"
ETCD_INITIAL_CLUSTER_TOKEN="kubernetes"
ETCD_INITIAL_CLUSTER_STATE="new"
###########################################################################################################################
#[Member]
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.10.103:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.10.103:2379"
ETCD_NAME="node1"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.103:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.103:2379"
ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380"
ETCD_INITIAL_CLUSTER_TOKEN="kubernetes"
ETCD_INITIAL_CLUSTER_STATE="new"
###########################################################################################################################
#[Member]
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.10.104:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.10.104:2379"
ETCD_NAME="node2"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.104:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.104:2379"
ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380"
ETCD_INITIAL_CLUSTER_TOKEN="kubernetes"
ETCD_INITIAL_CLUSTER_STATE="new"
###########################################################################################################################
#[Member]
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.10.105:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.10.105:2379"
ETCD_NAME="node3"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.105:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.105:2379"
ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380"
ETCD_INITIAL_CLUSTER_TOKEN="kubernetes"
ETCD_INITIAL_CLUSTER_STATE="new"
###########################################################################################################################
#[Member]
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.10.106:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.10.106:2379"
ETCD_NAME="node4"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.106:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.106:2379"
ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380"
ETCD_INITIAL_CLUSTER_TOKEN="kubernetes"
ETCD_INITIAL_CLUSTER_STATE="new"

4、重启 etcd 服务

systemctl daemon-reload
systemctl restart etcd
systemctl status etcd

# 在所有的 kubernetes节点重复上面的步骤,直到所有机器的 etcd 服务都已启动。


5、验证服务

[root@master ~]# etcdctl member list
33e828919ccd6e00: name=etcd3 peerURLs=http://192.168.10.104:2380 clientURLs=http://127.0.0.1:2379,http://192.168.10.104:2379 isLeader=false
63f491cf242db4b3: name=etcd1 peerURLs=http://192.168.10.102:2380 clientURLs=http://127.0.0.1:2379,http://192.168.10.102:2379 isLeader=true
d8703b918bb9d544: name=etcd2 peerURLs=http://192.168.10.103:2380 clientURLs=http://127.0.0.1:2379,http://192.168.10.103:2379 isLeader=false

2.png

[root@master ~]# etcdctl cluster-health
member 33e828919ccd6e00 is healthy: got healthy result from http://127.0.0.1:2379
member 63f491cf242db4b3 is healthy: got healthy result from http://127.0.0.1:2379
member d8703b918bb9d544 is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy

3.png

6、问题总结

①、如果验证输出以下内容

[root@master ~]# etcdctl member list
8e9e05c52164694d: name=etcd1 peerURLs=http://localhost:2380 clientURLs=http://127.0.0.1:2379,http://192.168.10.102:2379 isLeader=true

4.png

此问题是在搭建集群前,先进行了单实例的etcd部署测试, 即 etcd2、etcd3 节点分别进行了单实例部署测试,也就是历史数据的影响。删除历史数据即可。

rm -rf /var/lib/etcd/*

②、如果有其他问题,则需要将 etcd.service 中的 User=etcd 修改为 User=root ,且 ExecStart 中不能有空格

③、一个集群中只有一个Leader,如果不是一个就是报错。

以下内容为搭建带认证的 etcd 集群,可忽略

7、创建 etcd 密钥文件

cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF

生成 etcd-ca-csr.json 文件

cat >etcd-ca-csr.json<<EOF
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "etcd",
      "OU": "System"
    }
  ],
    "ca": {
       "expiry": "87600h"
    }
}
EOF

生成 etcd 的证书签名请求文件 etcd-csr.json

cat > etcd-csr.json << EOF
  {
    "CN": "etcd",
    "hosts": [
      "127.0.0.1",
      "192.168.10.102",
      "192.168.10.103",
      "192.168.10.104",
      "etcd1",
      "etcd1",
      "etcd2",
      "etcd3"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "etcd",
            "OU": "System"
        }
    ]
}
EOF

生成 etcd 客户端证书和密钥

cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

8、修改 etcd 服务文件 etcd.service ,在所有节点加入以下标红的内容

[root@master ~]# vim /lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
User=etcd
# set GOMAXPROCS to number of processors
# ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\""
ExecStart=/usr/bin/etcd \
    --name=${ETCD_NAME} \
    --data-dir=${ETCD_DATA_DIR} \
    --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS} \
    --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
    --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
    --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
    --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
    --initial-cluster-state=${ETCD_INITIAL_CLUSTER_STATE} \
    --initial-cluster=${ETCD_INITIAL_CLUSTER} \
    --cert-file=${ETCD_CERT_FILE} \
    --key-file=${ETCD_KEY_FILE} \
    --peer-cert-file=${ETCD_CERT_FILE} \
    --peer-key-file=${ETCD_KEY_FILE} \
    --trusted-ca-file=${ETCD_TRUSTED_CA_FILE} \
    --peer-trusted-ca-file=${ETCD_PEER_TRUSTED_CA_FILE}
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

9、修改 etcd 配置文件 etcd.conf ,加入以下内容

#[Security]
ETCD_CERT_FILE="/etc/kubernetes/ssl/kubernetes.pem"
ETCD_KEY_FILE="/etc/kubernetes/ssl/kubernetes-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/kubernetes.pem"
ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/kubernetes.pem"
ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/kubernetes-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/kubernetes.pem"

10、验证集群

[root@master ~]# etcdctl --ca-file=/etc/kubernetes/ssl/ca.pem --cert-file=/etc/kubernetes/ssl/kubernetes.pem --key-file=/etc/kubernetes/ssl/kubernetes-key.pem --endpoints=https://192.168.10.102:2379 member list
5590395074cc9636: name=node2 peerURLs=https://192.168.10.104:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.104:2379 isLeader=false
5c724b8641f4fa1c: name=master peerURLs=https://192.168.10.102:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.102:2379 isLeader=false
748537d982cf5c99: name=node3 peerURLs=https://192.168.10.105:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.105:2379 isLeader=false
c05394aa51d11fda: name=node4 peerURLs=https://192.168.10.106:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.106:2379 isLeader=false
d3993eda148039ba: name=node1 peerURLs=https://192.168.10.103:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.103:2379 isLeader=true

5.png

[root@master ~]# etcdctl --ca-file=/etc/kubernetes/ssl/ca.pem --cert-file=/etc/kubernetes/ssl/kubernetes.pem --key-file=/etc/kubernetes/ssl/kubernetes-key.pem --endpoints=https://192.168.10.102:2379 cluster-health
member 5590395074cc9636 is healthy: got healthy result from https://192.168.10.104:2379
member 5c724b8641f4fa1c is healthy: got healthy result from https://192.168.10.102:2379
member 748537d982cf5c99 is healthy: got healthy result from https://192.168.10.105:2379
member c05394aa51d11fda is healthy: got healthy result from https://192.168.10.106:2379
member d3993eda148039ba is healthy: got healthy result from https://192.168.10.103:2379
cluster is healthy

6.png

十二、Kubernetes 集群部署

经过上面的步骤,前期准备工作完成,下面开始正式部署 Kubernetes 。以下操作均在 master 节点进行

1、修改 kube-apiserver.service 服务文件

[root@master ~]# vim /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
After=etcd.service
[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/apiserver
User=kube
ExecStart=/usr/bin/kube-apiserver \
            $KUBE_LOGTOSTDERR \
            $KUBE_LOG_LEVEL \
            $KUBE_ETCD_SERVERS \
            $KUBE_API_ADDRESS \
            $KUBE_API_PORT \
            $KUBELET_PORT \
            $KUBE_ALLOW_PRIV \
            $KUBE_SERVICE_ADDRESSES \
            $KUBE_ADMISSION_CONTROL \
            $KUBE_API_ARGS
Restart=on-failure
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

2、修改kube-apiserver配置文件

[root@master ~]# vim /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=http://127.0.0.1:8080"
############################# 将以上默认配置文件修改为以下内容 #############################
KUBE_LOGTOSTDERR="--logtostderr=false"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=https://192.168.10.102:6443"

说明:该配置文件同时被kube-apiserver、kube-controller-manager、kube-scheduler、kubelet、kube-proxy使用。


3、修改 apiserver 文件

[root@master ~]# vim /etc/kubernetes/apiserver
###
# kubernetes system config
#
# The following values are used to configure the kube-apiserver
#
# The address on the local server to listen to.
KUBE_API_ADDRESS="--insecure-bind-address=192.168.10.102"
# The port on the local server to listen on.
# KUBE_API_PORT="--port=8080"
# Port minions listen on
# KUBELET_PORT="--kubelet-port=10250"
# Comma separated list of nodes in the etcd cluster
KUBE_ETCD_SERVERS="--etcd-servers=https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379"
# Address range to use for services
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
# default admission control policies
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
# Add your own!
KUBE_API_ARGS="--authorization-mode=AlwaysAllow --advertise-address=192.168.10.102 --bind-address=192.168.10.102 --secure-port=6443 --insecure-port=8080 --kubelet-https=true --service-node-port-range=1-65535 --storage-backend=etcd3 --etcd-servers=https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379 --token-auth-file=/etc/kubernetes/token.csv --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem --client-ca-file=/etc/kubernetes/ssl/ca.pem --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem --etcd-cafile=/etc/kubernetes/ssl/ca.pem --etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem --etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem --enable-swagger-ui=true --allow-privileged=true --apiserver-count=1 --audit-log-maxage=30 --audit-log-maxbackup=3 --audit-log-maxsize=100 --audit-log-path=/var/log/kubernetes/apiserver/audit.log --log-dir=/var/log/kubernetes --v=0 --event-ttl=1h"

参数说明:

# --admission-control:kuberneres集群的准入控制机制,各控制模块以插件的形式依次生效,集群时必须包含ServiceAccount;

AlwaysAdmit, AlwaysDeny, AlwaysPullImages, DefaultStorageClass, DefaultTolerationSeconds, DenyEscalatingExec, DenyExecOnPrivileged, EventRateLimit,    ExtendedResourceToleration, ImagePolicyWebhook, Initializers, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodPreset, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook # --bind-address:不能为 127.0.0.1;在本地址的6443端口开启https服务,默认值0.0.0.0;

# --insecure-port=0:禁用不安全的http服务,默认开启,端口8080,设置为0禁用;

# --secure-port=6443:https安全端口,默认即6443,0表示禁用;

# --authorization-mode:在安全端口使用 RBAC 授权模式,未通过授权的请求拒绝;

# --service-cluster-ip-range:指定 Service Cluster IP 地址段,该地址段外部路由不可达;

# --service-node-port-range:指定 NodePort 的端口范围;

# --storage-backend:持久化存储类型,v1.6版本后默认即etcd3;

# --enable-swagger-ui:设置为true时,启用swagger-ui网页,可通过apiserver的usl/swagger-ui访问,默认为false;

# --allow-privileged:设置为true时,kubernetes允许在Pod中运行拥有系统特权的容器应用;

# --audit-log-*:审计日志相关;

# --event-ttl:apiserver中各时间保留时间,默认即1h,通常用于审计与追踪;

# --logtostderr:默认为true,输出到stderr,不输出到日志;

# --log-dir:日志目录;

# --v:日志级别


创建日志目录及相关日志文件并授权

[root@master ~]# mkdir -p /var/log/kubernetes/apiserver/
[root@master ~]# touch /var/log/kubernetes/apiserver/audit.log
[root@master ~]# chown -R kube.kube /var/log/kubernetes

重启 apiserver 服务

systemctl daemon-reload
systemctl enable kube-apiserver
systemctl restart kube-apiserver
systemctl status kube-apiserver

4、修改 kube-control-manager.service 服务文件

[root@master ~]# vim /usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/controller-manager
User=kube
ExecStart=/usr/bin/kube-controller-manager \
            $KUBE_LOGTOSTDERR \
            $KUBE_LOG_LEVEL \
            $KUBE_MASTER \
            $KUBE_CONTROLLER_MANAGER_ARGS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

5、修改 controller-manager 配置文件

[root@master ~]# vim /etc/kubernetes/controller-manager
###
# The following values are used to configure the kubernetes controller-manager
# defaults from config and apiserver should be adequate
# Add your own!
KUBE_CONTROLLER_MANAGER_ARGS="--address=127.0.0.1 --service-cluster-ip-range=10.254.0.0/16 --cluster-name=kubernetes --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem --root-ca-file=/etc/kubernetes/ssl/ca.pem --leader-elect=true"

重启 kube-control-manager 服务

systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl restart kube-controller-manager
systemctl status kube-controller-manager

6、修改 kube-scheduler.service 服务文件

[root@master ~]# vim /usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler Plugin
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/scheduler
User=kube
ExecStart=/usr/bin/kube-scheduler \
            $KUBE_LOGTOSTDERR \
            $KUBE_LOG_LEVEL \
            $KUBE_MASTER \
            $KUBE_SCHEDULER_ARGS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

7、修改 scheduler 文件

[root@master ~]# vim /etc/kubernetes/scheduler
###
# kubernetes scheduler config
# default config should be adequate
# Add your own!
KUBE_SCHEDULER_ARGS="--leader-elect=true --address=127.0.0.1"

重启 scheduler 服务

systemctl daemon-reload
systemctl enable kube-scheduler
systemctl restart kube-scheduler
systemctl status kube-scheduler

8、修改修改 kubelet.service 服务文件

[root@master ~]# vim /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/bin/kubelet \
            $KUBE_LOGTOSTDERR \
            $KUBE_LOG_LEVEL \
            $KUBELET_API_SERVER \
            $KUBELET_ADDRESS \
            $KUBELET_PORT \
            $KUBELET_HOSTNAME \
            $KUBE_ALLOW_PRIV \
            $KUBELET_POD_INFRA_CONTAINER \
            $KUBELET_ARGS
Restart=on-failure
[Install]
WantedBy=multi-user.target

9、修改 kubelet 配置文件

[root@master ~]# vim /etc/kubernetes/kubelet
###
# kubernetes kubelet (minion) config
# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
KUBELET_ADDRESS="--address=192.168.10.102"
# The port for the info server to serve on
KUBELET_PORT="--port=10250"
# You may leave this blank to use the actual hostname
KUBELET_HOSTNAME="--hostname-override=192.168.10.102"
# location of the api-server
KUBELET_API_SERVER="--api-servers=https://192.168.10.102:6443"
# pod infrastructure container
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
# Add your own!
###
# kubernetes kubelet (minion) config
# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
KUBELET_ADDRESS="--address=192.168.10.102"
# The port for the info server to serve on
# KUBELET_PORT="--port=10250"
# You may leave this blank to use the actual hostname
KUBELET_HOSTNAME="--hostname-override=192.168.10.102"
# location of the api-server
KUBELET_API_SERVER="--api-servers=https://192.168.10.102:6443"
# pod infrastructure container
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
# Add your own!
KUBELET_ARGS="--pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 --kubeconfig=/etc/kubernetes/kubelet --cert-dir=/etc/kubernetes/ssl --cluster-dns=10.254.0.1 --cluster-domain=cluster.k8s. --allow-privileged=true --serialize-image-pulls=false --logtostderr=false --log-dir=/var/log/kubernetes/kubelet --v=0"
# KUBELET_ARGS="--pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 --kubeconfig=/etc/kubernetes/kubelet --cluster-dns=10.254.0.1 --cluster-domain=cluster.k8s. --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem  --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem --root-ca-file=/etc/kubernetes/ssl/ca.pem --allow-privileged=true --serialize-image-pulls=false --logtostderr=false --log-dir=/var/log/kubernetes/kubelet --v=0"

重启 kubelet 服务

systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet
systemctl status kubelet

10、验证 Master 节点功能

[root@master kubernetes]# kubectl get cs
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler            Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}   
etcd-4               Healthy   {"health":"true"}   
etcd-3               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}

7.png

如果出现以下错误:

[root@master kubernetes]# kubectl get cs
NAME                 STATUS      MESSAGE                                                                                        ERROR
scheduler             Unhealthy   Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: getsockopt: connection refused   
controller-manager         Unhealthy   Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: getsockopt: connection refused   
etcd-2               Healthy     {"health":"true"}                                                                              
etcd-1               Healthy     {"health":"true"}                                                                              
etcd-3               Healthy     {"health":"true"}                                                                              
etcd-0               Healthy     {"health":"true"}                                                                              
etcd-4               Healthy     {"health":"true"}

8.png

则需要将scheduler文件和controller-manager文件中的address修改为127.0.0.1,因为当前kube-apiserver期望scheduler和controller-manager在同一台服务器。

9.png

10.png

此处也可使用 kubectl get cs ,如果报错 Unable to

connect to the server: x509: certificate signed by unknown authority ,则使用上面的方式进行验证,或者做如下操作

从〜/.kube/config中删除嵌入的根证书并运行此config命令:

kubectl config set-cluster ${KUBE_CONTEXT} --insecure-skip-tls-verify=true --server=${KUBE_CONTEXT}

说明:${KUBE_CONTEXT} 此参数可自定义,此处为 kubernetes


十三、所有节点部署 Flannel

所有的node节点都需要安装Flannel网络插件,才能让所有的Pod加入到同一个局域网中,所以下面的操作在所有节点都需要执行一遍。建议直接使用yum安装flanneld,除非对版本有特殊需求,默认安装的是0.7.1版本的flannel。


1、yum 安装 flannel 插件

[root@master ~]# yum -y install flannel

2、修改 flanneld.service 服务启动文件,加入下面标红的两

[root@master ~]# vim /usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/etc/sysconfig/flanneld
EnvironmentFile=-/etc/sysconfig/docker-network
ExecStart=/usr/bin/flanneld-start -etcd-endpoints=${FLANNEL_ETCD_ENDPOINTS} -etcd-prefix=${FLANNEL_ETCD_PREFIX} $FLANNEL_OPTIONS
ExecStartPost=/usr/libexec/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure
[Install]
WantedBy=multi-user.target
WantedBy=docker.service

3、修改 /etc/sysconfig/flanneld 配置文件

[root@master ~]# vim /etc/sysconfig/flanneld
# Flanneld configuration options  
# etcd url location.  Point this to the server where etcd runs
FLANNEL_ETCD_ENDPOINTS="https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379"
# etcd config key.  This is the configuration key that flannel queries
# For address range assignment
FLANNEL_ETCD_PREFIX="/atomic.io/network"
# FLANNEL_ETCD_PREFIX="/kube-centos/network"
# Any additional options that you want to pass
FLANNEL_OPTIONS="-etcd-cafile=/etc/kubernetes/ssl/ca.pem -etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem -etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem"

注意:如果是多网卡(例如vagran环境),则需要在 FLANNEL_OPENTION 中添加指定的外网出口的网卡,例如iface=eth0


4、在 etcd 中创建网络配置(这个配置只在 master 节点操作即可)

[root@master ~]# etcdctl --endpoints https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379 \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/kubernetes/ssl/kubernetes.pem \
--key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
mkdir /kube-centos/network
[root@master ~]# etcdctl --endpoints https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379 \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/kubernetes/ssl/kubernetes.pem \
--key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
mk /atomic.io/network/config '{"Network":"10.254.0.0/16"}'

说明:如果此处想使用 vxlan ,在可以直接将 host-gw 修改为 vxlan 即可。


5、启动 flannel

systemctl daemon-reload
systemctl enable flanneld
systemctl start flanneld
systemctl status flanneld

6、查看flannel状态

[root@master ~]# systemctl status flanneld

11.png

如果有如上图提示,证明配置成功

十四、部署 node 节点

到此为止我们已经完成了Master节点服务、etcd集群、flannel集群都已经搭建完成,下面我们来部署 node 节点的服务。首先需要确认下node节点的flannel、docker、etcd是否启动,其次检查下/etc/kubernetes/下的证书和配置文件是否在,具体操作这里就不再赘述了。


1、修改 docker 配置,使其可以使用 flannel 网络

使用systemctl命令启动flanneld后,会自动执行./mk-docker-opts.sh -i在/run/flannel/目录下生成如下两个文件环境变量文件:

[root@master ~]# ll /run/flannel/
总用量 8
-rw-r--r-- 1 root root 174 7月  22 20:44 docker
-rw-r--r-- 1 root root  98 7月  22 20:44 subnet.env

Docker将会读取这两个环境变量文件作为容器启动参数,修改 docker 的配置文件 /usr/lib/systemd/system/docker.service,增加一条环境变量配置:

EnvironmentFile=-/run/flannel/docker

3、安装 conntrack

[root@master ~]# yum -y install conntrack-tools

4、修改 kube-proxy 的 service 配置文件 /usr/lib/systemd/system/kube-proxy.service

[root@master ~]# vim /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/proxy
ExecStart=/usr/bin/kube-proxy \
            $KUBE_LOGTOSTDERR \
            $KUBE_LOG_LEVEL \
            $KUBE_MASTER \
            $KUBE_PROXY_ARGS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

5、修改 kube-proxy 配置文件 /etc/kubernetes/proxy ,在每个节点都进行修改,这里只记录一条,其余的修改IP即可

[root@master ~]# vim /etc/kubernetes/proxy
KUBE_PROXY_ARGS="--bind-address=192.168.10.102 --hostname-override=192.168.10.102 --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig --cluster-cidr=10.254.0.0/16"

重启 kube-proxy 服务

systemctl daemon-reload
systemctl enable kube-proxy
systemctl restart kube-proxy
systemctl status kube-proxy

重启 docker 服务

systemctl daemon-reload
systemctl enable docker
systemctl restart docker
systemctl status docker

6、集群测试

[root@master ~]# kubectl --insecure-skip-tls-verify=true run nginx --replicas=2 --labels="run=load-balancer-example" --image=nginx  --port=80
deployment "nginx" created
[root@master ~]# kubectl expose deployment nginx --type=NodePort --name=example-service
service "example-service" exposed
[root@master kubernetes]# kubectl describe svc example-service
Name:example-service
Namespace:default
Labels:run=load-balancer-example
Selector:run=load-balancer-example
Type:NodePort
IP:10.254.253.193
Port:<unset>80/TCP
NodePort:<unset>57593/TCP
Endpoints:<none>
Session Affinity:None
No events.

12.png

spacer.gif