[root@szy-k8s-master log]# kubectl describe pod prometheus-544bf54848-b97rn --namespace=kube-system
Name: prometheus-544bf54848-b97rn
Namespace: kube-system
Node: szy-k8s-node1/10.10.31.203
Start Time: Tue, 26 Jun 2018 17:05:08 +0800
Labels: app=prometheus
pod-template-hash=1006910404
Annotations: <none>
Status: Pending
IP:
Controlled By: ReplicaSet/prometheus-544bf54848
Containers:
prometheus:
Container ID:
Image: prom/prometheus:latest
Image ID:
Port: 9090/TCP
Host Port: 0/TCP
State: Waiting
Reason: ContainerCreating
Ready: False
Restart Count: 0
Environment: <none>
Mounts:
/etc/prometheus/prometheus.yml from prometheus-conf-volume (rw)
/etc/prometheus/rules from prometheus-rules-volume (rw)
/prometheus from prometheus-data-volume (rw)
/var/run/secrets/kubernetes.io/serviceaccount from prometheus-token-6lmsv (ro)
Conditions:
Type Status
Initialized True
Ready False
PodScheduled True
Volumes:
prometheus-data-volume:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: prometheus-data-pvc
ReadOnly: false
prometheus-conf-volume:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: prometheus-conf
Optional: false
prometheus-rules-volume:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: prometheus-rules
Optional: false
prometheus-token-6lmsv:
Type: Secret (a volume populated by a Secret)
SecretName: prometheus-token-6lmsv
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node-role.kubernetes.io/master:NoSchedule
node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning FailedMount 30m (x348 over 16h) kubelet, szy-k8s-node1 Unable to mount volumes for pod "prometheus-544bf54848-b97rn_kube-system(068ba024-7920-11e8-89d1-00505681317e)": timeout expired waiting for volumes to attach or mount for pod "kube-system"/"prometheus-544bf54848-b97rn". list of unmounted volumes=[prometheus-data-volume]. list of unattached volumes=[prometheus-data-volume prometheus-conf-volume prometheus-rules-volume prometheus-token-6lmsv]
Warning FailedMount 36s (x573 over 16h) kubelet, szy-k8s-node1 (combined from similar events): MountVolume.SetUp failed for volume "prometheus-data-pv" : mount failed: exit status 32
Mounting command: systemd-run
Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/068ba024-7920-11e8-89d1-00505681317e/volumes/kubernetes.io~nfs/prometheus-data-pv --scope -- mount -t nfs 10.10.31.202:/nfs/prometheus/data /var/lib/kubelet/pods/068ba024-7920-11e8-89d1-00505681317e/volumes/kubernetes.io~nfs/prometheus-data-pv
Output: Running scope as unit run-26056.scope.
mount.nfs: requested NFS version or transport protocol is not supported
解决方法：搭建NFS服务
#在master安装
[root@szy-k8s-master /]# yum install -y nfs-utils rpcbind
#在每个node安装
[root@szy-k8s-node1 /]# yum install -y nfs-utils
#在每个node安装
[root@szy-k8s-node2 /]# yum install -y nfs-utils
配置：
#NFS服务的配置文件 /etc/exports。 这个文件可能不会存在，需要新建
[root@szy-k8s-master /]# cat /etc/exports
/nfs/prometheus/data/ 10.10.31.0/24(rw,no_root_squash,no_all_squash,sync)
#exports中的配置的内容，需要创建下/nfs/prometheus/data/
[root@szy-k8s-master /]# mkdir -p /nfs/prometheus/data/
#修改权限
[root@szy-k8s-master /]# chmod -R 777 /nfs/prometheus/data/
#验证配置的/nfs/prometheus/data/是否正确
[root@szy-k8s-master /]# exportfs -r
10.10.31.0/24：这个是运行访问NFS的IP范围，也就是10.10.31开头的IP，24是掩码长度。 根据自己的k8s主机网段设置。
(rw,no_root_squash,no_all_squash,sync)：
可以设定的参数主要有以下这些：
rw：可读写的权限；
ro：只读的权限；
no_root_squash：登入到NFS主机的用户如果是root，该用户即拥有root权限；
root_squash：登入NFS主机的用户如果是root，该用户权限将被限定为匿名使用者nobody；
all_squash：不管登陆NFS主机的用户是何权限都会被重新设定为匿名使用者nobody。
anonuid：将登入NFS主机的用户都设定成指定的user id，此ID必须存在于/etc/passwd中。
anongid：同anonuid，但是变成group ID就是了！
sync：资料同步写入存储器中。
async：资料会先暂时存放在内存中，不会直接写入硬盘。
insecure：允许从这台机器过来的非授权访问。
启动服务：
[root@szy-k8s-master /]# systemctl start rpcbind
[root@szy-k8s-master /]# systemctl status rpcbind
● rpcbind.service - RPC bind service
Loaded: loaded (/usr/lib/systemd/system/rpcbind.service; disabled; vendor preset: enabled)
Active: active (running) since Wed 2018-06-27 11:05:29 CST; 1h 51min ago
Process: 4111 ExecStart=/sbin/rpcbind -w $RPCBIND_ARGS (code=exited, status=0/SUCCESS)
Main PID: 4112 (rpcbind)
Memory: 580.0K
CGroup: /system.slice/rpcbind.service
└─4112 /sbin/rpcbind -w
Jun 27 11:05:29 szy-k8s-master systemd[1]: Starting RPC bind service...
Jun 27 11:05:29 szy-k8s-master systemd[1]: Started RPC bind service.
[root@szy-k8s-master /]# systemctl start nfs
[root@szy-k8s-master /]# systemctl status nfs
● nfs-server.service - NFS server and services
Loaded: loaded (/usr/lib/systemd/system/nfs-server.service; disabled; vendor preset: disabled)
Active: active (exited) since Wed 2018-06-27 11:26:54 CST; 1h 30min ago
Process: 11859 ExecStart=/usr/sbin/rpc.nfsd $RPCNFSDARGS (code=exited, status=0/SUCCESS)
Process: 11853 ExecStartPre=/bin/sh -c /bin/kill -HUP `cat /run/gssproxy.pid` (code=exited, status=0/SUCCESS)
Process: 11851 ExecStartPre=/usr/sbin/exportfs -r (code=exited, status=0/SUCCESS)
Main PID: 11859 (code=exited, status=0/SUCCESS)
Memory: 0B
CGroup: /system.slice/nfs-server.service
Jun 27 11:26:54 szy-k8s-master systemd[1]: Starting NFS server and services...
Jun 27 11:26:54 szy-k8s-master systemd[1]: Started NFS server and services.
检验：
[root@szy-k8s-master /]# showmount -e 10.10.31.202
Export list for 10.10.31.202:
/nfs/prometheus/data 10.10.31.0/24
[root@szy-k8s-master /]#
NFS客户端的操作：
1、showmout命令对于NFS的操作和查错有很大的帮助，所以我们先来看一下showmount的用法
showmout
-a ：这个参数是一般在NFS SERVER上使用，是用来显示已经mount上本机nfs目录的cline机器。
-e ：显示指定的NFS SERVER上export出来的目录。
2、mount nfs目录的方法：
mount -t nfs hostname(orIP):/directory /mount/point
再次查看问题，已经是成功，可挂载了。
[root@szy-k8s-master /]# kubectl describe pod prometheus-544bf54848-b97rn --namespace kube-system
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node-role.kubernetes.io/master:NoSchedule
node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning FailedMount 36m (x664 over 19h) kubelet, szy-k8s-node1 (combined from similar events): MountVolume.SetUp failed for volume "prometheus-data-pv" : mount failed: exit status 32
Mounting command: systemd-run
Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/068ba024-7920-11e8-89d1-00505681317e/volumes/kubernetes.io~nfs/prometheus-data-pv --scope -- mount -t nfs 10.10.31.202:/nfs/prometheus/data /var/lib/kubelet/pods/068ba024-7920-11e8-89d1-00505681317e/volumes/kubernetes.io~nfs/prometheus-data-pv
Output: Running scope as unit run-17572.scope.
mount.nfs: access denied by server while mounting 10.10.31.202:/nfs/prometheus/data
Warning FailedMount 20m (x412 over 19h) kubelet, szy-k8s-node1 Unable to mount volumes for pod "prometheus-544bf54848-b97rn_kube-system(068ba024-7920-11e8-89d1-00505681317e)": timeout expired waiting for volumes to attach or mount for pod "kube-system"/"prometheus-544bf54848-b97rn". list of unmounted volumes=[prometheus-data-volume]. list of unattached volumes=[prometheus-data-volume prometheus-conf-volume prometheus-rules-volume prometheus-token-6lmsv]
Normal SuccessfulMountVolume 15m kubelet, szy-k8s-node1 MountVolume.SetUp succeeded for volume "prometheus-data-pv"
Normal Pulled 10m kubelet, szy-k8s-node1 Successfully pulled image "prom/prometheus:latest"
[root@szy-k8s-master /]#
######挂载NFS报access denied by server :
查看日志信息cat /var/log/messages | grep mount
挂载NFS报access denied by server :
access denied by server while mounting 10.10.31.202:/nfs/grafana/data
[root@szy-k8s-master monitor]# kubectl describe pod grafana-865bdd58bc-bbfrm --namespace kube-system
......
Warning FailedMount 47s (x3 over 5m) kubelet, szy-k8s-node1 Unable to mount volumes for pod "grafana-865bdd58bc-bbfrm_kube-system(a0c7217e-79ca-11e8-89d1-00505681317e)": timeout expired waiting for volumes to attach or mount for pod "kube-system"/"grafana-865bdd58bc-bbfrm". list of unmounted volumes=[grafana-data-volume]. list of unattached volumes=[grafana-data-volume default-token-q8gkz]
#查看日志信息cat /var/log/messages | grep mount
[root@szy-k8s-master monitor]# cat /var/log/messages | grep mount
Jun 22 17:18:18 localhost kernel: XFS (dm-0): Ending clean mount
Jun 22 17:18:20 localhost kernel: XFS (sda1): Ending clean mount
Jun 22 17:18:56 localhost dracut: *** Including module: usrmount ***
Jun 25 15:24:27 szy-k8s-master kernel: XFS (dm-0): Ending clean mount
Jun 25 15:24:28 szy-k8s-master systemd: Started Remount Root and Kernel File Systems.
Jun 25 15:24:29 szy-k8s-master kernel: XFS (sda1): Ending clean mount
Jun 25 15:24:47 szy-k8s-master systemd: Unmounting RPC Pipe File System...
Jun 25 15:25:01 szy-k8s-master kernel: XFS (dm-0): Ending clean mount
Jun 25 15:25:02 szy-k8s-master systemd: Started Remount Root and Kernel File Systems.
Jun 25 15:25:03 szy-k8s-master kernel: XFS (sda1): Ending clean mount
Jun 25 17:31:34 szy-k8s-master systemd: Started Kubernetes transient mount for /var/lib/kubelet/pods/8cd706bc-785a-11e8-87c1-00505681317e/volumes/kubernetes.io~secret/kube-proxy-token-hwpp2.
Jun 25 17:31:34 szy-k8s-master systemd: Starting Kubernetes transient mount for /var/lib/kubelet/pods/8cd706bc-785a-11e8-87c1-00505681317e/volumes/kubernetes.io~secret/kube-proxy-token-hwpp2.
Jun 25 17:37:41 szy-k8s-master systemd: Started Kubernetes transient mount for /var/lib/kubelet/pods/67f013fa-785b-11e8-87c1-00505681317e/volumes/kubernetes.io~secret/flannel-token-mth94.
Jun 25 17:37:41 szy-k8s-master systemd: Starting Kubernetes transient mount for /var/lib/kubelet/pods/67f013fa-785b-11e8-87c1-00505681317e/volumes/kubernetes.io~secret/flannel-token-mth94.
Jun 25 17:37:46 szy-k8s-master systemd: Started Kubernetes transient mount for /var/lib/kubelet/pods/8ca12ba6-785a-11e8-87c1-00505681317e/volumes/kubernetes.io~secret/kube-dns-token-dcl8h.
Jun 25 17:37:46 szy-k8s-master systemd: Starting Kubernetes transient mount for /var/lib/kubelet/pods/8ca12ba6-785a-11e8-87c1-00505681317e/volumes/kubernetes.io~secret/kube-dns-token-dcl8h.
Jun 25 17:49:46 szy-k8s-master systemd: Started Kubernetes transient mount for /var/lib/kubelet/pods/17f3830f-785d-11e8-89d1-00505681317e/volumes/kubernetes.io~secret/kube-proxy-token-j9tc9.
Jun 25 17:49:46 szy-k8s-master systemd: Starting Kubernetes transient mount for /var/lib/kubelet/pods/17f3830f-785d-11e8-89d1-00505681317e/volumes/kubernetes.io~secret/kube-proxy-token-j9tc9.
Jun 25 17:57:56 szy-k8s-master systemd: Started Kubernetes transient mount for /var/lib/kubelet/pods/3beff963-785e-11e8-89d1-00505681317e/volumes/kubernetes.io~secret/flannel-token-4r9f4.
Jun 25 17:57:56 szy-k8s-master systemd: Starting Kubernetes transient mount for /var/lib/kubelet/pods/3beff963-785e-11e8-89d1-00505681317e/volumes/kubernetes.io~secret/flannel-token-4r9f4.
Jun 25 17:58:05 szy-k8s-master systemd: Started Kubernetes transient mount for /var/lib/kubelet/pods/17db1f20-785d-11e8-89d1-00505681317e/volumes/kubernetes.io~secret/kube-dns-token-qt6ch.
Jun 25 17:58:05 szy-k8s-master systemd: Starting Kubernetes transient mount for /var/lib/kubelet/pods/17db1f20-785d-11e8-89d1-00505681317e/volumes/kubernetes.io~secret/kube-dns-token-qt6ch.
Jun 26 16:38:24 szy-k8s-master systemd: Started Kubernetes transient mount for /var/lib/kubelet/pods/4a420702-791c-11e8-89d1-00505681317e/volumes/kubernetes.io~secret/default-token-q8gkz.
Jun 26 16:38:24 szy-k8s-master systemd: Starting Kubernetes transient mount for /var/lib/kubelet/pods/4a420702-791c-11e8-89d1-00505681317e/volumes/kubernetes.io~secret/default-token-q8gkz.
Jun 27 11:26:54 szy-k8s-master rpc.mountd[11849]: Version 1.3.0 starting
Jun 27 11:27:00 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:29:02 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:31:05 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:33:07 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:35:09 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:37:11 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:39:13 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:41:17 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:43:19 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:45:21 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:47:23 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:49:25 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:51:28 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:53:30 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:55:32 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:57:34 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 11:59:36 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:01:38 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:03:41 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:05:43 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:07:45 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:09:47 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:11:49 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:13:51 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:15:54 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:17:56 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:19:58 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:22:00 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:24:02 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:26:04 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:28:06 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:30:09 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:32:11 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/): not exported
Jun 27 12:34:13 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/prometheus/data (/nfs/prometheus/data): unmatched host
Jun 27 13:26:22 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:26:23 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:26:24 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:26:26 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:26:31 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:26:39 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:26:55 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:27:27 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:28:31 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:30:33 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
Jun 27 13:32:35 szy-k8s-master rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported
发现问题：rpc.mountd[11849]: refused mount request from 10.10.31.203 for /nfs/grafana/data (/): not exported.
