基于storageclass动态生成pv
1、创建rbd的供应商 provisioner ceph.com/rbd（环境变量里面的值）
#把rbd-provisioner.tar.gz上传，手动解压，这里面封装的是镜像
[root@master ceph]# docker load -i rbd-provisioner.tar.gz
1d31b5806ba4: Loading layer [==================================================>] 208.3MB/208.3MB
499d93e0e038: Loading layer [==================================================>] 164.1MB/164.1MB
7c9bb3d61493: Loading layer [==================================================>] 44.52MB/44.52MB
Loaded image: quay.io/xianchao/external_storage/rbd-provisioner:v1
[root@master rbd-provisioner]# cat rbd-provisioner.yaml
kind: ClusterRole #定义了一个ClusterRole，可以对哪些资源做操作
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-provisioner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
- apiGroups: [""]
resources: ["services"]
resourceNames: ["kube-dns","coredns"]
verbs: ["list", "get"]
---
kind: ClusterRoleBinding #定义了一个clusterrolebinding，将下面的serviceAccount: rbd-provisioner 绑定
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-provisioner
subjects:
- kind: ServiceAccount
name: rbd-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: rbd-provisioner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: rbd-provisioner
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rbd-provisioner
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rbd-provisioner
subjects:
- kind: ServiceAccount
name: rbd-provisioner
namespace: default
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rbd-provisioner
spec:
selector:
matchLabels:
app: rbd-provisioner
replicas: 1
strategy:
type: Recreate
template:
metadata:
labels:
app: rbd-provisioner
spec:
containers:
- name: rbd-provisioner
image: quay.io/xianchao/external_storage/rbd-provisioner:v1
imagePullPolicy: IfNotPresent
env:
- name: PROVISIONER_NAME #供应商名字
value: ceph.com/rbd
serviceAccount: rbd-provisioner
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rbd-provisioner
[root@master rbd-provisioner]# kubectl get pod
NAME READY STATUS RESTARTS AGE
rbd-provisioner-6bbc95cd74-g6lcd 1/1 Running 0 9s
2、创建ceph-secret
#创建pool池
[root@master1-admin ceph]# ceph osd pool create k8stest1 56
pool 'k8stest1' created
[root@master1-admin ceph]# ceph osd pool ls
rbd
cephfs_data
cephfs_metadata
k8srbd1
k8stest
k8stest1
You have new mail in /var/spool/mail/root
[root@master1-admin ~]# ceph auth get-key client.admin | base64
QVFDOWF4eGhPM0UzTlJBQUJZZnVCMlZISVJGREFCZHN0UGhMc3c9PQ==
[root@master rbd-provisioner]# cat ceph-secret-1.yaml
apiVersion: v1
kind: Secret
metadata:
name: ceph-secret-1
type: "ceph.com/rbd"
data:
key: QVFDOWF4eGhPM0UzTlJBQUJZZnVCMlZISVJGREFCZHN0UGhMc3c9PQ==
[root@master rbd-provisioner]# kubectl get secret
NAME TYPE DATA AGE
ceph-secret Opaque 1 19h
ceph-secret-1 ceph.com/rbd 1 2m33s
default-token-cwbdx kubernetes.io/service-account-token 3 91d
nfs-client-provisioner-token-plww9 kubernetes.io/service-account-token 3 19d
qingcloud kubernetes.io/dockerconfigjson 1 91d
rbd-provisioner-token-82bql kubernetes.io/service-account-token 3 10m
3、创建storageclass
[root@master ceph]# cat rbd-provisioner/storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: k8s-rbd
provisioner: ceph.com/rbd
parameters:
monitors: 192.168.0.5:6789,192.168.0.6:6789,192.168.0.7:6789
adminId: admin
adminSecretName: ceph-secret-1
pool: k8stest1
userId: admin
userSecretName: ceph-secret-1
fsType: xfs
imageFormat: "2"
imageFeatures: "layering"
# pool: k8stest1 这个pool池子是之前创建的
[root@master rbd-provisioner]# kubectl apply -f storageclass.yaml
storageclass.storage.k8s.io/k8s-rbd created
[root@master rbd-provisioner]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
k8s-rbd ceph.com/rbd Delete Immediate false 7s
这个时候存储类在申请pv的时候就会找到ceph.com/rbd供应商，然后通过该供应商从集群里面去划分出来一个pv。
4、创建pvc
invalid AccessModes [ReadWriteMany]: only AccessModes [ReadWriteOnce ReadOnlyMany] are supported(可以看到rbd不支持[ReadWriteMany)
[root@master rbd-provisioner]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ceph-pvc Bound ceph-pv 1Gi RWX 18h
example-local-claim Bound example-pv 5Gi RWO local-storage 63d
rbd-pvc Pending k8s-rbd 5s
[root@master rbd-provisioner]# kubectl describe pvc rbd-pvc
Name: rbd-pvc
Namespace: default
StorageClass: k8s-rbd
Status: Pending
Volume:
Labels: <none>
Annotations: volume.beta.kubernetes.io/storage-provisioner: ceph.com/rbd
Finalizers: [kubernetes.io/pvc-protection]
Capacity:
Access Modes:
VolumeMode: Filesystem
Mounted By: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Provisioning 10s (x2 over 25s) ceph.com/rbd_rbd-provisioner-6bbc95cd74-g6lcd_ee50d24f-015f-11ec-a6a6-9e54668c01e4 External provisioner is provisioning volume for claim "default/rbd-pvc"
Warning ProvisioningFailed 10s (x2 over 25s) ceph.com/rbd_rbd-provisioner-6bbc95cd74-g6lcd_ee50d24f-015f-11ec-a6a6-9e54668c01e4 failed to provision volume with StorageClass "k8s-rbd": invalid AccessModes [ReadWriteMany]: only AccessModes [ReadWriteOnce ReadOnlyMany] are supported
Normal ExternalProvisioning 10s (x3 over 25s) persistentvolume-controller waiting for a volume to be created, either by external provisioner "ceph.com/rbd" or manually created by system administrator
[root@master rbd-provisioner]# kubectl delete -f
ceph-secret-1.yaml rbd-provisioner.yaml rbd-pvc.yaml storageclass.yaml
[root@master rbd-provisioner]# kubectl delete -f rbd-pvc.yaml
persistentvolumeclaim "rbd-pvc" deleted
[root@master rbd-provisioner]# vim rbd-pvc.yaml
[root@master rbd-provisioner]# cat rbd-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: rbd-pvc
spec:
accessModes:
- ReadWriteOnce
volumeMode: Filesystem
resources:
requests:
storage: 1Gi
storageClassName: k8s-rbd
[root@master rbd-provisioner]# kubectl apply -f rbd-pvc.yaml
persistentvolumeclaim/rbd-pvc created
[root@master rbd-provisioner]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ceph-pvc Bound ceph-pv 1Gi RWX 19h
example-local-claim Bound example-pv 5Gi RWO local-storage 63d
rbd-pvc Bound pvc-5eef6286-c89f-421a-88d1-1a6593423087 1Gi RWO k8s-rbd 6s
5、创建pod，挂载pvc
[root@master rbd-provisioner]# cat pod-sto.yaml
apiVersion: v1
kind: Pod
metadata:
labels:
test: rbd-pod
name: ceph-rbd-pod
spec:
containers:
- name: ceph-rbd-nginx
image: nginx
imagePullPolicy: IfNotPresent
volumeMounts:
- name: ceph-rbd
mountPath: /mnt
readOnly: false
volumes:
- name: ceph-rbd
persistentVolumeClaim:
claimName: rbd-pvc
[root@master rbd-provisioner]# kubectl get pod
NAME READY STATUS RESTARTS AGE
ceph-rbd-pod 1/1 Running 0 26s
