一.使用pvc和pv关联ceph集群

1.ceph集群创建镜像设备

[root@ceph141 ~]# rbd create -s 2G yinzhengjie-k8s/rbd-pv01 --image-feature layering,exclusive-lock
[root@ceph141 ~]# 
[root@ceph141 ~]# rbd create -s 4G yinzhengjie-k8s/rbd-pv02 --image-feature layering,exclusive-lock
[root@ceph141 ~]# 
[root@ceph141 ~]# rbd ls -p yinzhengjie-k8s | grep rbd
rbd-pv01
rbd-pv02
[root@ceph141 ~]#

2.查看ceph集群的admin账号的对应的KEY并进行base64编码

[root@ceph141 ~]# grep key /etc/ceph/ceph.client.admin.keyring | awk '{printf "%s", $NF}' | base64 
QVFEakZycGx5dkZDRGhBQXBKZzExMVlNSUdRNi9GL3gvWStxcFE9PQ==
[root@ceph141 ~]#

3.删除默认的sc避免影响实验

[root@master231 pv-pvc]# kubectl get sc
NAME                         PROVISIONER       RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
yinzhengjie-nfs-sc (default)   yinzhengjie/linux   Delete          Immediate           false                  15d
[root@master231 pv-pvc]# 
[root@master231 pv-pvc]# kubectl delete sc --all
storageclass.storage.k8s.io "yinzhengjie-nfs-sc" deleted
[root@master231 pv-pvc]# 
[root@master231 pv-pvc]# kubectl get sc
No resources found
[root@master231 pv-pvc]#

4.编写资源清单

[root@master231 pv-pvc]# cat  01-pv-pvc-deploy-svc-ing-ceph.yaml 
apiVersion: v1
kind: Secret
metadata:
  name: ceph-admin-secret
type: "kubernetes.io/rbd"
data:
  #  指定ceph的admin的KEY，将其进行base64编码，此处需要修改! 
  key: QVFEakZycGx5dkZDRGhBQXBKZzExMVlNSUdRNi9GL3gvWStxcFE9PQ==

---

apiVersion: v1
kind: PersistentVolume
metadata:
  name: yinzhengjie-ceph-01
  labels:
    school: yinzhengjie
spec:
   accessModes:
   - ReadWriteMany
   persistentVolumeReclaimPolicy: Retain
   rbd:
     image: rbd-pv01
     monitors:
     - 10.0.0.141:6789
     - 10.0.0.142:6789
     - 10.0.0.143:6789
     pool: yinzhengjie-k8s
     secretRef:
       name: ceph-admin-secret
     user: admin
   capacity:
     storage: 2Gi

---

apiVersion: v1
kind: PersistentVolume
metadata:
  name: yinzhengjie-ceph-02
  labels:
    school: yinzhengjie
spec:
   accessModes:
   - ReadWriteMany
   persistentVolumeReclaimPolicy: Retain
   rbd:
     image: rbd-pv02
     monitors:
     - 10.0.0.141:6789
     - 10.0.0.142:6789
     - 10.0.0.143:6789
     pool: yinzhengjie-k8s
     secretRef:
       name: ceph-admin-secret
     user: admin
   capacity:
     storage: 3Gi

---

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: yinzhengjie-ceph-pvc-01
spec:
  #  引用指定的pv
  volumeName: yinzhengjie-ceph-01
  accessModes:
  - ReadWriteMany
  resources:
    limits:
       storage: 2Gi
    requests:
       storage: 1Gi

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: deploy-volume-pvc
spec:
  replicas: 1
  selector:
    matchLabels:
      apps: ceph-pvc
  template:
    metadata:
      labels:
        apps: ceph-pvc
    spec:
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: yinzhengjie-ceph-pvc-01
      containers:
      - name: c1
        image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1
        volumeMounts:
        - name: data
          mountPath: /yinzhengjie-data
        ports:
        - containerPort: 80

---

apiVersion: v1
kind: Service
metadata:
  name: svc-ceph-pvc
spec:
  selector:
    apps: ceph-pvc
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80

---

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-ceph-pvc
  annotations:
    #  指定Ingress controller的类型
    kubernetes.io/ingress.class: traefik
spec:
  #  指定Ingress controller的名称
  ingressClassName: mytraefik
  rules:
  - host: v1.yinzhengjie.com
    http:
      paths:
      - backend:
          service:
            name: svc-ceph-pvc
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

[root@master231 pv-pvc]#

5.创建资源

[root@master231 pv-pvc]# kubectl get po,svc,ing,pv,pvc
NAME                                     READY   STATUS    RESTARTS   AGE
pod/deploy-volume-pvc-5db454bd94-7l7c8   1/1     Running   0          23s

NAME                   TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
service/kubernetes     ClusterIP   10.200.0.1      <none>        443/TCP   4h14m
service/svc-ceph-pvc   ClusterIP   10.200.23.116   <none>        80/TCP    23s

NAME                                               CLASS       HOSTS              ADDRESS   PORTS   AGE
ingress.networking.k8s.io/ingress-ceph-pvc         <none>      v1.yinzhengjie.com             80      23s
ingress.networking.k8s.io/yinzhengjie-traefik-apps   mytraefik   v1.yinzhengjie.com             80      11d

NAME                                 CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                           STORAGECLASS   REASON   AGE
persistentvolume/yinzhengjie-ceph-01   2Gi        RWX            Retain           Bound       default/yinzhengjie-ceph-pvc-01                           23s
persistentvolume/yinzhengjie-ceph-02   3Gi        RWX            Retain           Available                                                           23s

NAME                                          STATUS   VOLUME              CAPACITY   ACCESS MODES   STORAGECLASS   AGE
persistentvolumeclaim/yinzhengjie-ceph-pvc-01   Bound    yinzhengjie-ceph-01   2Gi        RWX                           23s
[root@master231 pv-pvc]# 
[root@master231 pv-pvc]# kubectl describe ingress.networking.k8s.io/ingress-ceph-pvc 
Name:             ingress-ceph-pvc
Labels:           <none>
Namespace:        default
Address:          
Default backend:  default-http-backend:80 (<error: endpoints "default-http-backend" not found>)
Rules:
  Host              Path  Backends
  ----              ----  --------
  v1.yinzhengjie.com  
                    /   svc-ceph-pvc:80 (10.100.2.43:80)
Annotations:        kubernetes.io/ingress.class: traefik
Events:             <none>
[root@master231 pv-pvc]#

6.访问测试

http://v1.yinzhengjie.com/

二.k8s使用rbd的动态存储类关联ceph集群

推荐阅读:
    https://github.com/ceph/ceph-csi/tree/release-v3.7/deploy/rbd/kubernetes

1.目录结构如下

[root@master231 rbd]# ll
total 24
-rw-r--r-- 1 root root 454 Nov 15 16:17 ceph-config-map.yaml
-rw-r--r-- 1 root root 392 Nov 15 16:18 csi-config-map.yaml
-rw-r--r-- 1 root root 358 Nov 15 16:20 csi-kms-config-map.yaml
-rw-r--r-- 1 root root 370 Nov 15 16:22 csi-rbd-secret.yaml
drwxr-xr-x 3 root root  17 Nov 15 15:22 deploy
-rw-r--r-- 1 root root 389 Nov 15 16:22 pvc.yaml
-rw-r--r-- 1 root root 833 Nov 15 16:22 storageclass.yaml
[root@master231 rbd]#

2.资源清单列表如下

https://gitee.com/jasonyin2020/cloud-computing-stack/tree/master/kubernetes/projects/ceph-cluster/sc

温馨提示:
  需要根据自己的ceph集群环境，修改对应的集群配置即可。

3.清空default集群的所有资源，(可选操作，主要是看起来方便，生产环境别这么玩，测试环境可以这样搞)

kubectl delete all --all

4.安装rbd的sc

[root@master231 rbd]# pwd
/yinzhengjie/manifests/cloud-computing-stack/linux89/manifests/23-projects/06-ceph/sc/rbd
[root@master231 rbd]# 
[root@master231 rbd]#  kubectl apply -f deploy/rbd/kubernetes/
configmap/ceph-csi-config created
serviceaccount/rbd-csi-nodeplugin created
clusterrole.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-nodeplugin created
serviceaccount/rbd-csi-provisioner created
clusterrole.rbac.authorization.k8s.io/rbd-external-provisioner-runner created
clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role created
role.rbac.authorization.k8s.io/rbd-external-provisioner-cfg created
rolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role-cfg created
service/csi-rbdplugin-provisioner created
deployment.apps/csi-rbdplugin-provisioner created
daemonset.apps/csi-rbdplugin created
service/csi-metrics-rbdplugin created
csidriver.storage.k8s.io/rbd.csi.ceph.com created
[root@master231 rbd]# 
[root@master231 rbd]# 
[root@master231 rbd]#  kubectl apply -f .
configmap/ceph-config created
configmap/ceph-csi-config configured
configmap/ceph-csi-encryption-kms-config created
secret/csi-rbd-secret created
persistentvolumeclaim/rbd-pvc01 created
persistentvolumeclaim/rbd-pvc02 created
storageclass.storage.k8s.io/csi-rbd-sc created
[root@master231 rbd]#

5.查看数据

[root@master231 rbd]# kubectl get sc
NAME         PROVISIONER        RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
csi-rbd-sc   rbd.csi.ceph.com   Delete          Immediate           true                   4m55s
[root@master231 rbd]# 
[root@master231 rbd]# kubectl get po,pv,pvc
NAME                                             READY   STATUS    RESTARTS   AGE
pod/csi-rbdplugin-lqsxt                          3/3     Running   0          50s
pod/csi-rbdplugin-provisioner-5dfcf67885-5m9sj   7/7     Running   0          50s
pod/csi-rbdplugin-provisioner-5dfcf67885-92djh   7/7     Running   0          50s
pod/csi-rbdplugin-wmnm6                          3/3     Running   0          50s

NAME                                                        CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM               STORAGECLASS   REASON   AGE
persistentvolume/pvc-754a5284-5239-43be-99dd-03a5068f4b27   4Gi        RWO            Delete           Bound    default/rbd-pvc02   csi-rbd-sc              12s
persistentvolume/pvc-e6266518-9a17-42ac-aa60-e3c1f18f4696   2Gi        RWO            Delete           Bound    default/rbd-pvc01   csi-rbd-sc              12s

NAME                              STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
persistentvolumeclaim/rbd-pvc01   Bound    pvc-e6266518-9a17-42ac-aa60-e3c1f18f4696   2Gi        RWO            csi-rbd-sc     35s
persistentvolumeclaim/rbd-pvc02   Bound    pvc-754a5284-5239-43be-99dd-03a5068f4b27   4Gi        RWO            csi-rbd-sc     35s
[root@master231 rbd]#

6.验证ceph集群是否动态创建了pv和ceph对应的镜像文件

[root@ceph141 ~]# rbd ls -p yinzhengjie-k8s
csi-vol-027f1235-c26b-11ee-b24f-f65a0eab89b3
csi-vol-027f580e-c26b-11ee-b24f-f65a0eab89b3
nginx-web
rbd-pv01
rbd-pv02
[root@ceph141 ~]#