╰ kubectl get pv --all-namespaces -o wide
No resources found
╰ kubectl get pvc --all-namespaces -o wide
NAMESPACE   NAME                            STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   AGE    VOLUMEMODE
default     rbd-pvc                         Pending                                      csi-rbd-sc     90s    Filesystem

I0331 15:32:03.961881       1 controller.go:1199] provision "default/ceph-rbd-pvc" class "ceph-rbd": started
W0331 15:32:03.965791       1 controller.go:443] "fstype" is deprecated and will be removed in a future release, please use "csi.storage.k8s.io/fstype" instead
I0331 15:32:03.965817       1 controller.go:494] CreateVolumeRequest {Name:pvc-22e2ca16-5b0c-4c29-a3ba-9e7d5325c498 CapacityRange:required_bytes:1073741824  VolumeCapabilities:[mount:<fs_type:"ext4" > access_mode:<mode:SINGLE_NODE_WRITER > ] Parameters:map[clusterID:00f829de-9c40-4a0b-b8f2-8e270d3bded7 csi.storage.k8s.io/node-stage-secret-name:csi-rbd-secret csi.storage.k8s.io/node-stage-secret-namespace:default csi.storage.k8s.io/provisioner-secret-name:csi-rbd-secret csi.storage.k8s.io/provisioner-secret-namespace:default fsType:ext4 imageFeatures:layering imageFormat:2 monitors:10.30.3.68:6789,10.30.3.68:6789:6789,10.30.3.68:6789:6789 pool:rbd userId:kubernetes] Secrets:map[] VolumeContentSource:<nil> AccessibilityRequirements:<nil> XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0}
I0331 15:32:03.966014       1 event.go:255] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"default", Name:"ceph-rbd-pvc", UID:"22e2ca16-5b0c-4c29-a3ba-9e7d5325c498", APIVersion:"v1", ResourceVersion:"5971213", FieldPath:""}): type: 'Normal' reason: 'Provisioning' External provisioner is provisioning volume for claim "default/ceph-rbd-pvc"
I0331 15:32:03.969014       1 connection.go:180] GRPC call: /csi.v1.Controller/CreateVolume
I0331 15:32:03.969047       1 connection.go:181] GRPC request: {"capacity_range":{"required_bytes":1073741824},"name":"pvc-22e2ca16-5b0c-4c29-a3ba-9e7d5325c498","parameters":{"clusterID":"00f829de-9c40-4a0b-b8f2-8e270d3bded7","fsType":"ext4","imageFeatures":"layering","imageFormat":"2","monitors":"10.30.3.68:6789,10.30.3.68:6789:6789,10.30.3.68:6789:6789","pool":"rbd","userId":"kubernetes"},"secrets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}}]}
I0331 15:32:05.022841       1 connection.go:183] GRPC response: {}
I0331 15:32:05.023633       1 connection.go:184] GRPC error: rpc error: code = Internal desc = failed to get IOContext: failed to get connection: connecting failed: rados: ret=1, Operation not permitted
I0331 15:32:05.023700       1 controller.go:1016] Final error received, removing PVC 22e2ca16-5b0c-4c29-a3ba-9e7d5325c498 from claims in progress
W0331 15:32:05.023715       1 controller.go:887] Retrying syncing claim "22e2ca16-5b0c-4c29-a3ba-9e7d5325c498", failure 9
E0331 15:32:05.023740       1 controller.go:910] error syncing claim "22e2ca16-5b0c-4c29-a3ba-9e7d5325c498": failed to provision volume with StorageClass "ceph-rbd": rpc error: code = Internal desc = failed to get IOContext: failed to get connection: connecting failed: rados: ret=1, Operation not permitted
I0331 15:32:05.023961       1 event.go:255] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"default", Name:"ceph-rbd-pvc", UID:"22e2ca16-5b0c-4c29-a3ba-9e7d5325c498", APIVersion:"v1", ResourceVersion:"5971213", FieldPath:""}): type: 'Warning' reason: 'ProvisioningFailed' failed to provision volume with StorageClass "ceph-rbd": rpc error: code = Internal desc = failed to get IOContext: failed to get connection: connecting failed: rados: ret=1, Operation not permitted