diff --git a/csi-driver-nfs/csi-nfs-controller.yaml b/csi-driver-nfs/csi-nfs-controller.yaml new file mode 100644 index 0000000..bca5d22 --- /dev/null +++ b/csi-driver-nfs/csi-nfs-controller.yaml @@ -0,0 +1,197 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-nfs-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: csi-nfs-controller + template: + metadata: + labels: + app: csi-nfs-controller + spec: + hostNetwork: true # controller also needs to mount nfs to create dir + dnsPolicy: ClusterFirstWithHostNet # available values: Default, ClusterFirstWithHostNet, ClusterFirst + serviceAccountName: csi-nfs-controller-sa + nodeSelector: + kubernetes.io/os: linux # add "kubernetes.io/role: master" to run controller on master node + priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v5.3.0 + args: + - "-v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace=$(POD_NAMESPACE)" + - "--extra-create-metadata=true" + - "--feature-gates=HonorPVReclaimPolicy=true" + - "--timeout=1200s" + - "--retry-interval-max=30m" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + capabilities: + drop: + - ALL + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.14.0 + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - "--leader-election-namespace=$(POD_NAMESPACE)" + - '-handle-volume-inuse-error=false' + env: + - name: ADDRESS + value: /csi/csi.sock + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + capabilities: + drop: + - ALL + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.3.0 + args: + - "--v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election-namespace=$(POD_NAMESPACE)" + - "--leader-election" + - "--timeout=1200s" + - "--retry-interval-max=30m" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + capabilities: + drop: + - ALL + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.17.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --http-endpoint=localhost:29652 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + capabilities: + drop: + - ALL + - name: nfs + image: registry.k8s.io/sig-storage/nfsplugin:v4.12.1 + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + drop: + - ALL + allowPrivilegeEscalation: true + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + livenessProbe: + failureThreshold: 5 + httpGet: + host: localhost + path: /healthz + port: 29652 + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + volumeMounts: + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: socket-dir + emptyDir: {} diff --git a/csi-driver-nfs/csi-nfs-driverinfo.yaml b/csi-driver-nfs/csi-nfs-driverinfo.yaml new file mode 100644 index 0000000..ce1f04f --- /dev/null +++ b/csi-driver-nfs/csi-nfs-driverinfo.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: nfs.csi.k8s.io +spec: + attachRequired: false + volumeLifecycleModes: + - Persistent + fsGroupPolicy: File diff --git a/csi-driver-nfs/csi-nfs-node.yaml b/csi-driver-nfs/csi-nfs-node.yaml new file mode 100644 index 0000000..b15b0fd --- /dev/null +++ b/csi-driver-nfs/csi-nfs-node.yaml @@ -0,0 +1,134 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-nfs-node + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-nfs-node + template: + metadata: + labels: + app: csi-nfs-node + spec: + hostNetwork: true # original nfs connection would be broken without hostNetwork setting + dnsPolicy: ClusterFirstWithHostNet # available values: Default, ClusterFirstWithHostNet, ClusterFirst + serviceAccountName: csi-nfs-node-sa + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: "Exists" + containers: + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.17.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --http-endpoint=localhost:29653 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + capabilities: + drop: + - ALL + - name: node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.15.0 + args: + - --v=2 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + securityContext: + capabilities: + drop: + - ALL + - name: nfs + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + drop: + - ALL + allowPrivilegeEscalation: true + image: registry.k8s.io/sig-storage/nfsplugin:v4.12.1 + args: + - "-v=5" + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + livenessProbe: + failureThreshold: 5 + httpGet: + host: localhost + path: /healthz + port: 29653 + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + resources: + limits: + memory: 300Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir diff --git a/csi-driver-nfs/kustomization.yaml b/csi-driver-nfs/kustomization.yaml new file mode 100644 index 0000000..f4112df --- /dev/null +++ b/csi-driver-nfs/kustomization.yaml @@ -0,0 +1,6 @@ +resources: + - ./csi-nfs-controller.yaml + - ./csi-nfs-driverinfo.yaml + - ./csi-nfs-node.yaml + - ./rbac-csi-nfs.yaml + - ./storageclass.yaml \ No newline at end of file diff --git a/csi-driver-nfs/rbac-csi-nfs.yaml b/csi-driver-nfs/rbac-csi-nfs.yaml new file mode 100644 index 0000000..21e36ef --- /dev/null +++ b/csi-driver-nfs/rbac-csi-nfs.yaml @@ -0,0 +1,102 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nfs-controller-sa + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nfs-node-sa + namespace: kube-system +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-nfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: nfs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-external-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-csi-resizer-role +subjects: + - kind: ServiceAccount + name: csi-nfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: nfs-external-resizer-role + apiGroup: rbac.authorization.k8s.io diff --git a/csi-driver-nfs/storageclass.yaml b/csi-driver-nfs/storageclass.yaml new file mode 100644 index 0000000..73465a8 --- /dev/null +++ b/csi-driver-nfs/storageclass.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: nfs-csi +provisioner: nfs.csi.k8s.io +parameters: + server: 10.0.0.57 + share: /volume1/docker/ontime-k8s + # csi.storage.k8s.io/provisioner-secret is only needed for providing mountOptions in DeleteVolume + # csi.storage.k8s.io/provisioner-secret-name: "mount-options" + # csi.storage.k8s.io/provisioner-secret-namespace: "default" +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - nfsvers=4.1 diff --git a/kustomization.yaml b/kustomization.yaml index 5b110ce..b606445 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -1,3 +1,4 @@ resources: - ./ingress-nginx.yaml - - ./ontime-operator \ No newline at end of file + - ./ontime-operator + - ./csi-driver-nfs \ No newline at end of file