add csi nfs driver and ontime operator

clusters/ontime-dev/configs/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - storageclass.yaml

clusters/ontime-dev/infrastructure.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: infrastructure
+  namespace: flux-system
+spec:
+  interval: 1h
+  retryInterval: 1m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./infrastructure
+  prune: true
+  wait: true

csi-driver-nfs/csi-nfs-controller.yaml

@@ -1,197 +0,0 @@
----
-kind: Deployment
-apiVersion: apps/v1
-metadata:
-  name: csi-nfs-controller
-  namespace: kube-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: csi-nfs-controller
-  template:
-    metadata:
-      labels:
-        app: csi-nfs-controller
-    spec:
-      hostNetwork: true  # controller also needs to mount nfs to create dir
-      dnsPolicy: ClusterFirstWithHostNet  # available values: Default, ClusterFirstWithHostNet, ClusterFirst
-      serviceAccountName: csi-nfs-controller-sa
-      nodeSelector:
-        kubernetes.io/os: linux  # add "kubernetes.io/role: master" to run controller on master node
-      priorityClassName: system-cluster-critical
-      securityContext:
-        seccompProfile:
-          type: RuntimeDefault
-      tolerations:
-        - key: "node-role.kubernetes.io/master"
-          operator: "Exists"
-          effect: "NoSchedule"
-        - key: "node-role.kubernetes.io/controlplane"
-          operator: "Exists"
-          effect: "NoSchedule"
-        - key: "node-role.kubernetes.io/control-plane"
-          operator: "Exists"
-          effect: "NoSchedule"
-        - key: "CriticalAddonsOnly"
-          operator: "Exists"
-          effect: "NoSchedule"
-      containers:
-        - name: csi-provisioner
-          image: registry.k8s.io/sig-storage/csi-provisioner:v5.3.0
-          args:
-            - "-v=2"
-            - "--csi-address=$(ADDRESS)"
-            - "--leader-election"
-            - "--leader-election-namespace=$(POD_NAMESPACE)"
-            - "--extra-create-metadata=true"
-            - "--feature-gates=HonorPVReclaimPolicy=true"
-            - "--timeout=1200s"
-            - "--retry-interval-max=30m"
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          volumeMounts:
-            - mountPath: /csi
-              name: socket-dir
-          resources:
-            limits:
-              memory: 400Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-          securityContext:
-            capabilities:
-              drop:
-                - ALL
-        - name: csi-resizer
-          image: registry.k8s.io/sig-storage/csi-resizer:v1.14.0
-          args:
-            - "-csi-address=$(ADDRESS)"
-            - "-v=2"
-            - "-leader-election"
-            - "--leader-election-namespace=$(POD_NAMESPACE)"
-            - '-handle-volume-inuse-error=false'
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-          resources:
-            limits:
-              memory: 400Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-          securityContext:
-            capabilities:
-              drop:
-                - ALL
-        - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.3.0
-          args:
-            - "--v=2"
-            - "--csi-address=$(ADDRESS)"
-            - "--leader-election-namespace=$(POD_NAMESPACE)"
-            - "--leader-election"
-            - "--timeout=1200s"
-            - "--retry-interval-max=30m"
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-          resources:
-            limits:
-              memory: 200Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-          securityContext:
-            capabilities:
-              drop:
-                - ALL
-        - name: liveness-probe
-          image: registry.k8s.io/sig-storage/livenessprobe:v2.17.0
-          args:
-            - --csi-address=/csi/csi.sock
-            - --probe-timeout=3s
-            - --http-endpoint=localhost:29652
-            - --v=2
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-          resources:
-            limits:
-              memory: 100Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-          securityContext:
-            capabilities:
-              drop:
-                - ALL
-        - name: nfs
-          image: registry.k8s.io/sig-storage/nfsplugin:v4.12.1
-          securityContext:
-            privileged: true
-            capabilities:
-              add: ["SYS_ADMIN"]
-              drop:
-                - ALL
-            allowPrivilegeEscalation: true
-          imagePullPolicy: IfNotPresent
-          args:
-            - "-v=5"
-            - "--nodeid=$(NODE_ID)"
-            - "--endpoint=$(CSI_ENDPOINT)"
-          env:
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi.sock
-          livenessProbe:
-            failureThreshold: 5
-            httpGet:
-              host: localhost
-              path: /healthz
-              port: 29652
-            initialDelaySeconds: 30
-            timeoutSeconds: 10
-            periodSeconds: 30
-          volumeMounts:
-            - name: pods-mount-dir
-              mountPath: /var/lib/kubelet/pods
-              mountPropagation: "Bidirectional"
-            - mountPath: /csi
-              name: socket-dir
-          resources:
-            limits:
-              memory: 200Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-      volumes:
-        - name: pods-mount-dir
-          hostPath:
-            path: /var/lib/kubelet/pods
-            type: Directory
-        - name: socket-dir
-          emptyDir: {}

csi-driver-nfs/csi-nfs-driverinfo.yaml

@@ -1,10 +0,0 @@
----
-apiVersion: storage.k8s.io/v1
-kind: CSIDriver
-metadata:
-  name: nfs.csi.k8s.io
-spec:
-  attachRequired: false
-  volumeLifecycleModes:
-    - Persistent
-  fsGroupPolicy: File

csi-driver-nfs/csi-nfs-node.yaml

@@ -1,134 +0,0 @@
----
-kind: DaemonSet
-apiVersion: apps/v1
-metadata:
-  name: csi-nfs-node
-  namespace: kube-system
-spec:
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate
-  selector:
-    matchLabels:
-      app: csi-nfs-node
-  template:
-    metadata:
-      labels:
-        app: csi-nfs-node
-    spec:
-      hostNetwork: true  # original nfs connection would be broken without hostNetwork setting
-      dnsPolicy: ClusterFirstWithHostNet  # available values: Default, ClusterFirstWithHostNet, ClusterFirst
-      serviceAccountName: csi-nfs-node-sa
-      priorityClassName: system-node-critical
-      securityContext:
-        seccompProfile:
-          type: RuntimeDefault
-      nodeSelector:
-        kubernetes.io/os: linux
-      tolerations:
-        - operator: "Exists"
-      containers:
-        - name: liveness-probe
-          image: registry.k8s.io/sig-storage/livenessprobe:v2.17.0
-          args:
-            - --csi-address=/csi/csi.sock
-            - --probe-timeout=3s
-            - --http-endpoint=localhost:29653
-            - --v=2
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-          resources:
-            limits:
-              memory: 100Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-          securityContext:
-            capabilities:
-              drop:
-                - ALL
-        - name: node-driver-registrar
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.15.0
-          args:
-            - --v=2
-            - --csi-address=/csi/csi.sock
-            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
-          env:
-            - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
-            - name: KUBE_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-            - name: registration-dir
-              mountPath: /registration
-          resources:
-            limits:
-              memory: 100Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-          securityContext:
-            capabilities:
-              drop:
-                - ALL
-        - name: nfs
-          securityContext:
-            privileged: true
-            capabilities:
-              add: ["SYS_ADMIN"]
-              drop:
-                - ALL
-            allowPrivilegeEscalation: true
-          image: registry.k8s.io/sig-storage/nfsplugin:v4.12.1
-          args:
-            - "-v=5"
-            - "--nodeid=$(NODE_ID)"
-            - "--endpoint=$(CSI_ENDPOINT)"
-          env:
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi.sock
-          livenessProbe:
-            failureThreshold: 5
-            httpGet:
-              host: localhost
-              path: /healthz
-              port: 29653
-            initialDelaySeconds: 30
-            timeoutSeconds: 10
-            periodSeconds: 30
-          imagePullPolicy: "IfNotPresent"
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-            - name: pods-mount-dir
-              mountPath: /var/lib/kubelet/pods
-              mountPropagation: "Bidirectional"
-          resources:
-            limits:
-              memory: 300Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-      volumes:
-        - name: socket-dir
-          hostPath:
-            path: /var/lib/kubelet/plugins/csi-nfsplugin
-            type: DirectoryOrCreate
-        - name: pods-mount-dir
-          hostPath:
-            path: /var/lib/kubelet/pods
-            type: Directory
-        - hostPath:
-            path: /var/lib/kubelet/plugins_registry
-            type: Directory
-          name: registration-dir

csi-driver-nfs/kustomization.yaml

@@ -1,6 +0,0 @@
-resources:
-    - ./csi-nfs-controller.yaml 
-    - ./csi-nfs-driverinfo.yaml 
-    - ./csi-nfs-node.yaml 
-    - ./rbac-csi-nfs.yaml 
-    - ./storageclass.yaml

csi-driver-nfs/rbac-csi-nfs.yaml

@@ -1,102 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: csi-nfs-controller-sa
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: csi-nfs-node-sa
-  namespace: kube-system
----
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: nfs-external-provisioner-role
-rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "patch", "delete"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotclasses", "volumesnapshots"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents/status"]
-    verbs: ["get", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["get", "list", "watch", "create", "update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["csinodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "list", "watch", "create", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get"]
----
-
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: nfs-csi-provisioner-binding
-subjects:
-  - kind: ServiceAccount
-    name: csi-nfs-controller-sa
-    namespace: kube-system
-roleRef:
-  kind: ClusterRole
-  name: nfs-external-provisioner-role
-  apiGroup: rbac.authorization.k8s.io
----
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: nfs-external-resizer-role
-rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["list", "watch", "create", "update", "patch"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "list", "watch", "create", "update", "patch"]
----
-
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: nfs-csi-resizer-role
-subjects:
-  - kind: ServiceAccount
-    name: csi-nfs-controller-sa
-    namespace: kube-system
-roleRef:
-  kind: ClusterRole
-  name: nfs-external-resizer-role
-  apiGroup: rbac.authorization.k8s.io

infrastructure/csi-driver-nfs/csi-driver-nfs.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: csi-driver-nfs
+  namespace: kube-system
+spec:
+  interval: 24h
+  url: https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: csi-driver-nfs
+  namespace: kube-system
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: csi-driver-nfs
+      version: "4.12.1"
+      sourceRef:
+        kind: HelmRepository
+        name: csi-driver-nfs
+        namespace: kube-system
+      interval: 12h

infrastructure/ontime-operator/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+    - ./deploy.yaml

ontime-operator/kustomization.yaml

@@ -1,2 +0,0 @@
-resources:
-    - ./deploy.yaml