From 44da0c102282d8c5da2bc022c7e2824a35f23b14 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Wed, 27 Dec 2023 14:23:29 -0500 Subject: [PATCH] feat!: replace rancher/local-path-provisioner with democratic-csi in local-hostpath mode Signed-off-by: Devin Buhl --- .github/renovate.json5 | 10 -- README.md | 2 +- .../addons/grafana/app/helmrelease.yaml.j2 | 2 +- .../app/helmvalues.yaml.j2 | 2 +- .../inventory/group_vars/master/main.yaml.j2 | 2 +- .../app/helmrelease.yaml.j2 | 107 +++++++++--------- .../repositories/git/kustomization.yaml.j2 | 3 +- .../git/local-path-provisioner.yaml.j2 | 16 --- .../repositories/helm/democratic-csi.yaml.j2 | 10 ++ .../repositories/helm/kustomization.yaml.j2 | 1 + .../flux/repositories/kustomization.yaml.j2 | 2 +- .../kubernetes/flux/repositories/oci/.gitkeep | 0 .../repositories/oci/kustomization.yaml.j2 | 4 + bootstrap/vars/config.sample.yaml | 6 +- 14 files changed, 78 insertions(+), 89 deletions(-) delete mode 100644 bootstrap/templates/kubernetes/flux/repositories/git/local-path-provisioner.yaml.j2 create mode 100644 bootstrap/templates/kubernetes/flux/repositories/helm/democratic-csi.yaml.j2 delete mode 100644 bootstrap/templates/kubernetes/flux/repositories/oci/.gitkeep create mode 100644 bootstrap/templates/kubernetes/flux/repositories/oci/kustomization.yaml.j2 diff --git a/.github/renovate.json5 b/.github/renovate.json5 index a45c5beb..be552301 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -67,16 +67,6 @@ }, "separateMinorPatch": true }, - { - "description": "Local Path Provisioner Group", - "groupName": "Local Path Provisioner", - "matchPackagePatterns": ["local-path-provisioner"], - "matchDatasources": ["docker", "github-tags"], - "group": { - "commitMessageTopic": "{{{groupName}}} group" - }, - "separateMinorPatch": true - }, { "description": "System Upgrade Controller Group", "groupName": "System Upgrade Controller", diff --git a/README.md b/README.md index dc89dfbd..c696822e 100644 --- a/README.md +++ b/README.md @@ -502,7 +502,7 @@ To browse or get ideas on applications people are running, community member [@wh #### Storage -The included CSI (`local-path-provisioner`) is a great start for storage but soon you might find you need more features like replicated block storage, or to connect to a NFS/SMB/iSCSI server. If you need any of those features be sure to check out the projects like [rook-ceph](https://github.com/rook/rook), [longhorn](https://github.com/longhorn/longhorn), [openebs](https://github.com/openebs/openebs), [democratic-csi](https://github.com/democratic-csi/democratic-csi), [csi-driver-nfs](https://github.com/kubernetes-csi/csi-driver-nfs), +The included CSI (democratic-csi in local-hostpath mode) is a great start for storage but soon you might find you need more features like replicated block storage, or to connect to a NFS/SMB/iSCSI server. If you need any of those features be sure to check out the projects like [rook-ceph](https://github.com/rook/rook), [longhorn](https://github.com/longhorn/longhorn), [openebs](https://github.com/openebs/openebs), [democratic-csi](https://github.com/democratic-csi/democratic-csi), [csi-driver-nfs](https://github.com/kubernetes-csi/csi-driver-nfs), and [synology-csi](https://github.com/SynologyOpenSource/synology-csi). #### Authenticate Flux over SSH diff --git a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 index 76083767..db807b31 100644 --- a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 @@ -168,6 +168,6 @@ spec: - *host persistence: enabled: true - storageClassName: local-path + storageClassName: local-hostpath testFramework: enabled: false diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 index 60a40edc..645df885 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 @@ -116,7 +116,7 @@ data: storageSpec: volumeClaimTemplate: spec: - storageClassName: local-path + storageClassName: local-hostpath resources: requests: storage: 10Gi diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 index ed384310..a2739b91 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 @@ -17,7 +17,7 @@ k3s_server: disable: - coredns # Disable coredns - replaced with Coredns Helm Chart - flannel # Disable flannel - replaced with Cilium Helm Chart - - local-storage # Disable local-path-provisioner - installed with Flux + - local-storage # Disable local-path-provisioner - replaced with democratic-csi - metrics-server # Disable metrics-server - installed with Flux - servicelb # Disable servicelb - replaced with Cilium Helm Chart - traefik # Disable traefik - replaced with ingress-nginx and installed with Flux diff --git a/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 index b9453d8c..d6415c71 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2 @@ -7,12 +7,12 @@ spec: interval: 30m chart: spec: - chart: ./deploy/chart/local-path-provisioner + chart: democratic-csi + version: 0.14.3 sourceRef: - kind: GitRepository - name: local-path-provisioner + name: democratic-csi + kind: HelmRepository namespace: flux-system - maxHistory: 2 install: remediation: retries: 3 @@ -23,51 +23,54 @@ spec: uninstall: keepHistory: false values: - image: - repository: docker.io/rancher/local-path-provisioner - tag: v0.0.26 - helperImage: - repository: public.ecr.aws/docker/library/busybox - tag: latest - storageClass: - defaultClass: false - nodePathMap: - - node: DEFAULT_PATH_FOR_NON_LISTED_NODES - paths: ["{{ bootstrap_local_path_provisioner_path | default('/var/lib/rancher/k3s/storage', true) }}"] - # NOTE: Do not enable Flux variable substitution on this HelmRelease - configmap: - setup: |- - #!/bin/sh - while getopts "m:s:p:" opt - do - case $opt in - p) - absolutePath=$OPTARG - ;; - s) - sizeInBytes=$OPTARG - ;; - m) - volMode=$OPTARG - ;; - esac - done - mkdir -m 0777 -p ${absolutePath} - chmod 701 ${absolutePath}/.. - teardown: |- - #!/bin/sh - while getopts "m:s:p:" opt - do - case $opt in - p) - absolutePath=$OPTARG - ;; - s) - sizeInBytes=$OPTARG - ;; - m) - volMode=$OPTARG - ;; - esac - done - rm -rf ${absolutePath} + fullnameOverride: local-path-provisioner + controller: + strategy: node + externalProvisioner: + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.3 + extraArgs: + - --leader-election=false + - --node-deployment=true + - --node-deployment-immediate-binding=false + - --feature-gates=Topology=true + - --strict-topology=true + - --enable-capacity=true + - --capacity-ownerref-level=1 + externalResizer: + enabled: false + externalAttacher: + enabled: false + externalSnapshotter: + enabled: false + csiDriver: + name: local-hostpath.cluster.local + storageCapacity: true + attachRequired: false + fsGroupPolicy: File + storageClasses: + - name: local-hostpath + defaultClass: false + reclaimPolicy: Delete + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + driver: + config: + driver: local-hostpath + local-hostpath: + shareBasePath: &storagePath "{{ bootstrap_local_path_provisioner_path | default('/var/lib/rancher/k3s/local-hostpath', true) }}" + controllerBasePath: *storagePath + dirPermissionsMode: "0770" + dirPermissionsUser: 0 + dirPermissionsGroup: 0 + node: + driver: + image: ghcr.io/democratic-csi/democratic-csi:v1.8.4 + extraVolumeMounts: + - name: local-hostpath + mountPath: *storagePath + mountPropagation: Bidirectional + extraVolumes: + - name: local-hostpath + hostPath: + path: *storagePath + type: DirectoryOrCreate diff --git a/bootstrap/templates/kubernetes/flux/repositories/git/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/git/kustomization.yaml.j2 index bb78ecf0..fe0f332a 100644 --- a/bootstrap/templates/kubernetes/flux/repositories/git/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/repositories/git/kustomization.yaml.j2 @@ -1,5 +1,4 @@ --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: - - ./local-path-provisioner.yaml +resources: [] diff --git a/bootstrap/templates/kubernetes/flux/repositories/git/local-path-provisioner.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/git/local-path-provisioner.yaml.j2 deleted file mode 100644 index 25d4b147..00000000 --- a/bootstrap/templates/kubernetes/flux/repositories/git/local-path-provisioner.yaml.j2 +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: source.toolkit.fluxcd.io/v1 -kind: GitRepository -metadata: - name: local-path-provisioner - namespace: flux-system -spec: - interval: 30m - url: https://github.com/rancher/local-path-provisioner - ref: - tag: v0.0.26 - ignore: | - # exclude all - /* - # include kubernetes directory - !/deploy/chart/local-path-provisioner diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/democratic-csi.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/democratic-csi.yaml.j2 new file mode 100644 index 00000000..63091fd9 --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/democratic-csi.yaml.j2 @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1beta2.json +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: democratic-csi + namespace: flux-system +spec: + interval: 1h + url: https://democratic-csi.github.io/charts/ diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 index df330874..52bb16d6 100644 --- a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 @@ -8,6 +8,7 @@ resources: - ./cilium.yaml - ./coredns.yaml - ./csi-driver-nfs.yaml + - ./democratic-csi.yaml - ./external-dns.yaml - ./grafana.yaml - ./hajimari.yaml diff --git a/bootstrap/templates/kubernetes/flux/repositories/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/kustomization.yaml.j2 index cb57fda1..d158d426 100644 --- a/bootstrap/templates/kubernetes/flux/repositories/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/repositories/kustomization.yaml.j2 @@ -4,4 +4,4 @@ kind: Kustomization resources: - ./git - ./helm - # - ./oci + - ./oci diff --git a/bootstrap/templates/kubernetes/flux/repositories/oci/.gitkeep b/bootstrap/templates/kubernetes/flux/repositories/oci/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/bootstrap/templates/kubernetes/flux/repositories/oci/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/oci/kustomization.yaml.j2 new file mode 100644 index 00000000..fe0f332a --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/oci/kustomization.yaml.j2 @@ -0,0 +1,4 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [] diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index 1a62f2cb..73a595a9 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -62,10 +62,8 @@ bootstrap_cluster_cidr: 10.42.0.0/16 # (Advanced) For ipv6 use format 10.43.0.0/16,fd78:c889:47fb:e0::/112 bootstrap_service_cidr: 10.43.0.0/16 -# (Advanced) Path for local-path-provisioner to use for storage on ALL nodes -# If changing this value to anything other than the default -# make sure the path exists and is owned by root:root with 701 permissions -bootstrap_local_path_provisioner_path: /var/lib/rancher/k3s/storage +# (Advanced) Path for democratic-csi local-hostpath to use for storage on ALL nodes +bootstrap_local_path_provisioner_path: /var/lib/rancher/k3s/local-hostpath # Node information bootstrap_nodes: