diff --git a/.github/tests/addons.yaml b/.github/tests/addons.yaml index ad7ac3d4..786cb02b 100644 --- a/.github/tests/addons.yaml +++ b/.github/tests/addons.yaml @@ -35,3 +35,6 @@ volsync: spegel: enabled: true + +longhorn: + enabled: true diff --git a/bootstrap/templates/kubernetes/apps/longhorn-system/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/longhorn-system/kustomization.yaml.j2 new file mode 100644 index 00000000..f4a1d1e3 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/longhorn-system/kustomization.yaml.j2 @@ -0,0 +1,8 @@ +#% if longhorn.enabled | default(false) %# +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./longhorn/ks.yaml +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..e33596e0 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/helmrelease.yaml.j2 @@ -0,0 +1,48 @@ +#% if longhorn.enabled | default(false) %# +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: longhorn +spec: + interval: 30m + chart: + spec: + chart: longhorn + version: 1.5.3 + sourceRef: + kind: HelmRepository + name: longhorn + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + monitoring: + enabled: true + createPrometheusRules: true + defaultSettings: + defaultReplicaCount: 3 + backupstorePollInterval: 0 + createDefaultDiskLabeledNodes: true + restoreVolumeRecurringJobs: true + storageOverProvisioningPercentage: 100 + storageMinimalAvailablePercentage: 1 + guaranteedEngineManagerCPU: 20 + guaranteedReplicaManagerCPU: 20 + orphanAutoDeletion: true + concurrentAutomaticEngineUpgradePerNodeLimit: 3 + defaultLonghornStaticStorageClass: longhorn + nodeDownPodDeletionPolicy: delete-both-statefulset-and-deployment-pod + ingress: + enabled: true + ingressClassName: internal + host: longhorn.${SECRET_DOMAIN} + tls: true +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/kustomization.yaml.j2 new file mode 100644 index 00000000..fe61c7f7 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/kustomization.yaml.j2 @@ -0,0 +1,8 @@ +#% if longhorn.enabled | default(false) %# +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml + - ./snapshot.yaml +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/snapshot.yaml.j2 b/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/snapshot.yaml.j2 new file mode 100644 index 00000000..df15c5d1 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/snapshot.yaml.j2 @@ -0,0 +1,16 @@ +#% if longhorn.enabled | default(false) %# +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +reclaimPolicy: Delete +provisioner: driver.longhorn.io +parameters: + dataLocality: disabled + numberOfReplicas: "1" # Faster restores from the snapshotclass + replicaAutoBalance: best-effort + staleReplicaTimeout: "30" +allowVolumeExpansion: true +volumeBindingMode: Immediate +metadata: + name: longhorn-snapshot +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/ks.yaml.j2 new file mode 100644 index 00000000..0a1b19f6 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/ks.yaml.j2 @@ -0,0 +1,22 @@ +#% if longhorn.enabled | default(false) %# +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app longhorn + namespace: flux-system +spec: + targetNamespace: longhorn-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/longhorn-system/longhorn/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/longhorn-system/namespace.yaml.j2 b/bootstrap/templates/kubernetes/apps/longhorn-system/namespace.yaml.j2 new file mode 100644 index 00000000..920ec7df --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/longhorn-system/namespace.yaml.j2 @@ -0,0 +1,9 @@ +#% if longhorn.enabled | default(false) %# +--- +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system + labels: + kustomize.toolkit.fluxcd.io/prune: disabled +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/storage/volsync/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/storage/volsync/ks.yaml.j2 index 76f82c77..fc7e9a2a 100644 --- a/bootstrap/templates/kubernetes/apps/storage/volsync/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/storage/volsync/ks.yaml.j2 @@ -10,6 +10,10 @@ spec: commonMetadata: labels: app.kubernetes.io/name: *app + #% if longhorn.enabled | default(false) %# + dependsOn: + - name: longhorn + #% endif %# path: ./kubernetes/apps/storage/volsync/snapshot-controller prune: true sourceRef: diff --git a/bootstrap/templates/kubernetes/apps/storage/volsync/snapshot-controller/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/storage/volsync/snapshot-controller/helmrelease.yaml.j2 index 29c24951..40f06adb 100644 --- a/bootstrap/templates/kubernetes/apps/storage/volsync/snapshot-controller/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/storage/volsync/snapshot-controller/helmrelease.yaml.j2 @@ -27,6 +27,17 @@ spec: keepHistory: false values: controller: + #% if longhorn.enabled | default(false) %# + volumeSnapshotClasses: + - name: longhorn-snapclass + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" + driver: driver.longhorn.io + deletionPolicy: Delete + # Ref: https://github.com/longhorn/longhorn/issues/2534#issuecomment-1010508714 + parameters: + type: snap + #% endif %# serviceMonitor: create: true webhook: diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 index 784ac62b..03d1929b 100644 --- a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 @@ -15,6 +15,7 @@ resources: - ./jetstack.yaml - ./k8s-gateway.yaml - ./kubernetes-dashboard.yaml + - ./longhorn.yaml - ./metrics-server.yaml - ./openebs.yaml - ./piraeus.yaml diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/longhorn.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/longhorn.yaml.j2 new file mode 100644 index 00000000..bc41510b --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/longhorn.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: longhorn + namespace: flux-system +spec: + interval: 1h + url: https://charts.longhorn.io diff --git a/bootstrap/vars/addons.sample.yaml b/bootstrap/vars/addons.sample.yaml index 851dc1f4..1a08f8d4 100644 --- a/bootstrap/vars/addons.sample.yaml +++ b/bootstrap/vars/addons.sample.yaml @@ -38,7 +38,7 @@ csi_driver_nfs: # https://github.com/rancher/system-upgrade-controller system_upgrade_controller: - # WARNING: Only enable this if you also track the version of k3s in the + # IMPORTANT: Only enable this if you also track the version of k3s in the # ansible configuration files. Running ansible against an already provisioned # cluster with this enabled might cause your cluster to be downgraded. # Note: If bootstrap_distribution is set to k0s or talos this will be ignored. @@ -59,3 +59,11 @@ volsync: spegel: # Note: This only applies to k0s at the moment enabled: false + +# https://github.com/longhorn/longhorn +longhorn: + # IMPORTANT: Only enable this if you have 3 or more nodes in your cluster + # and you have enough IOPS on disk to support the number of replicas. + # Note: Consider enabling volsync if you would like to configure snapshots + # and backups to durable storage. + enabled: false