feat!: Remove core and crds fluxtomizations and use raw helm chart to install CRs

Signed-off-by: Devin Buhl <devin@buhl.casa>

.secretlintignore

@@ -1,2 +1,3 @@
 megalinter-reports
+tmpl
 README.md

README.md

@@ -103,10 +103,8 @@ The Git repository contains the following directories under `cluster` and are or
 ```sh
 📁 cluster      # k8s cluster defined as code
 ├─📁 flux       # flux, gitops operator, loaded before everything
-├─📁 crds       # custom resources, loaded before 📁 core and 📁 apps
+├─📁 charts     # helm chart repos
-├─📁 charts     # helm repos, loaded before 📁 core and 📁 apps
+├─📁 config     # cluster config
-├─📁 config     # cluster config, loaded before 📁 core and 📁 apps
-├─📁 core       # crucial apps, namespaced dir tree, loaded before 📁 apps
 └─📁 apps       # regular apps, namespaced dir tree, loaded last
 ```
 

cluster/apps/default/kustomization.yaml

@@ -5,3 +5,4 @@ resources:
   - namespace.yaml
   - echo-server
   - hajimari
+  - reloader

cluster/apps/kube-system/kustomization.yaml

@@ -3,7 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
-  - cert-manager
-  # - kured
   - metrics-server
-  - reloader

cluster/apps/kustomization.yaml

@@ -6,4 +6,4 @@ resources:
   - flux-system
   - kube-system
   - networking
-  - system-upgrade
+  # - system-upgrade

cluster/apps/networking/cert-manager/helm-release.yaml

@@ -3,7 +3,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:
   name: cert-manager
-  namespace: kube-system
+  namespace: networking
 spec:
   interval: 15m
   chart:
@@ -23,7 +23,7 @@ spec:
     remediation:
       retries: 5
   values:
-    installCRDs: false
+    installCRDs: true
     webhook:
       enabled: true
     extraArgs:

cluster/apps/networking/cert-manager/issuers/helm-release.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: cert-manager-issuers
+  namespace: networking
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: raw
+      version: v0.3.1
+      sourceRef:
+        kind: HelmRepository
+        name: dysnix
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: cert-manager
+      namespace: networking
+  values:
+    resources:
+      - apiVersion: cert-manager.io/v1
+        kind: ClusterIssuer
+        metadata:
+          name: letsencrypt-production
+        spec:
+          acme:
+            server: https://acme-v02.api.letsencrypt.org/directory
+            email: "${SECRET_CLOUDFLARE_EMAIL}"
+            privateKeySecretRef:
+              name: letsencrypt-production
+            solvers:
+              - dns01:
+                  cloudflare:
+                    email: "${SECRET_CLOUDFLARE_EMAIL}"
+                    apiKeySecretRef:
+                      name: cloudflare-api-key
+                      key: api-key
+                selector:
+                  dnsZones:
+                    - ${SECRET_DOMAIN}
+
+      - apiVersion: cert-manager.io/v1
+        kind: ClusterIssuer
+        metadata:
+          name: letsencrypt-staging
+        spec:
+          acme:
+            server: https://acme-staging-v02.api.letsencrypt.org/directory
+            email: "${SECRET_CLOUDFLARE_EMAIL}"
+            privateKeySecretRef:
+              name: letsencrypt-staging
+            solvers:
+              - dns01:
+                  cloudflare:
+                    email: "${SECRET_CLOUDFLARE_EMAIL}"
+                    apiKeySecretRef:
+                      name: cloudflare-api-key
+                      key: api-key
+                selector:
+                  dnsZones:
+                    - ${SECRET_DOMAIN}

cluster/apps/networking/cert-manager/issuers/kustomization.yaml

@@ -1,5 +1,5 @@
----
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - crds.yaml
+  - helm-release.yaml
+  - secret.sops.yaml

cluster/apps/networking/cert-manager/kustomization.yaml

@@ -1,5 +1,5 @@
----
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - cluster-issuers
+  - helm-release.yaml
+  - issuers

cluster/apps/networking/cloudflare-ddns/cron-job.yaml

@@ -2,41 +2,41 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: cloudflare-ddns
+  name: &app cloudflare-ddns
-  namespace: networking
+  namespace: default
 spec:
-  schedule: "0 * * * *"
+  schedule: "@hourly"
-  concurrencyPolicy: "Forbid"
+  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 3
+  successfulJobsHistoryLimit: 1
-  failedJobsHistoryLimit: 5
+  failedJobsHistoryLimit: 2
   jobTemplate:
     spec:
-      backoffLimit: 3
+      ttlSecondsAfterFinished: 3600
-      ttlSecondsAfterFinished: 300
       template:
         spec:
-          restartPolicy: Never
+          automountServiceAccountToken: false
+          restartPolicy: OnFailure
           containers:
-            - name: cloudflare-ddns
+            - name: *app
-              image: ghcr.io/onedr0p/kubernetes-kubectl:1.25.3
+              image: ghcr.io/onedr0p/kubernetes-kubectl:1.25.3@sha256:e1f54416c4f8779f748ba4a8d7d48d489b584e8963ee8b84310e19d6e2f0430e
               envFrom:
                 - secretRef:
-                    name: cloudflare-ddns
+                    name: *app
               command:
                 - "/bin/bash"
-                - "/app/cloudflare-ddns.sh"
+                - &scriptPath "/app/cloudflare-ddns.sh"
               volumeMounts:
-                - name: cloudflare-ddns
+                - name: *app
-                  mountPath: /app/cloudflare-ddns.sh
+                  mountPath: *scriptPath
-                  subPath: cloudflare-ddns.sh
+                  subPath: &scriptName cloudflare-ddns.sh
                   readOnly: true
           volumes:
-            - name: cloudflare-ddns
+            - name: *app
               projected:
                 defaultMode: 0775
                 sources:
                   - configMap:
-                      name: cloudflare-ddns
+                      name: *app
                       items:
-                        - key: cloudflare-ddns.sh
+                        - key: *scriptName
-                          path: cloudflare-ddns.sh
+                          path: *scriptName

cluster/apps/networking/ingress-nginx/helm-release.yaml

@@ -24,7 +24,9 @@ spec:
       retries: 5
   dependsOn:
     - name: cert-manager
-      namespace: kube-system
+      namespace: networking
+    - name: cert-manager-issuers
+      namespace: networking
   values:
     controller:
       replicaCount: 1

cluster/apps/networking/metallb/config.yaml

@@ -1,18 +0,0 @@
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: l2-ip
-  namespace: networking
-spec:
-  ipAddressPools:
-    - default-pool
----
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
-  name: default-pool
-  namespace: networking
-spec:
-  addresses:
-    - "${METALLB_LB_RANGE}"

cluster/apps/networking/metallb/config/helm-release.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: metallb-config
+  namespace: networking
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: raw
+      version: v0.3.1
+      sourceRef:
+        kind: HelmRepository
+        name: dysnix
+        namespace: flux-system
+  install:
+    createNamespace: true
+    remediation:
+      retries: 5
+  upgrade:
+    remediation:
+      retries: 5
+  dependsOn:
+    - name: metallb
+      namespace: networking
+  values:
+    resources:
+      - apiVersion: metallb.io/v1beta1
+        kind: L2Advertisement
+        metadata:
+          name: l2-ip
+          namespace: networking
+        spec:
+          ipAddressPools:
+            - default-pool
+
+      - apiVersion: metallb.io/v1beta1
+        kind: IPAddressPool
+        metadata:
+          name: default-pool
+          namespace: networking
+        spec:
+          addresses:
+            - "${METALLB_LB_RANGE}"

cluster/apps/networking/metallb/helm-release.yaml

@@ -22,7 +22,7 @@ spec:
     remediation:
       retries: 5
   values:
+    crds:
+      enabled: true
     psp:
       create: false
-    crds:
-      enabled: false

cluster/apps/networking/metallb/kustomization.yaml

@@ -3,4 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - helm-release.yaml
-  - config.yaml
+  - config

cluster/apps/system-upgrade/kured/helm-release.yaml

@@ -3,7 +3,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:
   name: kured
-  namespace: kube-system
+  namespace: system-upgrade
 spec:
   interval: 15m
   chart:

cluster/apps/system-upgrade/kustomization.yaml

@@ -3,4 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
+  # renovate: datasource=docker image=rancher/system-upgrade-controller
+  - https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/crd.yaml
   - system-upgrade-controller
+  - kured

cluster/apps/system-upgrade/system-upgrade-controller/kustomization.yaml

@@ -6,11 +6,7 @@ resources:
   - plans
 images:
   - name: rancher/system-upgrade-controller
-    newName: docker.io/rancher/system-upgrade-controller
     newTag: v0.9.1
-commonLabels:
-  app.kubernetes.io/name: system-upgrade-controller
-  app.kubernetes.io/instance: system-upgrade-controller
 patchesStrategicMerge:
   # Delete namespace resource
   - |
@@ -19,3 +15,13 @@ patchesStrategicMerge:
     kind: Namespace
     metadata:
       name: system-upgrade
+  # Add labels
+  - |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: system-upgrade-controller
+      namespace: system-upgrade
+      labels:
+        app.kubernetes.io/name: system-upgrade-controller
+        app.kubernetes.io/instance: system-upgrade-controller

cluster/charts/dysnix-charts.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: dysnix
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://dysnix.github.io/charts

cluster/charts/kustomization.yaml

@@ -4,6 +4,7 @@ kind: Kustomization
 resources:
   - bitnami-charts.yaml
   - bjw-s-charts.yaml
+  - dysnix-charts.yaml
   - external-dns-charts.yaml
   - hajimari-charts.yaml
   - ingress-nginx-charts.yaml

cluster/core/cluster-issuers/kustomization.yaml

@@ -1,6 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - letsencrypt-production.yaml
-  - letsencrypt-staging.yaml
-  - secret.sops.yaml

cluster/core/cluster-issuers/letsencrypt-production.yaml

@@ -1,21 +0,0 @@
----
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: letsencrypt-production
-spec:
-  acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: "${SECRET_CLOUDFLARE_EMAIL}"
-    privateKeySecretRef:
-      name: letsencrypt-production
-    solvers:
-      - dns01:
-          cloudflare:
-            email: "${SECRET_CLOUDFLARE_EMAIL}"
-            apiKeySecretRef:
-              name: cloudflare-api-key
-              key: api-key
-        selector:
-          dnsZones:
-            - ${SECRET_DOMAIN}

cluster/core/cluster-issuers/letsencrypt-staging.yaml

@@ -1,21 +0,0 @@
----
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: letsencrypt-staging
-spec:
-  acme:
-    server: https://acme-staging-v02.api.letsencrypt.org/directory
-    email: "${SECRET_CLOUDFLARE_EMAIL}"
-    privateKeySecretRef:
-      name: letsencrypt-staging
-    solvers:
-      - dns01:
-          cloudflare:
-            email: "${SECRET_CLOUDFLARE_EMAIL}"
-            apiKeySecretRef:
-              name: cloudflare-api-key
-              key: api-key
-        selector:
-          dnsZones:
-            - ${SECRET_DOMAIN}

cluster/crds/cert-manager/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  # renovate: datasource=docker image=quay.io/jetstack/cert-manager-controller
-  - https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.crds.yaml

cluster/crds/kustomization.yaml

@@ -1,7 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - cert-manager
-  - system-upgrade-controller
-  - metallb

cluster/crds/metallb/crds.yaml

@@ -1,30 +0,0 @@
----
-apiVersion: source.toolkit.fluxcd.io/v1beta2
-kind: GitRepository
-metadata:
-  name: metallb-source
-  namespace: flux-system
-spec:
-  interval: 12h
-  url: https://github.com/metallb/metallb.git
-  ref:
-    # renovate: registryUrl=https://metallb.github.io/metallb chart=metallb
-    tag: 0.13.7
-  ignore: |
-    # exclude all
-    /*
-    # include crd directory
-    !/config/crd
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
-kind: Kustomization
-metadata:
-  name: crds-metallb
-  namespace: flux-system
-spec:
-  interval: 30m
-  prune: false
-  wait: true
-  sourceRef:
-    kind: GitRepository
-    name: metallb-source

cluster/crds/system-upgrade-controller/kustomization.yaml

@@ -1,6 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  # renovate: datasource=docker image=rancher/system-upgrade-controller
-  - https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/crd.yaml

cluster/flux/apps.yaml

@@ -5,14 +5,15 @@ metadata:
   name: apps
   namespace: flux-system
 spec:
-  interval: 10m
+  interval: 30m
   path: ./cluster/apps
   prune: true
   sourceRef:
     kind: GitRepository
     name: flux-cluster
   dependsOn:
-    - name: core
+    - name: config
+    - name: charts
   decryption:
     provider: sops
     secretRef:

cluster/flux/charts.yaml

@@ -5,7 +5,7 @@ metadata:
   name: charts
   namespace: flux-system
 spec:
-  interval: 10m
+  interval: 30m
   path: ./cluster/charts
   prune: true
   sourceRef:

cluster/flux/config.yaml

@@ -5,7 +5,7 @@ metadata:
   name: config
   namespace: flux-system
 spec:
-  interval: 10m
+  interval: 30m
   path: ./cluster/config
   prune: true
   sourceRef:

cluster/flux/core.yaml

@@ -1,27 +0,0 @@
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
-kind: Kustomization
-metadata:
-  name: core
-  namespace: flux-system
-spec:
-  interval: 10m
-  path: ./cluster/core
-  prune: false
-  sourceRef:
-    kind: GitRepository
-    name: flux-cluster
-  dependsOn:
-    - name: charts
-    - name: config
-    - name: crds
-  decryption:
-    provider: sops
-    secretRef:
-      name: sops-age
-  postBuild:
-    substituteFrom:
-      - kind: ConfigMap
-        name: cluster-settings
-      - kind: Secret
-        name: cluster-secrets

cluster/flux/crds.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
-kind: Kustomization
-metadata:
-  name: crds
-  namespace: flux-system
-spec:
-  interval: 10m
-  path: ./cluster/crds
-  prune: false
-  sourceRef:
-    kind: GitRepository
-    name: flux-cluster

configure

@@ -53,14 +53,14 @@ main() {
         envsubst < "${PROJECT_DIR}/tmpl/cluster/cluster-secrets.sops.yaml" \
             > "${PROJECT_DIR}/cluster/config/cluster-secrets.sops.yaml"
         envsubst < "${PROJECT_DIR}/tmpl/cluster/cert-manager-secret.sops.yaml" \
-            > "${PROJECT_DIR}/cluster/core/cluster-issuers/secret.sops.yaml"
+            > "${PROJECT_DIR}/cluster/apps/networking/cert-manager/issuers/secret.sops.yaml"
         envsubst < "${PROJECT_DIR}/tmpl/cluster/cloudflare-ddns-secret.sops.yaml" \
             > "${PROJECT_DIR}/cluster/apps/networking/cloudflare-ddns/secret.sops.yaml"
         envsubst < "${PROJECT_DIR}/tmpl/cluster/external-dns-secret.sops.yaml" \
             > "${PROJECT_DIR}/cluster/apps/networking/external-dns/secret.sops.yaml"
         # encrypt cluster secrets
         sops --encrypt --in-place "${PROJECT_DIR}/cluster/config/cluster-secrets.sops.yaml"
-        sops --encrypt --in-place "${PROJECT_DIR}/cluster/core/cluster-issuers/secret.sops.yaml"
+        sops --encrypt --in-place "${PROJECT_DIR}/cluster/apps/networking/cert-manager/issuers/secret.sops.yaml"
         sops --encrypt --in-place "${PROJECT_DIR}/cluster/apps/networking/cloudflare-ddns/secret.sops.yaml"
         sops --encrypt --in-place "${PROJECT_DIR}/cluster/apps/networking/external-dns/secret.sops.yaml"
         # generate terraform secrets

provision/ansible/inventory/group_vars/kubernetes/os.yml

@@ -9,6 +9,7 @@ packages:
   - "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-{{ ansible_distribution_major_version }}.noarch.rpm"
   - "https://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-{{ ansible_distribution_major_version }}.noarch.rpm"
   - dnf-automatic
+  - dnf-plugin-system-upgrade
   - dnf-utils
   - hdparm
   - htop

tmpl/cluster/flux-cluster.yaml

@@ -5,7 +5,7 @@ metadata:
   name: flux-cluster
   namespace: flux-system
 spec:
-  interval: 10m
+  interval: 30m
   ref:
     branch: main
   url: ${BOOTSTRAP_GIT_REPOSITORY}
@@ -16,7 +16,7 @@ metadata:
   name: flux-cluster
   namespace: flux-system
 spec:
-  interval: 10m
+  interval: 30m
   path: ./cluster/flux
   prune: true
   wait: false