refactor(cluster)!: Update flux installation and upgrade method

Other changes include:
- Rename cluster/base to cluster/flux
- Add config and charts fluxtomizations
- Update scripts and taskfiles to reflect changes

.github/renovate.json5

@@ -8,15 +8,6 @@
   "schedule": [
     "every weekend"
   ],
-  // set up ignored deps
-  "ignoreDeps": [
-    "ghcr.io/fluxcd/helm-controller",
-    "ghcr.io/fluxcd/image-automation-controller",
-    "ghcr.io/fluxcd/image-reflector-controller",
-    "ghcr.io/fluxcd/kustomize-controller",
-    "ghcr.io/fluxcd/notification-controller",
-    "ghcr.io/fluxcd/source-controller"
-  ],
   "pre-commit": {
     "enabled": true
   },

.taskfiles/ClusterTasks.yml

@@ -3,20 +3,18 @@ version: "3"
 
 tasks:
 
-  flux:verify:
+  verify:
     desc: Verify flux meets the prerequisites
     cmds:
       - flux check --pre
 
-  flux:namespace:
+  install:
-    desc: Create the flux-system namespace
+    desc: Install Flux into your cluster
-    cmds:
-      - kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f -
-
-  flux:secret:
-    desc: Create the Age secret in the flux-system namespace
     cmds:
+      - kubectl apply --kustomize {{.CLUSTER_DIR}}/bootstrap/
       - cat {{.SOPS_AGE_KEY_FILE}} | kubectl -n flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin
+      - kubectl apply --kustomize {{.CLUSTER_DIR}}/flux/flux-system/
+      - task: reconcile
     preconditions:
       - sh: test -f {{.SOPS_AGE_KEY_FILE}}
         msg: |
@@ -24,15 +22,11 @@ tasks:
     vars:
       SOPS_AGE_KEY_FILE: ~/.config/sops/age/keys.txt
 
-  flux:install:
-    desc: Install Flux into your cluster, run this again if it fails
-    cmds:
-      - kubectl apply -k {{.CLUSTER_DIR}}/base/flux-system
-
   reconcile:
     desc: Force update Flux to pull in changes from your Git repository
     cmds:
-      - flux reconcile source git flux-system {{.CLI_ARGS | default "-n flux-system"}}
+      - flux reconcile -n flux-system source git flux-cluster
+      - flux reconcile -n flux-system source kustomization flux-cluster
 
   nodes:
     desc: List all the nodes in your cluster

README.md

@@ -94,29 +94,14 @@ It is advisable to install [pre-commit](https://pre-commit.com/) and the pre-com
 
 The Git repository contains the following directories under `cluster` and are ordered below by how Flux will apply them.
 
-- **base** directory is the entrypoint to Flux
+```sh
-- **crds** directory contains custom resource definitions (CRDs) that need to exist globally in your cluster before anything else exists
+📁 cluster      # k8s cluster defined as code
-- **core** directory (depends on **crds**) are important infrastructure applications (grouped by namespace) that should never be pruned by Flux
+├─📁 flux       # flux, gitops operator, loaded before everything
-- **apps** directory (depends on **core**) is where your common applications (grouped by namespace) could be placed, Flux will prune resources here if they are not tracked by Git anymore
+├─📁 crds       # custom resources, loaded before 📁 core and 📁 apps
-
+├─📁 charts     # helm repos, loaded before 📁 core and 📁 apps
-```
+├─📁 config     # cluster config, loaded before 📁 core and 📁 apps
-cluster
+├─📁 core       # crucial apps, namespaced dir tree, loaded before 📁 apps
-├── apps
+└─📁 apps       # regular apps, namespaced dir tree, loaded last
-│   ├── default
-│   ├── kube-system
-│   ├── networking
-│   └── system-upgrade
-├── base
-│   └── flux-system
-├── core
-│   ├── cert-manager
-│   ├── kube-system
-│   ├── metallb-system
-│   └── namespaces
-└── crds
-    ├── cert-manager
-    ├── system-upgrade-controller
-    └── traefik
 ```
 
 ## 🚀 Lets go
@@ -297,50 +282,32 @@ The cluster application [external-dns](https://github.com/kubernetes-sigs/extern
 1. Verify Flux can be installed
 
     ```sh
-    task cluster:flux:verify
+    task cluster:verify
     # ► checking prerequisites
     # ✔ kubectl 1.21.5 >=1.18.0-0
     # ✔ Kubernetes 1.21.5+k3s1 >=1.16.0-0
     # ✔ prerequisites checks passed
     ```
 
-2. Create the `flux-system` namespace
+2. Push you changes to git
-
-    ```sh
-    task cluster:flux:namespace
-    ```
-
-3. Add the Age key to your cluster as a secret in-order for Flux to decrypt SOPS secrets
-
-    ```sh
-    task cluster:flux:secret
-    ```
-
-    📍 Variables defined in `./cluster/base/cluster-secrets.sops.yaml` and `./cluster/base/cluster-settings.yaml` will be usable anywhere in your YAML manifests under `./cluster` except `./cluster/base`
-
-4. Push you changes to git
 
     📍 **Verify** all the `*.sops.yaml` and `*.sops.yml` files under the `./cluster` and `./provision` folders are **encrypted** with SOPS
 
     ```sh
     git add -A
-    git commit -m "initial commit"
+    git commit -m "Initial commit :rocket:"
     git push
     ```
 
-5. Install Flux
+3. Install Flux and sync the cluster to the Git repository
-
-    📍 Due to race conditions with the Flux CRDs you will have to run the below command twice. There should be no errors on this second run.
 
     ```sh
-    task cluster:flux:install
+    task cluster:install
     # namespace/flux-system configured
     # customresourcedefinition.apiextensions.k8s.io/alerts.notification.toolkit.fluxcd.io created
-    # ...
-    # unable to recognize "./cluster/base/flux-system": no matches for kind "HelmRepository" in version "source.toolkit.fluxcd.io/v1beta1"
     ```
 
-6. Verify Flux components are running in the cluster
+4. Verify Flux components are running in the cluster
 
     ```sh
     task cluster:pods -- -n flux-system
@@ -427,7 +394,7 @@ Once you have confirmed there are no issues requesting your certificates replace
 
 ### 🤖 Renovatebot
 
-[Renovatebot](https://www.whitesourcesoftware.com/free-developer-tools/renovate/) will scan your repository and offer PRs when it finds dependencies out of date. Common dependencies it will discover and update are Flux, Ansible Galaxy Roles, Terraform Providers, Kubernetes Helm Charts, Kubernetes Container Images, Pre-commit hooks updates, and more!
+[Renovatebot](https://www.mend.io/free-developer-tools/renovate/) will scan your repository and offer PRs when it finds dependencies out of date. Common dependencies it will discover and update are Flux, Ansible Galaxy Roles, Terraform Providers, Kubernetes Helm Charts, Kubernetes Container Images, Pre-commit hooks updates, and more!
 
 The base Renovate configuration provided in your repository can be view at [.github/renovate.json5](https://github.com/k8s-at-home/flux-cluster-template/blob/main/.github/renovate.json5). If you notice this only runs on weekends and you can [change the schedule to anything you want](https://docs.renovatebot.com/presets-schedule/) or simply remove it.
 
@@ -526,10 +493,10 @@ The benefits of a public repository include:
       apiVersion: source.toolkit.fluxcd.io/v1beta2
       kind: GitRepository
       metadata:
-        name: flux-system
+        name: flux-installation
         namespace: flux-system
       spec:
-        interval: 5m0s
+        interval: 10m
         # 6a: Change this to your user and repo names
         url: ssh://git@github.com/$user/$repo
         ref:

cluster/base/flux-system/gotk-patches.yaml

@@ -1,24 +0,0 @@
----
-# Delete the allow-egress NetworkPolicy
-$patch: delete
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: allow-egress
-  namespace: flux-system
----
-# Delete the allow-scraping NetworkPolicy
-$patch: delete
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: allow-scraping
-  namespace: flux-system
----
-# Delete the allow-webhooks NetworkPolicy
-$patch: delete
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: allow-webhooks
-  namespace: flux-system

cluster/base/flux-system/kustomization.yaml

@@ -1,9 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - gotk-components.yaml
-  - gotk-sync.yaml
-  - charts
-patchesStrategicMerge:
-  - ./gotk-patches.yaml

cluster/bootstrap/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - github.com/fluxcd/flux2/manifests/install?ref=v0.30.2

cluster/config/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - cluster-settings.yaml
+  - cluster-secrets.sops.yaml

cluster/flux/apps.yaml

@@ -6,13 +6,13 @@ metadata:
   namespace: flux-system
 spec:
   interval: 10m
-  dependsOn:
-    - name: core
   path: ./cluster/apps
   prune: true
   sourceRef:
     kind: GitRepository
-    name: flux-system
+    name: flux-cluster
+  dependsOn:
+    - name: core
   decryption:
     provider: sops
     secretRef:

cluster/flux/charts.yaml

@@ -10,4 +10,4 @@ spec:
   prune: true
   sourceRef:
     kind: GitRepository
-    name: flux-system
+    name: flux-cluster

cluster/flux/config.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: config
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./cluster/config
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-cluster
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-age

cluster/flux/core.yaml

@@ -6,14 +6,15 @@ metadata:
   namespace: flux-system
 spec:
   interval: 10m
-  dependsOn:
-    - name: charts
-    - name: crds
   path: ./cluster/core
   prune: false
   sourceRef:
     kind: GitRepository
-    name: flux-system
+    name: flux-cluster
+  dependsOn:
+    - name: charts
+    - name: config
+    - name: crds
   decryption:
     provider: sops
     secretRef:

cluster/flux/crds.yaml

@@ -2,7 +2,7 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
 kind: Kustomization
 metadata:
-  name: crds
+  name: flux-crds
   namespace: flux-system
 spec:
   interval: 10m
@@ -10,4 +10,4 @@ spec:
   prune: false
   sourceRef:
     kind: GitRepository
-    name: flux-system
+    name: flux-cluster

cluster/flux/flux-system/flux-installation.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: GitRepository
+metadata:
+  name: flux-installation
+  namespace: flux-system
+spec:
+  interval: 10m
+  ref:
+    # renovate: datasource=github-releases depName=fluxcd/flux2
+    tag: v0.30.2
+  url: https://github.com/fluxcd/flux2
+  ignore: |
+    # exclude all
+    /*
+    # path to manifests
+    !/manifests
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: flux-installation
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./manifests/install
+  prune: true
+  wait: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-installation

cluster/flux/flux-system/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - flux-installation.yaml
+  - flux-cluster.yaml

configure

@@ -42,14 +42,14 @@ main() {
             > "${PROJECT_DIR}/.sops.yaml"
         # generate cluster settings
         envsubst < "${PROJECT_DIR}/tmpl/cluster/cluster-settings.yaml" \
-            > "${PROJECT_DIR}/cluster/base/cluster-settings.yaml"
+            > "${PROJECT_DIR}/cluster/config/cluster-settings.yaml"
-        envsubst < "${PROJECT_DIR}/tmpl/cluster/gotk-sync.yaml" \
+        envsubst < "${PROJECT_DIR}/tmpl/cluster/flux-cluster.yaml" \
-            > "${PROJECT_DIR}/cluster/base/flux-system/gotk-sync.yaml"
+            > "${PROJECT_DIR}/cluster/flux/flux-system/flux-cluster.yaml"
         envsubst < "${PROJECT_DIR}/tmpl/cluster/kube-vip-daemonset.yaml" \
             > "${PROJECT_DIR}/cluster/core/kube-system/kube-vip/daemon-set.yaml"
         # generate cluster secrets
         envsubst < "${PROJECT_DIR}/tmpl/cluster/cluster-secrets.sops.yaml" \
-            > "${PROJECT_DIR}/cluster/base/cluster-secrets.sops.yaml"
+            > "${PROJECT_DIR}/cluster/config/cluster-secrets.sops.yaml"
         envsubst < "${PROJECT_DIR}/tmpl/cluster/cert-manager-secret.sops.yaml" \
             > "${PROJECT_DIR}/cluster/core/cert-manager/secret.sops.yaml"
         envsubst < "${PROJECT_DIR}/tmpl/cluster/cloudflare-ddns-secret.sops.yaml" \
@@ -57,7 +57,7 @@ main() {
         envsubst < "${PROJECT_DIR}/tmpl/cluster/external-dns-secret.sops.yaml" \
             > "${PROJECT_DIR}/cluster/apps/networking/external-dns/secret.sops.yaml"
         # encrypt cluster secrets
-        sops --encrypt --in-place "${PROJECT_DIR}/cluster/base/cluster-secrets.sops.yaml"
+        sops --encrypt --in-place "${PROJECT_DIR}/cluster/flux/cluster-secrets.sops.yaml"
         sops --encrypt --in-place "${PROJECT_DIR}/cluster/core/cert-manager/secret.sops.yaml"
         sops --encrypt --in-place "${PROJECT_DIR}/cluster/apps/networking/cloudflare-ddns/secret.sops.yaml"
         sops --encrypt --in-place "${PROJECT_DIR}/cluster/apps/networking/external-dns/secret.sops.yaml"

tmpl/cluster/flux-cluster.yaml

@@ -2,10 +2,10 @@
 apiVersion: source.toolkit.fluxcd.io/v1beta2
 kind: GitRepository
 metadata:
-  name: flux-system
+  name: flux-cluster
   namespace: flux-system
 spec:
-  interval: 5m0s
+  interval: 10m
   ref:
     branch: main
   url: ${BOOTSTRAP_GIT_REPOSITORY}
@@ -13,16 +13,13 @@ spec:
 apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
 kind: Kustomization
 metadata:
-  name: flux-system
+  name: flux-cluster
   namespace: flux-system
 spec:
-  interval: 10m0s
+  interval: 10m
-  path: ./cluster/base
+  path: ./cluster/flux
   prune: true
+  wait: false
   sourceRef:
     kind: GitRepository
-    name: flux-system
+    name: flux-cluster
-  decryption:
-    provider: sops
-    secretRef:
-      name: sops-age