feat: allow flux to take over cilium (#755)

.config.sample.env

@@ -16,6 +16,14 @@ export BOOTSTRAP_FLUX_GITHUB_WEBHOOK_SECRET="generated" # NOTE: Must only contai
 # Set this to any other string and it will be used for the secret
 export BOOTSTRAP_WEAVE_GITOPS_ADMIN_PASSWORD="generated" # NOTE: Must only contain alphanumeric characters and dashes
 
+# Age Public Key - string should start with age
+# e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta
+export BOOTSTRAP_AGE_PUBLIC_KEY=""
+
+# Choose your timezone
+# e.g. America/New_York
+export BOOTSTRAP_TIMEZONE="Etc/UTC"
+
 # Choose one of your cloudflare domains
 # e.g. onedr0p.com
 export BOOTSTRAP_CLOUDFLARE_DOMAIN=""
@@ -42,21 +50,16 @@ export BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR=""
 # that doesn't conflict with any other IP addresses here
 # e.g. 192.168.1.221
 export BOOTSTRAP_METALLB_INGRESS_ADDR=""
-
-# Age Public Key - string should start with age
-# e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta
-export BOOTSTRAP_AGE_PUBLIC_KEY=""
-
 # The IP Address to use with kube-vip
 # Pick a unused IP that is on the same network as your nodes
 # and outside the ${BOOTSTRAP_METALLB_LB_RANGE} range
 # and doesn't conflict with any other IP addresses here
 # e.g. 192.168.1.254
 export BOOTSTRAP_KUBE_VIP_ADDR=""
-
+# Choose your cluster or service cidrs
-# Choose your timezone
+# Leave this unchanged unless you know what you are doing
-# e.g. America/New_York
+export BOOTSTRAP_CLUSTER_CIDR="10.42.0.0/16"
-export BOOTSTRAP_TIMEZONE="Etc/UTC"
+export BOOTSTRAP_SERVICE_CIDR="10.43.0.0/16"
 
 #
 # Ansible related variables

ansible/inventory/group_vars/kubernetes/k3s.yml

@@ -42,7 +42,7 @@ k3s_server_manifests_urls:
 # (list) A flat list of templates to deploy on the primary control plane node
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
-  - cilium-helmchart.yaml.j2
+  - custom-cilium-helmchart.yaml.j2
 
 # (list) A flat list of templates to deploy as static pods on all the control plane nodes
 # /var/lib/rancher/k3s/agent/pod-manifests

ansible/inventory/group_vars/master/k3s.yml

@@ -29,9 +29,9 @@ k3s_server:
   disable-kube-proxy: true
   write-kubeconfig-mode: "644"
   # Network CIDR to use for pod IPs
-  cluster-cidr: "10.42.0.0/16"
+  cluster-cidr: "{{ cluster_cidr }}"
   # Network CIDR to use for service IPs
-  service-cidr: "10.43.0.0/16"
+  service-cidr: "{{ service_cidr }}"
   kube-controller-manager-arg:
     # Required to monitor kube-controller-manager with kube-prometheus-stack
     - "bind-address=0.0.0.0"

ansible/playbooks/cluster-installation.yml

@@ -57,33 +57,65 @@
         regexp: "https://127.0.0.1:6443"
         replace: "https://{{ k3s_registration_address }}:6443"
 
-    - name: Resource Readiness Check
+    # Unmanaging and removing the Cilium HelmChart is required for 
+    # flux to take over managing the lifecycle of Cilium
+
+    - name: Post installation of custom manifests tasks
       run_once: true
-      kubernetes.core.k8s_info:
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-        kind: "{{ item.kind }}"
-        name: "{{ item.name }}"
-        namespace: "{{ item.namespace | default('') }}"
-        wait: true
-        wait_sleep: 10
-        wait_timeout: 360
-      loop:
-        - name: cilium
-          kind: HelmChart
-          namespace: kube-system
-        - name: podmonitors.monitoring.coreos.com
-          kind: CustomResourceDefinition
-        - name: prometheusrules.monitoring.coreos.com
-          kind: CustomResourceDefinition
-        - name: servicemonitors.monitoring.coreos.com
-          kind: CustomResourceDefinition
       when:
         - k3s_server_manifests_templates | length > 0
             or k3s_server_manifests_urls | length > 0
         - k3s_control_node is defined
         - k3s_control_node
+      block:
+        - name: Wait for custom manifests to rollout
+          kubernetes.core.k8s_info:
+            kubeconfig: /etc/rancher/k3s/k3s.yaml
+            kind: "{{ item.kind }}"
+            name: "{{ item.name }}"
+            namespace: "{{ item.namespace | default('') }}"
+            wait: true
+            wait_sleep: 10
+            wait_timeout: 360
+          loop:
+            - name: cilium
+              kind: HelmChart
+              namespace: kube-system
+            - name: podmonitors.monitoring.coreos.com
+              kind: CustomResourceDefinition
+            - name: prometheusrules.monitoring.coreos.com
+              kind: CustomResourceDefinition
+            - name: servicemonitors.monitoring.coreos.com
+              kind: CustomResourceDefinition
+        - name: Wait for Cilium to rollout
+          kubernetes.core.k8s_info:
+            kubeconfig: /etc/rancher/k3s/k3s.yaml
+            kind: Job
+            name: helm-install-cilium
+            namespace: kube-system
+            wait: true
+            wait_condition:
+              type: Complete
+              status: true
+            wait_timeout: 360
+        - name: Patch the Cilium HelmChart to unmanage it
+          kubernetes.core.k8s_json_patch:
+            kubeconfig: /etc/rancher/k3s/k3s.yaml
+            name: cilium
+            kind: HelmChart
+            namespace: kube-system
+            patch:
+              - op: add
+                path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged
+                value: "true"
+        - name: Remove the Cilium HelmChart CR
+          kubernetes.core.k8s:
+            kubeconfig: /etc/rancher/k3s/k3s.yaml
+            name: cilium
+            kind: HelmChart
+            namespace: kube-system
+            state: absent
 
-    # NOTE
     # Cleaning up certain manifests from the /var/lib/rancher/k3s/server/manifests directory
     # is needed because k3s has an awesome "feature" to always re-deploy them when the k3s
     # service is restarted. Removing them does not uninstall the manifests from your cluster.

ansible/playbooks/cluster-nuke.yml

@@ -22,6 +22,11 @@
       ansible.builtin.pause:
         seconds: 5
   tasks:
+    - name: Reboot
+      ansible.builtin.reboot:
+        msg: Rebooting nodes
+        reboot_timeout: 3600
+
     - name: Uninstall k3s
       ansible.builtin.include_role:
         name: xanmanning.k3s

ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2

@@ -41,7 +41,7 @@ spec:
     ipam:
       mode: cluster-pool
       operator:
-        clusterPoolIPv4PodCIDRList: ["{{ k3s_server['cluster-cidr'] }}"]
+        clusterPoolIPv4PodCIDRList: ["{{ cluster_cidr }}"]
         clusterPoolIPv4MaskSize: 24
     k8sServiceHost: "{{ kubevip_address }}"
     k8sServicePort: 6443

configure

@@ -30,6 +30,7 @@ main() {
         verify_ansible_hosts
         verify_metallb
         verify_kubevip
+        verify_cluster_service_cidrs
         verify_addressing
         verify_age
         verify_git_repository
@@ -252,6 +253,12 @@ verify_metallb() {
     _has_valid_ip "${BOOTSTRAP_METALLB_INGRESS_ADDR}" "BOOTSTRAP_METALLB_INGRESS_ADDR"
 }
 
+verify_cluster_service_cidrs() {
+    #TODO: Validate input is a CIDR range
+    _has_envar "BOOTSTRAP_CLUSTER_CIDR"
+    _has_envar "BOOTSTRAP_SERVICE_CIDR"
+}
+
 verify_git_repository() {
     _has_envar "BOOTSTRAP_GIT_REPOSITORY"
     export GIT_TERMINAL_PROMPT=0

kubernetes/apps/kube-system/cilium/app/helmrelease.yaml

@@ -0,0 +1,88 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: cilium
+  namespace: kube-system
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: cilium
+      version: 1.13.2
+      sourceRef:
+        kind: HelmRepository
+        name: cilium
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    bpf:
+      masquerade: true
+    cluster:
+      name: home-cluster
+      id: 1
+    containerRuntime:
+      integration: containerd
+      socketPath: /var/run/k3s/containerd/containerd.sock
+    hubble:
+      enabled: true
+      metrics:
+        enabled:
+          - dns:query;ignoreAAAA
+          - drop
+          - tcp
+          - flow
+          - port-distribution
+          - icmp
+          - http
+        serviceMonitor:
+          enabled: true
+      relay:
+        enabled: true
+        rollOutPods: true
+        prometheus:
+          serviceMonitor:
+            enabled: true
+      ui:
+        enabled: true
+        rollOutPods: true
+        ingress:
+          enabled: true
+          className: nginx
+          hosts:
+            - &host "hubble.${SECRET_DOMAIN}"
+          tls:
+            - hosts:
+                - *host
+    ipam:
+      mode: cluster-pool
+      operator:
+        clusterPoolIPv4PodCIDRList: ["${CLUSTER_CIDR}"]
+        clusterPoolIPv4MaskSize: 24
+    k8sServiceHost: "${KUBE_VIP_ADDR}"
+    k8sServicePort: 6443
+    kubeProxyReplacement: strict
+    kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+    operator:
+      enabled: true
+      replicas: 1
+      rollOutPods: true
+      prometheus:
+        enabled: true
+        serviceMonitor:
+          enabled: true
+    prometheus:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+    rollOutCiliumPods: true

kubernetes/apps/kube-system/cilium/app/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kube-system
+resources:
+  - ./helmrelease.yaml

kubernetes/apps/kube-system/cilium/ks.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-cilium
+  namespace: flux-system
+spec:
+  path: ./kubernetes/apps/kube-system/cilium/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m

kubernetes/apps/kube-system/kustomization.yaml

@@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
+  - ./cilium/ks.yaml
   - ./local-path-provisioner/ks.yaml
   - ./metrics-server/ks.yaml
   - ./reloader/ks.yaml

kubernetes/flux/repositories/helm/cilium.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: cilium
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://helm.cilium.io

kubernetes/flux/repositories/helm/kustomization.yaml

@@ -4,6 +4,7 @@ kind: Kustomization
 resources:
   - ./bitnami.yaml
   - ./bjw-s.yaml
+  - ./cilium.yaml
   - ./external-dns.yaml
   - ./hajimari.yaml
   - ./ingress-nginx.yaml

tmpl/ansible/kube-vip.yml

@@ -1,3 +1,5 @@
 ---
 # (string) The ARP address kube-vip broadcasts
 kubevip_address: "${BOOTSTRAP_KUBE_VIP_ADDR}"
+cluster_cidr: "${BOOTSTRAP_CLUSTER_CIDR}"
+service_cidr: "${BOOTSTRAP_SERVICE_CIDR}"

tmpl/kubernetes/flux/cluster-settings.yaml

@@ -9,3 +9,6 @@ data:
   METALLB_LB_RANGE: "${BOOTSTRAP_METALLB_LB_RANGE}"
   METALLB_K8S_GATEWAY_ADDR: "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}"
   METALLB_INGRESS_ADDR: "${BOOTSTRAP_METALLB_INGRESS_ADDR}"
+  KUBE_VIP_ADDR: "${BOOTSTRAP_KUBE_VIP_ADDR}"
+  CLUSTER_CIDR: "${BOOTSTRAP_CLUSTER_CIDR}"
+  SERVICE_CIDR: "${BOOTSTRAP_SERVICE_CIDR}"