feat: allow flux to take over cilium (#755)

ansible/inventory/group_vars/kubernetes/k3s.yml

@@ -42,7 +42,7 @@ k3s_server_manifests_urls:
 # (list) A flat list of templates to deploy on the primary control plane node
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
-  - cilium-helmchart.yaml.j2
+  - custom-cilium-helmchart.yaml.j2
 
 # (list) A flat list of templates to deploy as static pods on all the control plane nodes
 # /var/lib/rancher/k3s/agent/pod-manifests

ansible/inventory/group_vars/master/k3s.yml

@@ -29,9 +29,9 @@ k3s_server:
   disable-kube-proxy: true
   write-kubeconfig-mode: "644"
   # Network CIDR to use for pod IPs
-  cluster-cidr: "10.42.0.0/16"
+  cluster-cidr: "{{ cluster_cidr }}"
   # Network CIDR to use for service IPs
-  service-cidr: "10.43.0.0/16"
+  service-cidr: "{{ service_cidr }}"
   kube-controller-manager-arg:
     # Required to monitor kube-controller-manager with kube-prometheus-stack
     - "bind-address=0.0.0.0"

ansible/playbooks/cluster-installation.yml

@@ -57,33 +57,65 @@
         regexp: "https://127.0.0.1:6443"
         replace: "https://{{ k3s_registration_address }}:6443"
 
-    - name: Resource Readiness Check
+    # Unmanaging and removing the Cilium HelmChart is required for 
+    # flux to take over managing the lifecycle of Cilium
+
+    - name: Post installation of custom manifests tasks
       run_once: true
-      kubernetes.core.k8s_info:
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-        kind: "{{ item.kind }}"
-        name: "{{ item.name }}"
-        namespace: "{{ item.namespace | default('') }}"
-        wait: true
-        wait_sleep: 10
-        wait_timeout: 360
-      loop:
-        - name: cilium
-          kind: HelmChart
-          namespace: kube-system
-        - name: podmonitors.monitoring.coreos.com
-          kind: CustomResourceDefinition
-        - name: prometheusrules.monitoring.coreos.com
-          kind: CustomResourceDefinition
-        - name: servicemonitors.monitoring.coreos.com
-          kind: CustomResourceDefinition
       when:
         - k3s_server_manifests_templates | length > 0
             or k3s_server_manifests_urls | length > 0
         - k3s_control_node is defined
         - k3s_control_node
+      block:
+        - name: Wait for custom manifests to rollout
+          kubernetes.core.k8s_info:
+            kubeconfig: /etc/rancher/k3s/k3s.yaml
+            kind: "{{ item.kind }}"
+            name: "{{ item.name }}"
+            namespace: "{{ item.namespace | default('') }}"
+            wait: true
+            wait_sleep: 10
+            wait_timeout: 360
+          loop:
+            - name: cilium
+              kind: HelmChart
+              namespace: kube-system
+            - name: podmonitors.monitoring.coreos.com
+              kind: CustomResourceDefinition
+            - name: prometheusrules.monitoring.coreos.com
+              kind: CustomResourceDefinition
+            - name: servicemonitors.monitoring.coreos.com
+              kind: CustomResourceDefinition
+        - name: Wait for Cilium to rollout
+          kubernetes.core.k8s_info:
+            kubeconfig: /etc/rancher/k3s/k3s.yaml
+            kind: Job
+            name: helm-install-cilium
+            namespace: kube-system
+            wait: true
+            wait_condition:
+              type: Complete
+              status: true
+            wait_timeout: 360
+        - name: Patch the Cilium HelmChart to unmanage it
+          kubernetes.core.k8s_json_patch:
+            kubeconfig: /etc/rancher/k3s/k3s.yaml
+            name: cilium
+            kind: HelmChart
+            namespace: kube-system
+            patch:
+              - op: add
+                path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged
+                value: "true"
+        - name: Remove the Cilium HelmChart CR
+          kubernetes.core.k8s:
+            kubeconfig: /etc/rancher/k3s/k3s.yaml
+            name: cilium
+            kind: HelmChart
+            namespace: kube-system
+            state: absent
 
-    # NOTE
     # Cleaning up certain manifests from the /var/lib/rancher/k3s/server/manifests directory
     # is needed because k3s has an awesome "feature" to always re-deploy them when the k3s
     # service is restarted. Removing them does not uninstall the manifests from your cluster.

ansible/playbooks/cluster-nuke.yml

@@ -22,6 +22,11 @@
       ansible.builtin.pause:
         seconds: 5
   tasks:
+    - name: Reboot
+      ansible.builtin.reboot:
+        msg: Rebooting nodes
+        reboot_timeout: 3600
+
     - name: Uninstall k3s
       ansible.builtin.include_role:
         name: xanmanning.k3s

ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2

@@ -41,7 +41,7 @@ spec:
     ipam:
       mode: cluster-pool
       operator:
-        clusterPoolIPv4PodCIDRList: ["{{ k3s_server['cluster-cidr'] }}"]
+        clusterPoolIPv4PodCIDRList: ["{{ cluster_cidr }}"]
         clusterPoolIPv4MaskSize: 24
     k8sServiceHost: "{{ kubevip_address }}"
     k8sServicePort: 6443