feat!: Drop Metallb in favor of Cilium L2Announcements (#821)

ansible/inventory/group_vars/kubernetes/kubernetes.yml

@@ -10,7 +10,7 @@ k3s_install_hard_links: true
 k3s_become: true
 k3s_etcd_datastore: true
 k3s_use_unsupported_config: true
-k3s_registration_address: "{{ kubevip_address }}"
+k3s_registration_address: "{{ kube_vip_addr }}"
 k3s_server_manifests_urls:
   # Kube-vip RBAC
   - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml

ansible/inventory/group_vars/master/kubernetes.yml

@@ -6,15 +6,15 @@ k3s_control_node: true
 k3s_server:
   node-ip: "{{ ansible_host }}"
   tls-san:
-    - "{{ kubevip_address }}"
+    - "{{ kube_vip_addr }}"
   docker: false
   flannel-backend: "none"             # This needs to be in quotes
   disable:
-    - coredns                         # Disable coredns                 - replaced with Helm Chart
-    - flannel                         # Disable flannel                 - replaced with Cilium
+    - coredns                         # Disable coredns                 - replaced with Coredns Helm Chart
+    - flannel                         # Disable flannel                 - replaced with Cilium Helm Chart
     - local-storage                   # Disable local-path-provisioner  - installed with Flux
     - metrics-server                  # Disable metrics-server          - installed with Flux
-    - servicelb                       # Disable servicelb               - replaced with metallb and installed with Flux
+    - servicelb                       # Disable servicelb               - replaced with Cilium Helm Chart
     - traefik                         # Disable traefik                 - replaced with ingress-nginx and installed with Flux
   disable-network-policy: true
   disable-cloud-controller: true

ansible/playbooks/cluster-installation.yml

@@ -47,9 +47,11 @@
       loop:
         - { name: cilium, kind: HelmChart, namespace: kube-system }
         - { name: coredns, kind: HelmChart, namespace: kube-system }
-        - { name: podmonitors.monitoring.coreos.com, kind: CustomResourceDefinition }
-        - { name: prometheusrules.monitoring.coreos.com, kind: CustomResourceDefinition }
-        - { name: servicemonitors.monitoring.coreos.com, kind: CustomResourceDefinition }
+        - { name: policy, kind: CiliumL2AnnouncementPolicy }
+        - { name: pool, kind: CiliumLoadBalancerIPPool }
+        - { name: podmonitors, kind: CustomResourceDefinition }
+        - { name: prometheusrules, kind: CustomResourceDefinition }
+        - { name: servicemonitors, kind: CustomResourceDefinition }
 
     - name: Coredns
       when:

ansible/playbooks/templates/cilium-l2.yaml.j2

@@ -0,0 +1,22 @@
+---
+# https://docs.cilium.io/en/latest/network/l2-announcements
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: policy
+spec:
+  loadBalancerIPs: true
+  # NOTE: This might need to be set if you have more than one active NIC on your nodes
+  # interfaces:
+  #   - ^eno[0-9]+
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/os: linux
+---
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: pool
+spec:
+  cidrs:
+    - cidr: "{{ (ansible_default_ipv4.network + '/' + ansible_default_ipv4.netmask) | ansible.utils.ipaddr('network/prefix') }}"

ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2

@@ -1,4 +1,5 @@
 ---
+# https://docs.k3s.io/helm
 apiVersion: helm.cattle.io/v1
 kind: HelmChart
 metadata:
@@ -8,25 +9,41 @@ spec:
   repo: https://helm.cilium.io/
   chart: cilium
   # renovate: datasource=helm depName=cilium registryUrl=https://helm.cilium.io
-  version: "1.13.4"
+  version: "1.14.0-rc.0"
   targetNamespace: kube-system
   bootstrap: true
   valuesContent: |-
+    autoDirectNodeRoutes: true
     bpf:
       masquerade: true
+    bgp:
+      enabled: false
     cluster:
       name: home-cluster
       id: 1
     containerRuntime:
       integration: containerd
       socketPath: /var/run/k3s/containerd/containerd.sock
+    endpointRoutes:
+      enabled: true
     hubble:
       enabled: false
     ipam:
       mode: kubernetes
-    k8sServiceHost: "{{ kubevip_address }}"
+    ipv4NativeRoutingCIDR: "{{ k3s_server['cluster-cidr'] }}"
+    k8sServiceHost: "{{ kube_vip_addr }}"
     k8sServicePort: 6443
     kubeProxyReplacement: strict
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+    l2announcements:
+      enabled: true
+    loadBalancer:
+      algorithm: maglev
+      mode: dsr
+    localRedirectPolicy: true
     operator:
-      replicas: 1
+      rollOutPods: true
+    rollOutCiliumPods: true
+    securityContext:
+      privileged: true
+    tunnel: disabled

ansible/playbooks/templates/kube-vip-static-pod.yaml.j2

@@ -15,7 +15,7 @@ spec:
       args: ["manager"]
       env:
         - name: address
-          value: "{{ kubevip_address }}"
+          value: "{{ kube_vip_addr }}"
         - name: vip_arp
           value: "true"
         - name: port