feat(talos): Add cilium bgpControlPlane support and CNI security settings (#1296)

* fix: Use cilium cni install values for Talos

* fix: Use cilium cni install values for Talos

* feat: Add cilium bgpControlPlane support for Talos

* feat: Add cilium bgpControlPlane support for Talos

* feat: Add LB cidr var for Talos BGPCP

* feat: Add jinja template for cilium-bgp.yaml

* Add cilium bgpControlPlane support for Talos

* feat: Add cilium bgpControlPlane example config for Talos

* fix: Remove invalid fields from bgp peering policy

* fix: Add serviceSelector to bgp peering policy

* Update bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

* Update bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

* Update bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

* Update bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

---------

Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com>

bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

@@ -1,6 +1,14 @@
 autoDirectNodeRoutes: true
+{% if distribution.talos.bgp.enabled %}
+bgpControlPlane:
+  enabled: true
+{% endif %}
 bpf:
   masquerade: true
+cgroup:
+  automount:
+    enabled: false
+  hostRoot: /sys/fs/cgroup
 cluster:
   name: home-kubernetes
   id: 1
@@ -68,7 +76,8 @@ k8sServicePort: 7445
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
 l2announcements:
-  {% if feature_gates.dual_stack_ipv4_first %}
+  {% if ( (distribution.talos.bgp.enabled) or
+          (feature_gates.dual_stack_ipv4_first) ) %}
   # https://github.com/cilium/cilium/issues/28985
   enabled: false
   {% else %}
@@ -101,4 +110,20 @@ dashboards:
 rollOutCiliumPods: true
 routingMode: native
 securityContext:
-  privileged: true
+  capabilities:
+    ciliumAgent:
+      - CHOWN
+      - KILL
+      - NET_ADMIN
+      - NET_RAW
+      - IPC_LOCK
+      - SYS_ADMIN
+      - SYS_RESOURCE
+      - DAC_OVERRIDE
+      - FOWNER
+      - SETGID
+      - SETUID
+    cleanCiliumState:
+      - NET_ADMIN
+      - SYS_ADMIN
+      - SYS_RESOURCE

bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

@@ -1,6 +1,14 @@
 autoDirectNodeRoutes: true
+{% if distribution.talos.bgp.enabled %}
+bgpControlPlane:
+  enabled: true
+{% endif %}
 bpf:
   masquerade: true
+cgroup:
+  automount:
+    enabled: false
+  hostRoot: /sys/fs/cgroup
 cluster:
   name: home-kubernetes
   id: 1
@@ -31,7 +39,8 @@ k8sServicePort: 7445
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
 l2announcements:
-  {% if feature_gates.dual_stack_ipv4_first %}
+  {% if ( (distribution.talos.bgp.enabled) or
+          (feature_gates.dual_stack_ipv4_first) ) %}
   # https://github.com/cilium/cilium/issues/28985
   enabled: false
   {% else %}
@@ -47,4 +56,20 @@ operator:
 rollOutCiliumPods: true
 routingMode: native
 securityContext:
-  privileged: true
+  capabilities:
+    ciliumAgent:
+      - CHOWN
+      - KILL
+      - NET_ADMIN
+      - NET_RAW
+      - IPC_LOCK
+      - SYS_ADMIN
+      - SYS_RESOURCE
+      - DAC_OVERRIDE
+      - FOWNER
+      - SETGID
+      - SETUID
+    cleanCiliumState:
+      - NET_ADMIN
+      - SYS_ADMIN
+      - SYS_RESOURCE