feat(talos): Add cilium bgpControlPlane support and CNI security settings (#1296)

* fix: Use cilium cni install values for Talos

* fix: Use cilium cni install values for Talos

* feat: Add cilium bgpControlPlane support for Talos

* feat: Add cilium bgpControlPlane support for Talos

* feat: Add LB cidr var for Talos BGPCP

* feat: Add jinja template for cilium-bgp.yaml

* Add cilium bgpControlPlane support for Talos

* feat: Add cilium bgpControlPlane example config for Talos

* fix: Remove invalid fields from bgp peering policy

* fix: Add serviceSelector to bgp peering policy

* Update bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

* Update bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

* Update bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

* Update bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

---------

Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com>

bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2

@@ -0,0 +1,33 @@
+---
+# https://docs.cilium.io/en/latest/network/bgp-control-plane/
+apiVersion: cilium.io/v2alpha1
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: policy
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/os: linux
+  virtualRouters:
+    - localASN: {{ distribution.talos.bgp.local_asn }}
+      neighbors:
+        {% if distribution.talos.bgp.peers %}
+        {% for item in distribution.talos.bgp.peers %}
+        - peerAddress: "{{ item }}/32"
+          peerASN: {{ distribution.talos.bgp.peer_asn }}
+        {% endfor %}
+        {% else %}
+        - peerAddress: "{{ nodes.host_network | nthhost(1) }}/32"
+          peerASN: {{ distribution.talos.bgp.peer_asn }}
+        {% endif %}
+      serviceSelector:
+        matchExpressions:
+          - {key: somekey, operator: NotIn, values: ['never-used-value']}
+---
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: pool
+spec:
+  cidrs:
+    - cidr: "${LOADBALANCER_CIDR}"

bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2

@@ -2,7 +2,11 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  {% if not feature_gates.dual_stack_ipv4_first %}
+  {% if distribution.talos.bgp.enabled %}
+  - ./cilium-bgp.yaml
+  {% endif %}
+  {% if ( (not distribution.talos.bgp.enabled) and
+          (feature_gates.dual_stack_ipv4_first) ) %}
   - ./cilium-l2.yaml
   {% endif %}
   - ./helmrelease.yaml

bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2

@@ -11,3 +11,6 @@ data:
   {% if feature_gates.dual_stack_ipv4_first %}
   CLUSTER_CIDR_V6: "{{ cluster.pod_network.split(',')[1] }}"
   {% endif %}
+  {% if distribution.talos.bgp.enabled %}
+  LOADBALANCER_CIDR: "{{ distribution.talos.bgp.loadbalancer_network }}"
+  {% endif %}

bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

@@ -1,6 +1,14 @@
 autoDirectNodeRoutes: true
+{% if distribution.talos.bgp.enabled %}
+bgpControlPlane:
+  enabled: true
+{% endif %}
 bpf:
   masquerade: true
+cgroup:
+  automount:
+    enabled: false
+  hostRoot: /sys/fs/cgroup
 cluster:
   name: home-kubernetes
   id: 1
@@ -68,7 +76,8 @@ k8sServicePort: 7445
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
 l2announcements:
-  {% if feature_gates.dual_stack_ipv4_first %}
+  {% if ( (distribution.talos.bgp.enabled) or
+          (feature_gates.dual_stack_ipv4_first) ) %}
   # https://github.com/cilium/cilium/issues/28985
   enabled: false
   {% else %}
@@ -101,4 +110,20 @@ dashboards:
 rollOutCiliumPods: true
 routingMode: native
 securityContext:
-  privileged: true
+  capabilities:
+    ciliumAgent:
+      - CHOWN
+      - KILL
+      - NET_ADMIN
+      - NET_RAW
+      - IPC_LOCK
+      - SYS_ADMIN
+      - SYS_RESOURCE
+      - DAC_OVERRIDE
+      - FOWNER
+      - SETGID
+      - SETUID
+    cleanCiliumState:
+      - NET_ADMIN
+      - SYS_ADMIN
+      - SYS_RESOURCE

bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

@@ -1,6 +1,14 @@
 autoDirectNodeRoutes: true
+{% if distribution.talos.bgp.enabled %}
+bgpControlPlane:
+  enabled: true
+{% endif %}
 bpf:
   masquerade: true
+cgroup:
+  automount:
+    enabled: false
+  hostRoot: /sys/fs/cgroup
 cluster:
   name: home-kubernetes
   id: 1
@@ -31,7 +39,8 @@ k8sServicePort: 7445
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
 l2announcements:
-  {% if feature_gates.dual_stack_ipv4_first %}
+  {% if ( (distribution.talos.bgp.enabled) or
+          (feature_gates.dual_stack_ipv4_first) ) %}
   # https://github.com/cilium/cilium/issues/28985
   enabled: false
   {% else %}
@@ -47,4 +56,20 @@ operator:
 rollOutCiliumPods: true
 routingMode: native
 securityContext:
-  privileged: true
+  capabilities:
+    ciliumAgent:
+      - CHOWN
+      - KILL
+      - NET_ADMIN
+      - NET_RAW
+      - IPC_LOCK
+      - SYS_ADMIN
+      - SYS_RESOURCE
+      - DAC_OVERRIDE
+      - FOWNER
+      - SETGID
+      - SETUID
+    cleanCiliumState:
+      - NET_ADMIN
+      - SYS_ADMIN
+      - SYS_RESOURCE

config.sample.yaml

@@ -28,6 +28,22 @@ distribution:
     # # (Optional) Add vlan tag to network master device
     # #   See: https://www.talos.dev/latest/advanced/advanced-networking/#vlans
     # vlan: 1
+    # # (Optional) Use cilium BGP control plane when L2 announcements won't traverse VLAN network segments.
+    # #   Needs a BGP capable router setup with the node IPs as peers.
+    # #   See: https://docs.cilium.io/en/latest/network/bgp-control-plane/
+    # bgp:
+    #   enabled: true
+    #   # (Optional) If using multiple BGP peers add them here.
+    #   #   Default is .1 derrived from host_network: ['x.x.x.1']
+    #   peers: []
+    #   # (Required) Set the BGP Autonomous System Number for the router(s) and nodes.
+    #   #  If these match, iBGP will be used. If not, eBGP will be used.
+    #   peer_asn: 64512   # Router(s) AS
+    #   local_asn: 64512  # Node(s) AS
+    #   # (Required) The loadbalancer CIDR for the cluster, this must NOT overlap with any
+    #   #   existing networks and is usually a /16 (64K IPs).
+    #   # If you want to use IPv6 check the advanced flags below
+    #   loadbalancer_network: 10.123.0.0/16
 
 #
 # (Required) Timezone is your IANA formatted timezone (e.g. America/New_York)