From b275b781fddef78d708db8540fa9e004e8db2729 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 18 Mar 2024 19:06:19 -0400 Subject: [PATCH] feat(talos): change from `net.ifnames` to a macaddr `deviceSelector` (#1376) * feat(talos): change from net.ifnames to a macaddr deviceSelector Signed-off-by: Devin Buhl * fix: derpge Signed-off-by: Devin Buhl * fix: logic is easier to follow now questionmark Signed-off-by: Devin Buhl * fix: logic is easier to follow now questionmark Signed-off-by: Devin Buhl --------- Signed-off-by: Devin Buhl --- .github/tests/config-talos.yaml | 2 ++ .../cilium/app/kustomization.yaml.j2 | 6 ---- .../cilium/{app => config}/cilium-l2.yaml.j2 | 12 +++++--- .../cilium-l3.yaml.j2} | 9 ++++-- .../cilium/config/kustomization.yaml.j2 | 11 ++++++++ .../apps/kube-system/cilium/ks.yaml.j2 | 22 +++++++++++++++ .../bootstrap/talos/talconfig.yaml.j2 | 28 ++++++------------- .../cilium-values-full.partial.yaml.j2 | 5 ++-- .../cilium-values-init.partial.yaml.j2 | 5 ++-- config.sample.yaml | 3 +- 10 files changed, 63 insertions(+), 40 deletions(-) rename bootstrap/templates/kubernetes/apps/kube-system/cilium/{app => config}/cilium-l2.yaml.j2 (56%) rename bootstrap/templates/kubernetes/apps/kube-system/cilium/{app/cilium-bgp.yaml.j2 => config/cilium-l3.yaml.j2} (90%) create mode 100644 bootstrap/templates/kubernetes/apps/kube-system/cilium/config/kustomization.yaml.j2 diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml index 545a5469..387c606d 100644 --- a/.github/tests/config-talos.yaml +++ b/.github/tests/config-talos.yaml @@ -12,10 +12,12 @@ bootstrap_node_inventory: address: 10.10.10.100 controller: true talos_disk: fake + talos_nic: fake - name: k8s-worker-0 address: 10.10.10.101 controller: false talos_disk: fake + talos_nic: fake bootstrap_dns_servers: ["1.1.1.1"] bootstrap_search_domain: "fake" bootstrap_pod_network: 10.69.0.0/16 diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 index 7da0fa0d..5dd7baca 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 @@ -2,10 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - #% if bootstrap_bgp.enabled %# - - ./cilium-bgp.yaml - #% endif %# - #% if ((not bootstrap_bgp.enabled) and (not bootstrap_feature_gates.dual_stack_ipv4_first)) %# - - ./cilium-l2.yaml - #% endif %# - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/config/cilium-l2.yaml.j2 similarity index 56% rename from bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 rename to bootstrap/templates/kubernetes/apps/kube-system/cilium/config/cilium-l2.yaml.j2 index caa35cab..7065f1fa 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/config/cilium-l2.yaml.j2 @@ -1,14 +1,16 @@ +#% if ((not bootstrap_bgp.enabled) and (not bootstrap_feature_gates.dual_stack_ipv4_first)) %# --- # https://docs.cilium.io/en/latest/network/l2-announcements apiVersion: cilium.io/v2alpha1 kind: CiliumL2AnnouncementPolicy metadata: - name: policy + name: l2-policy spec: loadBalancerIPs: true - # NOTE: This might need to be set if you have more than one active NIC on your hosts + # NOTE: interfaces might need to be set if you have more than one active NIC on your hosts # interfaces: # - ^eno[0-9]+ + # - ^eth[0-9]+ nodeSelector: matchLabels: kubernetes.io/os: linux @@ -16,7 +18,9 @@ spec: apiVersion: cilium.io/v2alpha1 kind: CiliumLoadBalancerIPPool metadata: - name: pool + name: l2-pool spec: - cidrs: + allowFirstLastIPs: "Yes" + blocks: - cidr: "${NODE_CIDR}" +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/config/cilium-l3.yaml.j2 similarity index 90% rename from bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 rename to bootstrap/templates/kubernetes/apps/kube-system/cilium/config/cilium-l3.yaml.j2 index 7e15b6f6..fa51e2da 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-bgp.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/config/cilium-l3.yaml.j2 @@ -1,9 +1,10 @@ +#% if bootstrap_bgp.enabled %# --- # https://docs.cilium.io/en/latest/network/bgp-control-plane/ apiVersion: cilium.io/v2alpha1 kind: CiliumBGPPeeringPolicy metadata: - name: policy + name: l3-policy spec: nodeSelector: matchLabels: @@ -31,7 +32,9 @@ spec: apiVersion: cilium.io/v2alpha1 kind: CiliumLoadBalancerIPPool metadata: - name: pool + name: l3-pool spec: - cidrs: + allowFirstLastIPs: "Yes" + blocks: - cidr: "${BGP_ADVERTISED_CIDR}" +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/config/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/config/kustomization.yaml.j2 new file mode 100644 index 00000000..4fc169b4 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/config/kustomization.yaml.j2 @@ -0,0 +1,11 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + #% if bootstrap_bgp.enabled %# + - ./cilium-l3.yaml + #% elif not bootstrap_feature_gates.dual_stack_ipv4_first %# + - ./cilium-l2.yaml + #% else %# + [] + #% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 index 34fbd9a9..2522f1df 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2 @@ -11,6 +11,28 @@ spec: app.kubernetes.io/name: *app path: ./kubernetes/apps/kube-system/cilium/app prune: false # never should be deleted + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app cilium-config + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: cilium + path: ./kubernetes/apps/kube-system/cilium/config + prune: false # never should be deleted sourceRef: kind: GitRepository name: home-kubernetes diff --git a/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2 b/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2 index 502efdd5..b3cf21b5 100644 --- a/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2 +++ b/bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2 @@ -40,7 +40,8 @@ nodes: #% endif %# controlPlane: #{ (item.controller) | string | lower }# networkInterfaces: - - interface: eth0 + - deviceSelector: + hardwareAddr: "#{ item.talos_nic }#" dhcp: false #% if bootstrap_talos.vlan %# vlans: @@ -122,8 +123,7 @@ patches: image-gc-high-threshold: 55 rotate-server-certificates: true nodeIP: - validSubnets: - - "#{ bootstrap_node_network }#" + validSubnets: ["#{ bootstrap_node_network }#"] # Force nameserver - |- @@ -139,8 +139,7 @@ patches: machine: time: disabled: false - servers: - - time.cloudflare.com + servers: ["time.cloudflare.com"] # Custom sysctl settings - |- @@ -158,17 +157,7 @@ patches: - destination: /var/openebs/local type: bind source: /var/openebs/local - options: - - bind - - rshared - - rw - - # Disable predictable NIC naming - - |- - machine: - install: - extraKernelArgs: - - net.ifnames=0 + options: ["bind", "rshared", "rw"] #% if bootstrap_talos.secureboot.enabled and bootstrap_talos.secureboot.encrypt_disk_with_tpm %# # Encrypt system disk with TPM @@ -186,6 +175,7 @@ patches: - slot: 0 tpm: {} #% endif %# + #% if bootstrap_talos.user_patches %# # User specified global patches - "@./patches/global.yaml" @@ -226,10 +216,8 @@ controlPlane: features: kubernetesTalosAPIAccess: enabled: true - allowedRoles: - - os:admin - allowedKubernetesNamespaces: - - system-upgrade + allowedRoles: ["os:admin"] + allowedKubernetesNamespaces: ["system-upgrade"] #% if bootstrap_talos.user_patches %# # User specified controlPlane patches diff --git a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 index 111b031b..3a17c8c8 100644 --- a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 @@ -15,9 +15,8 @@ containerRuntime: #% if bootstrap_distribution in ["k3s"] %# socketPath: /var/run/k3s/containerd/containerd.sock #% endif %# -# NOTE: This might need to be set if you have more than one active NIC on your hosts -# devices: -# - eno0 +# NOTE: devices might need to be set if you have more than one active NIC on your hosts +# devices: eno+ eth+ endpointRoutes: enabled: true #% if bootstrap_cloudflare.enabled %# diff --git a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 index 8ec51ebf..9dae5d5a 100644 --- a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 @@ -15,9 +15,8 @@ containerRuntime: #% if bootstrap_distribution in ["k3s"] %# socketPath: /var/run/k3s/containerd/containerd.sock #% endif %# -# NOTE: This might need to be set if you have more than one active NIC on your hosts -# devices: -# - eno0 +# NOTE: devices might need to be set if you have more than one active NIC on your hosts +# devices: eno+ eth+ endpointRoutes: enabled: true hubble: diff --git a/config.sample.yaml b/config.sample.yaml index 1ac69f88..ef071dae 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -49,7 +49,8 @@ bootstrap_node_inventory: [] # - name: "" # (Required) Name of the node (must match [a-z0-9-\.]+) # address: "" # (Required) IP address of the node # controller: true # (Required) Set to true if this is a controller node - # talos_disk: "" # (Required: Talos) Device path or serial number of the disk for this node + # talos_disk: "" # (Required: Talos) Device path or serial number of the disk for this node (talosctl disks -n --insecure) + # talos_nic: "" # (Required: Talos) MAC address of the NIC for this node (talosctl get links -n --insecure) # ssh_user: "" # (Required: k3s) SSH username of the node # ssh_key: "" # (Optional: k3s) Set specific SSH key for this node # ...