feat: talos_patches makejinja func and move certain patches to patches directory (#1509)

bootstrap/scripts/plugin.py

@@ -11,6 +11,20 @@ import makejinja
 import validation
 
 
+# Return the filename of a path without the j2 extension
+def basename(value: str) -> str:
+    return Path(value).stem
+
+
+# Return a list of files in the talos patches directory
+def talos_patches(value: str) -> list[str]:
+    path = Path(f'bootstrap/templates/kubernetes/bootstrap/talos/patches/{value}')
+    if not path.is_dir():
+        return []
+    return [str(f) for f in path.glob('*.yaml.j2') if f.is_file()]
+
+
+# Return the nth host in a CIDR range
 def nthhost(value: str, query: int) -> str:
     value = IPNetwork(value)
     try:
@@ -50,7 +64,11 @@ class Plugin(makejinja.plugin.Plugin):
 
 
     def filters(self) -> makejinja.plugin.Filters:
-        return [nthhost]
+        return [basename, nthhost]
+
+
+    def functions(self) -> makejinja.plugin.Functions:
+        return [talos_patches]
 
 
     def path_filters(self):

bootstrap/templates/kubernetes/bootstrap/talos/patches/README.md.j2

@@ -0,0 +1,15 @@
+# Talos Patching
+
+This directory contains Kustomization patches that are added to the talhelper configuration file.
+
+<https://www.talos.dev/v1.7/talos-guides/configuration/patching/>
+
+## Patch Directories
+
+Under this `patches` directory, there are several sub-directories that can contain patches that are added to the talhelper configuration file.
+Each directory is optional and therefore might not created by default.
+
+- `global/`: patches that are applied to both the controller and worker configurations
+- `controller/`: patches that are applied to the controller configurations
+- `worker/`: patches that are applied to the worker configurations
+- `${node-hostname}/`: patches that are applied to the node with the specified name

bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/api-access.yaml.j2

@@ -0,0 +1,8 @@
+machine:
+  features:
+    kubernetesTalosAPIAccess:
+      enabled: true
+      allowedRoles:
+        - os:admin
+      allowedKubernetesNamespaces:
+        - system-upgrade

bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/cluster.yaml.j2

@@ -0,0 +1,12 @@
+cluster:
+  allowSchedulingOnControlPlanes: true
+  controllerManager:
+    extraArgs:
+      bind-address: 0.0.0.0
+  coreDNS:
+    disabled: true
+  proxy:
+    disabled: true
+  scheduler:
+    extraArgs:
+      bind-address: 0.0.0.0

bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/disable-admission-controller.yaml.j2

@@ -0,0 +1,2 @@
+- op: remove
+  path: /cluster/apiServer/admissionControl

bootstrap/templates/kubernetes/bootstrap/talos/patches/controller/etcd.yaml.j2

@@ -0,0 +1,6 @@
+cluster:
+  etcd:
+    extraArgs:
+      listen-metrics-urls: http://0.0.0.0:2381
+    advertisedSubnets:
+      - #{ bootstrap_node_network }#

bootstrap/templates/kubernetes/bootstrap/talos/patches/global/cluster-discovery.yaml.j2

@@ -0,0 +1,7 @@
+cluster:
+  discovery:
+    registries:
+      kubernetes:
+        disabled: false
+      service:
+        disabled: false

bootstrap/templates/kubernetes/bootstrap/talos/patches/global/containerd.yaml.j2

@@ -0,0 +1,12 @@
+machine:
+  files:
+    - op: create
+      path: /etc/cri/conf.d/20-customization.part
+      content: |-
+        [plugins."io.containerd.grpc.v1.cri"]
+          enable_unprivileged_ports = true
+          enable_unprivileged_icmp = true
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+          discard_unpacked_layers = false
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          discard_unpacked_layers = false

bootstrap/templates/kubernetes/bootstrap/talos/patches/global/disable-search-domain.yaml.j2

@@ -0,0 +1,3 @@
+machine:
+  network:
+    disableSearchDomain: true

bootstrap/templates/kubernetes/bootstrap/talos/patches/global/hostdns.yaml.j2

@@ -0,0 +1,6 @@
+machine:
+  features:
+    hostDNS:
+      enabled: true
+      resolveMemberNames: true
+      forwardKubeDNSToHost: false

bootstrap/templates/kubernetes/bootstrap/talos/patches/global/kubelet.yaml.j2

@@ -0,0 +1,7 @@
+machine:
+  kubelet:
+    extraArgs:
+      rotate-server-certificates: true
+    nodeIP:
+      validSubnets:
+        - #{ bootstrap_node_network }#

bootstrap/templates/kubernetes/bootstrap/talos/patches/global/openebs-local.yaml.j2

@@ -0,0 +1,10 @@
+machine:
+  kubelet:
+    extraMounts:
+      - destination: /var/openebs/local
+        type: bind
+        source: /var/openebs/local
+        options:
+          - bind
+          - rshared
+          - rw

bootstrap/templates/kubernetes/bootstrap/talos/patches/global/sysctl.yaml.j2

@@ -0,0 +1,7 @@
+machine:
+  sysctls:
+    fs.inotify.max_queued_events: "65536"
+    fs.inotify.max_user_watches: "524288"
+    fs.inotify.max_user_instances: "8192"
+    net.core.rmem_max: "2500000"
+    net.core.wmem_max: "2500000"

bootstrap/templates/kubernetes/bootstrap/talos/talconfig.yaml.j2

@@ -82,10 +82,6 @@ nodes:
           ip: "#{ bootstrap_controller_vip }#"
         #% endif %#
         #% endif %#
-    #% if bootstrap_user_patches %#
-    patches:
-      - "@./patches/node_#{ item.name }#.yaml"
-    #% endif %#
     #% if item.manifests %#
     extraManifests:
       #% for manifest in item.manifests %#
@@ -110,62 +106,19 @@ nodes:
         #% endif %#
       #% endfor %#
     #% endif %#
+    #% for file in talos_patches('%s' % (item.name)) %#
+    #% if loop.index == 1 %#
+    patches:
+    #% endif %#
+      - "@./patches/#{ item.name }#/#{ file | basename }#"
+    #% endfor %#
   #% endfor %#
 
+# Global patches
 patches:
-  # Configure containerd
-  - |-
-    machine:
-      files:
-        - op: create
-          path: /etc/cri/conf.d/20-customization.part
-          content: |-
-            [plugins."io.containerd.grpc.v1.cri"]
-              enable_unprivileged_ports = true
-              enable_unprivileged_icmp = true
-            [plugins."io.containerd.grpc.v1.cri".containerd]
-              discard_unpacked_layers = false
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-              discard_unpacked_layers = false
-
-  # Disable search domain everywhere
-  - |-
-    machine:
-      network:
-        disableSearchDomain: true
-
-  # Enable cluster discovery
-  - |-
-    cluster:
-      discovery:
-        registries:
-          kubernetes:
-            disabled: false
-          service:
-            disabled: false
-
-  # Configure kubelet
-  - |-
-    machine:
-      kubelet:
-        extraArgs:
-          rotate-server-certificates: true
-        nodeIP:
-          validSubnets:
-            - #{ bootstrap_node_network }#
-
-  # Enable host dns
-  - |-
-    machine:
-      features:
-        hostDNS:
-          enabled: true
-          resolveMemberNames: true
-          forwardKubeDNSToHost: false
-
   #% if bootstrap_dns_servers | length %#
-  # Force nameserver
+  - # Force nameserver
-  - |-
+    |-
     machine:
       network:
         nameservers:
@@ -173,10 +126,9 @@ patches:
           - #{ item }#
           #% endfor %#
   #% endif %#
-
   #% if bootstrap_ntp_servers | length %#
-  # Configure NTP
+  - # Configure NTP
-  - |-
+    |-
     machine:
       time:
         disabled: false
@@ -185,33 +137,9 @@ patches:
           - #{ item }#
           #% endfor %#
   #% endif %#
-
-  # Custom sysctl settings
-  - |-
-    machine:
-      sysctls:
-        fs.inotify.max_queued_events: "65536"
-        fs.inotify.max_user_watches: "524288"
-        fs.inotify.max_user_instances: "8192"
-        net.core.rmem_max: "2500000"
-        net.core.wmem_max: "2500000"
-
-  # Mount openebs-hostpath in kubelet
-  - |-
-    machine:
-      kubelet:
-        extraMounts:
-          - destination: /var/openebs/local
-            type: bind
-            source: /var/openebs/local
-            options:
-              - bind
-              - rshared
-              - rw
-
   #% if bootstrap_secureboot.enabled and bootstrap_secureboot.encrypt_disk_with_tpm %#
-  # Encrypt system disk with TPM
+  - # Encrypt system disk with TPM
-  - |-
+    |-
     machine:
       systemDiskEncryption:
         ephemeral:
@@ -225,60 +153,26 @@ patches:
             - slot: 0
               tpm: {}
   #% endif %#
+  #% for file in talos_patches('global') %#
+  - "@./patches/global/#{ file | basename }#"
+  #% endfor %#
 
-  #% if bootstrap_user_patches %#
+#% for file in talos_patches('controller') %#
-  # User specified global patches
+#% if loop.index == 1 %#
-  - "@./patches/global.yaml"
+# Controller patches
-  #% endif %#
-
 controlPlane:
   patches:
-    # Cluster configuration
+#% endif %#
-    - |-
+    - "@./patches/controller/#{ file | basename }#"
-      cluster:
+#% endfor %#
-        allowSchedulingOnControlPlanes: true
-        controllerManager:
-          extraArgs:
-            bind-address: 0.0.0.0
-        coreDNS:
-          disabled: true
-        proxy:
-          disabled: true
-        scheduler:
-          extraArgs:
-            bind-address: 0.0.0.0
 
-    # ETCD configuration
+#% if (bootstrap_node_inventory | selectattr('controller', 'equalto', False) | list | length) and (talos_patches('worker') | length) %#
-    - |-
+#% for file in talos_patches('worker') %#
-      cluster:
+#% if loop.index == 1 %#
-        etcd:
+# Worker patches
-          extraArgs:
-            listen-metrics-urls: http://0.0.0.0:2381
-          advertisedSubnets:
-            - #{ bootstrap_node_network }#
-
-    # Disable default API server admission plugins.
-    - |-
-      - op: remove
-        path: /cluster/apiServer/admissionControl
-
-    # Enable K8s Talos API Access
-    - |-
-      machine:
-        features:
-          kubernetesTalosAPIAccess:
-            enabled: true
-            allowedRoles:
-              - os:admin
-            allowedKubernetesNamespaces:
-              - system-upgrade
-    #% if bootstrap_user_patches %#
-    # User specified controlPlane patches
-    - "@./patches/controlPlane.yaml"
-    #% endif %#
-#% if ((bootstrap_user_patches) and (bootstrap_node_inventory | selectattr('controller', 'equalto', False) | list | length)) %#
 worker:
   patches:
-    # User specified worker patches
+#% endif %#
-    - "@./patches/worker.yaml"
+    - "@./patches/worker/#{ file | basename }#"
+#% endfor %#
 #% endif %#

config.sample.yaml

@@ -111,16 +111,6 @@ bootstrap_secureboot:
   #   See: https://www.talos.dev/v1.6/talos-guides/install/bare-metal-platforms/secureboot/#disk-encryption-with-tpm
   encrypt_disk_with_tpm: false
 
-# (Optional) Add includes for user provided patches to generated talconfig.yaml.
-#   See: https://github.com/budimanjojo/talhelper/blob/179ba9ed42f70069c7842109bea24f769f7af6eb/example/extraKernelArgs-patch.yaml
-#   Patches are applied in this order. (global overrides cp/worker which overrides node-specific).
-#   Create these files to allow talos:bootstrap-genconfig to complete (empty files are ok).
-#     kubernetes/bootstrap/talos/patches/node_<name>.yaml  # Patches for individual nodes
-#     kubernetes/bootstrap/talos/patches/controlPlane.yaml # Patches for controlplane nodes
-#     kubernetes/bootstrap/talos/patches/worker.yaml       # Patches for worker nodes
-#     kubernetes/bootstrap/talos/patches/global.yaml       # Patches for ALL nodes
-bootstrap_user_patches: false
-
 #
 # 2. (Required) Flux details - Flux is used to manage the cluster configuration.
 #
@@ -219,6 +209,4 @@ bootstrap_cloudflare:
 #   # IMPORTANT: I am looking for people to help maintain IPv6 support since I cannot test it.
 #   #   Ref: https://github.com/onedr0p/cluster-template/issues/1148
 #   # Keep in mind that Cilium does not currently support IPv6 L2 announcements.
-#   # Make sure you set cluster.pod_cidr and cluster.service_cidr
-#   #   to a valid dual stack CIDRs, e.g. "10.42.0.0/16,fd00:10:244::/64"
 #   dual_stack_ipv4_first: false