feat: add SUC for talos and k3s and move schematics to cluster.nodes.talos

Signed-off-by: Devin Buhl <devin@buhl.casa>

.github/tests/config-talos.yaml

@@ -18,6 +18,17 @@ cluster:
         address: 10.10.10.101
         controller: false
         talos_disk_device: fake
+    talos:
+      schematics:
+        enabled: true
+        id: df491c50a5acc05b977ef00c32050e1ceb0df746e40b33c643ac8a9bfb7c7263
+        customization:
+          extraKernelArgs:
+            - net.ifnames=0
+          systemExtensions:
+            officialExtensions:
+              - siderolabs/intel-ucode
+              - siderolabs/i915-ucode
   pod_network: 10.69.0.0/16
   service_network: 10.96.0.0/16
   endpoint_vip: 10.10.10.254

bootstrap/templates/kubernetes/apps/system-upgrade/.mjfilter.py

@@ -0,0 +1 @@
+main = lambda data: data.get("distribution") in ['k3s', 'talos']

bootstrap/templates/kubernetes/apps/system-upgrade/app/helmrelease.yaml.j2

@@ -0,0 +1,103 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: &app system-upgrade-controller
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 2.5.0
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    controllers:
+      main:
+        strategy: RollingUpdate
+        containers:
+          main:
+            image:
+              repository: docker.io/rancher/system-upgrade-controller
+              tag: v0.13.2
+            env:
+              SYSTEM_UPGRADE_CONTROLLER_DEBUG: false
+              SYSTEM_UPGRADE_CONTROLLER_THREADS: 2
+              SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: 900
+              SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: 99
+              SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: IfNotPresent
+              SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: docker.io/rancher/kubectl:v1.29.0
+              SYSTEM_UPGRADE_JOB_PRIVILEGED: true
+              SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: 900
+              SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m
+              SYSTEM_UPGRADE_CONTROLLER_NAME: *app
+              SYSTEM_UPGRADE_CONTROLLER_NAMESPACE:
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.namespace
+        pod:
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65534
+            runAsGroup: 65534
+            allowPrivilegeEscalation: false
+            seccompProfile:
+              type: RuntimeDefault
+            capabilities:
+              drop: ["ALL"]
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                      - key: node-role.kubernetes.io/control-plane
+                        operator: Exists
+          tolerations:
+            - {key: CriticalAddonsOnly, operator: Exists}
+            - {key: node-role.kubernetes.io/master, operator: Exists, effect: NoSchedule}
+            - {key: node-role.kubernetes.io/controlplane, operator: Exists, effect: NoSchedule}
+            - {key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}
+            - {key: node-role.kubernetes.io/etcd, operator: Exists, effect: NoExecute}
+    serviceAccount:
+      create: true
+      name: system-upgrade
+    service:
+      main:
+        enabled: false
+    persistence:
+      tmp:
+        type: emptyDir
+        globalMounts:
+          - path: /tmp
+      etc-ssl:
+        type: hostPath
+        hostPath: /etc/ssl
+        hostPathType: DirectoryOrCreate
+        globalMounts:
+          - path: /etc/ssl
+            readOnly: true
+      etc-pki:
+        type: hostPath
+        hostPath: /etc/pki
+        hostPathType: DirectoryOrCreate
+        globalMounts:
+          - path: /etc/pki
+            readOnly: true
+      etc-ca-certificates:
+        type: hostPath
+        hostPath: /etc/ca-certificates
+        hostPathType: DirectoryOrCreate
+        globalMounts:
+          - path: /etc/ca-certificates
+            readOnly: true

bootstrap/templates/kubernetes/apps/system-upgrade/app/kustomization.yaml.j2

@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # renovate: datasource=github-releases depName=rancher/system-upgrade-controller
+  - https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.2/crd.yaml
+  - helmrelease.yaml
+  - rbac.yaml

bootstrap/templates/kubernetes/apps/system-upgrade/app/rbac.yaml.j2

@@ -0,0 +1,23 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system-upgrade
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: system-upgrade
+    namespace: system-upgrade
+{% if distribution in ['talos'] %}
+---
+apiVersion: talos.dev/v1alpha1
+kind: ServiceAccount
+metadata:
+  name: talos
+spec:
+  roles:
+    - os:operator
+{% endif %}

bootstrap/templates/kubernetes/apps/system-upgrade/ks.yaml.j2

@@ -0,0 +1,44 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app system-upgrade-controller
+  namespace: flux-system
+spec:
+  targetNamespace: system-upgrade
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/system-upgrade/system-upgrade-controller/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+{% if distribution in ['k3s'] or (distribution in ['talos'] and cluster.nodes.talos.schematics.enabled) %}
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app system-upgrade-controller-plans
+  namespace: flux-system
+spec:
+  targetNamespace: system-upgrade
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  dependsOn:
+    - name: system-upgrade-controller
+  path: ./kubernetes/apps/system-upgrade/system-upgrade-controller/plans
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+{% endif %}

bootstrap/templates/kubernetes/apps/system-upgrade/plans/k3s.yaml.j2

@@ -0,0 +1,54 @@
+{% if distribution in ['k3s'] %}
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: controllers
+spec:
+  # renovate: datasource=github-releases depName=k3s-io/k3s
+  version: v1.29.0+k3s1
+  upgrade:
+    image: rancher/k3s-upgrade
+  serviceAccountName: system-upgrade
+  concurrency: 1
+  cordon: true
+  nodeSelector:
+    matchExpressions:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+  tolerations:
+    - effect: NoSchedule
+      operator: Exists
+    - effect: NoExecute
+      operator: Exists
+    - key: node-role.kubernetes.io/control-plane
+      effect: NoSchedule
+      operator: Exists
+    - key: node-role.kubernetes.io/master
+      effect: NoSchedule
+      operator: Exists
+    - key: node-role.kubernetes.io/etcd
+      effect: NoExecute
+      operator: Exists
+    - key: CriticalAddonsOnly
+      operator: Exists
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: workers
+spec:
+  # renovate: datasource=github-releases depName=k3s-io/k3s
+  version: v1.29.0+k3s1
+  serviceAccountName: system-upgrade
+  concurrency: 1
+  nodeSelector:
+    matchExpressions:
+      - key: node-role.kubernetes.io/control-plane
+        operator: DoesNotExist
+  prepare:
+    image: rancher/k3s-upgrade
+    args: ["prepare", "server"]
+  upgrade:
+    image: rancher/k3s-upgrade
+{% endif %}

bootstrap/templates/kubernetes/apps/system-upgrade/plans/kustomization.yaml.j2

@@ -0,0 +1,11 @@
+{% if distribution in ['k3s'] or (distribution in ['talos'] and cluster.nodes.talos.schematics.enabled) %}
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  {% if distribution in ['k3s'] %}
+  - ./k3s.yaml
+  {% elif distribution in ['talos'] and cluster.nodes.talos.schematics.enabled %}
+  - ./talos.yaml
+  {% endif %}
+{% endif %}

bootstrap/templates/kubernetes/apps/system-upgrade/plans/talos.yaml.j2

@@ -0,0 +1,98 @@
+{% if distribution in ['talos'] and cluster.nodes.talos.schematics.enabled %}
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: kubernetes
+spec:
+  # renovate: datasource=docker depName=ghcr.io/siderolabs/kubelet
+  version: v1.29.1
+  serviceAccountName: system-upgrade
+  secrets:
+    - name: talos
+      path: /var/run/secrets/talos.dev
+      ignoreUpdates: true
+  concurrency: 1
+  exclusive: true
+  nodeSelector:
+    matchExpressions:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+  prepare: &prepare
+    image: ghcr.io/siderolabs/talosctl:v1.6.3
+    envs:
+      - name: NODE_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.hostIP
+    args:
+      - --nodes=$(NODE_IP)
+      - health
+      - --server=false
+  upgrade:
+    <<: *prepare
+    args:
+      - --nodes=$(NODE_IP)
+      - upgrade-k8s
+      - --to=$(SYSTEM_UPGRADE_PLAN_LATEST_VERSION)
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: talos
+spec:
+  # renovate: datasource=docker depName=ghcr.io/siderolabs/installer
+  version: v1.6.3
+  serviceAccountName: system-upgrade
+  secrets:
+    - name: talos
+      path: /var/run/secrets/talos.dev
+      ignoreUpdates: true
+  concurrency: 1
+  cordon: true
+  drain:
+    deleteLocalData: true
+    disableEviction: false
+    ignoreDaemonSets: true
+  exclusive: true
+  nodeSelector:
+    matchExpressions:
+      - key: kubernetes.io/os
+        operator: In
+        values:
+          - linux
+  tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    - key: node-role.kubernetes.io/master
+      operator: Exists
+      effect: NoSchedule
+    - key: node-role.kubernetes.io/controlplane
+      operator: Exists
+      effect: NoSchedule
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+    - key: node-role.kubernetes.io/etcd
+      operator: Exists
+      effect: NoSchedule
+  prepare: &prepare
+    image: ghcr.io/siderolabs/talosctl
+    envs:
+      - name: NODE_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.hostIP
+    args:
+      - --nodes=$(NODE_IP)
+      - health
+      - --server=false
+  upgrade:
+    <<: *prepare
+    args:
+      - --nodes=$(NODE_IP)
+      - upgrade
+      - --image=factory.talos.dev/installer/{{ cluster.nodes.talos.schematics.id }}:$(SYSTEM_UPGRADE_PLAN_LATEST_VERSION)
+      - --preserve=true
+      - --wait=false
+{% endif %}

bootstrap/templates/kubernetes/talos/talconfig.yaml.j2

@@ -55,6 +55,11 @@ nodes:
   {% endfor %}
 
 controlPlane:
+{% if cluster.nodes.talos.schematics.enabled %}
+  schematic:
+    customization:
+{{ cluster.nodes.talos.schematics.customization | indent(6, first=True) }}
+{% endif %}
   patches:
     # Configure containerd
     - &containerdPatch |-
@@ -157,7 +162,23 @@ controlPlane:
       - op: remove
         path: /cluster/apiServer/admissionControl
 
+    # Enable K8s Talos API Access
+    - |-
+      machine:
+        features:
+          kubernetesTalosAPIAccess:
+            enabled: true
+            allowedRoles:
+              - os:operator
+            allowedKubernetesNamespaces:
+              - system-upgrade
+
 worker:
+{% if cluster.nodes.talos.schematics.enabled %}
+  schematic:
+    customization:
+{{ cluster.nodes.talos.schematics.customization | indent(6, first=True) }}
+{% endif %}
   patches:
     - *containerdPatch
     - *disableSearchDomainPatch

config.sample.yaml

@@ -32,13 +32,25 @@ cluster:
       #   controller: true       # (Required) Set to true if this is a controller node
       #   ssh_username: ""       # (Required: k3s/k0s) SSH username of the node
       #   talos_disk_device: ""  # (Required: Talos) Device Path or Serial number of Disk for this node
-      #   talos_node_config: |-  # (Optional: Talos) Additional NodeConfigs to apply to this node
-      #     schematic:           #  See: https://budimanjojo.github.io/talhelper/latest/reference/configuration/#nodeconfigs
-      #       customization:
-      #         systemExtensions:
-      #           officialExtensions:
-      #             - siderolabs/intel-ucode
       # ...
+    # (Optional) Talos Specific Options
+    talos: {}
+      # schematics:
+      #   # (Required) Disabling the schematic will disable the system-upgrade-controller and any
+      #   #   additional customizations you have defined.
+      #   enabled: false
+      #   # (Required) For use with the system-upgrade-controller generate a schematic ID
+      #   #   based on your System Extension requirements https://factory.talos.dev/
+      #   id: ""
+      #   # (Required) Additional NodeConfigs to apply to all nodes
+      #   #   See: https://budimanjojo.github.io/talhelper/latest/reference/configuration/#nodeconfigs
+      #   customization:
+      #     extraKernelArgs:
+      #       - net.ifnames=0
+      #     systemExtensions:
+      #       officialExtensions:
+      #         - siderolabs/intel-ucode
+      #         - siderolabs/i915-ucode
   # (Required) The pod CIDR for the cluster, this must NOT overlap with any
   #   existing networks and is usually a /16 (64K IPs).
   # If you want to use IPv6 check the advanced flags below