From 2267494b982a04ffd0da383f3a1691798f7192d0 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Tue, 30 Jan 2024 09:10:46 -0500 Subject: [PATCH] feat: add SUC for talos and k3s and move schematics to cluster.nodes.talos Signed-off-by: Devin Buhl --- .github/tests/config-talos.yaml | 11 ++ .../apps/system-upgrade/.mjfilter.py | 1 + .../system-upgrade/app/helmrelease.yaml.j2 | 103 ++++++++++++++++++ .../system-upgrade/app/kustomization.yaml.j2 | 8 ++ .../apps/system-upgrade/app/rbac.yaml.j2 | 23 ++++ .../kubernetes/apps/system-upgrade/ks.yaml.j2 | 44 ++++++++ .../apps/system-upgrade/plans/k3s.yaml.j2 | 54 +++++++++ .../plans/kustomization.yaml.j2 | 11 ++ .../apps/system-upgrade/plans/talos.yaml.j2 | 98 +++++++++++++++++ .../kubernetes/talos/talconfig.yaml.j2 | 21 ++++ config.sample.yaml | 24 +++- 11 files changed, 392 insertions(+), 6 deletions(-) create mode 100644 bootstrap/templates/kubernetes/apps/system-upgrade/.mjfilter.py create mode 100644 bootstrap/templates/kubernetes/apps/system-upgrade/app/helmrelease.yaml.j2 create mode 100644 bootstrap/templates/kubernetes/apps/system-upgrade/app/kustomization.yaml.j2 create mode 100644 bootstrap/templates/kubernetes/apps/system-upgrade/app/rbac.yaml.j2 create mode 100644 bootstrap/templates/kubernetes/apps/system-upgrade/ks.yaml.j2 create mode 100644 bootstrap/templates/kubernetes/apps/system-upgrade/plans/k3s.yaml.j2 create mode 100644 bootstrap/templates/kubernetes/apps/system-upgrade/plans/kustomization.yaml.j2 create mode 100644 bootstrap/templates/kubernetes/apps/system-upgrade/plans/talos.yaml.j2 diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml index 1b1a3afa..4d38ee39 100644 --- a/.github/tests/config-talos.yaml +++ b/.github/tests/config-talos.yaml @@ -18,6 +18,17 @@ cluster: address: 10.10.10.101 controller: false talos_disk_device: fake + talos: + schematics: + enabled: true + id: df491c50a5acc05b977ef00c32050e1ceb0df746e40b33c643ac8a9bfb7c7263 + customization: + extraKernelArgs: + - net.ifnames=0 + systemExtensions: + officialExtensions: + - siderolabs/intel-ucode + - siderolabs/i915-ucode pod_network: 10.69.0.0/16 service_network: 10.96.0.0/16 endpoint_vip: 10.10.10.254 diff --git a/bootstrap/templates/kubernetes/apps/system-upgrade/.mjfilter.py b/bootstrap/templates/kubernetes/apps/system-upgrade/.mjfilter.py new file mode 100644 index 00000000..7b935772 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/system-upgrade/.mjfilter.py @@ -0,0 +1 @@ +main = lambda data: data.get("distribution") in ['k3s', 'talos'] diff --git a/bootstrap/templates/kubernetes/apps/system-upgrade/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/system-upgrade/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..897a1784 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/system-upgrade/app/helmrelease.yaml.j2 @@ -0,0 +1,103 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: &app system-upgrade-controller +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 2.5.0 + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + controllers: + main: + strategy: RollingUpdate + containers: + main: + image: + repository: docker.io/rancher/system-upgrade-controller + tag: v0.13.2 + env: + SYSTEM_UPGRADE_CONTROLLER_DEBUG: false + SYSTEM_UPGRADE_CONTROLLER_THREADS: 2 + SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: 900 + SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: 99 + SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: IfNotPresent + SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: docker.io/rancher/kubectl:v1.29.0 + SYSTEM_UPGRADE_JOB_PRIVILEGED: true + SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: 900 + SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m + SYSTEM_UPGRADE_CONTROLLER_NAME: *app + SYSTEM_UPGRADE_CONTROLLER_NAMESPACE: + valueFrom: + fieldRef: + fieldPath: metadata.namespace + pod: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ["ALL"] + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - {key: CriticalAddonsOnly, operator: Exists} + - {key: node-role.kubernetes.io/master, operator: Exists, effect: NoSchedule} + - {key: node-role.kubernetes.io/controlplane, operator: Exists, effect: NoSchedule} + - {key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule} + - {key: node-role.kubernetes.io/etcd, operator: Exists, effect: NoExecute} + serviceAccount: + create: true + name: system-upgrade + service: + main: + enabled: false + persistence: + tmp: + type: emptyDir + globalMounts: + - path: /tmp + etc-ssl: + type: hostPath + hostPath: /etc/ssl + hostPathType: DirectoryOrCreate + globalMounts: + - path: /etc/ssl + readOnly: true + etc-pki: + type: hostPath + hostPath: /etc/pki + hostPathType: DirectoryOrCreate + globalMounts: + - path: /etc/pki + readOnly: true + etc-ca-certificates: + type: hostPath + hostPath: /etc/ca-certificates + hostPathType: DirectoryOrCreate + globalMounts: + - path: /etc/ca-certificates + readOnly: true diff --git a/bootstrap/templates/kubernetes/apps/system-upgrade/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/system-upgrade/app/kustomization.yaml.j2 new file mode 100644 index 00000000..74d66703 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/system-upgrade/app/kustomization.yaml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # renovate: datasource=github-releases depName=rancher/system-upgrade-controller + - https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.2/crd.yaml + - helmrelease.yaml + - rbac.yaml diff --git a/bootstrap/templates/kubernetes/apps/system-upgrade/app/rbac.yaml.j2 b/bootstrap/templates/kubernetes/apps/system-upgrade/app/rbac.yaml.j2 new file mode 100644 index 00000000..8da574dd --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/system-upgrade/app/rbac.yaml.j2 @@ -0,0 +1,23 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-upgrade +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: system-upgrade + namespace: system-upgrade +{% if distribution in ['talos'] %} +--- +apiVersion: talos.dev/v1alpha1 +kind: ServiceAccount +metadata: + name: talos +spec: + roles: + - os:operator +{% endif %} diff --git a/bootstrap/templates/kubernetes/apps/system-upgrade/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/system-upgrade/ks.yaml.j2 new file mode 100644 index 00000000..50b0382c --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/system-upgrade/ks.yaml.j2 @@ -0,0 +1,44 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app system-upgrade-controller + namespace: flux-system +spec: + targetNamespace: system-upgrade + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/system-upgrade/system-upgrade-controller/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m +{% if distribution in ['k3s'] or (distribution in ['talos'] and cluster.nodes.talos.schematics.enabled) %} +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app system-upgrade-controller-plans + namespace: flux-system +spec: + targetNamespace: system-upgrade + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: system-upgrade-controller + path: ./kubernetes/apps/system-upgrade/system-upgrade-controller/plans + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m +{% endif %} diff --git a/bootstrap/templates/kubernetes/apps/system-upgrade/plans/k3s.yaml.j2 b/bootstrap/templates/kubernetes/apps/system-upgrade/plans/k3s.yaml.j2 new file mode 100644 index 00000000..9f93eb22 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/system-upgrade/plans/k3s.yaml.j2 @@ -0,0 +1,54 @@ +{% if distribution in ['k3s'] %} +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: controllers +spec: + # renovate: datasource=github-releases depName=k3s-io/k3s + version: v1.29.0+k3s1 + upgrade: + image: rancher/k3s-upgrade + serviceAccountName: system-upgrade + concurrency: 1 + cordon: true + nodeSelector: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: Exists + - key: node-role.kubernetes.io/etcd + effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: workers +spec: + # renovate: datasource=github-releases depName=k3s-io/k3s + version: v1.29.0+k3s1 + serviceAccountName: system-upgrade + concurrency: 1 + nodeSelector: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + prepare: + image: rancher/k3s-upgrade + args: ["prepare", "server"] + upgrade: + image: rancher/k3s-upgrade +{% endif %} diff --git a/bootstrap/templates/kubernetes/apps/system-upgrade/plans/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/system-upgrade/plans/kustomization.yaml.j2 new file mode 100644 index 00000000..f49d9593 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/system-upgrade/plans/kustomization.yaml.j2 @@ -0,0 +1,11 @@ +{% if distribution in ['k3s'] or (distribution in ['talos'] and cluster.nodes.talos.schematics.enabled) %} +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + {% if distribution in ['k3s'] %} + - ./k3s.yaml + {% elif distribution in ['talos'] and cluster.nodes.talos.schematics.enabled %} + - ./talos.yaml + {% endif %} +{% endif %} diff --git a/bootstrap/templates/kubernetes/apps/system-upgrade/plans/talos.yaml.j2 b/bootstrap/templates/kubernetes/apps/system-upgrade/plans/talos.yaml.j2 new file mode 100644 index 00000000..5b705945 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/system-upgrade/plans/talos.yaml.j2 @@ -0,0 +1,98 @@ +{% if distribution in ['talos'] and cluster.nodes.talos.schematics.enabled %} +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: kubernetes +spec: + # renovate: datasource=docker depName=ghcr.io/siderolabs/kubelet + version: v1.29.1 + serviceAccountName: system-upgrade + secrets: + - name: talos + path: /var/run/secrets/talos.dev + ignoreUpdates: true + concurrency: 1 + exclusive: true + nodeSelector: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + prepare: &prepare + image: ghcr.io/siderolabs/talosctl:v1.6.3 + envs: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + args: + - --nodes=$(NODE_IP) + - health + - --server=false + upgrade: + <<: *prepare + args: + - --nodes=$(NODE_IP) + - upgrade-k8s + - --to=$(SYSTEM_UPGRADE_PLAN_LATEST_VERSION) +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: talos +spec: + # renovate: datasource=docker depName=ghcr.io/siderolabs/installer + version: v1.6.3 + serviceAccountName: system-upgrade + secrets: + - name: talos + path: /var/run/secrets/talos.dev + ignoreUpdates: true + concurrency: 1 + cordon: true + drain: + deleteLocalData: true + disableEviction: false + ignoreDaemonSets: true + exclusive: true + nodeSelector: + matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/controlplane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/etcd + operator: Exists + effect: NoSchedule + prepare: &prepare + image: ghcr.io/siderolabs/talosctl + envs: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + args: + - --nodes=$(NODE_IP) + - health + - --server=false + upgrade: + <<: *prepare + args: + - --nodes=$(NODE_IP) + - upgrade + - --image=factory.talos.dev/installer/{{ cluster.nodes.talos.schematics.id }}:$(SYSTEM_UPGRADE_PLAN_LATEST_VERSION) + - --preserve=true + - --wait=false +{% endif %} diff --git a/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 index 24cf01c7..5fc240a1 100644 --- a/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 +++ b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 @@ -55,6 +55,11 @@ nodes: {% endfor %} controlPlane: +{% if cluster.nodes.talos.schematics.enabled %} + schematic: + customization: +{{ cluster.nodes.talos.schematics.customization | indent(6, first=True) }} +{% endif %} patches: # Configure containerd - &containerdPatch |- @@ -157,7 +162,23 @@ controlPlane: - op: remove path: /cluster/apiServer/admissionControl + # Enable K8s Talos API Access + - |- + machine: + features: + kubernetesTalosAPIAccess: + enabled: true + allowedRoles: + - os:operator + allowedKubernetesNamespaces: + - system-upgrade + worker: +{% if cluster.nodes.talos.schematics.enabled %} + schematic: + customization: +{{ cluster.nodes.talos.schematics.customization | indent(6, first=True) }} +{% endif %} patches: - *containerdPatch - *disableSearchDomainPatch diff --git a/config.sample.yaml b/config.sample.yaml index f85d9a36..19d63ef3 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -32,13 +32,25 @@ cluster: # controller: true # (Required) Set to true if this is a controller node # ssh_username: "" # (Required: k3s/k0s) SSH username of the node # talos_disk_device: "" # (Required: Talos) Device Path or Serial number of Disk for this node - # talos_node_config: |- # (Optional: Talos) Additional NodeConfigs to apply to this node - # schematic: # See: https://budimanjojo.github.io/talhelper/latest/reference/configuration/#nodeconfigs - # customization: - # systemExtensions: - # officialExtensions: - # - siderolabs/intel-ucode # ... + # (Optional) Talos Specific Options + talos: {} + # schematics: + # # (Required) Disabling the schematic will disable the system-upgrade-controller and any + # # additional customizations you have defined. + # enabled: false + # # (Required) For use with the system-upgrade-controller generate a schematic ID + # # based on your System Extension requirements https://factory.talos.dev/ + # id: "" + # # (Required) Additional NodeConfigs to apply to all nodes + # # See: https://budimanjojo.github.io/talhelper/latest/reference/configuration/#nodeconfigs + # customization: + # extraKernelArgs: + # - net.ifnames=0 + # systemExtensions: + # officialExtensions: + # - siderolabs/intel-ucode + # - siderolabs/i915-ucode # (Required) The pod CIDR for the cluster, this must NOT overlap with any # existing networks and is usually a /16 (64K IPs). # If you want to use IPv6 check the advanced flags below