feat: add talos support (#1192)

Signed-off-by: Devin Buhl <devin@buhl.casa>

bootstrap/tasks/validation/cli.yaml

@@ -1,7 +1,6 @@
 ---
 - name: Check if required CLI tools are present
-  ansible.builtin.shell: |
-    command -v {{ item }} >/dev/null 2>&1
+  ansible.builtin.shell: command -v {{ item }} >/dev/null 2>&1
   loop: ["age", "cloudflared", "flux", "sops", "jq", "kubeconform", "kustomize"]
   changed_when: false
   check_mode: false
@@ -10,10 +9,18 @@
 
 - name: Check if required CLI tools are present for k0s
   when: bootstrap_distribution == "k0s"
-  ansible.builtin.shell: |
-    command -v {{ item }} >/dev/null 2>&1
+  ansible.builtin.shell: command -v {{ item }} >/dev/null 2>&1
   loop: ["k0sctl"]
   changed_when: false
   check_mode: false
   register: result
   failed_when: result.rc != 0 and result.rc != 127
+
+- name: Check if required CLI tools are present for talos
+  when: bootstrap_distribution == "talos"
+  ansible.builtin.shell: command -v {{ item }} >/dev/null 2>&1
+  loop: ["talhelper", "talosctl"]
+  changed_when: false
+  check_mode: false
+  register: result
+  failed_when: result.rc != 0 and result.rc != 127

bootstrap/tasks/validation/net.yaml

@@ -1,11 +1,4 @@
 ---
-- name: Set reachable address
-  ansible.builtin.set_fact:
-    current_address: "{{ item.external_address | default(item.address) }}"
-  loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
-  loop_control:
-    label: "{{ item.address }}"
-
 - name: Check if master node count is odd
   ansible.builtin.assert:
     that:
@@ -161,7 +154,6 @@
     fail_msg: Node address {{ item.address }} is not within {{ bootstrap_node_cidr }}.
     quiet: true
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
-  when: item.external_address is not defined
   loop_control:
     label: "{{ item.address }}"
 
@@ -193,14 +185,24 @@
     fail_msg: All node names are not unique.
     quiet: true
 
-- name: Check if nodes SSH ports are reachable
-  when: not ci_test | default(false)
+- name: Check if SSH ports are reachable
+  when: (not ci_test | default(false)) and (not bootstrap_distribution == "talos")
   ansible.builtin.wait_for:
-    host: "{{ current_address }}"
+    host: "{{ item.address }}"
     port: 22
-    search_regex: OpenSSH
     timeout: 10
   connection: local
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
   loop_control:
-    label: "{{ current_address }}"
+    label: "{{ item.address }}"
+
+- name: Check if Talos ports are reachable
+  when: (not ci_test | default(false)) and (bootstrap_distribution == "talos")
+  ansible.builtin.wait_for:
+    host: "{{ item.address }}"
+    port: 50000
+    timeout: 10
+  connection: local
+  loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
+  loop_control:
+    label: "{{ item.address }}"

bootstrap/tasks/validation/vars.yaml

@@ -33,7 +33,7 @@
 
 - name: Check if bootstrap distribution is valid
   ansible.builtin.assert:
-    that: bootstrap_distribution in ['k0s', 'k3s']
+    that: bootstrap_distribution in ['k0s', 'k3s', 'talos']
     success_msg: Distribution {{ bootstrap_distribution }} is valid
     fail_msg: Distribution {{ bootstrap_distribution }} is not valid
 

bootstrap/templates/.sops.yaml.j2

@@ -1,5 +1,10 @@
 ---
 creation_rules:
+  - # IMPORTANT: This rule MUST be above the others
+    path_regex: talos/.*\.sops\.ya?ml
+    key_groups:
+      - age:
+          - "#{ bootstrap_age_public_key }#"
   - path_regex: kubernetes/.*\.sops\.ya?ml
     encrypted_regex: "^(data|stringData)$"
     key_groups:

bootstrap/templates/ansible/inventory/hosts.yaml.j2

@@ -7,11 +7,7 @@ kubernetes:
       #% for item in bootstrap_nodes.master %#
         "#{ item.name }#":
           ansible_user: "#{ item.username }#"
-      #% if item.external_address is defined %#
-          ansible_host: "#{ item.external_address }#"
-      #% else %#
           ansible_host: "#{ item.address }#"
-      #% endif %#
       #% endfor %#
     #% if bootstrap_nodes.worker | default([]) | length > 0 %#
     worker:
@@ -19,11 +15,7 @@ kubernetes:
       #% for item in bootstrap_nodes.worker %#
         "#{ item.name }#":
           ansible_user: "#{ item.username }#"
-      #% if item.external_address is defined %#
-          ansible_host: "#{ item.external_address }#"
-      #% else %#
           ansible_host: "#{ item.address }#"
-      #% endif %#
       #% endfor %#
     #% endif %#
 #% endif %#

bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/helmrelease.yaml.j2

@@ -0,0 +1,29 @@
+#% if bootstrap_distribution == "talos" %#
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: kubelet-csr-approver
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: kubelet-csr-approver
+      version: 1.0.6
+      sourceRef:
+        kind: HelmRepository
+        name: postfinance
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    providerRegex: |
+      ^k(m|w)\d\d?$
+#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/kustomization.yaml.j2

@@ -0,0 +1,7 @@
+#% if bootstrap_distribution == "talos" %#
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/ks.yaml.j2

@@ -0,0 +1,22 @@
+#% if bootstrap_distribution == "talos" %#
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app kubelet-csr-approver
+  namespace: flux-system
+spec:
+  targetNamespace: kube-system
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/kube-system/kubelet-csr-approver/app
+  prune: false # never should be deleted
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2

@@ -7,7 +7,10 @@ resources:
   #% if bootstrap_distribution == "k3s" %#
   - ./coredns/ks.yaml
   #% endif %#
+  #% if bootstrap_distribution == "talos" %#
+  - ./kubelet-csr-approver/ks.yaml
+  #% endif %#
   - ./metrics-server/ks.yaml
-  #% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+  #% if (bootstrap_distribution == "k0s" or bootstrap_distribution == "talos") and spegel.enabled | default(false) %#
   - ./spegel/ks.yaml
   #% endif %#

bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+#% if (bootstrap_distribution == "k0s" or bootstrap_distribution == "talos") and spegel.enabled | default(false) %#
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2beta2
 kind: HelmRelease

bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+#% if (bootstrap_distribution == "k0s" or bootstrap_distribution == "talos") and spegel.enabled | default(false) %#
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+#% if (bootstrap_distribution == "k0s" or bootstrap_distribution == "talos") and spegel.enabled | default(false) %#
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization

bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2

@@ -18,6 +18,7 @@ resources:
   - ./metrics-server.yaml
   - ./openebs.yaml
   - ./piraeus.yaml
+  - ./postfinance.yaml
   - ./prometheus-community.yaml
   - ./stakater.yaml
   - ./weave-gitops.yaml

bootstrap/templates/kubernetes/flux/repositories/helm/postfinance.yaml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: postfinance
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://postfinance.github.io/kubelet-csr-approver

bootstrap/templates/kubernetes/talos/README.md.j2

@@ -0,0 +1,31 @@
+#% if bootstrap_distribution == 'talos' %#
+# Bootstrap
+
+## Talos
+
+### 1. Create Talos Secrets
+
+```sh
+task talos:gensecret
+task talos:genconfig
+```
+
+### 2. Apply Talos Config
+
+```sh
+task talos:apply-config
+```
+
+### 3. Boostrap Talos and get kubeconfig
+
+```sh
+task talos:bootstrap
+task talos:kubeconfig
+```
+
+### 4. Install Cilium and kubelet-csr-approver into the cluster
+
+```sh
+task talos:apply-extras
+```
+#% endif %#

bootstrap/templates/kubernetes/talos/cilium/.gitignore.j2

@@ -0,0 +1,3 @@
+#% if bootstrap_distribution == 'talos' %#
+charts/
+#% endif %#

bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2

@@ -0,0 +1,17 @@
+#% if bootstrap_distribution == 'talos' %#
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+helmCharts:
+  - name: cilium
+    repo: https://helm.cilium.io
+    version: 1.14.5
+    releaseName: cilium
+    namespace: kube-system
+    valuesFile: values.yaml
+commonAnnotations:
+  meta.helm.sh/release-name: cilium
+  meta.helm.sh/release-namespace: kube-system
+commonLabels:
+  app.kubernetes.io/managed-by: Helm
+#% endif %#

bootstrap/templates/kubernetes/talos/cilium/values.yaml.j2

@@ -0,0 +1,6 @@
+#% if bootstrap_distribution == 'talos' %#
+---
+#% filter indent(width=0, first=True) %#
+#% include 'partials/cilium-values-init.partial.yaml.j2' %#
+#% endfilter %#
+#% endif %#

bootstrap/templates/kubernetes/talos/kubelet-csr-approver/.gitignore.j2

@@ -0,0 +1,3 @@
+#% if bootstrap_distribution == 'talos' %#
+charts/
+#% endif %#

bootstrap/templates/kubernetes/talos/kubelet-csr-approver/kustomization.yaml

@@ -0,0 +1,20 @@
+#% if bootstrap_distribution == 'talos' %#
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kube-system
+helmCharts:
+  - name: kubelet-csr-approver
+    repo: https://postfinance.github.io/kubelet-csr-approver
+    version: 1.0.6
+    releaseName: kubelet-csr-approver
+    namespace: kube-system
+    valuesInline:
+      providerRegex: ^k(m|w)\d\d?$
+      bypassDnsResolution: true
+commonAnnotations:
+  meta.helm.sh/release-name: kubelet-csr-approver
+  meta.helm.sh/release-namespace: kube-system
+commonLabels:
+  app.kubernetes.io/managed-by: Helm
+#% endif %#

bootstrap/templates/kubernetes/talos/talconfig.yaml.j2

@@ -0,0 +1,192 @@
+#% if bootstrap_distribution == 'talos' %#
+# yaml-language-server: $schema=https://raw.githubusercontent.com/budimanjojo/talhelper/master/pkg/config/schemas/talconfig.json
+---
+# renovate: datasource=github-releases depName=siderolabs/talos
+talosVersion: v1.6.1
+# renovate: datasource=github-releases depName=kubernetes/kubernetes extractVersion=^v(?<version>.*)$
+kubernetesVersion: 1.29.1
+
+clusterName: &cluster home-kubernetes
+
+#% if bootstrap_kube_api_hostname | default('') %#
+endpoint: https://#{ bootstrap_kube_api_hostname }#:6443
+#% else %#
+endpoint: https://#{ bootstrap_kube_api_addr }#:6443
+#% endif %#
+
+clusterPodNets:
+  - "#{ bootstrap_cluster_cidr.split(',')[0] }#"
+clusterSvcNets:
+  - "#{ bootstrap_service_cidr.split(',')[0] }#"
+
+additionalApiServerCertSans: &sans
+  - "#{ bootstrap_kube_api_addr }#"
+  - 127.0.0.1 #KubePrism
+  #% if bootstrap_kube_api_hostname | default('') %#
+  - "#{ bootstrap_kube_api_hostname }#"
+  #% endif %#
+
+additionalMachineCertSans: *sans
+
+cniConfig:
+  name: none
+
+nodes:
+  #% for item in bootstrap_nodes.master %#
+  - hostname: "#{ item.name }#"
+    ipAddress: "#{ item.address }#"
+    installDiskSelector:
+      serial: "#{ item.diskSerial }#"
+    controlPlane: true
+    networkInterfaces:
+      - interface: eth0
+        dhcp: false
+        mtu: 1500
+        routes:
+          - network: 0.0.0.0/0
+            gateway: "#{ bootstrap_node_cidr | nthhost(1) }#"
+        vip:
+          ip: "#{ bootstrap_kube_api_addr }#"
+  #% endfor %#
+  #% if bootstrap_nodes.worker | default([]) | length > 0 %#
+  #% for item in bootstrap_nodes.worker %#
+  - hostname: "#{ item.name }#"
+    ipAddress: "#{ item.address }#"
+    installDiskSelector:
+      serial: "#{ item.diskSerial }#"
+    controlPlane: false
+    networkInterfaces:
+      - interface: eth0
+        dhcp: false
+        mtu: 1500
+        routes:
+          - network: 0.0.0.0/0
+            gateway: "#{ bootstrap_node_cidr | nthhost(1) }#"
+  #% endfor %#
+  #% endif %#
+
+controlPlane:
+  patches:
+    # Configure containerd
+    - &containerdPatch |-
+      machine:
+        files:
+          - op: create
+            path: /etc/cri/conf.d/20-customization.part
+            content: |-
+              [plugins."io.containerd.grpc.v1.cri"]
+                enable_unprivileged_ports = true
+                enable_unprivileged_icmp = true
+              #% if spegel.enabled | default(false) %#
+              [plugins."io.containerd.grpc.v1.cri".containerd]
+                discard_unpacked_layers = false
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                discard_unpacked_layers = false
+              #% endif %#
+
+    # Disable search domain everywhere
+    - &disableSearchDomainPatch |-
+      machine:
+        network:
+          disableSearchDomain: true
+
+    # Enable cluster discovery
+    - &discoveryPatch |-
+      cluster:
+        discovery:
+          registries:
+            kubernetes:
+              disabled: false
+            service:
+              disabled: false
+
+    # Configure kubelet
+    - &kubeletPatch |-
+      machine:
+        kubelet:
+          extraArgs:
+            rotate-server-certificates: true
+          nodeIP:
+            validSubnets:
+              - "#{ bootstrap_node_cidr }#"
+
+    # Enable KubePrism
+    - &kubePrismPatch |-
+      machine:
+        features:
+          kubePrism:
+            enabled: true
+            port: 7445
+
+    # Force nameserver
+    - &nameserverPatch |-
+      machine:
+        network:
+          nameservers:
+            - 1.1.1.1
+
+    # Configure NTP
+    - &ntpPatch |-
+      machine:
+        time:
+          disabled: false
+          servers:
+            - time.cloudflare.com
+
+    # Custom sysctl settings
+    - &sysctlPatch |-
+      machine:
+        sysctls:
+          fs.inotify.max_queued_events: 65536
+          fs.inotify.max_user_watches: 524288
+          fs.inotify.max_user_instances: 8192
+
+    # Cluster configuration
+    - |-
+      cluster:
+        allowSchedulingOnMasters: true
+        controllerManager:
+          extraArgs:
+            bind-address: 0.0.0.0
+        proxy:
+          disabled: true
+        scheduler:
+          extraArgs:
+            bind-address: 0.0.0.0
+
+    # ETCD configuration
+    - |-
+      cluster:
+        etcd:
+          extraArgs:
+            listen-metrics-urls: http://0.0.0.0:2381
+          advertisedSubnets:
+            - "#{ bootstrap_node_cidr }#"
+
+    # Disable default API server admission plugins.
+    - |-
+      - op: remove
+        path: /cluster/apiServer/admissionControl
+
+    # Enable K8s Talos API Access
+    - |-
+      machine:
+        features:
+          kubernetesTalosAPIAccess:
+            enabled: true
+            allowedRoles:
+              - os:admin
+            allowedKubernetesNamespaces:
+              - kube-system
+
+worker:
+  patches:
+    - *containerdPatch
+    - *disableSearchDomainPatch
+    - *discoveryPatch
+    - *kubeletPatch
+    - *kubePrismPatch
+    - *nameserverPatch
+    - *ntpPatch
+    - *sysctlPatch
+#% endif %#

bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

@@ -70,6 +70,9 @@ k8sServicePort: 6444
 #% elif bootstrap_distribution == "k0s"  %#
 k8sServiceHost: localhost
 k8sServicePort: 7443
+#% elif bootstrap_distribution == "talos"  %#
+k8sServiceHost: localhost
+k8sServicePort: 7445
 #% endif %#
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256

bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

@@ -31,6 +31,9 @@ k8sServicePort: 6444
 #% elif bootstrap_distribution == "k0s"  %#
 k8sServiceHost: localhost
 k8sServicePort: 7443
+#% elif bootstrap_distribution == "talos"  %#
+k8sServiceHost: localhost
+k8sServicePort: 7445
 #% endif %#
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256

bootstrap/vars/config.sample.yaml

@@ -3,7 +3,7 @@
 # Bootstrap configuration - config.yaml is gitignored
 #
 
-# Distribution can either be 'k3s' or 'k0s'
+# Distribution can either be 'k3s', 'k0s', or 'talos'
 bootstrap_distribution: k3s
 
 # Github username (e.g. onedr0p)
@@ -77,15 +77,15 @@ bootstrap_local_storage_path: /var/openebs/local
 bootstrap_nodes:
   # Use only 1, 3 or more odd master nodes, recommended is 3
   master:
-    # - name:     # name or hostname of the master node (must match [a-z0-9-\.]+)
-    #   address:  # ip address of the master node
-    #   username: # ssh username of the master node
-    #   role:     # (Optional) k0s only: can be controller+worker (default) or controller
-    #   # external_address: # (Optional) k3s only: Only use when SSH is not reachable on the IP specified in 'address' field
+    # - name:       # name or hostname of the master node (must match [a-z0-9-\.]+)
+    #   address:    # ip address of the master node
+    #   username:   # ssh username of the master node
+    #   role:       # (Optional: k0s) controller+worker (default) or controller
+    #   diskSerial: # (Required: Talos) Serial number of Disk for this master node
     # ...
   worker: # set to [] or omit if no workers are needed
-    # - name:     # name or hostname of the worker node (must match [a-z0-9-\.]+)
-    #   address:  # ip address of the worker node
-    #   username: # ssh username of the worker node
-    #   # external_address: # (Optional) k3s only: Only use when SSH is not reachable on the IP specified in 'address' field
+    # - name:       # name or hostname of the worker node (must match [a-z0-9-\.]+)
+    #   address:    # ip address of the worker node
+    #   username:   # ssh username of the worker node
+    #   diskSerial: # (Required: Talos) Serial number of Disk for this worker node
     # ...