fix: address talos comments (#1253)

* fix: update cni to cilium in taskfile

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: more updates

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: more cleanup

Signed-off-by: Devin Buhl <devin@buhl.casa>

* chore: housekeeping

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>

.envrc

@@ -15,5 +15,7 @@ export K8S_AUTH_KUBECONFIG="$(expand_path ./kubeconfig)"
 # k0s
 export DISABLE_TELEMETRY="true"
 export DISABLE_UPGRADE_CHECK="true"
+# Talos
+export TALOSCONFIG="$(expand_path ./kubernetes/talos/clusterconfig/talosconfig)"
 # Bin
 PATH_add "$(expand_path ./.bin)"

.github/tests/config-k0s.yaml

@@ -27,20 +27,26 @@ bootstrap_nodes:
   master:
     - name: k8s-0
       address: 10.10.10.100
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-1
       address: 10.10.10.101
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-2
       address: 10.10.10.102
-      username: fake
+      ssh:
+        username: fake
   worker:
     - name: k8s-3
       address: 10.10.10.103
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-4
       address: 10.10.10.104
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-5
       address: 10.10.10.105
-      username: fake
+      ssh:
+        username: fake

.github/tests/config-k3s-ipv4.yaml

@@ -27,20 +27,26 @@ bootstrap_nodes:
   master:
     - name: k8s-0
       address: 10.10.10.100
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-1
       address: 10.10.10.101
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-2
       address: 10.10.10.102
-      username: fake
+      ssh:
+        username: fake
   worker:
     - name: k8s-3
       address: 10.10.10.103
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-4
       address: 10.10.10.104
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-5
       address: 10.10.10.105
-      username: fake
+      ssh:
+        username: fake

.github/tests/config-k3s-ipv6.yaml

@@ -29,20 +29,26 @@ bootstrap_nodes:
   master:
     - name: k8s-0
       address: 10.10.10.100
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-1
       address: 10.10.10.101
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-2
       address: 10.10.10.102
-      username: fake
+      ssh:
+        username: fake
   worker:
     - name: k8s-3
       address: 10.10.10.103
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-4
       address: 10.10.10.104
-      username: fake
+      ssh:
+        username: fake
     - name: k8s-5
       address: 10.10.10.105
-      username: fake
+      ssh:
+        username: fake

.github/tests/config-talos.yaml

@@ -27,26 +27,26 @@ bootstrap_nodes:
   master:
     - name: k8s-0
       address: 10.10.10.100
-      username: fake
+      talos:
-      diskSerial: fake
+        disk_device: fake
     - name: k8s-1
       address: 10.10.10.101
-      username: fake
+      talos:
-      diskSerial: fake
+        disk_device: fake
     - name: k8s-2
       address: 10.10.10.102
-      username: fake
+      talos:
-      diskSerial: fake
+        disk_device: fake
   worker:
     - name: k8s-3
       address: 10.10.10.103
-      username: fake
+      talos:
-      diskSerial: fake
+        disk_device: fake
     - name: k8s-4
       address: 10.10.10.104
-      username: fake
+      talos:
-      diskSerial: fake
+        disk_device: fake
     - name: k8s-5
       address: 10.10.10.105
-      username: fake
+      talos:
-      diskSerial: fake
+        disk_device: fake

.taskfiles/Talos/Taskfile.yaml

@@ -7,6 +7,9 @@ vars:
   TALHELPER_SECRET_FILE: "{{.TALOS_DIR}}/talhelper.sops.yaml"
   TALHELPER_CONFIG_FILE: "{{.TALOS_DIR}}/talconfig.yaml"
 
+env:
+  TALOSCONFIG: "{{.TALOS_DIR}}/clusterconfig/talosconfig"
+
 tasks:
 
   gensecret:
@@ -46,13 +49,15 @@ tasks:
     desc: Generate talos kubeconfig
     prompt: This will overwrite the existing kubeconfig... continue?
     dir: "{{.TALOS_DIR}}"
-    cmd: talosctl kubeconfig {{.ROOT_DIR}} --force
+    cmd: talosctl --nodes {{.node}} kubeconfig {{.ROOT_DIR}} --force
+    requires:
+      vars: ["node"]
 
   apply-extras:
     desc: Apply extras
     dir: "{{.TALOS_DIR}}"
     cmds:
-      - kubectl --kubeconfig {{.KUBECONFIG_FILE}} kustomize --enable-helm ./cni | kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename -
+      - kubectl --kubeconfig {{.KUBECONFIG_FILE}} kustomize --enable-helm ./cilium | kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename -
       - kubectl --kubeconfig {{.KUBECONFIG_FILE}} kustomize --enable-helm ./kubelet-csr-approver | kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename -
     preconditions:
       - { msg: "Missing kubeconfig", sh: "test -f {{.KUBECONFIG_FILE}}" }

README.md

@@ -362,7 +362,7 @@ You have two different options for setting up your local workstation. First one
 
     ```sh
     task talos:bootstrap
-    task talos:kubeconfig
+    task talos:kubeconfig node=$master_node_ip_address
     ```
 
 4. Install Cilium and kubelet-csr-approver into the cluster

bootstrap/scripts/validation.py

@@ -58,10 +58,10 @@ def _validate_distribution(distribution: str) -> None:
 def _validate_node(node: dict, node_cidr: str, distribution: str) -> None:
     if not node.get("name"):
         raise ValueError(f"A node is missing a name")
-    if not node.get("username") and distribution not in ["k0s", "k3s"]:
+    if not node.get("ssh", {}).get("username") and distribution in ["k0s", "k3s"]:
-        raise ValueError(f"Node {node.get('name')} is missing a username")
+        raise ValueError(f"Node {node.get('name')} is missing a SSH username")
-    if not node.get("diskSerial") and distribution in ["talos"]:
+    if not node.get("disk_device") and distribution in ["talos"]:
-        raise ValueError(f"Node {node.get('name')} is missing a disk serial")
+        raise ValueError(f"Node {node.get('name')} is missing a disk device")
     if not re.match(r"^[a-z0-9-\.]+$", node.get('name')):
         raise ValueError(f"Node {node.get('name')} has an invalid name")
     ip = _validate_ip(node.get("address"))

bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2

@@ -1,14 +1,12 @@
 ---
-# https://github.com/PyratLabs/ansible-role-k3s
-# renovate: datasource=github-releases depName=k3s-io/k3s
-k3s_release_version: v1.29.0+k3s1
-k3s_install_hard_links: true
 k3s_become: true
 k3s_etcd_datastore: true
-k3s_use_unsupported_config: true
+k3s_install_hard_links: true
 k3s_registration_address: "#{ bootstrap_kube_api_addr }#"
-# /var/lib/rancher/k3s/server/manifests
+# renovate: datasource=github-releases depName=k3s-io/k3s
+k3s_release_version: v1.29.0+k3s1
 k3s_server_manifests_templates:
   - custom-cilium-helmchart.yaml
   - kube-vip-ds.yaml
   - kube-vip-rbac.yaml
+k3s_use_unsupported_config: true

bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2

@@ -1,27 +1,6 @@
 ---
-# https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/
-# https://github.com/PyratLabs/ansible-role-k3s
 k3s_control_node: true
 k3s_server:
-  #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
-  node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}"
-  #% else %#
-  node-ip: "{{ ansible_host }}"
-  #% endif %#
-  tls-san:
-    - "#{ bootstrap_kube_api_addr }#"
-    #% for item in bootstrap_kube_additional_tls_sans %#
-    - "{{ item }}"
-    #% endfor %#
-  docker: false
-  flannel-backend: "none" # quote
-  disable: ["flannel", "local-storage", "metrics-server", "servicelb", "traefik"]
-  disable-network-policy: true
-  disable-cloud-controller: true
-  disable-kube-proxy: true
-  write-kubeconfig-mode: "644"
-  pause-image: registry.k8s.io/pause:3.9
-  secrets-encryption: true
   #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
   cluster-cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#,#{ bootstrap_cluster_cidr.split(',')[1] }#"
   service-cidr: "#{ bootstrap_service_cidr.split(',')[0] }#,#{ bootstrap_service_cidr.split(',')[1] }#"
@@ -29,13 +8,32 @@ k3s_server:
   cluster-cidr: "#{ bootstrap_cluster_cidr }#"
   service-cidr: "#{ bootstrap_service_cidr }#"
   #% endif %#
-  etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack
+  disable: ["flannel", "local-storage", "metrics-server", "servicelb", "traefik"]
-  kube-controller-manager-arg:
+  disable-cloud-controller: true
-    - "bind-address=0.0.0.0" # Required to monitor kube-controller-manager with kube-prometheus-stack
+  disable-kube-proxy: true
-  kube-scheduler-arg:
+  disable-network-policy: true
-    - "bind-address=0.0.0.0" # Required to monitor kube-scheduler with kube-prometheus-stack
+  docker: false
+  etcd-expose-metrics: true
+  flannel-backend: "none"
   kube-apiserver-arg:
-    - "anonymous-auth=true" # Required for external LB health-checks
+    - "anonymous-auth=true"
+  kube-controller-manager-arg:
+    - "bind-address=0.0.0.0"
+  kube-scheduler-arg:
+    - "bind-address=0.0.0.0"
   kubelet-arg:
     - "image-gc-high-threshold=55"
     - "image-gc-low-threshold=50"
+  #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
+  node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}"
+  #% else %#
+  node-ip: "{{ ansible_host }}"
+  #% endif %#
+  pause-image: registry.k8s.io/pause:3.9
+  secrets-encryption: true
+  tls-san:
+    - "#{ bootstrap_kube_api_addr }#"
+    #% for item in bootstrap_kube_additional_tls_sans %#
+    - "{{ item }}"
+    #% endfor %#
+  write-kubeconfig-mode: "644"

bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2

@@ -1,14 +1,12 @@
 ---
-# https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/
-# https://github.com/PyratLabs/ansible-role-k3s
 k3s_control_node: false
 k3s_agent:
+  kubelet-arg:
+    - "image-gc-high-threshold=55"
+    - "image-gc-low-threshold=50"
   #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
   node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}"
   #% else %#
   node-ip: "{{ ansible_host }}"
   #% endif %#
   pause-image: registry.k8s.io/pause:3.9
-  kubelet-arg:
-    - "image-gc-high-threshold=55"
-    - "image-gc-low-threshold=50"

bootstrap/templates/ansible/inventory/hosts.yaml.j2

@@ -5,15 +5,15 @@ kubernetes:
       hosts:
       #% for item in bootstrap_nodes.master %#
         "#{ item.name }#":
-          ansible_user: "#{ item.username }#"
+          ansible_user: "#{ item.ssh.username }#"
           ansible_host: "#{ item.address }#"
       #% endfor %#
-    #% if bootstrap_nodes.worker | length > 0 %#
+    #% if bootstrap_nodes.worker|length > 0 %#
     worker:
       hosts:
       #% for item in bootstrap_nodes.worker %#
         "#{ item.name }#":
-          ansible_user: "#{ item.username }#"
+          ansible_user: "#{ item.ssh.username }#"
           ansible_host: "#{ item.address }#"
       #% endfor %#
     #% endif %#

bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2

@@ -30,10 +30,6 @@
 
     - name: User Configuration
       block:
-        - name: User Configuration | SSH keys
-          ansible.posix.authorized_key:
-            user: "{{ ansible_user }}"
-            key: "https://github.com/#{ bootstrap_github_username }#.keys"
         - name: User Configuration | Silence login
           ansible.builtin.file:
             dest: "{{ '/home/' + ansible_user if ansible_user != 'root' else '/root' }}/.hushlogin"

bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2

@@ -9,7 +9,7 @@ spec:
     - role: controller+worker
       ssh:
         address: "#{ item.address }#"
-        user: "#{ item.username }#"
+        user: "#{ item.ssh.username }#"
       installFlags:
         - --disable-components=metrics-server
         - --no-taints
@@ -24,12 +24,11 @@ spec:
           after:
             - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s -
     #% endfor %#
-    #% if bootstrap_nodes.worker | length > 0 %#
     #% for item in bootstrap_nodes.worker %#
     - role: worker
       ssh:
         address: "#{ item.address }#"
-        user: "#{ item.username }#"
+        user: "#{ item.ssh.username }#"
       hooks:
         apply:
           before:
@@ -40,7 +39,6 @@ spec:
           after:
             - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s -
     #% endfor %#
-    #% endif %#
   k0s:
     # renovate: datasource=github-releases depName=k0sproject/k0s
     version: v1.28.6+k0s.0
@@ -51,16 +49,14 @@ spec:
           enabled: false
         controllerManager:
           extraArgs:
-            # Required to monitor kube-controller-manager with kube-prometheus-stack
             bind-address: "0.0.0.0"
         scheduler:
           extraArgs:
-            # Required to monitor kube-scheduler with kube-prometheus-stack
             bind-address: "0.0.0.0"
         storage:
           etcd:
             extraArgs:
-              listen-metrics-urls: http://0.0.0.0:2381 # Required to monitor etcd with kube-prometheus-stack
+              listen-metrics-urls: http://0.0.0.0:2381
         api:
           sans:
             #% if bootstrap_kube_api_addr %#

bootstrap/templates/kubernetes/talos/talconfig.yaml.j2

@@ -25,12 +25,18 @@ nodes:
   #% for item in bootstrap_nodes.master %#
   - hostname: "#{ item.name }#"
     ipAddress: "#{ item.address }#"
+    #% if item.talos.disk_device.startswith('/') %#
+    installDisk: "#{ item.talos.disk_device }#"
+    #% else %#
     installDiskSelector:
-      serial: "#{ item.diskSerial }#"
+      serial: "#{ item.talos.disk_device }#"
+    #% endif %#
     controlPlane: true
     networkInterfaces:
       - interface: eth0
         dhcp: false
+        addresses:
+          - "#{ item.address }#/#{ bootstrap_node_cidr.split('/') | last }#"
         mtu: 1500
         routes:
           - network: 0.0.0.0/0
@@ -38,22 +44,26 @@ nodes:
         vip:
           ip: "#{ bootstrap_kube_api_addr }#"
   #% endfor %#
-  #% if bootstrap_nodes.worker | length > 0 %#
   #% for item in bootstrap_nodes.worker %#
   - hostname: "#{ item.name }#"
     ipAddress: "#{ item.address }#"
+    #% if item.talos.disk_device.startswith('/') %#
+    installDisk: "#{ item.talos.disk_device }#"
+    #% else %#
     installDiskSelector:
-      serial: "#{ item.diskSerial }#"
+      serial: "#{ item.talos.disk_device }#"
+    #% endif %#
     controlPlane: false
     networkInterfaces:
       - interface: eth0
         dhcp: false
+        addresses:
+          - "#{ item.address }#/#{ bootstrap_node_cidr.split('/') | last }#"
         mtu: 1500
         routes:
           - network: 0.0.0.0/0
             gateway: "#{ bootstrap_node_cidr | nthhost(1) }#"
   #% endfor %#
-  #% endif %#
 
 controlPlane:
   patches:

bootstrap/vars/config.sample.yaml

@@ -74,16 +74,20 @@ bootstrap_service_cidr: 10.43.0.0/16
 bootstrap_nodes:
   # Use only 1, 3 or more odd master nodes, recommended is 3
   master:
-    # - name:       # name or hostname of the master node (must match [a-z0-9-\.]+)
+    # - name:            # name or hostname of the node (must match [a-z0-9-\.]+)
-    #   address:    # ip address of the master node
+    #   address:         # ip address of the node
-    #   username:   # ssh username of the master node
+    #   ssh:
-    #   diskSerial: # (Required: Talos) Serial number of Disk for this master node
+    #     username:      # (Required: k3s/k0s) SSH username of the node
+    #   talos:
+    #     disk_device:   # (Required: Talos) Device Path or Serial number of Disk for this node
     # ...
   worker:           # set to [] or omit if no workers are needed
-    # - name:       # name or hostname of the worker node (must match [a-z0-9-\.]+)
+    # - name:            # name or hostname of the node (must match [a-z0-9-\.]+)
-    #   address:    # ip address of the worker node
+    #   address:         # ip address of the node
-    #   username:   # ssh username of the worker node
+    #   ssh:
-    #   diskSerial: # (Required: Talos) Serial number of Disk for this worker node
+    #     username:      # (Required: k3s/k0s) SSH username of the node
+    #   talos:
+    #     disk_device:   # (Required: Talos) Device Path or Serial number of Disk for this node
     # ...
 
 #