From 90ca5f026f8ca263aa8c1a7428cd1fea2e567275 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Fri, 26 Jan 2024 14:59:11 -0500 Subject: [PATCH] fix: address talos comments (#1253) * fix: update cni to cilium in taskfile Signed-off-by: Devin Buhl * fix: more updates Signed-off-by: Devin Buhl * fix: more cleanup Signed-off-by: Devin Buhl * chore: housekeeping Signed-off-by: Devin Buhl --------- Signed-off-by: Devin Buhl --- .envrc | 2 + .github/tests/config-k0s.yaml | 18 ++++--- .github/tests/config-k3s-ipv4.yaml | 18 ++++--- .github/tests/config-k3s-ipv6.yaml | 18 ++++--- .github/tests/config-talos.yaml | 24 ++++----- .taskfiles/Talos/Taskfile.yaml | 9 +++- README.md | 2 +- bootstrap/scripts/validation.py | 8 +-- .../group_vars/kubernetes/main.yaml.j2 | 10 ++-- .../inventory/group_vars/master/main.yaml.j2 | 52 +++++++++---------- .../inventory/group_vars/worker/main.yaml.j2 | 8 ++- .../templates/ansible/inventory/hosts.yaml.j2 | 6 +-- .../ansible/playbooks/cluster-prepare.yaml.j2 | 4 -- .../templates/kubernetes/k0s/k0sctl.yaml.j2 | 10 ++-- .../kubernetes/talos/talconfig.yaml.j2 | 18 +++++-- bootstrap/vars/config.sample.yaml | 20 ++++--- 16 files changed, 126 insertions(+), 101 deletions(-) diff --git a/.envrc b/.envrc index a2e7de09..4c954a23 100644 --- a/.envrc +++ b/.envrc @@ -15,5 +15,7 @@ export K8S_AUTH_KUBECONFIG="$(expand_path ./kubeconfig)" # k0s export DISABLE_TELEMETRY="true" export DISABLE_UPGRADE_CHECK="true" +# Talos +export TALOSCONFIG="$(expand_path ./kubernetes/talos/clusterconfig/talosconfig)" # Bin PATH_add "$(expand_path ./.bin)" diff --git a/.github/tests/config-k0s.yaml b/.github/tests/config-k0s.yaml index ae37dfb4..a14748a0 100644 --- a/.github/tests/config-k0s.yaml +++ b/.github/tests/config-k0s.yaml @@ -27,20 +27,26 @@ bootstrap_nodes: master: - name: k8s-0 address: 10.10.10.100 - username: fake + ssh: + username: fake - name: k8s-1 address: 10.10.10.101 - username: fake + ssh: + username: fake - name: k8s-2 address: 10.10.10.102 - username: fake + ssh: + username: fake worker: - name: k8s-3 address: 10.10.10.103 - username: fake + ssh: + username: fake - name: k8s-4 address: 10.10.10.104 - username: fake + ssh: + username: fake - name: k8s-5 address: 10.10.10.105 - username: fake + ssh: + username: fake diff --git a/.github/tests/config-k3s-ipv4.yaml b/.github/tests/config-k3s-ipv4.yaml index cbfef5a9..af0d9909 100644 --- a/.github/tests/config-k3s-ipv4.yaml +++ b/.github/tests/config-k3s-ipv4.yaml @@ -27,20 +27,26 @@ bootstrap_nodes: master: - name: k8s-0 address: 10.10.10.100 - username: fake + ssh: + username: fake - name: k8s-1 address: 10.10.10.101 - username: fake + ssh: + username: fake - name: k8s-2 address: 10.10.10.102 - username: fake + ssh: + username: fake worker: - name: k8s-3 address: 10.10.10.103 - username: fake + ssh: + username: fake - name: k8s-4 address: 10.10.10.104 - username: fake + ssh: + username: fake - name: k8s-5 address: 10.10.10.105 - username: fake + ssh: + username: fake diff --git a/.github/tests/config-k3s-ipv6.yaml b/.github/tests/config-k3s-ipv6.yaml index f4a00423..c77d3f81 100644 --- a/.github/tests/config-k3s-ipv6.yaml +++ b/.github/tests/config-k3s-ipv6.yaml @@ -29,20 +29,26 @@ bootstrap_nodes: master: - name: k8s-0 address: 10.10.10.100 - username: fake + ssh: + username: fake - name: k8s-1 address: 10.10.10.101 - username: fake + ssh: + username: fake - name: k8s-2 address: 10.10.10.102 - username: fake + ssh: + username: fake worker: - name: k8s-3 address: 10.10.10.103 - username: fake + ssh: + username: fake - name: k8s-4 address: 10.10.10.104 - username: fake + ssh: + username: fake - name: k8s-5 address: 10.10.10.105 - username: fake + ssh: + username: fake diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml index 5e713e98..3e3e41fb 100644 --- a/.github/tests/config-talos.yaml +++ b/.github/tests/config-talos.yaml @@ -27,26 +27,26 @@ bootstrap_nodes: master: - name: k8s-0 address: 10.10.10.100 - username: fake - diskSerial: fake + talos: + disk_device: fake - name: k8s-1 address: 10.10.10.101 - username: fake - diskSerial: fake + talos: + disk_device: fake - name: k8s-2 address: 10.10.10.102 - username: fake - diskSerial: fake + talos: + disk_device: fake worker: - name: k8s-3 address: 10.10.10.103 - username: fake - diskSerial: fake + talos: + disk_device: fake - name: k8s-4 address: 10.10.10.104 - username: fake - diskSerial: fake + talos: + disk_device: fake - name: k8s-5 address: 10.10.10.105 - username: fake - diskSerial: fake + talos: + disk_device: fake diff --git a/.taskfiles/Talos/Taskfile.yaml b/.taskfiles/Talos/Taskfile.yaml index b075723a..4ae83ada 100644 --- a/.taskfiles/Talos/Taskfile.yaml +++ b/.taskfiles/Talos/Taskfile.yaml @@ -7,6 +7,9 @@ vars: TALHELPER_SECRET_FILE: "{{.TALOS_DIR}}/talhelper.sops.yaml" TALHELPER_CONFIG_FILE: "{{.TALOS_DIR}}/talconfig.yaml" +env: + TALOSCONFIG: "{{.TALOS_DIR}}/clusterconfig/talosconfig" + tasks: gensecret: @@ -46,13 +49,15 @@ tasks: desc: Generate talos kubeconfig prompt: This will overwrite the existing kubeconfig... continue? dir: "{{.TALOS_DIR}}" - cmd: talosctl kubeconfig {{.ROOT_DIR}} --force + cmd: talosctl --nodes {{.node}} kubeconfig {{.ROOT_DIR}} --force + requires: + vars: ["node"] apply-extras: desc: Apply extras dir: "{{.TALOS_DIR}}" cmds: - - kubectl --kubeconfig {{.KUBECONFIG_FILE}} kustomize --enable-helm ./cni | kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename - + - kubectl --kubeconfig {{.KUBECONFIG_FILE}} kustomize --enable-helm ./cilium | kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename - - kubectl --kubeconfig {{.KUBECONFIG_FILE}} kustomize --enable-helm ./kubelet-csr-approver | kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename - preconditions: - { msg: "Missing kubeconfig", sh: "test -f {{.KUBECONFIG_FILE}}" } diff --git a/README.md b/README.md index df2bfc78..7e31b995 100644 --- a/README.md +++ b/README.md @@ -362,7 +362,7 @@ You have two different options for setting up your local workstation. First one ```sh task talos:bootstrap - task talos:kubeconfig + task talos:kubeconfig node=$master_node_ip_address ``` 4. Install Cilium and kubelet-csr-approver into the cluster diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py index c20bc34a..d4760079 100644 --- a/bootstrap/scripts/validation.py +++ b/bootstrap/scripts/validation.py @@ -58,10 +58,10 @@ def _validate_distribution(distribution: str) -> None: def _validate_node(node: dict, node_cidr: str, distribution: str) -> None: if not node.get("name"): raise ValueError(f"A node is missing a name") - if not node.get("username") and distribution not in ["k0s", "k3s"]: - raise ValueError(f"Node {node.get('name')} is missing a username") - if not node.get("diskSerial") and distribution in ["talos"]: - raise ValueError(f"Node {node.get('name')} is missing a disk serial") + if not node.get("ssh", {}).get("username") and distribution in ["k0s", "k3s"]: + raise ValueError(f"Node {node.get('name')} is missing a SSH username") + if not node.get("disk_device") and distribution in ["talos"]: + raise ValueError(f"Node {node.get('name')} is missing a disk device") if not re.match(r"^[a-z0-9-\.]+$", node.get('name')): raise ValueError(f"Node {node.get('name')} has an invalid name") ip = _validate_ip(node.get("address")) diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 index 2eee1a7c..eef11a47 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 @@ -1,14 +1,12 @@ --- -# https://github.com/PyratLabs/ansible-role-k3s -# renovate: datasource=github-releases depName=k3s-io/k3s -k3s_release_version: v1.29.0+k3s1 -k3s_install_hard_links: true k3s_become: true k3s_etcd_datastore: true -k3s_use_unsupported_config: true +k3s_install_hard_links: true k3s_registration_address: "#{ bootstrap_kube_api_addr }#" -# /var/lib/rancher/k3s/server/manifests +# renovate: datasource=github-releases depName=k3s-io/k3s +k3s_release_version: v1.29.0+k3s1 k3s_server_manifests_templates: - custom-cilium-helmchart.yaml - kube-vip-ds.yaml - kube-vip-rbac.yaml +k3s_use_unsupported_config: true diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 index 708c1483..2cc1a415 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 @@ -1,27 +1,6 @@ --- -# https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/ -# https://github.com/PyratLabs/ansible-role-k3s k3s_control_node: true k3s_server: - #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# - node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}" - #% else %# - node-ip: "{{ ansible_host }}" - #% endif %# - tls-san: - - "#{ bootstrap_kube_api_addr }#" - #% for item in bootstrap_kube_additional_tls_sans %# - - "{{ item }}" - #% endfor %# - docker: false - flannel-backend: "none" # quote - disable: ["flannel", "local-storage", "metrics-server", "servicelb", "traefik"] - disable-network-policy: true - disable-cloud-controller: true - disable-kube-proxy: true - write-kubeconfig-mode: "644" - pause-image: registry.k8s.io/pause:3.9 - secrets-encryption: true #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# cluster-cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#,#{ bootstrap_cluster_cidr.split(',')[1] }#" service-cidr: "#{ bootstrap_service_cidr.split(',')[0] }#,#{ bootstrap_service_cidr.split(',')[1] }#" @@ -29,13 +8,32 @@ k3s_server: cluster-cidr: "#{ bootstrap_cluster_cidr }#" service-cidr: "#{ bootstrap_service_cidr }#" #% endif %# - etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack - kube-controller-manager-arg: - - "bind-address=0.0.0.0" # Required to monitor kube-controller-manager with kube-prometheus-stack - kube-scheduler-arg: - - "bind-address=0.0.0.0" # Required to monitor kube-scheduler with kube-prometheus-stack + disable: ["flannel", "local-storage", "metrics-server", "servicelb", "traefik"] + disable-cloud-controller: true + disable-kube-proxy: true + disable-network-policy: true + docker: false + etcd-expose-metrics: true + flannel-backend: "none" kube-apiserver-arg: - - "anonymous-auth=true" # Required for external LB health-checks + - "anonymous-auth=true" + kube-controller-manager-arg: + - "bind-address=0.0.0.0" + kube-scheduler-arg: + - "bind-address=0.0.0.0" kubelet-arg: - "image-gc-high-threshold=55" - "image-gc-low-threshold=50" + #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# + node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}" + #% else %# + node-ip: "{{ ansible_host }}" + #% endif %# + pause-image: registry.k8s.io/pause:3.9 + secrets-encryption: true + tls-san: + - "#{ bootstrap_kube_api_addr }#" + #% for item in bootstrap_kube_additional_tls_sans %# + - "{{ item }}" + #% endfor %# + write-kubeconfig-mode: "644" diff --git a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 index 1d35d6f6..181dc1c7 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 @@ -1,14 +1,12 @@ --- -# https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/ -# https://github.com/PyratLabs/ansible-role-k3s k3s_control_node: false k3s_agent: + kubelet-arg: + - "image-gc-high-threshold=55" + - "image-gc-low-threshold=50" #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}" #% else %# node-ip: "{{ ansible_host }}" #% endif %# pause-image: registry.k8s.io/pause:3.9 - kubelet-arg: - - "image-gc-high-threshold=55" - - "image-gc-low-threshold=50" diff --git a/bootstrap/templates/ansible/inventory/hosts.yaml.j2 b/bootstrap/templates/ansible/inventory/hosts.yaml.j2 index 6e423221..aa092f49 100644 --- a/bootstrap/templates/ansible/inventory/hosts.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/hosts.yaml.j2 @@ -5,15 +5,15 @@ kubernetes: hosts: #% for item in bootstrap_nodes.master %# "#{ item.name }#": - ansible_user: "#{ item.username }#" + ansible_user: "#{ item.ssh.username }#" ansible_host: "#{ item.address }#" #% endfor %# - #% if bootstrap_nodes.worker | length > 0 %# + #% if bootstrap_nodes.worker|length > 0 %# worker: hosts: #% for item in bootstrap_nodes.worker %# "#{ item.name }#": - ansible_user: "#{ item.username }#" + ansible_user: "#{ item.ssh.username }#" ansible_host: "#{ item.address }#" #% endfor %# #% endif %# diff --git a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 index 053579dc..f06c3d39 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 @@ -30,10 +30,6 @@ - name: User Configuration block: - - name: User Configuration | SSH keys - ansible.posix.authorized_key: - user: "{{ ansible_user }}" - key: "https://github.com/#{ bootstrap_github_username }#.keys" - name: User Configuration | Silence login ansible.builtin.file: dest: "{{ '/home/' + ansible_user if ansible_user != 'root' else '/root' }}/.hushlogin" diff --git a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 index 769cfe31..aebcf47f 100644 --- a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 +++ b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 @@ -9,7 +9,7 @@ spec: - role: controller+worker ssh: address: "#{ item.address }#" - user: "#{ item.username }#" + user: "#{ item.ssh.username }#" installFlags: - --disable-components=metrics-server - --no-taints @@ -24,12 +24,11 @@ spec: after: - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s - #% endfor %# - #% if bootstrap_nodes.worker | length > 0 %# #% for item in bootstrap_nodes.worker %# - role: worker ssh: address: "#{ item.address }#" - user: "#{ item.username }#" + user: "#{ item.ssh.username }#" hooks: apply: before: @@ -40,7 +39,6 @@ spec: after: - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s - #% endfor %# - #% endif %# k0s: # renovate: datasource=github-releases depName=k0sproject/k0s version: v1.28.6+k0s.0 @@ -51,16 +49,14 @@ spec: enabled: false controllerManager: extraArgs: - # Required to monitor kube-controller-manager with kube-prometheus-stack bind-address: "0.0.0.0" scheduler: extraArgs: - # Required to monitor kube-scheduler with kube-prometheus-stack bind-address: "0.0.0.0" storage: etcd: extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 # Required to monitor etcd with kube-prometheus-stack + listen-metrics-urls: http://0.0.0.0:2381 api: sans: #% if bootstrap_kube_api_addr %# diff --git a/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 index c77a6fdc..8254ff3a 100644 --- a/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 +++ b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 @@ -25,12 +25,18 @@ nodes: #% for item in bootstrap_nodes.master %# - hostname: "#{ item.name }#" ipAddress: "#{ item.address }#" + #% if item.talos.disk_device.startswith('/') %# + installDisk: "#{ item.talos.disk_device }#" + #% else %# installDiskSelector: - serial: "#{ item.diskSerial }#" + serial: "#{ item.talos.disk_device }#" + #% endif %# controlPlane: true networkInterfaces: - interface: eth0 dhcp: false + addresses: + - "#{ item.address }#/#{ bootstrap_node_cidr.split('/') | last }#" mtu: 1500 routes: - network: 0.0.0.0/0 @@ -38,22 +44,26 @@ nodes: vip: ip: "#{ bootstrap_kube_api_addr }#" #% endfor %# - #% if bootstrap_nodes.worker | length > 0 %# #% for item in bootstrap_nodes.worker %# - hostname: "#{ item.name }#" ipAddress: "#{ item.address }#" + #% if item.talos.disk_device.startswith('/') %# + installDisk: "#{ item.talos.disk_device }#" + #% else %# installDiskSelector: - serial: "#{ item.diskSerial }#" + serial: "#{ item.talos.disk_device }#" + #% endif %# controlPlane: false networkInterfaces: - interface: eth0 dhcp: false + addresses: + - "#{ item.address }#/#{ bootstrap_node_cidr.split('/') | last }#" mtu: 1500 routes: - network: 0.0.0.0/0 gateway: "#{ bootstrap_node_cidr | nthhost(1) }#" #% endfor %# - #% endif %# controlPlane: patches: diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index e104f69c..18b27296 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -74,16 +74,20 @@ bootstrap_service_cidr: 10.43.0.0/16 bootstrap_nodes: # Use only 1, 3 or more odd master nodes, recommended is 3 master: - # - name: # name or hostname of the master node (must match [a-z0-9-\.]+) - # address: # ip address of the master node - # username: # ssh username of the master node - # diskSerial: # (Required: Talos) Serial number of Disk for this master node + # - name: # name or hostname of the node (must match [a-z0-9-\.]+) + # address: # ip address of the node + # ssh: + # username: # (Required: k3s/k0s) SSH username of the node + # talos: + # disk_device: # (Required: Talos) Device Path or Serial number of Disk for this node # ... worker: # set to [] or omit if no workers are needed - # - name: # name or hostname of the worker node (must match [a-z0-9-\.]+) - # address: # ip address of the worker node - # username: # ssh username of the worker node - # diskSerial: # (Required: Talos) Serial number of Disk for this worker node + # - name: # name or hostname of the node (must match [a-z0-9-\.]+) + # address: # ip address of the node + # ssh: + # username: # (Required: k3s/k0s) SSH username of the node + # talos: + # disk_device: # (Required: Talos) Device Path or Serial number of Disk for this node # ... #