diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 31e40bdc..418f08a3 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -228,7 +228,7 @@ "matchStrings": [ // Example: // k3s_release_version: "v1.27.3+k3s1" - "datasource=(?\\S+) depName=(?\\S+)( versioning=(?\\S+))?\n.*?\"(?.*)\"\n", + "datasource=(?\\S+) depName=(?\\S+)( versioning=(?\\S+))?( extractVersion=(?\\S+))?\n.*?\"(?.*)\"\n", // Example: // - https://github.com/rancher/system-upgrade-controller/releases/download/v0.11.0/crd.yaml "datasource=(?\\S+) depName=(?\\S+)( versioning=(?\\S+))?\n.*?-\\s(.*?)\/(?[^/]+)\/[^/]+\n", diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml new file mode 100644 index 00000000..964a725f --- /dev/null +++ b/.github/tests/config-talos.yaml @@ -0,0 +1,55 @@ +--- +ci_test: true + +bootstrap_distribution: talos +bootstrap_github_username: onedr0p +bootstrap_github_repository_name: flux-cluster-template +bootstrap_github_repository_branch: main +bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY +bootstrap_timezone: Etc/UTC +bootstrap_acme_email: fake +bootstrap_acme_production_enabled: false +bootstrap_flux_github_webhook_token: fake +bootstrap_cloudflare_domain: fake +bootstrap_cloudflare_token: fake +bootstrap_cloudflare_account_tag: fake +bootstrap_cloudflare_tunnel_secret: fake +bootstrap_cloudflare_tunnel_id: fake +bootstrap_node_cidr: 10.10.10.0/24 +bootstrap_kube_api_addr: 10.10.10.254 +bootstrap_kube_api_hostname: fake +bootstrap_k8s_gateway_addr: 10.10.10.253 +bootstrap_external_ingress_addr: 10.10.10.252 +bootstrap_internal_ingress_addr: 10.10.10.251 +bootstrap_cilium_loadbalancer_mode: dsr +bootstrap_ipv6_enabled: false +bootstrap_cluster_cidr: 10.42.0.0/16 +bootstrap_service_cidr: 10.43.0.0/16 +bootstrap_local_storage_path: /var/openebs/local +bootstrap_nodes: + master: + - name: k8s-0 + address: 10.10.10.100 + username: fake + diskSerial: fake + - name: k8s-1 + address: 10.10.10.101 + username: fake + diskSerial: fake + - name: k8s-2 + address: 10.10.10.102 + username: fake + diskSerial: fake + worker: + - name: k8s-3 + address: 10.10.10.103 + username: fake + diskSerial: fake + - name: k8s-4 + address: 10.10.10.104 + username: fake + diskSerial: fake + - name: k8s-5 + address: 10.10.10.105 + username: fake + diskSerial: fake diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 5297ee00..22642cb6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -41,6 +41,7 @@ jobs: - k3s-ipv4 - k3s-ipv6 - k3s-no-kube-vip + - talos addon-files: ["addons"] steps: - name: Checkout @@ -112,6 +113,13 @@ jobs: shell: bash run: task configure --yes + - name: Run Talos configure + if: ${{ steps.config-env.outputs.distribution == 'talos' }} + shell: bash + run: | + task talos:gensecret --yes + task talos:genconfig + - name: Run Ansible lint if: ${{ steps.config-env.outputs.distribution == 'k3s' || steps.config-env.outputs.distribution == 'k0s' }} shell: bash diff --git a/.taskfiles/K0s/Taskfile.yaml b/.taskfiles/K0s/Taskfile.yaml index 4f737937..37554593 100644 --- a/.taskfiles/K0s/Taskfile.yaml +++ b/.taskfiles/K0s/Taskfile.yaml @@ -1,4 +1,5 @@ --- +# yaml-language-server: $schema=https://taskfile.dev/schema.json version: "3" env: diff --git a/.taskfiles/Talos/Taskfile.yaml b/.taskfiles/Talos/Taskfile.yaml new file mode 100644 index 00000000..d17c7f34 --- /dev/null +++ b/.taskfiles/Talos/Taskfile.yaml @@ -0,0 +1,72 @@ +--- +# yaml-language-server: $schema=https://taskfile.dev/schema.json +version: "3" + +vars: + TALOS_DIR: "{{.ROOT_DIR}}/kubernetes/talos" + TALHELPER_SECRET_FILE: "{{.TALOS_DIR}}/talhelper.sops.yaml" + TALHELPER_CONFIG_FILE: "{{.TALOS_DIR}}/talconfig.yaml" + +tasks: + + gensecret: + desc: Generate talos secret + prompt: This will overwrite the existing secret... continue? + dir: "{{.TALOS_DIR}}" + cmds: + - talhelper gensecret > {{.TALHELPER_SECRET_FILE}} + - task: :sops:.encrypt-file + vars: + file: "{{.TALHELPER_SECRET_FILE}}" + preconditions: + - { msg: "Missing talhelper config file", sh: "test -f {{.TALHELPER_CONFIG_FILE}}" } + + genconfig: + desc: Generate talos config + dir: "{{.TALOS_DIR}}" + cmd: talhelper genconfig + preconditions: + - { msg: "Missing talhelper config file", sh: "test -f {{.TALHELPER_CONFIG_FILE}}" } + + apply-config: + desc: Apply talos config on a node + dir: "{{.TALOS_DIR}}" + cmd: talhelper gencommand apply --extra-flags=--insecure | bash + + bootstrap: + desc: Bootstrap the talos cluster + dir: "{{.TALOS_DIR}}" + cmd: talhelper gencommand bootstrap | bash + + kubeconfig: + desc: Generate talos kubeconfig + prompt: This will overwrite the existing kubeconfig... continue? + dir: "{{.TALOS_DIR}}" + cmd: talosctl kubeconfig {{.ROOT_DIR}} --force + + apply-extras: + desc: Apply extras + dir: "{{.TALOS_DIR}}" + cmds: + - kubectl --kubeconfig {{.KUBECONFIG_FILE}} kustomize --enable-helm ./cni | kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename - + - kubectl --kubeconfig {{.KUBECONFIG_FILE}} kustomize --enable-helm ./kubelet-csr-approver | kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename - + preconditions: + - { msg: "Missing kubeconfig", sh: "test -f {{.KUBECONFIG_FILE}}" } + + upgrade-talos: + desc: Upgrade talos on a node + cmd: talosctl --nodes {{.node}} upgrade --image {{.image}} --preserve=true --reboot-mode=powercycle + vars: + image: '{{ or .image (fail "Argument (image) is required") }}' + node: '{{ or .node (fail "Argument (node) is required") }}' + preconditions: + - { msg: "Node not found", sh: "talosctl --nodes {{.node}} get machineconfig" } + + upgrade-k8s: + desc: Upgrade k8s on a node + cmd: talosctl --nodes {{.node}} upgrade-k8s --to {{.to}} + vars: + node: '{{ or .node (fail "Argument (node) is required") }}' + to: '{{ or .to (fail "Argument (to) is required") }}' + preconditions: + - { msg: "Node not found", sh: "talosctl --nodes {{.node}} get machineconfig" } diff --git a/.taskfiles/Workstation/Archfile b/.taskfiles/Workstation/Archfile index 42722241..2e024a5a 100644 --- a/.taskfiles/Workstation/Archfile +++ b/.taskfiles/Workstation/Archfile @@ -14,3 +14,4 @@ moreutils sops stern-bin talhelper-bin +talosctl diff --git a/.taskfiles/Workstation/Brewfile b/.taskfiles/Workstation/Brewfile index 5d696965..46a03eae 100644 --- a/.taskfiles/Workstation/Brewfile +++ b/.taskfiles/Workstation/Brewfile @@ -1,6 +1,7 @@ tap "fluxcd/tap" tap "go-task/tap" tap "k0sproject/tap" +tap "siderolabs/talos" brew "age" brew "cloudflared" brew "direnv" @@ -16,4 +17,5 @@ brew "moreutils" brew "sops" brew "stern" brew "talhelper" +brew "talosctl" brew "yq" diff --git a/README.md b/README.md index 55d972ac..06e23553 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Deploy a Kubernetes cluster backed by Flux -Welcome to my highly opinionated template for deploying a single Kubernetes ([k3s](https://k3s.io) or [k0s](https://github.com/k0sproject/k0s)) cluster with [Ansible](https://www.ansible.com) and using [Flux](https://toolkit.fluxcd.io) to manage its state. +Welcome to my opinionated and extendable template for deploying a single Kubernetes cluster with [k0s](https://github.com/k0sproject/k0s), [k3s](https://k3s.io) or [Talos](https://github.com/siderolabs/talos) backed by [Flux](https://fluxcd.io/flux/) to manage its state. ## 👋 Introduction diff --git a/Taskfile.yaml b/Taskfile.yaml index b76be486..ca722a33 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -28,6 +28,7 @@ includes: repository: aliases: ["repo"] taskfile: .taskfiles/Repository/Taskfile.yaml + talos: .taskfiles/Talos/Taskfile.yaml sops: .taskfiles/Sops/Taskfile.yaml workstation: .taskfiles/Workstation/Taskfile.yaml diff --git a/bootstrap/tasks/validation/cli.yaml b/bootstrap/tasks/validation/cli.yaml index 930078d5..3147300b 100644 --- a/bootstrap/tasks/validation/cli.yaml +++ b/bootstrap/tasks/validation/cli.yaml @@ -1,7 +1,6 @@ --- - name: Check if required CLI tools are present - ansible.builtin.shell: | - command -v {{ item }} >/dev/null 2>&1 + ansible.builtin.shell: command -v {{ item }} >/dev/null 2>&1 loop: ["age", "cloudflared", "flux", "sops", "jq", "kubeconform", "kustomize"] changed_when: false check_mode: false @@ -10,10 +9,18 @@ - name: Check if required CLI tools are present for k0s when: bootstrap_distribution == "k0s" - ansible.builtin.shell: | - command -v {{ item }} >/dev/null 2>&1 + ansible.builtin.shell: command -v {{ item }} >/dev/null 2>&1 loop: ["k0sctl"] changed_when: false check_mode: false register: result failed_when: result.rc != 0 and result.rc != 127 + +- name: Check if required CLI tools are present for talos + when: bootstrap_distribution == "talos" + ansible.builtin.shell: command -v {{ item }} >/dev/null 2>&1 + loop: ["talhelper", "talosctl"] + changed_when: false + check_mode: false + register: result + failed_when: result.rc != 0 and result.rc != 127 diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml index 1a42ba28..640e6706 100644 --- a/bootstrap/tasks/validation/net.yaml +++ b/bootstrap/tasks/validation/net.yaml @@ -1,11 +1,4 @@ --- -- name: Set reachable address - ansible.builtin.set_fact: - current_address: "{{ item.external_address | default(item.address) }}" - loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" - loop_control: - label: "{{ item.address }}" - - name: Check if master node count is odd ansible.builtin.assert: that: @@ -161,7 +154,6 @@ fail_msg: Node address {{ item.address }} is not within {{ bootstrap_node_cidr }}. quiet: true loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" - when: item.external_address is not defined loop_control: label: "{{ item.address }}" @@ -193,14 +185,24 @@ fail_msg: All node names are not unique. quiet: true -- name: Check if nodes SSH ports are reachable - when: not ci_test | default(false) +- name: Check if SSH ports are reachable + when: (not ci_test | default(false)) and (not bootstrap_distribution == "talos") ansible.builtin.wait_for: - host: "{{ current_address }}" + host: "{{ item.address }}" port: 22 - search_regex: OpenSSH timeout: 10 connection: local loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" loop_control: - label: "{{ current_address }}" + label: "{{ item.address }}" + +- name: Check if Talos ports are reachable + when: (not ci_test | default(false)) and (bootstrap_distribution == "talos") + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: 50000 + timeout: 10 + connection: local + loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" + loop_control: + label: "{{ item.address }}" diff --git a/bootstrap/tasks/validation/vars.yaml b/bootstrap/tasks/validation/vars.yaml index 45036de2..467ceec3 100644 --- a/bootstrap/tasks/validation/vars.yaml +++ b/bootstrap/tasks/validation/vars.yaml @@ -33,7 +33,7 @@ - name: Check if bootstrap distribution is valid ansible.builtin.assert: - that: bootstrap_distribution in ['k0s', 'k3s'] + that: bootstrap_distribution in ['k0s', 'k3s', 'talos'] success_msg: Distribution {{ bootstrap_distribution }} is valid fail_msg: Distribution {{ bootstrap_distribution }} is not valid diff --git a/bootstrap/templates/.sops.yaml.j2 b/bootstrap/templates/.sops.yaml.j2 index 5da1af91..c44f9bad 100644 --- a/bootstrap/templates/.sops.yaml.j2 +++ b/bootstrap/templates/.sops.yaml.j2 @@ -1,5 +1,10 @@ --- creation_rules: + - # IMPORTANT: This rule MUST be above the others + path_regex: talos/.*\.sops\.ya?ml + key_groups: + - age: + - "#{ bootstrap_age_public_key }#" - path_regex: kubernetes/.*\.sops\.ya?ml encrypted_regex: "^(data|stringData)$" key_groups: diff --git a/bootstrap/templates/ansible/inventory/hosts.yaml.j2 b/bootstrap/templates/ansible/inventory/hosts.yaml.j2 index 9302adcc..ea0c7b08 100644 --- a/bootstrap/templates/ansible/inventory/hosts.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/hosts.yaml.j2 @@ -7,11 +7,7 @@ kubernetes: #% for item in bootstrap_nodes.master %# "#{ item.name }#": ansible_user: "#{ item.username }#" - #% if item.external_address is defined %# - ansible_host: "#{ item.external_address }#" - #% else %# ansible_host: "#{ item.address }#" - #% endif %# #% endfor %# #% if bootstrap_nodes.worker | default([]) | length > 0 %# worker: @@ -19,11 +15,7 @@ kubernetes: #% for item in bootstrap_nodes.worker %# "#{ item.name }#": ansible_user: "#{ item.username }#" - #% if item.external_address is defined %# - ansible_host: "#{ item.external_address }#" - #% else %# ansible_host: "#{ item.address }#" - #% endif %# #% endfor %# #% endif %# #% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/helmrelease.yaml.j2 new file mode 100644 index 00000000..32014eb1 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/helmrelease.yaml.j2 @@ -0,0 +1,29 @@ +#% if bootstrap_distribution == "talos" %# +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: kubelet-csr-approver +spec: + interval: 30m + chart: + spec: + chart: kubelet-csr-approver + version: 1.0.6 + sourceRef: + kind: HelmRepository + name: postfinance + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + providerRegex: | + ^k(m|w)\d\d?$ +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/kustomization.yaml.j2 new file mode 100644 index 00000000..6651d885 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/kustomization.yaml.j2 @@ -0,0 +1,7 @@ +#% if bootstrap_distribution == "talos" %# +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/ks.yaml.j2 new file mode 100644 index 00000000..43c19f0d --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/ks.yaml.j2 @@ -0,0 +1,22 @@ +#% if bootstrap_distribution == "talos" %# +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app kubelet-csr-approver + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/kube-system/kubelet-csr-approver/app + prune: false # never should be deleted + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m +#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 index d0da8c60..0f1ee8ae 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 @@ -7,7 +7,10 @@ resources: #% if bootstrap_distribution == "k3s" %# - ./coredns/ks.yaml #% endif %# + #% if bootstrap_distribution == "talos" %# + - ./kubelet-csr-approver/ks.yaml + #% endif %# - ./metrics-server/ks.yaml - #% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %# + #% if (bootstrap_distribution == "k0s" or bootstrap_distribution == "talos") and spegel.enabled | default(false) %# - ./spegel/ks.yaml #% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 index f8cbf7d8..10609f2a 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %# +#% if (bootstrap_distribution == "k0s" or bootstrap_distribution == "talos") and spegel.enabled | default(false) %# --- apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 index 96b4eb8e..aa8f0a5b 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %# +#% if (bootstrap_distribution == "k0s" or bootstrap_distribution == "talos") and spegel.enabled | default(false) %# --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 index cdb0566b..92479485 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %# +#% if (bootstrap_distribution == "k0s" or bootstrap_distribution == "talos") and spegel.enabled | default(false) %# --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 index 07fc265e..784ac62b 100644 --- a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 @@ -18,6 +18,7 @@ resources: - ./metrics-server.yaml - ./openebs.yaml - ./piraeus.yaml + - ./postfinance.yaml - ./prometheus-community.yaml - ./stakater.yaml - ./weave-gitops.yaml diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/postfinance.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/postfinance.yaml.j2 new file mode 100644 index 00000000..84312edd --- /dev/null +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/postfinance.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: postfinance + namespace: flux-system +spec: + interval: 1h + url: https://postfinance.github.io/kubelet-csr-approver diff --git a/bootstrap/templates/kubernetes/talos/README.md.j2 b/bootstrap/templates/kubernetes/talos/README.md.j2 new file mode 100644 index 00000000..d88d0538 --- /dev/null +++ b/bootstrap/templates/kubernetes/talos/README.md.j2 @@ -0,0 +1,31 @@ +#% if bootstrap_distribution == 'talos' %# +# Bootstrap + +## Talos + +### 1. Create Talos Secrets + +```sh +task talos:gensecret +task talos:genconfig +``` + +### 2. Apply Talos Config + +```sh +task talos:apply-config +``` + +### 3. Boostrap Talos and get kubeconfig + +```sh +task talos:bootstrap +task talos:kubeconfig +``` + +### 4. Install Cilium and kubelet-csr-approver into the cluster + +```sh +task talos:apply-extras +``` +#% endif %# diff --git a/bootstrap/templates/kubernetes/talos/cilium/.gitignore.j2 b/bootstrap/templates/kubernetes/talos/cilium/.gitignore.j2 new file mode 100644 index 00000000..4f4111e0 --- /dev/null +++ b/bootstrap/templates/kubernetes/talos/cilium/.gitignore.j2 @@ -0,0 +1,3 @@ +#% if bootstrap_distribution == 'talos' %# +charts/ +#% endif %# diff --git a/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 new file mode 100644 index 00000000..d7bdddab --- /dev/null +++ b/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 @@ -0,0 +1,17 @@ +#% if bootstrap_distribution == 'talos' %# +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +helmCharts: + - name: cilium + repo: https://helm.cilium.io + version: 1.14.5 + releaseName: cilium + namespace: kube-system + valuesFile: values.yaml +commonAnnotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system +commonLabels: + app.kubernetes.io/managed-by: Helm +#% endif %# diff --git a/bootstrap/templates/kubernetes/talos/cilium/values.yaml.j2 b/bootstrap/templates/kubernetes/talos/cilium/values.yaml.j2 new file mode 100644 index 00000000..7d75b310 --- /dev/null +++ b/bootstrap/templates/kubernetes/talos/cilium/values.yaml.j2 @@ -0,0 +1,6 @@ +#% if bootstrap_distribution == 'talos' %# +--- +#% filter indent(width=0, first=True) %# +#% include 'partials/cilium-values-init.partial.yaml.j2' %# +#% endfilter %# +#% endif %# diff --git a/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/.gitignore.j2 b/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/.gitignore.j2 new file mode 100644 index 00000000..4f4111e0 --- /dev/null +++ b/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/.gitignore.j2 @@ -0,0 +1,3 @@ +#% if bootstrap_distribution == 'talos' %# +charts/ +#% endif %# diff --git a/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/kustomization.yaml b/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/kustomization.yaml new file mode 100644 index 00000000..700801f1 --- /dev/null +++ b/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/kustomization.yaml @@ -0,0 +1,20 @@ +#% if bootstrap_distribution == 'talos' %# +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kube-system +helmCharts: + - name: kubelet-csr-approver + repo: https://postfinance.github.io/kubelet-csr-approver + version: 1.0.6 + releaseName: kubelet-csr-approver + namespace: kube-system + valuesInline: + providerRegex: ^k(m|w)\d\d?$ + bypassDnsResolution: true +commonAnnotations: + meta.helm.sh/release-name: kubelet-csr-approver + meta.helm.sh/release-namespace: kube-system +commonLabels: + app.kubernetes.io/managed-by: Helm +#% endif %# diff --git a/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 new file mode 100644 index 00000000..3dc0f28b --- /dev/null +++ b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 @@ -0,0 +1,192 @@ +#% if bootstrap_distribution == 'talos' %# +# yaml-language-server: $schema=https://raw.githubusercontent.com/budimanjojo/talhelper/master/pkg/config/schemas/talconfig.json +--- +# renovate: datasource=github-releases depName=siderolabs/talos +talosVersion: v1.6.1 +# renovate: datasource=github-releases depName=kubernetes/kubernetes extractVersion=^v(?.*)$ +kubernetesVersion: 1.29.1 + +clusterName: &cluster home-kubernetes + +#% if bootstrap_kube_api_hostname | default('') %# +endpoint: https://#{ bootstrap_kube_api_hostname }#:6443 +#% else %# +endpoint: https://#{ bootstrap_kube_api_addr }#:6443 +#% endif %# + +clusterPodNets: + - "#{ bootstrap_cluster_cidr.split(',')[0] }#" +clusterSvcNets: + - "#{ bootstrap_service_cidr.split(',')[0] }#" + +additionalApiServerCertSans: &sans + - "#{ bootstrap_kube_api_addr }#" + - 127.0.0.1 #KubePrism + #% if bootstrap_kube_api_hostname | default('') %# + - "#{ bootstrap_kube_api_hostname }#" + #% endif %# + +additionalMachineCertSans: *sans + +cniConfig: + name: none + +nodes: + #% for item in bootstrap_nodes.master %# + - hostname: "#{ item.name }#" + ipAddress: "#{ item.address }#" + installDiskSelector: + serial: "#{ item.diskSerial }#" + controlPlane: true + networkInterfaces: + - interface: eth0 + dhcp: false + mtu: 1500 + routes: + - network: 0.0.0.0/0 + gateway: "#{ bootstrap_node_cidr | nthhost(1) }#" + vip: + ip: "#{ bootstrap_kube_api_addr }#" + #% endfor %# + #% if bootstrap_nodes.worker | default([]) | length > 0 %# + #% for item in bootstrap_nodes.worker %# + - hostname: "#{ item.name }#" + ipAddress: "#{ item.address }#" + installDiskSelector: + serial: "#{ item.diskSerial }#" + controlPlane: false + networkInterfaces: + - interface: eth0 + dhcp: false + mtu: 1500 + routes: + - network: 0.0.0.0/0 + gateway: "#{ bootstrap_node_cidr | nthhost(1) }#" + #% endfor %# + #% endif %# + +controlPlane: + patches: + # Configure containerd + - &containerdPatch |- + machine: + files: + - op: create + path: /etc/cri/conf.d/20-customization.part + content: |- + [plugins."io.containerd.grpc.v1.cri"] + enable_unprivileged_ports = true + enable_unprivileged_icmp = true + #% if spegel.enabled | default(false) %# + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + discard_unpacked_layers = false + #% endif %# + + # Disable search domain everywhere + - &disableSearchDomainPatch |- + machine: + network: + disableSearchDomain: true + + # Enable cluster discovery + - &discoveryPatch |- + cluster: + discovery: + registries: + kubernetes: + disabled: false + service: + disabled: false + + # Configure kubelet + - &kubeletPatch |- + machine: + kubelet: + extraArgs: + rotate-server-certificates: true + nodeIP: + validSubnets: + - "#{ bootstrap_node_cidr }#" + + # Enable KubePrism + - &kubePrismPatch |- + machine: + features: + kubePrism: + enabled: true + port: 7445 + + # Force nameserver + - &nameserverPatch |- + machine: + network: + nameservers: + - 1.1.1.1 + + # Configure NTP + - &ntpPatch |- + machine: + time: + disabled: false + servers: + - time.cloudflare.com + + # Custom sysctl settings + - &sysctlPatch |- + machine: + sysctls: + fs.inotify.max_queued_events: 65536 + fs.inotify.max_user_watches: 524288 + fs.inotify.max_user_instances: 8192 + + # Cluster configuration + - |- + cluster: + allowSchedulingOnMasters: true + controllerManager: + extraArgs: + bind-address: 0.0.0.0 + proxy: + disabled: true + scheduler: + extraArgs: + bind-address: 0.0.0.0 + + # ETCD configuration + - |- + cluster: + etcd: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 + advertisedSubnets: + - "#{ bootstrap_node_cidr }#" + + # Disable default API server admission plugins. + - |- + - op: remove + path: /cluster/apiServer/admissionControl + + # Enable K8s Talos API Access + - |- + machine: + features: + kubernetesTalosAPIAccess: + enabled: true + allowedRoles: + - os:admin + allowedKubernetesNamespaces: + - kube-system + +worker: + patches: + - *containerdPatch + - *disableSearchDomainPatch + - *discoveryPatch + - *kubeletPatch + - *kubePrismPatch + - *nameserverPatch + - *ntpPatch + - *sysctlPatch +#% endif %# diff --git a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 index 78ceb0e7..9a173f5e 100644 --- a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 @@ -70,6 +70,9 @@ k8sServicePort: 6444 #% elif bootstrap_distribution == "k0s" %# k8sServiceHost: localhost k8sServicePort: 7443 +#% elif bootstrap_distribution == "talos" %# +k8sServiceHost: localhost +k8sServicePort: 7445 #% endif %# kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 diff --git a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 index 52e60eec..a0d96904 100644 --- a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 @@ -31,6 +31,9 @@ k8sServicePort: 6444 #% elif bootstrap_distribution == "k0s" %# k8sServiceHost: localhost k8sServicePort: 7443 +#% elif bootstrap_distribution == "talos" %# +k8sServiceHost: localhost +k8sServicePort: 7445 #% endif %# kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index 159eb0f4..70277eac 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -3,7 +3,7 @@ # Bootstrap configuration - config.yaml is gitignored # -# Distribution can either be 'k3s' or 'k0s' +# Distribution can either be 'k3s', 'k0s', or 'talos' bootstrap_distribution: k3s # Github username (e.g. onedr0p) @@ -77,15 +77,15 @@ bootstrap_local_storage_path: /var/openebs/local bootstrap_nodes: # Use only 1, 3 or more odd master nodes, recommended is 3 master: - # - name: # name or hostname of the master node (must match [a-z0-9-\.]+) - # address: # ip address of the master node - # username: # ssh username of the master node - # role: # (Optional) k0s only: can be controller+worker (default) or controller - # # external_address: # (Optional) k3s only: Only use when SSH is not reachable on the IP specified in 'address' field + # - name: # name or hostname of the master node (must match [a-z0-9-\.]+) + # address: # ip address of the master node + # username: # ssh username of the master node + # role: # (Optional: k0s) controller+worker (default) or controller + # diskSerial: # (Required: Talos) Serial number of Disk for this master node # ... worker: # set to [] or omit if no workers are needed - # - name: # name or hostname of the worker node (must match [a-z0-9-\.]+) - # address: # ip address of the worker node - # username: # ssh username of the worker node - # # external_address: # (Optional) k3s only: Only use when SSH is not reachable on the IP specified in 'address' field + # - name: # name or hostname of the worker node (must match [a-z0-9-\.]+) + # address: # ip address of the worker node + # username: # ssh username of the worker node + # diskSerial: # (Required: Talos) Serial number of Disk for this worker node # ...