From afbf097dacda39a7c3a9e5356a817062e17f54c4 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 22 Jan 2024 15:28:21 -0500 Subject: [PATCH] feat: update kube-vip ansible playbook for k0s and k3s - ipv6 config changes (#1234) * feat: update kube-vip ansible playbook for k0s and k3s Signed-off-by: Devin Buhl * fix: update renovate Signed-off-by: Devin Buhl * fix: move ipv6 to advanced options Signed-off-by: Devin Buhl * feat: kube-vip manifests to partials Signed-off-by: Devin Buhl * feat: kube vip samesies Signed-off-by: Devin Buhl * fix: update config sample Signed-off-by: Devin Buhl --------- Signed-off-by: Devin Buhl --- .github/renovate.json5 | 7 +- .github/tests/config-k0s.yaml | 1 - .github/tests/config-k3s-ipv4.yaml | 1 - .github/tests/config-k3s-ipv6.yaml | 3 +- .github/tests/config-talos.yaml | 1 - bootstrap/scripts/validation.py | 15 +-- bootstrap/templates/.sops.yaml.j2 | 2 +- bootstrap/templates/ansible/.ansible-lint.j2 | 2 +- .../group_vars/kubernetes/main.yaml.j2 | 11 +- .../inventory/group_vars/master/main.yaml.j2 | 6 +- .../inventory/group_vars/worker/main.yaml.j2 | 4 +- .../templates/ansible/inventory/hosts.yaml.j2 | 2 +- .../playbooks/cluster-installation.yaml.j2 | 2 +- .../playbooks/cluster-kube-vip.yaml.j2 | 13 +- .../ansible/playbooks/cluster-nuke.yaml.j2 | 2 +- .../ansible/playbooks/cluster-prepare.yaml.j2 | 2 +- .../ansible/playbooks/cluster-reboot.yaml.j2 | 2 +- .../playbooks/cluster-rollout-update.yaml.j2 | 18 +-- .../ansible/playbooks/tasks/cilium.yaml.j2 | 2 +- .../ansible/playbooks/tasks/cruft.yaml.j2 | 2 +- .../playbooks/tasks/kubeconfig.yaml.j2 | 2 +- .../templates/custom-cilium-helmchart.yaml.j2 | 4 +- .../playbooks/templates/kube-vip-ds.yaml.j2 | 77 +---------- .../playbooks/templates/kube-vip-rbac.yaml.j2 | 4 + .../templates/ansible/requirements.txt.j2 | 2 +- .../templates/ansible/requirements.yaml.j2 | 4 +- .../cilium/app/helmrelease.yaml.j2 | 2 +- .../cilium/app/kustomization.yaml.j2 | 2 +- .../app/helmrelease.yaml.j2 | 2 +- .../app/kustomization.yaml.j2 | 2 +- .../kubelet-csr-approver/ks.yaml.j2 | 2 +- .../apps/kube-system/kustomization.yaml.j2 | 2 +- .../spegel/app/helmrelease.yaml.j2 | 2 +- .../app/helmrelease.yaml.j2 | 8 +- .../csi-driver-nfs/app/helmrelease.yaml.j2 | 2 +- .../csi-driver-smb/app/helmrelease.yaml.j2 | 2 +- .../apps/tools/kustomization.yaml.j2 | 2 +- .../app/helmrelease.yaml.j2 | 2 +- .../app/kustomization.yaml.j2 | 2 +- .../app/rbac.yaml.j2 | 2 +- .../system-upgrade-controller/ks.yaml.j2 | 2 +- .../plans/agent.yaml.j2 | 2 +- .../plans/kustomization.yaml.j2 | 2 +- .../plans/server.yaml.j2 | 2 +- .../flux/vars/cluster-settings.yaml.j2 | 2 +- .../templates/kubernetes/k0s/k0sctl.yaml.j2 | 4 +- .../k0s/resources/apply-kube-vip.sh.j2 | 127 ++---------------- .../k0s/resources/apply-system.sh.j2 | 2 +- .../k0s/resources/reset-cilium.sh.j2 | 2 +- .../k0s/resources/reset-system.sh.j2 | 2 +- .../kubernetes/talos/cilium/.gitignore.j2 | 2 +- .../talos/cilium/kustomization.yaml.j2 | 2 +- .../kubernetes/talos/cilium/values.yaml.j2 | 2 +- .../talos/kubelet-csr-approver/.gitignore.j2 | 2 +- .../kustomization.yaml.j2 | 2 +- .../kubernetes/talos/talconfig.yaml.j2 | 9 +- .../cilium-values-full.partial.yaml.j2 | 15 ++- .../cilium-values-init.partial.yaml.j2 | 15 ++- .../partials/kube-vip-ds.partial.yaml.j2 | 74 ++++++++++ .../partials/kube-vip-rbac.partial.yaml.j2 | 41 ++++++ bootstrap/vars/config.sample.yaml | 15 ++- 61 files changed, 244 insertions(+), 307 deletions(-) create mode 100644 bootstrap/templates/ansible/playbooks/templates/kube-vip-rbac.yaml.j2 create mode 100644 bootstrap/templates/partials/kube-vip-ds.partial.yaml.j2 create mode 100644 bootstrap/templates/partials/kube-vip-rbac.partial.yaml.j2 diff --git a/.github/renovate.json5 b/.github/renovate.json5 index d5e74e5d..90cbfbcc 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -238,13 +238,12 @@ "matchStrings": [ // Example: // k3s_release_version: "v1.27.3+k3s1" - "datasource=(?\\S+) depName=(?\\S+)( versioning=(?\\S+))?( extractVersion=(?\\S+))?\n.*?\"(?.*)\"\n", + "datasource=(?\\S+) depName=(?\\S+)( repository=(?\\S+))?( extractVersion=(?\\S+))?\n.*?\"(?.*)\"\n", // Example: // - https://github.com/rancher/system-upgrade-controller/releases/download/v0.11.0/crd.yaml - "datasource=(?\\S+) depName=(?\\S+)( versioning=(?\\S+))?\n.*?-\\s(.*?)\/(?[^/]+)\/[^/]+\n", + "datasource=(?\\S+) depName=(?\\S+)\n.*?-\\s(.*?)\/(?[^/]+)\/[^/]+\n", ], - "datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}", - "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}" + "datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}" } ] } diff --git a/.github/tests/config-k0s.yaml b/.github/tests/config-k0s.yaml index 6b487d06..ae37dfb4 100644 --- a/.github/tests/config-k0s.yaml +++ b/.github/tests/config-k0s.yaml @@ -21,7 +21,6 @@ bootstrap_k8s_gateway_addr: 10.10.10.253 bootstrap_external_ingress_addr: 10.10.10.252 bootstrap_internal_ingress_addr: 10.10.10.251 bootstrap_dns_server: 1.1.1.1 -bootstrap_ipv6_enabled: false bootstrap_cluster_cidr: 10.42.0.0/16 bootstrap_service_cidr: 10.43.0.0/16 bootstrap_nodes: diff --git a/.github/tests/config-k3s-ipv4.yaml b/.github/tests/config-k3s-ipv4.yaml index 9416dd49..cbfef5a9 100644 --- a/.github/tests/config-k3s-ipv4.yaml +++ b/.github/tests/config-k3s-ipv4.yaml @@ -21,7 +21,6 @@ bootstrap_k8s_gateway_addr: 10.10.10.253 bootstrap_external_ingress_addr: 10.10.10.252 bootstrap_internal_ingress_addr: 10.10.10.251 bootstrap_dns_server: 1.1.1.1 -bootstrap_ipv6_enabled: false bootstrap_cluster_cidr: 10.42.0.0/16 bootstrap_service_cidr: 10.43.0.0/16 bootstrap_nodes: diff --git a/.github/tests/config-k3s-ipv6.yaml b/.github/tests/config-k3s-ipv6.yaml index 5a7ff117..f4a00423 100644 --- a/.github/tests/config-k3s-ipv6.yaml +++ b/.github/tests/config-k3s-ipv6.yaml @@ -21,7 +21,8 @@ bootstrap_k8s_gateway_addr: 10.10.10.253 bootstrap_external_ingress_addr: 10.10.10.252 bootstrap_internal_ingress_addr: 10.10.10.251 bootstrap_dns_server: 1.1.1.1 -bootstrap_ipv6_enabled: true +bootstrap_advanced_flags: + dual_stack_ipv4_first: true bootstrap_cluster_cidr: 10.42.0.0/16,fd7f:8f5:e87c:a::/64 bootstrap_service_cidr: 10.43.0.0/16,fd7f:8f5:e87c:e::/112 bootstrap_nodes: diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml index a47603fd..5e713e98 100644 --- a/.github/tests/config-talos.yaml +++ b/.github/tests/config-talos.yaml @@ -21,7 +21,6 @@ bootstrap_k8s_gateway_addr: 10.10.10.253 bootstrap_external_ingress_addr: 10.10.10.252 bootstrap_internal_ingress_addr: 10.10.10.251 bootstrap_dns_server: 1.1.1.1 -bootstrap_ipv6_enabled: false bootstrap_cluster_cidr: 10.42.0.0/16 bootstrap_service_cidr: 10.43.0.0/16 bootstrap_nodes: diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py index b4272f2d..c20bc34a 100644 --- a/bootstrap/scripts/validation.py +++ b/bootstrap/scripts/validation.py @@ -83,14 +83,14 @@ def validate_python_version() -> None: @required("bootstrap_distribution") def validate_cli_tools(distribution: str, **_) -> None: - distro = _validate_distribution(distribution) + _distribution = _validate_distribution(distribution) for tool in GLOBAL_CLI_TOOLS: if not which(tool): raise ValueError(f"Missing required CLI tool {tool}") - for tool in TALOS_CLI_TOOLS if distro == "talos" else []: + for tool in TALOS_CLI_TOOLS if _distribution in ["talos"] else []: if not which(tool): raise ValueError(f"Missing required CLI tool {tool}") - for tool in K0S_CLI_TOOLS if distro == "k0s" else []: + for tool in K0S_CLI_TOOLS if _distribution in ["k0s"] else []: if not which(tool): raise ValueError(f"Missing required CLI tool {tool}") @@ -125,15 +125,14 @@ def validate_timezone(timezone: str, **_) -> None: raise ValueError(f"Invalid timezone {timezone}") -@required("bootstrap_ipv6_enabled", "bootstrap_cluster_cidr", "bootstrap_service_cidr") -def validate_cluster_cidrs(ipv6_enabled: bool, cluster_cidr: str, service_cidr: str, **_) -> None: - if not isinstance(ipv6_enabled, bool): - raise ValueError(f"Invalid IPv6 enabled {ipv6_enabled}") +@required("bootstrap_cluster_cidr", "bootstrap_service_cidr", "bootstrap_advanced_flags") +def validate_cluster_cidrs(cluster_cidr: str, service_cidr: str, advanced_flags: dict, **_) -> None: + dual_stack_ipv4_first = advanced_flags.get("dual_stack_ipv4_first", False) if cluster_cidr == service_cidr: raise ValueError(f"Cluster CIDR {cluster_cidr} is the same as service CIDR {service_cidr}") - if ipv6_enabled: + if dual_stack_ipv4_first: if len(cluster_cidr.split(",")) != 2: raise ValueError(f"Invalid cluster CIDR {cluster_cidr}") if len(service_cidr.split(",")) != 2: diff --git a/bootstrap/templates/.sops.yaml.j2 b/bootstrap/templates/.sops.yaml.j2 index f3e32435..dfd6885f 100644 --- a/bootstrap/templates/.sops.yaml.j2 +++ b/bootstrap/templates/.sops.yaml.j2 @@ -1,6 +1,6 @@ --- creation_rules: - #% if bootstrap_distribution == "talos" %# + #% if bootstrap_distribution in ['talos'] %# - # IMPORTANT: This rule MUST be above the others path_regex: talos/.*\.sops\.ya?ml key_groups: diff --git a/bootstrap/templates/ansible/.ansible-lint.j2 b/bootstrap/templates/ansible/.ansible-lint.j2 index 7a02d8c5..0639cee3 100644 --- a/bootstrap/templates/ansible/.ansible-lint.j2 +++ b/bootstrap/templates/ansible/.ansible-lint.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution in ['k3s', 'k0s'] %# +#% if bootstrap_distribution in ['k0s', 'k3s'] %# skip_list: - yaml[commas] - yaml[line-length] diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 index 12598831..c28b6589 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- # https://github.com/PyratLabs/ansible-role-k3s # renovate: datasource=github-releases depName=k3s-io/k3s @@ -10,10 +10,7 @@ k3s_use_unsupported_config: true k3s_registration_address: "#{ bootstrap_kube_api_addr }#" # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_templates: - - custom-cilium-helmchart.yaml.j2 - - kube-vip-ds.yaml.j2 -# /var/lib/rancher/k3s/server/manifests -k3s_server_manifests_urls: - - url: https://kube-vip.io/manifests/rbac.yaml - filename: kube-vip-rbac.yaml + - custom-cilium-helmchart.yaml + - kube-vip-ds.yaml + - kube-vip-rbac.yaml #% endif %# diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 index 0654315a..a2c62087 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 @@ -1,10 +1,10 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- # https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/ # https://github.com/PyratLabs/ansible-role-k3s k3s_control_node: true k3s_server: - #% if bootstrap_ipv6_enabled %# + #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}" #% else %# node-ip: "{{ ansible_host }}" @@ -23,7 +23,7 @@ k3s_server: write-kubeconfig-mode: "644" pause-image: registry.k8s.io/pause:3.9 secrets-encryption: true - #% if bootstrap_ipv6_enabled %# + #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# cluster-cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#,#{ bootstrap_cluster_cidr.split(',')[1] }#" service-cidr: "#{ bootstrap_service_cidr.split(',')[0] }#,#{ bootstrap_service_cidr.split(',')[1] }#" #% else %# diff --git a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 index 47681fe8..57e62949 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 @@ -1,10 +1,10 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- # https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/ # https://github.com/PyratLabs/ansible-role-k3s k3s_control_node: false k3s_agent: - #% if bootstrap_ipv6_enabled %# + #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}" #% else %# node-ip: "{{ ansible_host }}" diff --git a/bootstrap/templates/ansible/inventory/hosts.yaml.j2 b/bootstrap/templates/ansible/inventory/hosts.yaml.j2 index be4757a5..5ceb634f 100644 --- a/bootstrap/templates/ansible/inventory/hosts.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/hosts.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution in ['k3s', 'k0s'] %# +#% if bootstrap_distribution in ['k0s', 'k3s'] %# --- kubernetes: children: diff --git a/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 index 43ee78b8..65732202 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- - name: Cluster Installation hosts: kubernetes diff --git a/bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2 index 42d71c1b..bb24461b 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k0s', 'k3s'] %# --- - name: Cluster kube-vip hosts: master @@ -11,16 +11,21 @@ ansible.builtin.pause: seconds: 5 tasks: + #% if bootstrap_distribution in ['k3s'] %# - name: Ensure Kubernetes is running ansible.builtin.include_role: name: xanmanning.k3s public: true vars: k3s_state: started - + #% endif %# - name: Upgrade kube-vip ansible.builtin.template: - src: templates/kube-vip-static-pod.yaml.j2 - dest: "{{ k3s_server_pod_manifests_dir }}/kube-vip-static-pod.yaml" + src: templates/kube-vip-ds.yaml.j2 + #% if bootstrap_distribution in ['k3s'] %# + dest: "{{ k3s_server_manifests_dir }}/kube-vip-ds.yaml" + #% elif bootstrap_distribution in ['k0s'] %# + dest: "/var/lib/k0s/manifests/kube-vip/ds.yaml" + #% endif %# mode: preserve #% endif %# diff --git a/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 index 21eabb0f..54243958 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- - name: Cluster Nuke hosts: kubernetes diff --git a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 index 61d5fc84..053579dc 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- - name: Prepare System hosts: kubernetes diff --git a/bootstrap/templates/ansible/playbooks/cluster-reboot.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-reboot.yaml.j2 index 2a695681..c42f4fff 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-reboot.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-reboot.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution in ['k3s', 'k0s'] %# +#% if bootstrap_distribution in ['k0s', 'k3s'] %# --- - name: Reboot hosts: kubernetes diff --git a/bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2 index e027c26a..8e4653b1 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution in ['k3s', 'k0s'] %# +#% if bootstrap_distribution in ['k0s', 'k3s'] %# --- # https://github.com/kevincoakley/ansible-role-k8s-rolling-update - name: Cluster rollout update @@ -13,9 +13,9 @@ seconds: 5 tasks: - name: Details - #% if bootstrap_distribution == 'k3s' %# + #% if bootstrap_distribution in ['k3s'] %# ansible.builtin.command: "k3s kubectl get node {{ inventory_hostname }} -o json" - #% elif bootstrap_distribution == 'k0s' %# + #% elif bootstrap_distribution in ['k0s'] %# ansible.builtin.command: "k0s kubectl get node {{ inventory_hostname }} -o json" #% endif %# register: kubectl_get_node @@ -33,9 +33,9 @@ - name: Cordon kubernetes.core.k8s_drain: name: "{{ inventory_hostname }}" - #% if bootstrap_distribution == 'k3s' %# + #% if bootstrap_distribution in ['k3s'] %# kubeconfig: /etc/rancher/k3s/k3s.yaml - #% elif bootstrap_distribution == 'k0s' %# + #% elif bootstrap_distribution in ['k0s'] %# kubeconfig: /var/lib/k0s/pki/admin.conf #% endif %# state: cordon @@ -44,9 +44,9 @@ - name: Drain kubernetes.core.k8s_drain: name: "{{ inventory_hostname }}" - #% if bootstrap_distribution == 'k3s' %# + #% if bootstrap_distribution in ['k3s'] %# kubeconfig: /etc/rancher/k3s/k3s.yaml - #% elif bootstrap_distribution == 'k0s' %# + #% elif bootstrap_distribution in ['k0s'] %# kubeconfig: /var/lib/k0s/pki/admin.conf #% endif %# state: drain @@ -80,9 +80,9 @@ - name: Uncordon kubernetes.core.k8s_drain: name: "{{ inventory_hostname }}" - #% if bootstrap_distribution == 'k3s' %# + #% if bootstrap_distribution in ['k3s'] %# kubeconfig: /etc/rancher/k3s/k3s.yaml - #% elif bootstrap_distribution == 'k0s' %# + #% elif bootstrap_distribution in ['k0s'] %# kubeconfig: /var/lib/k0s/pki/admin.conf #% endif %# state: uncordon diff --git a/bootstrap/templates/ansible/playbooks/tasks/cilium.yaml.j2 b/bootstrap/templates/ansible/playbooks/tasks/cilium.yaml.j2 index d7de64d2..c9a71622 100644 --- a/bootstrap/templates/ansible/playbooks/tasks/cilium.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/tasks/cilium.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- - name: Cilium block: diff --git a/bootstrap/templates/ansible/playbooks/tasks/cruft.yaml.j2 b/bootstrap/templates/ansible/playbooks/tasks/cruft.yaml.j2 index 8f646025..e0256933 100644 --- a/bootstrap/templates/ansible/playbooks/tasks/cruft.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/tasks/cruft.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- # https://github.com/k3s-io/k3s/issues/1971 - name: Cruft diff --git a/bootstrap/templates/ansible/playbooks/tasks/kubeconfig.yaml.j2 b/bootstrap/templates/ansible/playbooks/tasks/kubeconfig.yaml.j2 index 4bbb530d..79207673 100644 --- a/bootstrap/templates/ansible/playbooks/tasks/kubeconfig.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/tasks/kubeconfig.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- - name: Get absolute path to this Git repository # noqa: command-instead-of-module ansible.builtin.command: git rev-parse --show-toplevel diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 index 9d269d74..04243c29 100644 --- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- # https://docs.k3s.io/helm apiVersion: helm.cattle.io/v1 @@ -9,7 +9,7 @@ metadata: spec: repo: https://helm.cilium.io/ chart: cilium - # renovate: datasource=github-releases depName=cilium/cilium + # renovate: datasource=helm depName=cilium repository=https://helm.cilium.io version: "1.14.5" targetNamespace: kube-system bootstrap: true diff --git a/bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2 b/bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2 index 38c9f29d..16490fd2 100644 --- a/bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2 @@ -1,77 +1,4 @@ -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# --- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-vip - namespace: kube-system - labels: - app.kubernetes.io/name: kube-vip -spec: - selector: - matchLabels: - app.kubernetes.io/name: kube-vip - template: - metadata: - labels: - app.kubernetes.io/name: kube-vip - spec: - containers: - - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.6.4 - imagePullPolicy: IfNotPresent - args: ["manager"] - env: - - name: address - value: "#{ bootstrap_kube_api_addr }#" - - name: vip_arp - value: "true" - - name: lb_enable - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "false" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: prometheus_server - value: :2112 - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"] - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - serviceAccountName: kube-vip - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists +#% include 'partials/kube-vip-ds.partial.yaml.j2' %# #% endif %# diff --git a/bootstrap/templates/ansible/playbooks/templates/kube-vip-rbac.yaml.j2 b/bootstrap/templates/ansible/playbooks/templates/kube-vip-rbac.yaml.j2 new file mode 100644 index 00000000..f931d299 --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/templates/kube-vip-rbac.yaml.j2 @@ -0,0 +1,4 @@ +#% if bootstrap_distribution in ['k3s'] %# +--- +#% include 'partials/kube-vip-rbac.partial.yaml.j2' %# +#% endif %# diff --git a/bootstrap/templates/ansible/requirements.txt.j2 b/bootstrap/templates/ansible/requirements.txt.j2 index 3fcc5274..5d7cde18 100644 --- a/bootstrap/templates/ansible/requirements.txt.j2 +++ b/bootstrap/templates/ansible/requirements.txt.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution in ['k3s', 'k0s'] %# +#% if bootstrap_distribution in ['k0s', 'k3s'] %# # Ansible ansible-lint==6.22.2 ansible==9.1.0 diff --git a/bootstrap/templates/ansible/requirements.yaml.j2 b/bootstrap/templates/ansible/requirements.yaml.j2 index 4ca4646f..6a5e27e6 100644 --- a/bootstrap/templates/ansible/requirements.yaml.j2 +++ b/bootstrap/templates/ansible/requirements.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution in ['k3s', 'k0s'] %# +#% if bootstrap_distribution in ['k0s', 'k3s'] %# --- collections: - name: ansible.posix @@ -9,7 +9,7 @@ collections: version: 8.2.0 - name: kubernetes.core version: 3.0.0 -#% if bootstrap_distribution == 'k3s' %# +#% if bootstrap_distribution in ['k3s'] %# roles: - name: xanmanning.k3s src: https://github.com/PyratLabs/ansible-role-k3s diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 index 48450c34..c7311e3a 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 @@ -8,7 +8,7 @@ spec: chart: spec: chart: cilium - version: 1.14.6 + version: 1.14.5 sourceRef: kind: HelmRepository name: cilium diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 index 7e748a78..abfe4b79 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - #% if not bootstrap_ipv6_enabled %# + #% if not bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# - ./cilium-l2.yaml #% endif %# - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/helmrelease.yaml.j2 index b15d445a..53682bb5 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/helmrelease.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "talos" %# +#% if bootstrap_distribution in ['talos'] %# --- apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/kustomization.yaml.j2 index 6651d885..ba2b23b3 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/kustomization.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "talos" %# +#% if bootstrap_distribution in ['talos'] %# --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/ks.yaml.j2 index 43c19f0d..2ef6a5cf 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/ks.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "talos" %# +#% if bootstrap_distribution in ['talos'] %# --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 index 53945924..2336c26f 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 @@ -4,7 +4,7 @@ kind: Kustomization resources: - ./namespace.yaml - ./cilium/ks.yaml - #% if bootstrap_distribution == "talos" %# + #% if bootstrap_distribution in ['talos'] %# - ./kubelet-csr-approver/ks.yaml #% endif %# - ./metrics-server/ks.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 index 14f962a3..2b4d16a2 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 @@ -25,7 +25,7 @@ spec: keepHistory: false values: spegel: - #% if bootstrap_distribution == "k0s" %# + #% if bootstrap_distribution in ['k0s'] %# containerdSock: /run/k0s/containerd.sock containerdRegistryConfigPath: /var/lib/k0s/containerd/certs.d #% endif %# diff --git a/bootstrap/templates/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml.j2 index 62ff866e..3dcb59a6 100644 --- a/bootstrap/templates/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml.j2 @@ -53,7 +53,7 @@ spec: enabled: true serviceMonitor: metricRelabelings: - #% if bootstrap_distribution == 'k3s' %# + #% if bootstrap_distribution in ['k3s'] %# # Remove duplicate labels provided by k3s - action: keep sourceLabels: ["__name__"] @@ -74,7 +74,7 @@ spec: enabled: true serviceMonitor: metricRelabelings: - #% if bootstrap_distribution == 'k3s' %# + #% if bootstrap_distribution in ['k3s'] %# # Remove duplicate labels provided by k3s - action: keep sourceLabels: ["__name__"] @@ -93,7 +93,7 @@ spec: #% for item in bootstrap_nodes.master %# - "#{ item.address }#" #% endfor %# - #% if bootstrap_distribution == 'k3s' %# + #% if bootstrap_distribution in ['k3s'] %# serviceMonitor: metricRelabelings: # Remove duplicate labels provided by k3s @@ -107,7 +107,7 @@ spec: kubeScheduler: enabled: true endpoints: *endpoints - #% if bootstrap_distribution == 'k3s' %# + #% if bootstrap_distribution in ['k3s'] %# serviceMonitor: metricRelabelings: # Remove duplicate labels provided by k3s diff --git a/bootstrap/templates/kubernetes/apps/storage/csi-driver-nfs/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/storage/csi-driver-nfs/app/helmrelease.yaml.j2 index 9b52509b..5472ef69 100644 --- a/bootstrap/templates/kubernetes/apps/storage/csi-driver-nfs/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/storage/csi-driver-nfs/app/helmrelease.yaml.j2 @@ -26,7 +26,7 @@ spec: values: externalSnapshotter: enabled: false - #% if bootstrap_distribution == "k0s" %# + #% if bootstrap_distribution in ['k0s'] %# kubeletDir: /var/lib/k0s/kubelet #% endif %# #% endif %# diff --git a/bootstrap/templates/kubernetes/apps/storage/csi-driver-smb/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/storage/csi-driver-smb/app/helmrelease.yaml.j2 index 8c763e3a..2e49c564 100644 --- a/bootstrap/templates/kubernetes/apps/storage/csi-driver-smb/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/storage/csi-driver-smb/app/helmrelease.yaml.j2 @@ -25,7 +25,7 @@ spec: keepHistory: false values: linux: - #% if bootstrap_distribution == "k0s" %# + #% if bootstrap_distribution in ['k0s'] %# kubelet: /var/lib/k0s/kubelet #% else %# kubelet: /var/lib/kubelet diff --git a/bootstrap/templates/kubernetes/apps/tools/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/tools/kustomization.yaml.j2 index 13c9f83b..d8836c40 100644 --- a/bootstrap/templates/kubernetes/apps/tools/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/tools/kustomization.yaml.j2 @@ -5,6 +5,6 @@ resources: - ./namespace.yaml - ./descheduler/ks.yaml - ./reloader/ks.yaml - #% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %# + #% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %# - ./system-upgrade-controller/ks.yaml #% endif %# diff --git a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/helmrelease.yaml.j2 index f15939ec..597e0676 100644 --- a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/helmrelease.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %# +#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %# --- apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease diff --git a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/kustomization.yaml.j2 index 1e3f307f..9c7bb02a 100644 --- a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/kustomization.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %# +#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %# --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/rbac.yaml.j2 b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/rbac.yaml.j2 index 0a270eea..a69efbde 100644 --- a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/rbac.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/rbac.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %# +#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %# --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/ks.yaml.j2 index ec8c6fc3..b31a0144 100644 --- a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/ks.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %# +#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %# --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/agent.yaml.j2 b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/agent.yaml.j2 index a326911e..4f7260ec 100644 --- a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/agent.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/agent.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %# +#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %# --- apiVersion: upgrade.cattle.io/v1 kind: Plan diff --git a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/kustomization.yaml.j2 index 1fa55d79..b4e8d98e 100644 --- a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/kustomization.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %# +#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %# --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/server.yaml.j2 b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/server.yaml.j2 index 94c647ee..9def14a6 100644 --- a/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/server.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/server.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %# +#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %# --- apiVersion: upgrade.cattle.io/v1 kind: Plan diff --git a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 index 9722f399..9413471a 100644 --- a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 @@ -11,7 +11,7 @@ data: CLUSTER_CIDR: "#{ bootstrap_cluster_cidr.split(',')[0] }#" SERVICE_CIDR: "#{ bootstrap_service_cidr.split(',')[0] }#" NODE_CIDR: "#{ bootstrap_node_cidr }#" - #% if bootstrap_ipv6_enabled %# + #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# CLUSTER_CIDR_V6: "#{ bootstrap_cluster_cidr.split(',')[1] }#" SERVICE_CIDR_V6: "#{ bootstrap_service_cidr.split(',')[1] }#" #% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 index 6d778cd5..cfc557cc 100644 --- a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 +++ b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k0s' %# +#% if bootstrap_distribution in ['k0s'] %# --- apiVersion: k0sctl.k0sproject.io/v1beta1 kind: Cluster @@ -91,7 +91,7 @@ spec: charts: - name: cilium chartname: cilium/cilium - # renovate: datasource=github-releases depName=cilium/cilium + # renovate: datasource=helm depName=cilium repository=https://helm.cilium.io version: "1.14.5" namespace: kube-system values: |2 diff --git a/bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2 index a7bddc24..9addd1b1 100755 --- a/bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2 @@ -1,131 +1,20 @@ +#% if bootstrap_distribution in ['k0s'] %# #!/usr/bin/env bash set -e set -o noglob [ $(id -u) -eq 0 ] || exec sudo $0 $@ -# Create directory +# Create kube-vip directory mkdir -p /var/lib/k0s/manifests/kube-vip # Create kube-vip rbac -cat < /var/lib/k0s/manifests/kube-vip/rbac.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system +cat < /var/lib/k0s/manifests/kube-vip/kube-vip-rbac.yaml +#% include 'partials/kube-vip-rbac.partial.yaml.j2' %# EOF -cat < /var/lib/k0s/manifests/kube-vip/ds.yaml ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-vip - namespace: kube-system - labels: - app.kubernetes.io/name: kube-vip -spec: - selector: - matchLabels: - app.kubernetes.io/name: kube-vip - template: - metadata: - labels: - app.kubernetes.io/name: kube-vip - spec: - containers: - - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.6.4 - imagePullPolicy: IfNotPresent - args: ["manager"] - env: - - name: address - value: "#{ bootstrap_kube_api_addr }#" - - name: vip_arp - value: "true" - - name: lb_enable - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "false" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: prometheus_server - value: :2112 - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"] - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - serviceAccountName: kube-vip - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists +# Create kube-vip daemonset +cat < /var/lib/k0s/manifests/kube-vip/kube-vip-ds.yaml +#% include 'partials/kube-vip-ds.partial.yaml.j2' %# EOF +#% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/resources/apply-system.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/apply-system.sh.j2 index 2643da63..3faec6b9 100755 --- a/bootstrap/templates/kubernetes/k0s/resources/apply-system.sh.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/apply-system.sh.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k0s' %# +#% if bootstrap_distribution in ['k0s'] %# #!/usr/bin/env bash set -e set -o noglob diff --git a/bootstrap/templates/kubernetes/k0s/resources/reset-cilium.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/reset-cilium.sh.j2 index 4e8d17d3..0ec0423a 100755 --- a/bootstrap/templates/kubernetes/k0s/resources/reset-cilium.sh.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/reset-cilium.sh.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k0s' %# +#% if bootstrap_distribution in ['k0s'] %# #!/usr/bin/env bash set -e set -o noglob diff --git a/bootstrap/templates/kubernetes/k0s/resources/reset-system.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/reset-system.sh.j2 index 05e8c928..1a8e0ce5 100755 --- a/bootstrap/templates/kubernetes/k0s/resources/reset-system.sh.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/reset-system.sh.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k0s' %# +#% if bootstrap_distribution in ['k0s'] %# #!/usr/bin/env bash set -e set -o noglob diff --git a/bootstrap/templates/kubernetes/talos/cilium/.gitignore.j2 b/bootstrap/templates/kubernetes/talos/cilium/.gitignore.j2 index 4f4111e0..f780a20c 100644 --- a/bootstrap/templates/kubernetes/talos/cilium/.gitignore.j2 +++ b/bootstrap/templates/kubernetes/talos/cilium/.gitignore.j2 @@ -1,3 +1,3 @@ -#% if bootstrap_distribution == 'talos' %# +#% if bootstrap_distribution in ['talos'] %# charts/ #% endif %# diff --git a/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 index d7bdddab..62789974 100644 --- a/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'talos' %# +#% if bootstrap_distribution in ['talos'] %# --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/talos/cilium/values.yaml.j2 b/bootstrap/templates/kubernetes/talos/cilium/values.yaml.j2 index 7d75b310..8d4e7ce8 100644 --- a/bootstrap/templates/kubernetes/talos/cilium/values.yaml.j2 +++ b/bootstrap/templates/kubernetes/talos/cilium/values.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'talos' %# +#% if bootstrap_distribution in ['talos'] %# --- #% filter indent(width=0, first=True) %# #% include 'partials/cilium-values-init.partial.yaml.j2' %# diff --git a/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/.gitignore.j2 b/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/.gitignore.j2 index 4f4111e0..f780a20c 100644 --- a/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/.gitignore.j2 +++ b/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/.gitignore.j2 @@ -1,3 +1,3 @@ -#% if bootstrap_distribution == 'talos' %# +#% if bootstrap_distribution in ['talos'] %# charts/ #% endif %# diff --git a/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/kustomization.yaml.j2 index 7df9b942..cd04e7e3 100644 --- a/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/talos/kubelet-csr-approver/kustomization.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'talos' %# +#% if bootstrap_distribution in ['talos'] %# --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 index 1e427106..296eaf3a 100644 --- a/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 +++ b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 @@ -1,29 +1,24 @@ -#% if bootstrap_distribution == 'talos' %# +#% if bootstrap_distribution in ['talos'] %# # yaml-language-server: $schema=https://raw.githubusercontent.com/budimanjojo/talhelper/master/pkg/config/schemas/talconfig.json --- -# renovate: datasource=github-releases depName=siderolabs/talos +# renovate: depName=ghcr.io/siderolabs/installer datasource=docker extractVersion=^(?.*)$ talosVersion: v1.6.1 # renovate: datasource=github-releases depName=kubernetes/kubernetes extractVersion=^v(?.*)$ kubernetesVersion: 1.29.1 clusterName: &cluster home-kubernetes - endpoint: https://#{ bootstrap_kube_api_addr }#:6443 - clusterPodNets: - "#{ bootstrap_cluster_cidr.split(',')[0] }#" clusterSvcNets: - "#{ bootstrap_service_cidr.split(',')[0] }#" - additionalApiServerCertSans: &sans - "#{ bootstrap_kube_api_addr }#" - 127.0.0.1 # KubePrism #% for item in bootstrap_kube_additional_tls_sans %# - "#{ item }#" #% endfor %# - additionalMachineCertSans: *sans - cniConfig: name: none diff --git a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 index fc68e03b..10b73b1c 100644 --- a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 @@ -8,9 +8,9 @@ cluster: id: 1 containerRuntime: integration: containerd - #% if bootstrap_distribution == "k3s" %# + #% if bootstrap_distribution in ['k3s'] %# socketPath: /var/run/k3s/containerd/containerd.sock - #% elif bootstrap_distribution == "k0s" %# + #% elif bootstrap_distribution in ['k0s'] %# socketPath: /var/run/k0s/containerd.sock #% endif %# endpointRoutes: @@ -59,25 +59,26 @@ hubble: ipam: mode: kubernetes ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}" -#% if bootstrap_ipv6_enabled %# +#% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}" ipv6: enabled: true #% endif %# -#% if bootstrap_distribution == "k3s" %# +#% if bootstrap_distribution in ['k3s'] %# k8sServiceHost: 127.0.0.1 k8sServicePort: 6444 -#% elif bootstrap_distribution == "k0s" %# +#% elif bootstrap_distribution in ['k0s'] %# k8sServiceHost: localhost k8sServicePort: 7443 -#% elif bootstrap_distribution == "talos" %# +#% elif bootstrap_distribution in ['talos'] %# k8sServiceHost: localhost k8sServicePort: 7445 #% endif %# kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 l2announcements: - #% if bootstrap_ipv6_enabled %# + #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# + # https://github.com/cilium/cilium/issues/28985 enabled: false #% else %# enabled: true diff --git a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 index 219f3c35..4d58ca82 100644 --- a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 @@ -8,9 +8,9 @@ cluster: id: 1 containerRuntime: integration: containerd - #% if bootstrap_distribution == "k3s" %# + #% if bootstrap_distribution in ['k3s'] %# socketPath: /var/run/k3s/containerd/containerd.sock - #% elif bootstrap_distribution == "k0s" %# + #% elif bootstrap_distribution in ['k0s'] %# socketPath: /var/run/k0s/containerd.sock #% endif %# endpointRoutes: @@ -20,25 +20,26 @@ hubble: ipam: mode: kubernetes ipv4NativeRoutingCIDR: "#{ bootstrap_cluster_cidr }#" -#% if bootstrap_ipv6_enabled %# +#% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# ipv6NativeRoutingCIDR: "#{ bootstrap_cluster_cidr_v6 }#" ipv6: enabled: true #% endif %# -#% if bootstrap_distribution == "k3s" %# +#% if bootstrap_distribution in ['k3s'] %# k8sServiceHost: 127.0.0.1 k8sServicePort: 6444 -#% elif bootstrap_distribution == "k0s" %# +#% elif bootstrap_distribution in ['k0s'] %# k8sServiceHost: localhost k8sServicePort: 7443 -#% elif bootstrap_distribution == "talos" %# +#% elif bootstrap_distribution in ['talos'] %# k8sServiceHost: localhost k8sServicePort: 7445 #% endif %# kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 l2announcements: - #% if bootstrap_ipv6_enabled %# + #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %# + # https://github.com/cilium/cilium/issues/28985 enabled: false #% else %# enabled: true diff --git a/bootstrap/templates/partials/kube-vip-ds.partial.yaml.j2 b/bootstrap/templates/partials/kube-vip-ds.partial.yaml.j2 new file mode 100644 index 00000000..d2f0d480 --- /dev/null +++ b/bootstrap/templates/partials/kube-vip-ds.partial.yaml.j2 @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-vip + namespace: kube-system + labels: + app.kubernetes.io/name: kube-vip +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip + template: + metadata: + labels: + app.kubernetes.io/name: kube-vip + spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.6.4 + imagePullPolicy: IfNotPresent + args: ["manager"] + env: + - name: address + value: "#{ bootstrap_kube_api_addr }#" + - name: vip_arp + value: "true" + - name: lb_enable + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: prometheus_server + value: :2112 + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"] + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + serviceAccountName: kube-vip + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists diff --git a/bootstrap/templates/partials/kube-vip-rbac.partial.yaml.j2 b/bootstrap/templates/partials/kube-vip-rbac.partial.yaml.j2 new file mode 100644 index 00000000..d6ecc936 --- /dev/null +++ b/bootstrap/templates/partials/kube-vip-rbac.partial.yaml.j2 @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index 45f63e33..f964af81 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -59,11 +59,11 @@ bootstrap_kube_additional_tls_sans: [] # 3. you are not force redirecting DNS requests to it - this will break cert generation over DNS01 bootstrap_dns_server: 1.1.1.1 -# (Advanced) Enable ipv6 -bootstrap_ipv6_enabled: false -# For ipv6 use format "10.42.0.0/16,fd78:c889:47fb:10::/60", /60 IPv6 block is enough for 16 nodes +# The pod CIDR for the cluster, this must not overlap with any existing networks +# If you want to use IPv6 check the advanced flags below bootstrap_cluster_cidr: 10.42.0.0/16 -# For ipv6 use format "10.43.0.0/16,fd78:c889:47fb:e0::/112" +# The service CIDR for the cluster, this must not overlap with any existing networks +# If you want to use IPv6 check the advanced flags below bootstrap_service_cidr: 10.43.0.0/16 # Node information @@ -99,3 +99,10 @@ bootstrap_nodes: # cilium_loadbalancer_mode: dsr # # Path for openebs openebs-hostpath to use for storage on ALL nodes # local_storage_path: /var/openebs/local +# # Enable Dual Stack IPv4 first +# # IMPORTANT: I am looking for people to help maintain IPv6 support since I cannot test it. +# # https://github.com/onedr0p/flux-cluster-template/issues/1148 +# # Keep in mind that Cilium does not currently support IPv6 L2 announcements. +# # Make sure you set bootstrap_cluster_cidr and bootstrap_service_cidr +# # to a valid dual stack CIDRs, e.g. "10.42.0.0/16,fd00:10:244::/64" +# dual_stack_ipv4_first: false