feat: update kube-vip ansible playbook for k0s and k3s - ipv6 config changes (#1234)

* feat: update kube-vip ansible playbook for k0s and k3s

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: update renovate

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: move ipv6 to advanced options

Signed-off-by: Devin Buhl <devin@buhl.casa>

* feat: kube-vip manifests to partials

Signed-off-by: Devin Buhl <devin@buhl.casa>

* feat: kube vip samesies

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: update config sample

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>

.github/renovate.json5

@@ -238,13 +238,12 @@
       "matchStrings": [
         // Example:
         //   k3s_release_version: "v1.27.3+k3s1"
-        "datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)( versioning=(?<versioning>\\S+))?( extractVersion=(?<extractVersion>\\S+))?\n.*?\"(?<currentValue>.*)\"\n",
+        "datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)( repository=(?<registryUrl>\\S+))?( extractVersion=(?<extractVersion>\\S+))?\n.*?\"(?<currentValue>.*)\"\n",
         // Example:
         //   - https://github.com/rancher/system-upgrade-controller/releases/download/v0.11.0/crd.yaml
-        "datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)( versioning=(?<versioning>\\S+))?\n.*?-\\s(.*?)\/(?<currentValue>[^/]+)\/[^/]+\n",
+        "datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)\n.*?-\\s(.*?)\/(?<currentValue>[^/]+)\/[^/]+\n",
       ],
-      "datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}",
+      "datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}"
-      "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}"
     }
   ]
 }

.github/tests/config-k0s.yaml

@@ -21,7 +21,6 @@ bootstrap_k8s_gateway_addr: 10.10.10.253
 bootstrap_external_ingress_addr: 10.10.10.252
 bootstrap_internal_ingress_addr: 10.10.10.251
 bootstrap_dns_server: 1.1.1.1
-bootstrap_ipv6_enabled: false
 bootstrap_cluster_cidr: 10.42.0.0/16
 bootstrap_service_cidr: 10.43.0.0/16
 bootstrap_nodes:

.github/tests/config-k3s-ipv4.yaml

@@ -21,7 +21,6 @@ bootstrap_k8s_gateway_addr: 10.10.10.253
 bootstrap_external_ingress_addr: 10.10.10.252
 bootstrap_internal_ingress_addr: 10.10.10.251
 bootstrap_dns_server: 1.1.1.1
-bootstrap_ipv6_enabled: false
 bootstrap_cluster_cidr: 10.42.0.0/16
 bootstrap_service_cidr: 10.43.0.0/16
 bootstrap_nodes:

.github/tests/config-k3s-ipv6.yaml

@@ -21,7 +21,8 @@ bootstrap_k8s_gateway_addr: 10.10.10.253
 bootstrap_external_ingress_addr: 10.10.10.252
 bootstrap_internal_ingress_addr: 10.10.10.251
 bootstrap_dns_server: 1.1.1.1
-bootstrap_ipv6_enabled: true
+bootstrap_advanced_flags:
+  dual_stack_ipv4_first: true
 bootstrap_cluster_cidr: 10.42.0.0/16,fd7f:8f5:e87c:a::/64
 bootstrap_service_cidr: 10.43.0.0/16,fd7f:8f5:e87c:e::/112
 bootstrap_nodes:

.github/tests/config-talos.yaml

@@ -21,7 +21,6 @@ bootstrap_k8s_gateway_addr: 10.10.10.253
 bootstrap_external_ingress_addr: 10.10.10.252
 bootstrap_internal_ingress_addr: 10.10.10.251
 bootstrap_dns_server: 1.1.1.1
-bootstrap_ipv6_enabled: false
 bootstrap_cluster_cidr: 10.42.0.0/16
 bootstrap_service_cidr: 10.43.0.0/16
 bootstrap_nodes:

bootstrap/scripts/validation.py

@@ -83,14 +83,14 @@ def validate_python_version() -> None:
 
 @required("bootstrap_distribution")
 def validate_cli_tools(distribution: str, **_) -> None:
-    distro = _validate_distribution(distribution)
+    _distribution = _validate_distribution(distribution)
     for tool in GLOBAL_CLI_TOOLS:
         if not which(tool):
             raise ValueError(f"Missing required CLI tool {tool}")
-    for tool in TALOS_CLI_TOOLS if distro == "talos" else []:
+    for tool in TALOS_CLI_TOOLS if _distribution in ["talos"] else []:
         if not which(tool):
             raise ValueError(f"Missing required CLI tool {tool}")
-    for tool in K0S_CLI_TOOLS if distro == "k0s" else []:
+    for tool in K0S_CLI_TOOLS if _distribution in ["k0s"] else []:
         if not which(tool):
             raise ValueError(f"Missing required CLI tool {tool}")
 
@@ -125,15 +125,14 @@ def validate_timezone(timezone: str, **_) -> None:
         raise ValueError(f"Invalid timezone {timezone}")
 
 
-@required("bootstrap_ipv6_enabled", "bootstrap_cluster_cidr", "bootstrap_service_cidr")
+@required("bootstrap_cluster_cidr", "bootstrap_service_cidr", "bootstrap_advanced_flags")
-def validate_cluster_cidrs(ipv6_enabled: bool, cluster_cidr: str, service_cidr: str, **_) -> None:
+def validate_cluster_cidrs(cluster_cidr: str, service_cidr: str, advanced_flags: dict, **_) -> None:
-    if not isinstance(ipv6_enabled, bool):
+    dual_stack_ipv4_first = advanced_flags.get("dual_stack_ipv4_first", False)
-        raise ValueError(f"Invalid IPv6 enabled {ipv6_enabled}")
 
     if cluster_cidr == service_cidr:
         raise ValueError(f"Cluster CIDR {cluster_cidr} is the same as service CIDR {service_cidr}")
 
-    if ipv6_enabled:
+    if dual_stack_ipv4_first:
         if len(cluster_cidr.split(",")) != 2:
             raise ValueError(f"Invalid cluster CIDR {cluster_cidr}")
         if len(service_cidr.split(",")) != 2:

bootstrap/templates/.sops.yaml.j2

@@ -1,6 +1,6 @@
 ---
 creation_rules:
-  #% if bootstrap_distribution == "talos" %#
+  #% if bootstrap_distribution in ['talos'] %#
   - # IMPORTANT: This rule MUST be above the others
     path_regex: talos/.*\.sops\.ya?ml
     key_groups:

bootstrap/templates/ansible/.ansible-lint.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution in ['k3s', 'k0s'] %#
+#% if bootstrap_distribution in ['k0s', 'k3s'] %#
 skip_list:
   - yaml[commas]
   - yaml[line-length]

bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 # https://github.com/PyratLabs/ansible-role-k3s
 # renovate: datasource=github-releases depName=k3s-io/k3s
@@ -10,10 +10,7 @@ k3s_use_unsupported_config: true
 k3s_registration_address: "#{ bootstrap_kube_api_addr }#"
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
-  - custom-cilium-helmchart.yaml.j2
+  - custom-cilium-helmchart.yaml
-  - kube-vip-ds.yaml.j2
+  - kube-vip-ds.yaml
-# /var/lib/rancher/k3s/server/manifests
+  - kube-vip-rbac.yaml
-k3s_server_manifests_urls:
-  - url: https://kube-vip.io/manifests/rbac.yaml
-    filename: kube-vip-rbac.yaml
 #% endif %#

bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2

@@ -1,10 +1,10 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 # https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/
 # https://github.com/PyratLabs/ansible-role-k3s
 k3s_control_node: true
 k3s_server:
-  #% if bootstrap_ipv6_enabled %#
+  #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
   node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}"
   #% else %#
   node-ip: "{{ ansible_host }}"
@@ -23,7 +23,7 @@ k3s_server:
   write-kubeconfig-mode: "644"
   pause-image: registry.k8s.io/pause:3.9
   secrets-encryption: true
-  #% if bootstrap_ipv6_enabled %#
+  #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
   cluster-cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#,#{ bootstrap_cluster_cidr.split(',')[1] }#"
   service-cidr: "#{ bootstrap_service_cidr.split(',')[0] }#,#{ bootstrap_service_cidr.split(',')[1] }#"
   #% else %#

bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2

@@ -1,10 +1,10 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 # https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/
 # https://github.com/PyratLabs/ansible-role-k3s
 k3s_control_node: false
 k3s_agent:
-  #% if bootstrap_ipv6_enabled %#
+  #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
   node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}"
   #% else %#
   node-ip: "{{ ansible_host }}"

bootstrap/templates/ansible/inventory/hosts.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution in ['k3s', 'k0s'] %#
+#% if bootstrap_distribution in ['k0s', 'k3s'] %#
 ---
 kubernetes:
   children:

bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 - name: Cluster Installation
   hosts: kubernetes

bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k0s', 'k3s'] %#
 ---
 - name: Cluster kube-vip
   hosts: master
@@ -11,16 +11,21 @@
       ansible.builtin.pause:
         seconds: 5
   tasks:
+    #% if bootstrap_distribution in ['k3s'] %#
     - name: Ensure Kubernetes is running
       ansible.builtin.include_role:
         name: xanmanning.k3s
         public: true
       vars:
         k3s_state: started
-
+    #% endif %#
     - name: Upgrade kube-vip
       ansible.builtin.template:
-        src: templates/kube-vip-static-pod.yaml.j2
+        src: templates/kube-vip-ds.yaml.j2
-        dest: "{{ k3s_server_pod_manifests_dir }}/kube-vip-static-pod.yaml"
+        #% if bootstrap_distribution in ['k3s'] %#
+        dest: "{{ k3s_server_manifests_dir }}/kube-vip-ds.yaml"
+        #% elif bootstrap_distribution in ['k0s'] %#
+        dest: "/var/lib/k0s/manifests/kube-vip/ds.yaml"
+        #% endif %#
         mode: preserve
 #% endif %#

bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 - name: Cluster Nuke
   hosts: kubernetes

bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 - name: Prepare System
   hosts: kubernetes

bootstrap/templates/ansible/playbooks/cluster-reboot.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution in ['k3s', 'k0s'] %#
+#% if bootstrap_distribution in ['k0s', 'k3s'] %#
 ---
 - name: Reboot
   hosts: kubernetes

bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution in ['k3s', 'k0s'] %#
+#% if bootstrap_distribution in ['k0s', 'k3s'] %#
 ---
 # https://github.com/kevincoakley/ansible-role-k8s-rolling-update
 - name: Cluster rollout update
@@ -13,9 +13,9 @@
         seconds: 5
   tasks:
     - name: Details
-      #% if bootstrap_distribution == 'k3s' %#
+      #% if bootstrap_distribution in ['k3s'] %#
       ansible.builtin.command: "k3s kubectl get node {{ inventory_hostname }} -o json"
-      #% elif bootstrap_distribution == 'k0s' %#
+      #% elif bootstrap_distribution in ['k0s'] %#
       ansible.builtin.command: "k0s kubectl get node {{ inventory_hostname }} -o json"
       #% endif %#
       register: kubectl_get_node
@@ -33,9 +33,9 @@
         - name: Cordon
           kubernetes.core.k8s_drain:
             name: "{{ inventory_hostname }}"
-            #% if bootstrap_distribution == 'k3s' %#
+            #% if bootstrap_distribution in ['k3s'] %#
             kubeconfig: /etc/rancher/k3s/k3s.yaml
-            #% elif bootstrap_distribution == 'k0s' %#
+            #% elif bootstrap_distribution in ['k0s'] %#
             kubeconfig: /var/lib/k0s/pki/admin.conf
             #% endif %#
             state: cordon
@@ -44,9 +44,9 @@
         - name: Drain
           kubernetes.core.k8s_drain:
             name: "{{ inventory_hostname }}"
-            #% if bootstrap_distribution == 'k3s' %#
+            #% if bootstrap_distribution in ['k3s'] %#
             kubeconfig: /etc/rancher/k3s/k3s.yaml
-            #% elif bootstrap_distribution == 'k0s' %#
+            #% elif bootstrap_distribution in ['k0s'] %#
             kubeconfig: /var/lib/k0s/pki/admin.conf
             #% endif %#
             state: drain
@@ -80,9 +80,9 @@
         - name: Uncordon
           kubernetes.core.k8s_drain:
             name: "{{ inventory_hostname }}"
-            #% if bootstrap_distribution == 'k3s' %#
+            #% if bootstrap_distribution  in ['k3s'] %#
             kubeconfig: /etc/rancher/k3s/k3s.yaml
-            #% elif bootstrap_distribution == 'k0s' %#
+            #% elif bootstrap_distribution in ['k0s'] %#
             kubeconfig: /var/lib/k0s/pki/admin.conf
             #% endif %#
             state: uncordon

bootstrap/templates/ansible/playbooks/tasks/cilium.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 - name: Cilium
   block:

bootstrap/templates/ansible/playbooks/tasks/cruft.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 # https://github.com/k3s-io/k3s/issues/1971
 - name: Cruft

bootstrap/templates/ansible/playbooks/tasks/kubeconfig.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 - name: Get absolute path to this Git repository # noqa: command-instead-of-module
   ansible.builtin.command: git rev-parse --show-toplevel

bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
 # https://docs.k3s.io/helm
 apiVersion: helm.cattle.io/v1
@@ -9,7 +9,7 @@ metadata:
 spec:
   repo: https://helm.cilium.io/
   chart: cilium
-  # renovate: datasource=github-releases depName=cilium/cilium
+  # renovate: datasource=helm depName=cilium repository=https://helm.cilium.io
   version: "1.14.5"
   targetNamespace: kube-system
   bootstrap: true

bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2

@@ -1,77 +1,4 @@
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 ---
-apiVersion: apps/v1
+#% include 'partials/kube-vip-ds.partial.yaml.j2' %#
-kind: DaemonSet
-metadata:
-  name: kube-vip
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/name: kube-vip
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: kube-vip
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: kube-vip
-    spec:
-      containers:
-        - name: kube-vip
-          image: ghcr.io/kube-vip/kube-vip:v0.6.4
-          imagePullPolicy: IfNotPresent
-          args: ["manager"]
-          env:
-            - name: address
-              value: "#{ bootstrap_kube_api_addr }#"
-            - name: vip_arp
-              value: "true"
-            - name: lb_enable
-              value: "true"
-            - name: port
-              value: "6443"
-            - name: vip_cidr
-              value: "32"
-            - name: cp_enable
-              value: "true"
-            - name: cp_namespace
-              value: kube-system
-            - name: vip_ddns
-              value: "false"
-            - name: svc_enable
-              value: "false"
-            - name: vip_leaderelection
-              value: "true"
-            - name: vip_leaseduration
-              value: "15"
-            - name: vip_renewdeadline
-              value: "10"
-            - name: vip_retryperiod
-              value: "2"
-            - name: prometheus_server
-              value: :2112
-          securityContext:
-            capabilities:
-              add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"]
-      hostAliases:
-        - hostnames:
-            - kubernetes
-          ip: 127.0.0.1
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/master
-                    operator: Exists
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
 #% endif %#

bootstrap/templates/ansible/playbooks/templates/kube-vip-rbac.yaml.j2

@@ -0,0 +1,4 @@
+#% if bootstrap_distribution in ['k3s'] %#
+---
+#% include 'partials/kube-vip-rbac.partial.yaml.j2' %#
+#% endif %#

bootstrap/templates/ansible/requirements.txt.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution in ['k3s', 'k0s'] %#
+#% if bootstrap_distribution in ['k0s', 'k3s'] %#
 # Ansible
 ansible-lint==6.22.2
 ansible==9.1.0

bootstrap/templates/ansible/requirements.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution in ['k3s', 'k0s'] %#
+#% if bootstrap_distribution in ['k0s', 'k3s'] %#
 ---
 collections:
   - name: ansible.posix
@@ -9,7 +9,7 @@ collections:
     version: 8.2.0
   - name: kubernetes.core
     version: 3.0.0
-#% if bootstrap_distribution == 'k3s' %#
+#% if bootstrap_distribution in ['k3s'] %#
 roles:
   - name: xanmanning.k3s
     src: https://github.com/PyratLabs/ansible-role-k3s

bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2

@@ -8,7 +8,7 @@ spec:
   chart:
     spec:
       chart: cilium
-      version: 1.14.6
+      version: 1.14.5
       sourceRef:
         kind: HelmRepository
         name: cilium

bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2

@@ -2,7 +2,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  #% if not bootstrap_ipv6_enabled %#
+  #% if not bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
   - ./cilium-l2.yaml
   #% endif %#
   - ./helmrelease.yaml

bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/helmrelease.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "talos" %#
+#% if bootstrap_distribution in ['talos'] %#
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2beta2
 kind: HelmRelease

bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/app/kustomization.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "talos" %#
+#% if bootstrap_distribution in ['talos'] %#
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

bootstrap/templates/kubernetes/apps/kube-system/kubelet-csr-approver/ks.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "talos" %#
+#% if bootstrap_distribution in ['talos'] %#
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization

bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2

@@ -4,7 +4,7 @@ kind: Kustomization
 resources:
   - ./namespace.yaml
   - ./cilium/ks.yaml
-  #% if bootstrap_distribution == "talos" %#
+  #% if bootstrap_distribution in ['talos'] %#
   - ./kubelet-csr-approver/ks.yaml
   #% endif %#
   - ./metrics-server/ks.yaml

bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2

@@ -25,7 +25,7 @@ spec:
     keepHistory: false
   values:
     spegel:
-      #% if bootstrap_distribution == "k0s" %#
+      #% if bootstrap_distribution in ['k0s'] %#
       containerdSock: /run/k0s/containerd.sock
       containerdRegistryConfigPath: /var/lib/k0s/containerd/certs.d
       #% endif %#

bootstrap/templates/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml.j2

@@ -53,7 +53,7 @@ spec:
       enabled: true
       serviceMonitor:
         metricRelabelings:
-          #% if bootstrap_distribution == 'k3s' %#
+          #% if bootstrap_distribution in ['k3s'] %#
           # Remove duplicate labels provided by k3s
           - action: keep
             sourceLabels: ["__name__"]
@@ -74,7 +74,7 @@ spec:
       enabled: true
       serviceMonitor:
         metricRelabelings:
-          #% if bootstrap_distribution == 'k3s' %#
+          #% if bootstrap_distribution in ['k3s'] %#
           # Remove duplicate labels provided by k3s
           - action: keep
             sourceLabels: ["__name__"]
@@ -93,7 +93,7 @@ spec:
       #% for item in bootstrap_nodes.master %#
         - "#{ item.address }#"
       #% endfor %#
-      #% if bootstrap_distribution == 'k3s' %#
+      #% if bootstrap_distribution in ['k3s'] %#
       serviceMonitor:
         metricRelabelings:
           # Remove duplicate labels provided by k3s
@@ -107,7 +107,7 @@ spec:
     kubeScheduler:
       enabled: true
       endpoints: *endpoints
-      #% if bootstrap_distribution == 'k3s' %#
+      #% if bootstrap_distribution in ['k3s'] %#
       serviceMonitor:
         metricRelabelings:
           # Remove duplicate labels provided by k3s

bootstrap/templates/kubernetes/apps/storage/csi-driver-nfs/app/helmrelease.yaml.j2

@@ -26,7 +26,7 @@ spec:
   values:
     externalSnapshotter:
       enabled: false
-    #% if bootstrap_distribution == "k0s" %#
+    #% if bootstrap_distribution in ['k0s'] %#
     kubeletDir: /var/lib/k0s/kubelet
     #% endif %#
 #% endif %#

bootstrap/templates/kubernetes/apps/storage/csi-driver-smb/app/helmrelease.yaml.j2

@@ -25,7 +25,7 @@ spec:
     keepHistory: false
   values:
     linux:
-      #% if bootstrap_distribution == "k0s" %#
+      #% if bootstrap_distribution in ['k0s'] %#
       kubelet: /var/lib/k0s/kubelet
       #% else %#
       kubelet: /var/lib/kubelet

bootstrap/templates/kubernetes/apps/tools/kustomization.yaml.j2

@@ -5,6 +5,6 @@ resources:
   - ./namespace.yaml
   - ./descheduler/ks.yaml
   - ./reloader/ks.yaml
-  #% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %#
+  #% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %#
   - ./system-upgrade-controller/ks.yaml
   #% endif %#

bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/helmrelease.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %#
+#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %#
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2beta2
 kind: HelmRelease

bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/kustomization.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %#
+#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %#
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/app/rbac.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %#
+#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %#
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/ks.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %#
+#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %#
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization

bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/agent.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %#
+#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %#
 ---
 apiVersion: upgrade.cattle.io/v1
 kind: Plan

bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/kustomization.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %#
+#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %#
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

bootstrap/templates/kubernetes/apps/tools/system-upgrade-controller/plans/server.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == "k3s" and addon_system_upgrade_controller.enabled %#
+#% if bootstrap_distribution in ['k3s'] and addon_system_upgrade_controller.enabled %#
 ---
 apiVersion: upgrade.cattle.io/v1
 kind: Plan

bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2

@@ -11,7 +11,7 @@ data:
   CLUSTER_CIDR: "#{ bootstrap_cluster_cidr.split(',')[0] }#"
   SERVICE_CIDR: "#{ bootstrap_service_cidr.split(',')[0] }#"
   NODE_CIDR: "#{ bootstrap_node_cidr }#"
-  #% if bootstrap_ipv6_enabled %#
+  #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
   CLUSTER_CIDR_V6: "#{ bootstrap_cluster_cidr.split(',')[1] }#"
   SERVICE_CIDR_V6: "#{ bootstrap_service_cidr.split(',')[1] }#"
   #% endif %#

bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k0s' %#
+#% if bootstrap_distribution in ['k0s'] %#
 ---
 apiVersion: k0sctl.k0sproject.io/v1beta1
 kind: Cluster
@@ -91,7 +91,7 @@ spec:
             charts:
               - name: cilium
                 chartname: cilium/cilium
-                # renovate: datasource=github-releases depName=cilium/cilium
+                # renovate: datasource=helm depName=cilium repository=https://helm.cilium.io
                 version: "1.14.5"
                 namespace: kube-system
                 values: |2

bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2

@@ -1,131 +1,20 @@
+#% if bootstrap_distribution in ['k0s'] %#
 #!/usr/bin/env bash
 set -e
 set -o noglob
 
 [ $(id -u) -eq 0 ] || exec sudo $0 $@
 
-# Create directory
+# Create kube-vip directory
 mkdir -p /var/lib/k0s/manifests/kube-vip
 
 # Create kube-vip rbac
-cat <<EOF > /var/lib/k0s/manifests/kube-vip/rbac.yaml
+cat <<EOF > /var/lib/k0s/manifests/kube-vip/kube-vip-rbac.yaml
-apiVersion: v1
+#% include 'partials/kube-vip-rbac.partial.yaml.j2' %#
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services/status"]
-    verbs: ["update"]
-  - apiGroups: [""]
-    resources: ["services", "endpoints"]
-    verbs: ["list","get","watch", "update"]
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["list","get","watch", "update", "patch"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["list", "get", "watch", "update", "create"]
-  - apiGroups: ["discovery.k8s.io"]
-    resources: ["endpointslices"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
 EOF
 
-cat <<EOF > /var/lib/k0s/manifests/kube-vip/ds.yaml
+# Create kube-vip daemonset
----
+cat <<EOF > /var/lib/k0s/manifests/kube-vip/kube-vip-ds.yaml
-apiVersion: apps/v1
+#% include 'partials/kube-vip-ds.partial.yaml.j2' %#
-kind: DaemonSet
-metadata:
-  name: kube-vip
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/name: kube-vip
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: kube-vip
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: kube-vip
-    spec:
-      containers:
-        - name: kube-vip
-          image: ghcr.io/kube-vip/kube-vip:v0.6.4
-          imagePullPolicy: IfNotPresent
-          args: ["manager"]
-          env:
-            - name: address
-              value: "#{ bootstrap_kube_api_addr }#"
-            - name: vip_arp
-              value: "true"
-            - name: lb_enable
-              value: "true"
-            - name: port
-              value: "6443"
-            - name: vip_cidr
-              value: "32"
-            - name: cp_enable
-              value: "true"
-            - name: cp_namespace
-              value: kube-system
-            - name: vip_ddns
-              value: "false"
-            - name: svc_enable
-              value: "false"
-            - name: vip_leaderelection
-              value: "true"
-            - name: vip_leaseduration
-              value: "15"
-            - name: vip_renewdeadline
-              value: "10"
-            - name: vip_retryperiod
-              value: "2"
-            - name: prometheus_server
-              value: :2112
-          securityContext:
-            capabilities:
-              add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"]
-      hostAliases:
-        - hostnames:
-            - kubernetes
-          ip: 127.0.0.1
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/master
-                    operator: Exists
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
 EOF
+#% endif %#

bootstrap/templates/kubernetes/k0s/resources/apply-system.sh.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k0s' %#
+#% if bootstrap_distribution in ['k0s'] %#
 #!/usr/bin/env bash
 set -e
 set -o noglob

bootstrap/templates/kubernetes/k0s/resources/reset-cilium.sh.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k0s' %#
+#% if bootstrap_distribution in ['k0s'] %#
 #!/usr/bin/env bash
 set -e
 set -o noglob

bootstrap/templates/kubernetes/k0s/resources/reset-system.sh.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k0s' %#
+#% if bootstrap_distribution in ['k0s'] %#
 #!/usr/bin/env bash
 set -e
 set -o noglob

bootstrap/templates/kubernetes/talos/cilium/.gitignore.j2

@@ -1,3 +1,3 @@
-#% if bootstrap_distribution == 'talos' %#
+#% if bootstrap_distribution in ['talos'] %#
 charts/
 #% endif %#

bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'talos' %#
+#% if bootstrap_distribution in ['talos'] %#
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

bootstrap/templates/kubernetes/talos/cilium/values.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'talos' %#
+#% if bootstrap_distribution in ['talos'] %#
 ---
 #% filter indent(width=0, first=True) %#
 #% include 'partials/cilium-values-init.partial.yaml.j2' %#

bootstrap/templates/kubernetes/talos/kubelet-csr-approver/.gitignore.j2

@@ -1,3 +1,3 @@
-#% if bootstrap_distribution == 'talos' %#
+#% if bootstrap_distribution in ['talos'] %#
 charts/
 #% endif %#

bootstrap/templates/kubernetes/talos/kubelet-csr-approver/kustomization.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'talos' %#
+#% if bootstrap_distribution in ['talos'] %#
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

bootstrap/templates/kubernetes/talos/talconfig.yaml.j2

@@ -1,29 +1,24 @@
-#% if bootstrap_distribution == 'talos' %#
+#% if bootstrap_distribution in ['talos'] %#
 # yaml-language-server: $schema=https://raw.githubusercontent.com/budimanjojo/talhelper/master/pkg/config/schemas/talconfig.json
 ---
-# renovate: datasource=github-releases depName=siderolabs/talos
+# renovate: depName=ghcr.io/siderolabs/installer datasource=docker extractVersion=^(?<version>.*)$
 talosVersion: v1.6.1
 # renovate: datasource=github-releases depName=kubernetes/kubernetes extractVersion=^v(?<version>.*)$
 kubernetesVersion: 1.29.1
 
 clusterName: &cluster home-kubernetes
-
 endpoint: https://#{ bootstrap_kube_api_addr }#:6443
-
 clusterPodNets:
   - "#{ bootstrap_cluster_cidr.split(',')[0] }#"
 clusterSvcNets:
   - "#{ bootstrap_service_cidr.split(',')[0] }#"
-
 additionalApiServerCertSans: &sans
   - "#{ bootstrap_kube_api_addr }#"
   - 127.0.0.1 # KubePrism
   #% for item in bootstrap_kube_additional_tls_sans %#
   - "#{ item }#"
   #% endfor %#
-
 additionalMachineCertSans: *sans
-
 cniConfig:
   name: none
 

bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

@@ -8,9 +8,9 @@ cluster:
   id: 1
 containerRuntime:
   integration: containerd
-  #% if bootstrap_distribution == "k3s" %#
+  #% if bootstrap_distribution in ['k3s'] %#
   socketPath: /var/run/k3s/containerd/containerd.sock
-  #% elif bootstrap_distribution == "k0s" %#
+  #% elif bootstrap_distribution in ['k0s'] %#
   socketPath: /var/run/k0s/containerd.sock
   #% endif %#
 endpointRoutes:
@@ -59,25 +59,26 @@ hubble:
 ipam:
   mode: kubernetes
 ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}"
-#% if bootstrap_ipv6_enabled %#
+#% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
 ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}"
 ipv6:
   enabled: true
 #% endif %#
-#% if bootstrap_distribution == "k3s" %#
+#% if bootstrap_distribution in ['k3s'] %#
 k8sServiceHost: 127.0.0.1
 k8sServicePort: 6444
-#% elif bootstrap_distribution == "k0s"  %#
+#% elif bootstrap_distribution in ['k0s'] %#
 k8sServiceHost: localhost
 k8sServicePort: 7443
-#% elif bootstrap_distribution == "talos"  %#
+#% elif bootstrap_distribution in ['talos'] %#
 k8sServiceHost: localhost
 k8sServicePort: 7445
 #% endif %#
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
 l2announcements:
-  #% if bootstrap_ipv6_enabled %#
+  #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
+  # https://github.com/cilium/cilium/issues/28985
   enabled: false
   #% else %#
   enabled: true

bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

@@ -8,9 +8,9 @@ cluster:
   id: 1
 containerRuntime:
   integration: containerd
-  #% if bootstrap_distribution == "k3s" %#
+  #% if bootstrap_distribution in ['k3s'] %#
   socketPath: /var/run/k3s/containerd/containerd.sock
-  #% elif bootstrap_distribution == "k0s" %#
+  #% elif bootstrap_distribution in ['k0s'] %#
   socketPath: /var/run/k0s/containerd.sock
   #% endif %#
 endpointRoutes:
@@ -20,25 +20,26 @@ hubble:
 ipam:
   mode: kubernetes
 ipv4NativeRoutingCIDR: "#{ bootstrap_cluster_cidr }#"
-#% if bootstrap_ipv6_enabled %#
+#% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
 ipv6NativeRoutingCIDR: "#{ bootstrap_cluster_cidr_v6 }#"
 ipv6:
   enabled: true
 #% endif %#
-#% if bootstrap_distribution == "k3s" %#
+#% if bootstrap_distribution in ['k3s'] %#
 k8sServiceHost: 127.0.0.1
 k8sServicePort: 6444
-#% elif bootstrap_distribution == "k0s"  %#
+#% elif bootstrap_distribution in ['k0s'] %#
 k8sServiceHost: localhost
 k8sServicePort: 7443
-#% elif bootstrap_distribution == "talos"  %#
+#% elif bootstrap_distribution in ['talos'] %#
 k8sServiceHost: localhost
 k8sServicePort: 7445
 #% endif %#
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
 l2announcements:
-  #% if bootstrap_ipv6_enabled %#
+  #% if bootstrap_advanced_flags.dual_stack_ipv4_first|default(false) %#
+  # https://github.com/cilium/cilium/issues/28985
   enabled: false
   #% else %#
   enabled: true

bootstrap/templates/partials/kube-vip-ds.partial.yaml.j2

@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-vip
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kube-vip
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-vip
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kube-vip
+    spec:
+      containers:
+        - name: kube-vip
+          image: ghcr.io/kube-vip/kube-vip:v0.6.4
+          imagePullPolicy: IfNotPresent
+          args: ["manager"]
+          env:
+            - name: address
+              value: "#{ bootstrap_kube_api_addr }#"
+            - name: vip_arp
+              value: "true"
+            - name: lb_enable
+              value: "true"
+            - name: port
+              value: "6443"
+            - name: vip_cidr
+              value: "32"
+            - name: cp_enable
+              value: "true"
+            - name: cp_namespace
+              value: kube-system
+            - name: vip_ddns
+              value: "false"
+            - name: svc_enable
+              value: "false"
+            - name: vip_leaderelection
+              value: "true"
+            - name: vip_leaseduration
+              value: "15"
+            - name: vip_renewdeadline
+              value: "10"
+            - name: vip_retryperiod
+              value: "2"
+            - name: prometheus_server
+              value: :2112
+          securityContext:
+            capabilities:
+              add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"]
+      hostAliases:
+        - hostnames:
+            - kubernetes
+          ip: 127.0.0.1
+      hostNetwork: true
+      serviceAccountName: kube-vip
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/master
+                    operator: Exists
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists

bootstrap/templates/partials/kube-vip-rbac.partial.yaml.j2

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-vip
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: system:kube-vip-role
+rules:
+  - apiGroups: [""]
+    resources: ["services/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["services", "endpoints"]
+    verbs: ["list","get","watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list","get","watch", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["list", "get", "watch", "update", "create"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["list","get","watch", "update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:kube-vip-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-vip-role
+subjects:
+- kind: ServiceAccount
+  name: kube-vip
+  namespace: kube-system

bootstrap/vars/config.sample.yaml

@@ -59,11 +59,11 @@ bootstrap_kube_additional_tls_sans: []
 #   3. you are not force redirecting DNS requests to it - this will break cert generation over DNS01
 bootstrap_dns_server: 1.1.1.1
 
-# (Advanced) Enable ipv6
+# The pod CIDR for the cluster, this must not overlap with any existing networks
-bootstrap_ipv6_enabled: false
+# If you want to use IPv6 check the advanced flags below
-# For ipv6 use format "10.42.0.0/16,fd78:c889:47fb:10::/60", /60 IPv6 block is enough for 16 nodes
 bootstrap_cluster_cidr: 10.42.0.0/16
-# For ipv6 use format "10.43.0.0/16,fd78:c889:47fb:e0::/112"
+# The service CIDR for the cluster, this must not overlap with any existing networks
+# If you want to use IPv6 check the advanced flags below
 bootstrap_service_cidr: 10.43.0.0/16
 
 # Node information
@@ -99,3 +99,10 @@ bootstrap_nodes:
 #   cilium_loadbalancer_mode: dsr
 #   # Path for openebs openebs-hostpath to use for storage on ALL nodes
 #   local_storage_path: /var/openebs/local
+#   # Enable Dual Stack IPv4 first
+#   # IMPORTANT: I am looking for people to help maintain IPv6 support since I cannot test it.
+#   # https://github.com/onedr0p/flux-cluster-template/issues/1148
+#   # Keep in mind that Cilium does not currently support IPv6 L2 announcements.
+#   # Make sure you set bootstrap_cluster_cidr and bootstrap_service_cidr
+#   # to a valid dual stack CIDRs, e.g. "10.42.0.0/16,fd00:10:244::/64"
+#   dual_stack_ipv4_first: false