feat: more changes to templating (#1218)

* feat: clean up more jinja default references

Signed-off-by: Devin Buhl <devin@buhl.casa>

* feat: add additional tls sans force kube-vip on k3s

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>

.github/tests/config-k0s.yaml

@@ -17,7 +17,7 @@ bootstrap_cloudflare_tunnel_secret: fake
 bootstrap_cloudflare_tunnel_id: fake
 bootstrap_node_cidr: 10.10.10.0/24
 bootstrap_kube_api_addr: 10.10.10.254
-bootstrap_kube_api_hostname: fake
+bootstrap_kube_additional_tls_sans: ["fake"]
 bootstrap_k8s_gateway_addr: 10.10.10.253
 bootstrap_external_ingress_addr: 10.10.10.252
 bootstrap_internal_ingress_addr: 10.10.10.251

.github/tests/config-k3s-ipv4.yaml

@@ -17,7 +17,7 @@ bootstrap_cloudflare_tunnel_secret: fake
 bootstrap_cloudflare_tunnel_id: fake
 bootstrap_node_cidr: 10.10.10.0/24
 bootstrap_kube_api_addr: 10.10.10.254
-bootstrap_kube_api_hostname: fake
+bootstrap_kube_additional_tls_sans: ["fake"]
 bootstrap_k8s_gateway_addr: 10.10.10.253
 bootstrap_external_ingress_addr: 10.10.10.252
 bootstrap_internal_ingress_addr: 10.10.10.251

.github/tests/config-k3s-ipv6.yaml

@@ -17,7 +17,7 @@ bootstrap_cloudflare_tunnel_secret: fake
 bootstrap_cloudflare_tunnel_id: fake
 bootstrap_node_cidr: 10.10.10.0/24
 bootstrap_kube_api_addr: 10.10.10.254
-bootstrap_kube_api_hostname: fake
+bootstrap_kube_additional_tls_sans: ["fake"]
 bootstrap_k8s_gateway_addr: 10.10.10.253
 bootstrap_external_ingress_addr: 10.10.10.252
 bootstrap_internal_ingress_addr: 10.10.10.251

.github/tests/config-k3s-no-kube-vip.yaml

@@ -1,32 +0,0 @@
----
-skip_tests: true
-
-bootstrap_distribution: k3s
-bootstrap_github_username: onedr0p
-bootstrap_github_repository_name: flux-cluster-template
-bootstrap_github_repository_branch: main
-bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
-bootstrap_timezone: Etc/UTC
-bootstrap_acme_email: fake
-bootstrap_acme_production_enabled: false
-bootstrap_flux_github_webhook_token: fake
-bootstrap_cloudflare_domain: fake
-bootstrap_cloudflare_token: fake
-bootstrap_cloudflare_account_tag: fake
-bootstrap_cloudflare_tunnel_secret: fake
-bootstrap_cloudflare_tunnel_id: fake
-bootstrap_node_cidr: 10.10.10.0/24
-bootstrap_kube_api_addr: # left blank to not use kube-vip
-bootstrap_kube_api_hostname: fake
-bootstrap_k8s_gateway_addr: 10.10.10.253
-bootstrap_external_ingress_addr: 10.10.10.252
-bootstrap_internal_ingress_addr: 10.10.10.251
-bootstrap_dns_server: 1.1.1.1
-bootstrap_ipv6_enabled: false
-bootstrap_cluster_cidr: 10.42.0.0/16
-bootstrap_service_cidr: 10.43.0.0/16
-bootstrap_nodes:
-  master:
-    - name: k8s-0
-      address: 10.10.10.100
-      username: fake

.github/tests/config-talos.yaml

@@ -17,7 +17,7 @@ bootstrap_cloudflare_tunnel_secret: fake
 bootstrap_cloudflare_tunnel_id: fake
 bootstrap_node_cidr: 10.10.10.0/24
 bootstrap_kube_api_addr: 10.10.10.254
-bootstrap_kube_api_hostname: fake
+bootstrap_kube_additional_tls_sans: ["fake"]
 bootstrap_k8s_gateway_addr: 10.10.10.253
 bootstrap_external_ingress_addr: 10.10.10.252
 bootstrap_internal_ingress_addr: 10.10.10.251

.github/workflows/e2e.yaml

@@ -40,7 +40,6 @@ jobs:
           - k0s
           - k3s-ipv4
           - k3s-ipv6
-          - k3s-no-kube-vip
           - talos
         addon-files: ["addons"]
     steps:

bootstrap/scripts/loader.py

@@ -20,19 +20,8 @@ class Loader:
     def __init__(self, data):
         if data.get("skip_tests", False):
             return
-        validation.validate_python_version()
+
-        validation.validate_cli_tools(data)
+        validation.validate(data)
-        validation.validate_distribution(data)
-        validation.validate_github(data)
-        validation.validate_age(data)
-        validation.validate_timezone(data)
-        validation.validate_acme_email(data)
-        validation.validate_flux_github_webhook_token(data)
-        validation.validate_cloudflare(data)
-        validation.validate_host_network(data)
-        validation.validate_bootstrap_dns_server(data)
-        validation.validate_cluster_cidrs(data)
-        validation.validate_nodes(data)
 
     def filters(self):
         return [nthhost, encrypt]

bootstrap/scripts/validation.py

@@ -198,3 +198,18 @@ def validate_nodes(node_cidr: str, nodes: dict[list], distribution: str, **_) ->
     workers = nodes.get("worker", [])
     for node in workers:
         _validate_node(node, node_cidr, distribution)
+
+def validate(data: dict) -> None:
+    validate_python_version()
+    validate_cli_tools(data)
+    validate_distribution(data)
+    validate_github(data)
+    validate_age(data)
+    validate_timezone(data)
+    validate_acme_email(data)
+    validate_flux_github_webhook_token(data)
+    validate_cloudflare(data)
+    validate_host_network(data)
+    validate_bootstrap_dns_server(data)
+    validate_cluster_cidrs(data)
+    validate_nodes(data)

bootstrap/templates/ansible/.ansible-lint.j2

@@ -1,6 +1,5 @@
 #% if bootstrap_distribution in ['k3s', 'k0s'] %#
 skip_list:
-  - jinja[spacing]
   - yaml[commas]
   - yaml[line-length]
   - var-naming

bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2

@@ -11,7 +11,6 @@ k3s_registration_address: "{{ kube_api_addr }}"
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
   - custom-cilium-helmchart.yaml.j2
-#% if bootstrap_nodes.master | length > 1 or bootstrap_kube_api_addr|default('') %#
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_urls:
   - url: https://raw.githubusercontent.com/kube-vip/website/main/content/manifests/rbac.yaml
@@ -20,4 +19,3 @@ k3s_server_manifests_urls:
 k3s_server_pod_manifests_templates:
   - kube-vip-static-pod.yaml.j2
 #% endif %#
-#% endif %#

bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2

@@ -3,18 +3,11 @@
 timezone: "#{ bootstrap_timezone }#"
 github_username: "#{ bootstrap_github_username }#"
 coredns_addr: "#{ bootstrap_service_cidr.split(',')[0] | nthhost(10) }#"
-#% if bootstrap_nodes.master | length > 1 or bootstrap_kube_api_addr|default('') %#
 kube_api_addr: "#{ bootstrap_kube_api_addr }#"
-#% else %#
-kube_api_addr: "#{ bootstrap_nodes.master[0].address }#"
-#% endif %#
-#% if bootstrap_kube_api_hostname|default('') %#
-kube_api_hostname: "#{ bootstrap_kube_api_hostname }#"
-#% endif %#
 cluster_cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#"
 service_cidr: "#{ bootstrap_service_cidr.split(',')[0] }#"
 node_cidr: "#{ bootstrap_node_cidr }#"
-#% if bootstrap_ipv6_enabled|default(false) %#
+#% if bootstrap_ipv6_enabled %#
 cluster_cidr_v6: "#{ bootstrap_cluster_cidr.split(',')[1] }#"
 service_cidr_v6: "#{ bootstrap_service_cidr.split(',')[1] }#"
 #% endif %#

bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2

@@ -4,16 +4,16 @@
 # https://github.com/PyratLabs/ansible-role-k3s
 k3s_control_node: true
 k3s_server:
-  #% if bootstrap_ipv6_enabled|default(false) %#
+  #% if bootstrap_ipv6_enabled %#
   node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}"
   #% else %#
   node-ip: "{{ ansible_host }}"
   #% endif %#
   tls-san:
     - "{{ kube_api_addr }}"
-    #% if bootstrap_kube_api_hostname|default('') %#
+    #% for item in bootstrap_kube_additional_tls_sans %#
-    - "{{ kube_api_hostname }}"
+    - "{{ item }}"
-    #% endif %#
+    #% endfor %#
   docker: false
   flannel-backend: "none" # quote
   disable: ["flannel", "local-storage", "metrics-server", "servicelb", "traefik"]
@@ -23,7 +23,7 @@ k3s_server:
   write-kubeconfig-mode: "644"
   pause-image: registry.k8s.io/pause:3.9
   secrets-encryption: true
-  #% if bootstrap_ipv6_enabled|default(false) %#
+  #% if bootstrap_ipv6_enabled %#
   cluster-cidr: "{{ cluster_cidr }},{{ cluster_cidr_v6 }}"
   service-cidr: "{{ service_cidr }},{{ service_cidr_v6 }}"
   #% else %#

bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2

@@ -4,7 +4,7 @@
 # https://github.com/PyratLabs/ansible-role-k3s
 k3s_control_node: false
 k3s_agent:
-  #% if bootstrap_ipv6_enabled|default(false) %#
+  #% if bootstrap_ipv6_enabled %#
   node-ip: "{{ ansible_host }},{{ ansible_default_ipv6.address }}"
   #% else %#
   node-ip: "{{ ansible_host }}"

bootstrap/templates/ansible/inventory/hosts.yaml.j2

@@ -9,7 +9,7 @@ kubernetes:
           ansible_user: "#{ item.username }#"
           ansible_host: "#{ item.address }#"
       #% endfor %#
-    #% if bootstrap_nodes.worker|default([]) | length > 0 %#
+    #% if bootstrap_nodes.worker | length > 0 %#
     worker:
       hosts:
       #% for item in bootstrap_nodes.worker %#

bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2

@@ -41,7 +41,7 @@
         kubeconfig: /etc/rancher/k3s/k3s.yaml
         kind: "{{ item.kind }}"
         name: "{{ item.name }}"
-        namespace: "{{ item.namespace|default('') }}"
+        namespace: "{{ item.namespace | default('') }}"
         wait: true
         wait_sleep: 10
         wait_timeout: 360

bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2

@@ -16,7 +16,7 @@
       block:
         - name: Locale | Set timezone
           community.general.timezone:
-            name: "{{ timezone|default('Etc/UTC') }}"
+            name: "{{ timezone | default('Etc/UTC') }}"
 
     - name: Packages
       block:
@@ -62,7 +62,7 @@
               ff02::2 ip6-allrouters
             dest: /etc/hosts
             mode: preserve
-        #% if bootstrap_advanced_flags|default({}) and bootstrap_advanced_flags.update_resolv_conf|default(true) %#
+        #% if bootstrap_advanced_flags.update_resolv_conf %#
         # https://github.com/onedr0p/flux-cluster-template/discussions/635
         - name: Network Configuration | Remove immutable flag from /etc/resolv.conf
           ansible.builtin.file:
@@ -80,7 +80,7 @@
             dest: /etc/resolv.conf
             content: |
               search .
-              nameserver #{ bootstrap_dns_server|default('1.1.1.1', true) }#
+              nameserver #{ bootstrap_dns_server }#
         #% endif %#
 
     - name: System Configuration

bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' and (bootstrap_nodes.master | length > 1 or bootstrap_kube_api_addr|default('')) %#
+#% if bootstrap_distribution == 'k3s' %#
 ---
 apiVersion: v1
 kind: Pod

bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2

@@ -2,7 +2,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  #% if not bootstrap_ipv6_enabled|default(false) %#
+  #% if not bootstrap_ipv6_enabled %#
   - ./cilium-l2.yaml
   #% endif %#
   - ./helmrelease.yaml

bootstrap/templates/kubernetes/apps/network/nginx/certificates/kustomization.yaml.j2

@@ -3,6 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./staging.yaml
-  #% if bootstrap_acme_production_enabled|default(false) %#
+  #% if bootstrap_acme_production_enabled %#
   - ./production.yaml
   #% endif %#

bootstrap/templates/kubernetes/apps/network/nginx/external/helmrelease.yaml.j2

@@ -70,7 +70,7 @@ spec:
           namespaceSelector:
             any: true
       extraArgs:
-        #% if bootstrap_acme_production_enabled|default(false) %#
+        #% if bootstrap_acme_production_enabled %#
         default-ssl-certificate: "network/${SECRET_DOMAIN/./-}-production-tls"
         #% else %#
         default-ssl-certificate: "network/${SECRET_DOMAIN/./-}-staging-tls"

bootstrap/templates/kubernetes/apps/network/nginx/internal/helmrelease.yaml.j2

@@ -68,7 +68,7 @@ spec:
           namespaceSelector:
             any: true
       extraArgs:
-        #% if bootstrap_acme_production_enabled|default(false) %#
+        #% if bootstrap_acme_production_enabled %#
         default-ssl-certificate: "network/${SECRET_DOMAIN/./-}-production-tls"
         #% else %#
         default-ssl-certificate: "network/${SECRET_DOMAIN/./-}-staging-tls"

bootstrap/templates/kubernetes/apps/storage/csi-driver-smb/app/storageclass.yaml.j2

@@ -8,7 +8,7 @@ metadata:
 provisioner: smb.csi.k8s.io
 parameters:
   source: "#{ item.source }#"
-  csi.storage.k8s.io/node-stage-secret-name: "#{ item.existing_secret_name|default(item.name+"-secret") }#"
+  csi.storage.k8s.io/node-stage-secret-name: "#{ item.existing_secret_name|default(item.name + '-secret') }#"
   #% if item.existing_secret_name and item.existing_secret_namespace %#
   csi.storage.k8s.io/node-stage-secret-namespace: "#{ item.existing_secret_namespace }#"
   #% else %#

bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2

@@ -7,8 +7,8 @@ metadata:
 spec:
   interval: 30m
   ref:
-    branch: "#{ bootstrap_github_repository_branch|default('main', true) }#"
+    branch: "#{ bootstrap_github_repository_branch }#"
-  #% if bootstrap_private_github_repo|default(false) %#
+  #% if bootstrap_private_github_repo %#
   secretRef:
     name: github-deploy-key
   url: "ssh://github.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#"

bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2

@@ -7,15 +7,11 @@ metadata:
 data:
   TIMEZONE: "#{ bootstrap_timezone }#"
   COREDNS_ADDR: "#{ bootstrap_service_cidr.split(',')[0] | nthhost(10) }#"
-  #% if bootstrap_nodes.master | length > 1 or bootstrap_kube_api_addr|default('') %#
   KUBE_API_ADDR: "#{ bootstrap_kube_api_addr }#"
-  #% else %#
-  KUBE_API_ADDR: "#{ bootstrap_nodes.master[0].address }#"
-  #% endif %#
   CLUSTER_CIDR: "#{ bootstrap_cluster_cidr.split(',')[0] }#"
   SERVICE_CIDR: "#{ bootstrap_service_cidr.split(',')[0] }#"
   NODE_CIDR: "#{ bootstrap_node_cidr }#"
-  #% if bootstrap_ipv6_enabled|default(false) %#
+  #% if bootstrap_ipv6_enabled %#
   CLUSTER_CIDR_V6: "#{ bootstrap_cluster_cidr.split(',')[1] }#"
   SERVICE_CIDR_V6: "#{ bootstrap_service_cidr.split(',')[1] }#"
   #% endif %#

bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2

@@ -49,7 +49,7 @@ spec:
       hooks: *hooks
       #% endif %#
     #% endfor %#
-    #% if bootstrap_nodes.worker|default([]) | length > 0 %#
+    #% if bootstrap_nodes.worker | length > 0 %#
     #% for item in bootstrap_nodes.worker %#
     - role: worker
       ssh:
@@ -81,17 +81,14 @@ spec:
               listen-metrics-urls: http://0.0.0.0:2381 # Required to monitor etcd with kube-prometheus-stack
         api:
           sans:
+            #% if bootstrap_kube_api_addr %#
             - "#{ bootstrap_kube_api_addr }#"
-            #% if bootstrap_kube_api_hostname|default('') %#
-            - "#{ bootstrap_kube_api_hostname }#"
             #% endif %#
+            #% for item in bootstrap_kube_additional_tls_sans %#
+            - "#{ item }#"
+            #% endfor %#
             #% for item in bootstrap_nodes.master %#
-            #% if item.address != bootstrap_kube_api_addr %#
             - "#{ item.address }#"
-            #% endif %#
-            #% if (not bootstrap_kube_api_hostname|default('')) or (item.name != bootstrap_kube_api_hostname|default('')) %#
-            - "#{ item.name }#"
-            #% endif %#
             #% endfor %#
         network:
           podCIDR: "#{ bootstrap_cluster_cidr.split(',')[0] }#"

bootstrap/templates/kubernetes/k0s/resources/hooks/apply-system.sh.j2

@@ -16,13 +16,13 @@ apt-get install -y --no-install-recommends \
     nvme-cli open-iscsi parted psmisc python3 python3-apt python3-kubernetes python3-yaml \
     smartmontools socat software-properties-common unzip util-linux
 
-#% if bootstrap_advanced_flags|default({}) and bootstrap_advanced_flags.update_resolv_conf|default(true) %#
+#% if bootstrap_advanced_flags.update_resolv_conf %#
 # Update DNS
 chattr -i /etc/resolv.conf
 rm -f /etc/resolv.conf
 cat <<EOF > /etc/resolv.conf
 search .
-nameserver #{ bootstrap_dns_server|default('1.1.1.1', true) }#
+nameserver #{ bootstrap_dns_server }#
 EOF
 chattr +i /etc/resolv.conf
 #% endif %#

bootstrap/templates/kubernetes/talos/talconfig.yaml.j2

@@ -8,11 +8,7 @@ kubernetesVersion: 1.29.1
 
 clusterName: &cluster home-kubernetes
 
-#% if bootstrap_kube_api_hostname|default('') %#
-endpoint: https://#{ bootstrap_kube_api_hostname }#:6443
-#% else %#
 endpoint: https://#{ bootstrap_kube_api_addr }#:6443
-#% endif %#
 
 clusterPodNets:
   - "#{ bootstrap_cluster_cidr.split(',')[0] }#"
@@ -21,10 +17,10 @@ clusterSvcNets:
 
 additionalApiServerCertSans: &sans
   - "#{ bootstrap_kube_api_addr }#"
-  - 127.0.0.1 #KubePrism
+  - 127.0.0.1 # KubePrism
-  #% if bootstrap_kube_api_hostname|default('') %#
+  #% for item in bootstrap_kube_additional_tls_sans %#
-  - "#{ bootstrap_kube_api_hostname }#"
+  - "#{ item }#"
-  #% endif %#
+  #% endfor %#
 
 additionalMachineCertSans: *sans
 
@@ -48,7 +44,7 @@ nodes:
         vip:
           ip: "#{ bootstrap_kube_api_addr }#"
   #% endfor %#
-  #% if bootstrap_nodes.worker|default([]) | length > 0 %#
+  #% if bootstrap_nodes.worker | length > 0 %#
   #% for item in bootstrap_nodes.worker %#
   - hostname: "#{ item.name }#"
     ipAddress: "#{ item.address }#"
@@ -123,7 +119,7 @@ controlPlane:
       machine:
         network:
           nameservers:
-            - "#{ bootstrap_dns_server|default('1.1.1.1', true) }#"
+            - "#{ bootstrap_dns_server }#"
 
     # Configure NTP
     - &ntpPatch |-

bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

@@ -59,7 +59,7 @@ hubble:
 ipam:
   mode: kubernetes
 ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}"
-#% if bootstrap_ipv6_enabled|default(false) %#
+#% if bootstrap_ipv6_enabled %#
 ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}"
 ipv6:
   enabled: true
@@ -77,7 +77,7 @@ k8sServicePort: 7445
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
 l2announcements:
-  #% if bootstrap_ipv6_enabled|default(false) %#
+  #% if bootstrap_ipv6_enabled %#
   enabled: false
   #% else %#
   enabled: true

bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

@@ -20,7 +20,7 @@ hubble:
 ipam:
   mode: kubernetes
 ipv4NativeRoutingCIDR: "#{ bootstrap_cluster_cidr }#"
-#% if bootstrap_ipv6_enabled|default(false) %#
+#% if bootstrap_ipv6_enabled %#
 ipv6NativeRoutingCIDR: "#{ bootstrap_cluster_cidr_v6 }#"
 ipv6:
   enabled: true
@@ -38,7 +38,7 @@ k8sServicePort: 7445
 kubeProxyReplacement: true
 kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
 l2announcements:
-  #% if bootstrap_ipv6_enabled|default(false) %#
+  #% if bootstrap_ipv6_enabled %#
   enabled: false
   #% else %#
   enabled: true

bootstrap/vars/config.sample.yaml

@@ -42,12 +42,11 @@ bootstrap_cloudflare_tunnel_id:
 # CIDR your nodes are on (e.g. 192.168.1.0/24)
 bootstrap_node_cidr:
 # The IP address of the Kube API, choose an available IP in your nodes network that is not being used
-# Leave this blank if you are deploying a single master node, this will disable kube-vip in k3s
+# NOTE: For k0s set this to a IP you can then later use to manually setup a load balancer for the Kube API (e.g. Haproxy or Keepalived)
-# and keepalived in k0s and use the master node IP instead
+# Ref: https://github.com/k0sproject/k0s/issues/1150
 bootstrap_kube_api_addr:
-# (Optional) The hostname of the Kube API, set this if you intend to call the Kube API by
+# Add additional SANs to the Kube API cert, this is useful if you want to call the Kube API by hostname rather than IP
-# hostname rather than IP. If the hostname is set it is added to the Kube API cert sans
+bootstrap_kube_additional_tls_sans: []
-bootstrap_kube_api_hostname:
 # The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used
 bootstrap_k8s_gateway_addr:
 # The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used
@@ -62,13 +61,11 @@ bootstrap_internal_ingress_addr:
 #   3. you are not force redirecting DNS requests to it - this will break cert generation over DNS01
 bootstrap_dns_server: 1.1.1.1
 
-# Keep the next three options default unless you know what you are doing
 # (Advanced) Enable ipv6
 bootstrap_ipv6_enabled: false
-# (Advanced) For ipv6 use format 10.42.0.0/16,fd78:c889:47fb:10::/60
+# For ipv6 use format "10.42.0.0/16,fd78:c889:47fb:10::/60", /60 IPv6 block is enough for 16 nodes
-# /60 IPv6 block is enough for 16 nodes
 bootstrap_cluster_cidr: 10.42.0.0/16
-# (Advanced) For ipv6 use format 10.43.0.0/16,fd78:c889:47fb:e0::/112
+# For ipv6 use format "10.43.0.0/16,fd78:c889:47fb:e0::/112"
 bootstrap_service_cidr: 10.43.0.0/16
 
 # Node information