feat: IPv6 support 🚀 (#855)

Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com>

bootstrap/tasks/validation/net.yaml

@@ -13,17 +13,47 @@
     success_msg: "Node CIDR {{ bootstrap_node_cidr }} is valid."
     fail_msg: "Node CIDR {{ bootstrap_node_cidr }} is invalid."
 
-- name: Verify cluster CIDR
+- name: Verify cluster CIDR is ipv4 OR ipv6
+  when: not bootstrap_ipv6_enabled | default(false)
   ansible.builtin.assert:
-    that: ["{{ bootstrap_cluster_cidr is ansible.utils.ipv4 }}"]
+    that: bootstrap_cluster_cidr is ansible.utils.ipv4 or bootstrap_cluster_cidr is ansible.utils.ipv6
-    success_msg: "Cluster CIDR {{ bootstrap_cluster_cidr }} is valid."
+    success_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is valid.
-    fail_msg: "Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid."
+    fail_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid.
 
-- name: Verify service CIDR
+- name: Verify service CIDR is ipv4 OR ipv6
+  when: not bootstrap_ipv6_enabled | default(false)
   ansible.builtin.assert:
-    that: ["{{ bootstrap_service_cidr is ansible.utils.ipv4 }}"]
+    that: bootstrap_service_cidr is ansible.utils.ipv4 or bootstrap_service_cidr is ansible.utils.ipv6
-    success_msg: "Service CIDR {{ bootstrap_service_cidr }} is valid."
+    success_msg: Service CIDR {{ bootstrap_service_cidr }} is valid.
-    fail_msg: "Service CIDR {{ bootstrap_service_cidr }} is invalid."
+    fail_msg: Service CIDR {{ bootstrap_service_cidr }} is invalid.
+
+- name: Verify cluster CIDR is ipv4 AND ipv6
+  when: bootstrap_ipv6_enabled | default(false)
+  ansible.builtin.assert:
+    that: >
+      (
+        bootstrap_cluster_cidr.split(',')[0] is ansible.utils.ipv4 or
+          bootstrap_cluster_cidr.split(',')[1] is ansible.utils.ipv4
+      ) and (
+        bootstrap_cluster_cidr.split(',')[1] is ansible.utils.ipv6 or
+          bootstrap_cluster_cidr.split(',')[0] is ansible.utils.ipv6
+      )
+    success_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is valid.
+    fail_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid.
+
+- name: Verify service CIDR is ipv4 AND ipv6
+  when: bootstrap_ipv6_enabled | default(false)
+  ansible.builtin.assert:
+    that: >
+      (
+        bootstrap_service_cidr.split(',')[0] is ansible.utils.ipv4 or
+          bootstrap_service_cidr.split(',')[1] is ansible.utils.ipv4
+      ) and (
+        bootstrap_service_cidr.split(',')[1] is ansible.utils.ipv6 or
+          bootstrap_service_cidr.split(',')[0] is ansible.utils.ipv6
+      )
+    success_msg: Service CIDR {{ bootstrap_service_cidr }} is valid.
+    fail_msg: Service CIDR {{ bootstrap_service_cidr }} is invalid.
 
 - name: Verify k8s_gateway
   ansible.builtin.assert:
@@ -72,6 +102,16 @@
   loop_control:
      label: "{{ item.address }}"
 
+- name: Verify nodes are ipv4
+  ansible.builtin.assert:
+    that: item.address is ansible.utils.ipv4
+    success_msg: Node address {{ item.address }} is valid.
+    fail_msg: Node address {{ item.address }} is invalid.
+    quiet: true
+  loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
+  loop_control:
+    label: "{{ item.address }}"
+
 - name: Verify nodes are in node CIDR
   ansible.builtin.assert:
     that: ["{{ bootstrap_node_cidr | ansible.utils.network_in_usable(item.address) }}"]

bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2

@@ -1,10 +1,15 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 ---
 # https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/
 # https://github.com/PyratLabs/ansible-role-k3s
 
 k3s_control_node: true
 k3s_server:
+{% if bootstrap_ipv6_enabled | default(false) %}
+  node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
+{% else %}
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
+{% endif %}
   tls-san:
     - "{% raw %}{{ kube_vip_addr }}{% endraw %}"
   docker: false

bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2

@@ -1,7 +1,12 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 ---
 # https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/
 # https://github.com/PyratLabs/ansible-role-k3s
 
 k3s_control_node: false
 k3s_agent:
+{% if bootstrap_ipv6_enabled | default(false) %}
+  node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
+{% else %}
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
+{% endif %}

bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2

@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 ---
 # https://docs.k3s.io/helm
 apiVersion: helm.cattle.io/v1
@@ -30,17 +31,29 @@ spec:
       enabled: false
     ipam:
       mode: kubernetes
+    {% if bootstrap_ipv6_enabled | default(false) %}
+    ipv4NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'].split(',')[0] }}{% endraw %}"
+    ipv6NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'].split(',')[1] }}{% endraw %}"
+    ipv6:
+      enabled: true
+    {% else %}
     ipv4NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'] }}{% endraw %}"
+    {% endif %}
     k8sServiceHost: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
     k8sServicePort: 6443
     kubeProxyReplacement: strict
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+    {% if bootstrap_ipv6_enabled | default(false) %}
+    l2announcements:
+      enabled: false
+    {% else %}
     # https://github.com/cilium/cilium/issues/26586
     l2announcements:
       enabled: true
       leaseDuration: 120s
       leaseRenewDeadline: 60s
       leaseRetryPeriod: 1s
+    {% endif %}
     loadBalancer:
       algorithm: maglev
       mode: dsr

bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2

@@ -19,7 +19,11 @@ spec:
     service:
       name: kube-dns
       # Choose the 10th IP address from the start of the service-cidr
+      {% if bootstrap_ipv6_enabled | default(false) %}
+      clusterIP: "{% raw %}{{ k3s_server['service-cidr'].split(',')[0] | ansible.utils.nthhost(10) }}{% endraw %}"
+      {% else %}
       clusterIP: "{% raw %}{{ k3s_server['service-cidr'] | ansible.utils.nthhost(10) }}{% endraw %}"
+      {% endif %}
     serviceAccount:
       create: true
     deployment:

bootstrap/vars/config.sample.yaml

@@ -42,10 +42,17 @@ bootstrap_kube_vip_addr:
 bootstrap_k8s_gateway_addr:
 # The Load balancer IP for ingress-nginx, choose an available IP in your nodes network that is not being used
 bootstrap_ingress_nginx_addr:
-# Choose your cluster and service cidrs, leave these unchanged unless you know what you are doing
+
+# Keep the next three options default unless you know what you are doing
+# (Advanced) Enable ipv6
+bootstrap_ipv6_enabled: false
+# (Advanced) For ipv6 use format 10.42.0.0/16,fd78:c889:47fb:10::/60
+# /60 IPv6 block is enough for 16 nodes
 bootstrap_cluster_cidr: 10.42.0.0/16
+# (Advanced) For ipv6 use format 10.43.0.0/16,fd78:c889:47fb:e0::/112
 bootstrap_service_cidr: 10.43.0.0/16
 
+
 # Node information
 bootstrap_nodes:
   # Use only 1, 3 or more odd master nodes, recommended is 3