From 4dd5f9d5a85170aca87e5bc104fcb2d72bdca631 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 17 Jul 2023 20:52:12 -0400 Subject: [PATCH] feat: re-introduce flux vars for cluster network settings Signed-off-by: Devin Buhl --- bootstrap/defaults/main.yaml | 2 -- .../group_vars/kubernetes/supplemental.yaml.j2 | 11 ++++++++--- .../templates/custom-cilium-helmchart.yaml.j2.j2 | 15 ++++++--------- .../templates/custom-coredns-helmchart.yaml.j2.j2 | 3 +-- .../apps/kube-system/cilium/app/cilium-l2.yaml.j2 | 2 +- .../kube-system/cilium/app/helmrelease.yaml.j2 | 15 +++++++-------- .../kube-system/coredns/app/helmrelease.yaml.j2 | 2 +- .../kubernetes/flux/vars/cluster-settings.yaml.j2 | 10 ++++++++++ 8 files changed, 34 insertions(+), 26 deletions(-) delete mode 100644 bootstrap/defaults/main.yaml diff --git a/bootstrap/defaults/main.yaml b/bootstrap/defaults/main.yaml deleted file mode 100644 index d0eb3f0a..00000000 --- a/bootstrap/defaults/main.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -bootstrap_coredns_addr: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}" diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 index 6c2bcdc4..816dc3f5 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 @@ -1,8 +1,13 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True --- timezone: "{{ bootstrap_timezone }}" github_username: "{{ bootstrap_github_username }}" -coredns_addr: "{{ bootstrap_coredns_addr }}" +coredns_addr: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}" kube_vip_addr: "{{ bootstrap_kube_vip_addr }}" -cluster_cidr: "{{ bootstrap_cluster_cidr }}" -service_cidr: "{{ bootstrap_service_cidr }}" +cluster_cidr: "{{ bootstrap_cluster_cidr.split(',')[0] }}" +service_cidr: "{{ bootstrap_service_cidr.split(',')[0] }}" node_cidr: "{{ bootstrap_node_cidr }}" +{% if bootstrap_ipv6_enabled | default(false) %} +cluster_cidr_v6: "{{ bootstrap_cluster_cidr.split(',')[1] }}" +service_cidr_v6: "{{ bootstrap_service_cidr.split(',')[1] }}" +{% endif %} diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 index ee72855a..13ba3dda 100644 --- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 +++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 @@ -31,29 +31,26 @@ spec: enabled: false ipam: mode: kubernetes + ipv4NativeRoutingCIDR: "{% raw %}{{ cluster_cidr }}{% endraw %}" {% if bootstrap_ipv6_enabled | default(false) %} - ipv4NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'].split(',')[0] }}{% endraw %}" - ipv6NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'].split(',')[1] }}{% endraw %}" + ipv6NativeRoutingCIDR: "{% raw %}{{ cluster_cidr_v6 }}{% endraw %}" ipv6: enabled: true - {% else %} - ipv4NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'] }}{% endraw %}" {% endif %} k8sServiceHost: "{% raw %}{{ kube_vip_addr }}{% endraw %}" k8sServicePort: 6443 kubeProxyReplacement: strict kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 - {% if bootstrap_ipv6_enabled | default(false) %} l2announcements: + {% if bootstrap_ipv6_enabled | default(false) %} enabled: false - {% else %} - # https://github.com/cilium/cilium/issues/26586 - l2announcements: + {% else %} enabled: true + # https://github.com/cilium/cilium/issues/26586 leaseDuration: 120s leaseRenewDeadline: 60s leaseRetryPeriod: 1s - {% endif %} + {% endif %} loadBalancer: algorithm: maglev mode: dsr diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 index b0a74544..7b00db44 100644 --- a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 +++ b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 @@ -19,8 +19,7 @@ spec: k8sAppLabelOverride: kube-dns service: name: kube-dns - # Choose the 10th IP address from the start of the service-cidr - clusterIP: "{% raw %}{{ k3s_server['service-cidr'].split(',')[0] | ansible.utils.nthhost(10) }}{% endraw %}" + clusterIP: "{% raw %}{{ coredns_addr }}{% endraw %}" serviceAccount: create: true deployment: diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 index 6a5cdbeb..e8eba26e 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2 @@ -19,4 +19,4 @@ metadata: name: pool spec: cidrs: - - cidr: "{{ bootstrap_node_cidr }}" + - cidr: "${NODE_CIDR}" diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 index 447f67b8..7c2c0dab 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 @@ -76,27 +76,26 @@ spec: - *host ipam: mode: kubernetes - ipv4NativeRoutingCIDR: "{{ bootstrap_cluster_cidr.split(',')[0] }}" + ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}" {% if bootstrap_ipv6_enabled | default(false) %} - ipv6NativeRoutingCIDR: "{{ bootstrap_cluster_cidr.split(',')[1] }}" + ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}" ipv6: enabled: true {% endif %} - k8sServiceHost: "{{ bootstrap_kube_vip_addr }}" + k8sServiceHost: "${KUBE_VIP_ADDR}" k8sServicePort: 6443 kubeProxyReplacement: strict kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 - {% if bootstrap_ipv6_enabled | default(false) %} l2announcements: + {% if bootstrap_ipv6_enabled | default(false) %} enabled: false - {% else %} - # https://github.com/cilium/cilium/issues/26586 - l2announcements: + {% else %} enabled: true + # https://github.com/cilium/cilium/issues/26586 leaseDuration: 120s leaseRenewDeadline: 60s leaseRetryPeriod: 1s - {% endif %} + {% endif %} loadBalancer: algorithm: maglev mode: dsr diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 index 1ef80888..2088a9a7 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 @@ -32,7 +32,7 @@ spec: k8sAppLabelOverride: kube-dns service: name: kube-dns - clusterIP: "{{ bootstrap_coredns_addr }}" + clusterIP: "${COREDNS_ADDR}" serviceAccount: create: true deployment: diff --git a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 index ca76a0cf..511c2bd2 100644 --- a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 @@ -1,3 +1,4 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True --- apiVersion: v1 kind: ConfigMap @@ -6,3 +7,12 @@ metadata: namespace: flux-system data: TIMEZONE: "{{ bootstrap_timezone }}" + COREDNS_ADDR: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}" + KUBE_VIP_ADDR: "{{ bootstrap_kube_vip_addr }}" + CLUSTER_CIDR: "{{ bootstrap_cluster_cidr.split(',')[0] }}" + SERVICE_CIDR: "{{ bootstrap_service_cidr.split(',')[0] }}" + NODE_CIDR: "{{ bootstrap_node_cidr }}" + {% if bootstrap_ipv6_enabled | default(false) %} + CLUSTER_CIDR_V6: "{{ bootstrap_cluster_cidr.split(',')[1] }}" + SERVICE_CIDR_V6: "{{ bootstrap_service_cidr.split(',')[1] }}" + {% endif %}