From 9373065338f06340435f90f9b4691b73176b19c5 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 22 Jan 2024 12:45:11 -0500 Subject: [PATCH] feat: implement kube-vip for k0s & k3s in daemonset mode (#1230) * feat: implement kube-vip for k0s Signed-off-by: Devin Buhl * feat: implement kube-vip for k0s Signed-off-by: Devin Buhl * fix: k3s to ds as well Signed-off-by: Devin Buhl * fix: no j2j2 nonsense Signed-off-by: Devin Buhl * upadte: readme and taskfile Signed-off-by: Devin Buhl * update: renovate config Signed-off-by: Devin Buhl --------- Signed-off-by: Devin Buhl --- .github/renovate.json5 | 27 ++-- .taskfiles/Repository/Taskfile.yaml | 2 +- .../group_vars/kubernetes/main.yaml.j2 | 8 +- .../kubernetes/supplemental.yaml.j2 | 14 -- .../inventory/group_vars/master/main.yaml.j2 | 10 +- .../ansible/playbooks/cluster-prepare.yaml.j2 | 4 +- ....j2.j2 => custom-cilium-helmchart.yaml.j2} | 4 +- .../playbooks/templates/kube-vip-ds.yaml.j2 | 77 ++++++++++ .../templates/kube-vip-static-pod.yaml.j2.j2 | 61 -------- .../templates/kubernetes/k0s/k0sctl.yaml.j2 | 1 + .../k0s/resources/apply-kube-vip.sh.j2 | 131 ++++++++++++++++++ .../talos/cilium/kustomization.yaml.j2 | 2 +- bootstrap/vars/config.sample.yaml | 18 +-- 13 files changed, 243 insertions(+), 116 deletions(-) delete mode 100644 bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 rename bootstrap/templates/ansible/playbooks/templates/{custom-cilium-helmchart.yaml.j2.j2 => custom-cilium-helmchart.yaml.j2} (83%) create mode 100644 bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2 delete mode 100644 bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 create mode 100755 bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2 diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 54dea8dd..d5e74e5d 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -15,35 +15,35 @@ "schedule": ["on saturday"], "flux": { "fileMatch": [ - "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$", - "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$" + "(^|/)ansible/.+\\.ya?ml(\\.j2)?$", + "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$" ] }, "helm-values": { "fileMatch": [ - "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$", - "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$" + "(^|/)ansible/.+\\.ya?ml(\\.j2)?$", + "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$" ] }, "kubernetes": { "fileMatch": [ - "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$", - "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$" + "(^|/)ansible/.+\\.ya?ml(\\.j2)?$", + "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$" ] }, "kustomize": { "fileMatch": [ - "(^|/)kustomization\\.ya?ml(\\.j2)?(\\.j2)?$" + "(^|/)kustomization\\.ya?ml(\\.j2)?$" ] }, "pip_requirements": { "fileMatch": [ - "(^|/)[\\w-]*requirements(-\\w+)?\\.(txt|pip)(\\.j2)?(\\.j2)?$" + "(^|/)[\\w-]*requirements(-\\w+)?\\.(txt|pip)(\\.j2)?$" ] }, "ansible-galaxy": { "fileMatch": [ - "(^|/)(galaxy|requirements)(\\.ansible)?\\.ya?ml(\\.j2)?(\\.j2)?$" + "(^|/)(galaxy|requirements)(\\.ansible)?\\.ya?ml(\\.j2)?$" ] }, // commit message topics @@ -232,8 +232,8 @@ "description": "Process various other dependencies", "fileMatch": [ "(^|/).taskfiles/.+\\.ya?ml$", - "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$", - "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$" + "(^|/)ansible/.+\\.ya?ml(\\.j2)?$", + "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$" ], "matchStrings": [ // Example: @@ -242,11 +242,6 @@ // Example: // - https://github.com/rancher/system-upgrade-controller/releases/download/v0.11.0/crd.yaml "datasource=(?\\S+) depName=(?\\S+)( versioning=(?\\S+))?\n.*?-\\s(.*?)\/(?[^/]+)\/[^/]+\n", - // Example: - // repo: https://helm.cilium.io - // chart: cilium - // version: 1.14.5 - "datasource=(?\\S+)\n.*?repo: (?\\S+)\n.*?chart: (?\\S+)\n.*?version: (?\\S+)\n" ], "datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}", "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}" diff --git a/.taskfiles/Repository/Taskfile.yaml b/.taskfiles/Repository/Taskfile.yaml index dade642f..0b020436 100644 --- a/.taskfiles/Repository/Taskfile.yaml +++ b/.taskfiles/Repository/Taskfile.yaml @@ -14,7 +14,7 @@ tasks: - mv {{.BOOTSTRAP_DIR}} {{.PRIVATE_DIR}}/bootstrap-{{now | date "150405"}} - mv {{.MAKEJINJA_CONFIG_FILE}} {{.PRIVATE_DIR}}/makejinja-{{now | date "150405"}}.toml # Update renovate.json5 - - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5 + - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5 preconditions: - { msg: "Missing bootstrap directory", sh: "test -d {{.BOOTSTRAP_DIR}}" } - { msg: "Missing private directory", sh: "test -d {{.PRIVATE_DIR}}" } diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 index fff67163..12598831 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 @@ -7,15 +7,13 @@ k3s_install_hard_links: true k3s_become: true k3s_etcd_datastore: true k3s_use_unsupported_config: true -k3s_registration_address: "{{ kube_api_addr }}" +k3s_registration_address: "#{ bootstrap_kube_api_addr }#" # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_templates: - custom-cilium-helmchart.yaml.j2 + - kube-vip-ds.yaml.j2 # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_urls: - - url: https://raw.githubusercontent.com/kube-vip/website/main/content/manifests/rbac.yaml + - url: https://kube-vip.io/manifests/rbac.yaml filename: kube-vip-rbac.yaml -# /var/lib/rancher/k3s/agent/pod-manifests -k3s_server_pod_manifests_templates: - - kube-vip-static-pod.yaml.j2 #% endif %# diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 deleted file mode 100644 index 100b988e..00000000 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 +++ /dev/null @@ -1,14 +0,0 @@ -#% if bootstrap_distribution in ['k3s', 'k0s'] %# ---- -timezone: "#{ bootstrap_timezone }#" -github_username: "#{ bootstrap_github_username }#" -coredns_addr: "#{ bootstrap_service_cidr.split(',')[0] | nthhost(10) }#" -kube_api_addr: "#{ bootstrap_kube_api_addr }#" -cluster_cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#" -service_cidr: "#{ bootstrap_service_cidr.split(',')[0] }#" -node_cidr: "#{ bootstrap_node_cidr }#" -#% if bootstrap_ipv6_enabled %# -cluster_cidr_v6: "#{ bootstrap_cluster_cidr.split(',')[1] }#" -service_cidr_v6: "#{ bootstrap_service_cidr.split(',')[1] }#" -#% endif %# -#% endif %# diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 index b0e67f04..0654315a 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 @@ -10,7 +10,7 @@ k3s_server: node-ip: "{{ ansible_host }}" #% endif %# tls-san: - - "{{ kube_api_addr }}" + - "#{ bootstrap_kube_api_addr }#" #% for item in bootstrap_kube_additional_tls_sans %# - "{{ item }}" #% endfor %# @@ -24,11 +24,11 @@ k3s_server: pause-image: registry.k8s.io/pause:3.9 secrets-encryption: true #% if bootstrap_ipv6_enabled %# - cluster-cidr: "{{ cluster_cidr }},{{ cluster_cidr_v6 }}" - service-cidr: "{{ service_cidr }},{{ service_cidr_v6 }}" + cluster-cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#,#{ bootstrap_cluster_cidr.split(',')[1] }#" + service-cidr: "#{ bootstrap_service_cidr.split(',')[0] }#,#{ bootstrap_service_cidr.split(',')[1] }#" #% else %# - cluster-cidr: "{{ cluster_cidr }}" - service-cidr: "{{ service_cidr }}" + cluster-cidr: "#{ bootstrap_cluster_cidr }#" + service-cidr: "#{ bootstrap_service_cidr }#" #% endif %# etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack kube-controller-manager-arg: diff --git a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 index 1a26992a..61d5fc84 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 @@ -16,7 +16,7 @@ block: - name: Locale | Set timezone community.general.timezone: - name: "{{ timezone | default('Etc/UTC') }}" + name: "#{ bootstrap_timezone }#" - name: Packages block: @@ -33,7 +33,7 @@ - name: User Configuration | SSH keys ansible.posix.authorized_key: user: "{{ ansible_user }}" - key: "https://github.com/{{ github_username }}.keys" + key: "https://github.com/#{ bootstrap_github_username }#.keys" - name: User Configuration | Silence login ansible.builtin.file: dest: "{{ '/home/' + ansible_user if ansible_user != 'root' else '/root' }}/.hushlogin" diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 similarity index 83% rename from bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 rename to bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 index 9d655485..9d269d74 100644 --- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 +++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 @@ -7,10 +7,10 @@ metadata: name: cilium namespace: kube-system spec: - # renovate: datasource=helm repo: https://helm.cilium.io/ chart: cilium - version: 1.14.6 + # renovate: datasource=github-releases depName=cilium/cilium + version: "1.14.5" targetNamespace: kube-system bootstrap: true valuesContent: |- diff --git a/bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2 b/bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2 new file mode 100644 index 00000000..38c9f29d --- /dev/null +++ b/bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2 @@ -0,0 +1,77 @@ +#% if bootstrap_distribution == 'k3s' %# +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-vip + namespace: kube-system + labels: + app.kubernetes.io/name: kube-vip +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip + template: + metadata: + labels: + app.kubernetes.io/name: kube-vip + spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.6.4 + imagePullPolicy: IfNotPresent + args: ["manager"] + env: + - name: address + value: "#{ bootstrap_kube_api_addr }#" + - name: vip_arp + value: "true" + - name: lb_enable + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: prometheus_server + value: :2112 + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"] + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + serviceAccountName: kube-vip + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists +#% endif %# diff --git a/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 deleted file mode 100644 index 6525b14f..00000000 --- a/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 +++ /dev/null @@ -1,61 +0,0 @@ -#% if bootstrap_distribution == 'k3s' %# ---- -apiVersion: v1 -kind: Pod -metadata: - name: kube-vip - namespace: kube-system - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip -spec: - containers: - - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.6.4 - imagePullPolicy: IfNotPresent - args: ["manager"] - env: - - name: address - value: "{{ kube_api_addr }}" - - name: vip_arp - value: "true" - - name: lb_enable - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "false" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: prometheus_server - value: :2112 - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - name: kubeconfig - hostPath: - path: /etc/rancher/k3s/k3s.yaml -#% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 index c345e4a9..6d778cd5 100644 --- a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 +++ b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 @@ -18,6 +18,7 @@ spec: apply: before: - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-system.sh | INSTALL_K0S_HOSTNAME="#{ item.name }#" bash -s - + - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-kube-vip.sh | bash -s - reset: before: - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-cilium.sh | bash -s - diff --git a/bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2 new file mode 100755 index 00000000..a7bddc24 --- /dev/null +++ b/bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2 @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -e +set -o noglob + +[ $(id -u) -eq 0 ] || exec sudo $0 $@ + +# Create directory +mkdir -p /var/lib/k0s/manifests/kube-vip + +# Create kube-vip rbac +cat < /var/lib/k0s/manifests/kube-vip/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system +EOF + +cat < /var/lib/k0s/manifests/kube-vip/ds.yaml +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-vip + namespace: kube-system + labels: + app.kubernetes.io/name: kube-vip +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip + template: + metadata: + labels: + app.kubernetes.io/name: kube-vip + spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.6.4 + imagePullPolicy: IfNotPresent + args: ["manager"] + env: + - name: address + value: "#{ bootstrap_kube_api_addr }#" + - name: vip_arp + value: "true" + - name: lb_enable + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: prometheus_server + value: :2112 + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"] + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + serviceAccountName: kube-vip + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists +EOF diff --git a/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 index 1f119260..d7bdddab 100644 --- a/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2 @@ -5,7 +5,7 @@ kind: Kustomization helmCharts: - name: cilium repo: https://helm.cilium.io - version: 1.14.6 + version: 1.14.5 releaseName: cilium namespace: kube-system valuesFile: values.yaml diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index 3f9344d5..45f63e33 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -39,18 +39,18 @@ bootstrap_cloudflare_tunnel_id: # CIDR your nodes are on (e.g. 192.168.1.0/24) bootstrap_node_cidr: -# The IP address of the Kube API, choose an available IP in your nodes network that is not being used -# NOTE: For k0s set this to a IP you can then later use to manually setup a load balancer for the Kube API (e.g. Haproxy or Keepalived) -# Ref: https://github.com/k0sproject/k0s/issues/1150 + +# The IP address of the Kube API, choose an available IP in your nodes network that is not being used. This is announced over L2. bootstrap_kube_api_addr: +# The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used. This is announced over L2. +bootstrap_k8s_gateway_addr: +# The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used. This is announced over L2. +bootstrap_external_ingress_addr: +# The Load balancer IP for internal ingress, choose an available IP in your nodes network that is not being used. This is announced over L2. +bootstrap_internal_ingress_addr: + # Add additional SANs to the Kube API cert, this is useful if you want to call the Kube API by hostname rather than IP bootstrap_kube_additional_tls_sans: [] -# The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used -bootstrap_k8s_gateway_addr: -# The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used -bootstrap_external_ingress_addr: -# The Load balancer IP for internal ingress, choose an available IP in your nodes network that is not being used -bootstrap_internal_ingress_addr: # The DNS server to use for the cluster, this can be an existing local DNS server or a public one # If using a local DNS server make sure it meets the following requirements: