diff --git a/bootstrap/configure.yaml b/bootstrap/configure.yaml index 33bfff27..9b3d9cb8 100644 --- a/bootstrap/configure.yaml +++ b/bootstrap/configure.yaml @@ -17,6 +17,12 @@ ansible.builtin.set_fact: repository_path: "{{ repository.stdout }}" + - name: Override kube-vip address when there is a single master node and no address is defined + when: bootstrap_nodes.master | length == 1 and not bootstrap_kube_vip_addr + ansible.builtin.set_fact: + bootstrap_kube_vip_enabled: false + bootstrap_kube_vip_addr: "{{ bootstrap_nodes.master[0].address }}" + - name: Verify configuration ansible.builtin.include_tasks: tasks/validation/main.yaml diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml index 81bec16f..7128b737 100644 --- a/bootstrap/tasks/validation/net.yaml +++ b/bootstrap/tasks/validation/net.yaml @@ -1,6 +1,6 @@ --- - name: Set reachable address - set_fact: + ansible.builtin.set_fact: current_address: "{{ item.external_address | default(item.address) }}" loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" loop_control: @@ -122,7 +122,19 @@ success_msg: All IP addresses are unique. fail_msg: All IP addresses are not unique. +- name: Verify nodes are not the same IPs as k8s_gateway or ingress external/internal + when: not bootstrap_kube_vip_enabled | default(true) + ansible.builtin.assert: + that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr) + success_msg: Node address {{ item.address }} is different than k8s_gateway or ingress-nginx. + fail_msg: Node address {{ item.address }} is not different than k8s_gateway or ingress-nginx. + quiet: true + loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" + loop_control: + label: "{{ item.address }}" + - name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kube-vip + when: bootstrap_kube_vip_enabled | default(true) ansible.builtin.assert: that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kube_vip_addr) success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip. @@ -153,6 +165,34 @@ loop_control: label: "{{ item.address }}" +- name: Verify node IP addresses are unique + ansible.builtin.assert: + that: > + ( + (bootstrap_nodes.master + bootstrap_nodes.worker | default([])) | map(attribute='address') | list + ) | unique | length + == + ( + (bootstrap_nodes.master + bootstrap_nodes.worker | default([])) | map(attribute='address') | list + ) | length + success_msg: All node IP addresses are unique. + fail_msg: All node IP addresses are not unique. + quiet: true + +- name: Verify node names are unique + ansible.builtin.assert: + that: > + ( + (bootstrap_nodes.master + bootstrap_nodes.worker | default([])) | map(attribute='name') | list + ) | unique | length + == + ( + (bootstrap_nodes.master + bootstrap_nodes.worker | default([])) | map(attribute='name') | list + ) | length + success_msg: All node names are unique. + fail_msg: All node names are not unique. + quiet: true + - name: Verify SSH port is reachable ansible.builtin.wait_for: host: "{{ current_address }}" diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 index 645df885..073cc199 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 @@ -61,7 +61,7 @@ data: regex: (apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket) kubeControllerManager: enabled: true - endpoints: + endpoints: &endpoints {% for item in bootstrap_nodes.master %} - {{ item.address }} {% endfor %} @@ -72,17 +72,15 @@ data: sourceLabels: ["__name__"] regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|attachdetach_controller|authenticated_user|authentication|cronjob_controller|disabled_metric|endpoint_slice|ephemeral_volume|garbagecollector_controller|get_token|go|hidden_metric|job_controller|kubernetes_build|kubernetes_feature|leader_election|node_collector|node_ipam|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|pv_collector|registered_metric|replicaset_controller|rest_client|retroactive_storageclass|root_ca|running_managed|scrape_duration|scrape_samples|scrape_series|service_controller|storage_count|storage_operation|ttl_after|volume_operation|workqueue)_(.+)" kubeEtcd: + {% if bootstrap_nodes.master | length > 1 %} enabled: true - endpoints: - {% for item in bootstrap_nodes.master %} - - {{ item.address }} - {% endfor %} + {% else %} + enabled: false + {% endif %} + endpoints: *endpoints kubeScheduler: enabled: true - endpoints: - {% for item in bootstrap_nodes.master %} - - {{ item.address }} - {% endfor %} + endpoints: *endpoints serviceMonitor: metricRelabelings: # Remove duplicate labels provided by k3s diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 index 3e6c62d0..69591d24 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 @@ -9,13 +9,18 @@ k3s_release_version: "v1.29.0+k3s1" k3s_install_hard_links: true k3s_become: true +{% if bootstrap_nodes.master | length > 1 %} k3s_etcd_datastore: true -k3s_use_unsupported_config: true +{% else %} +k3s_etcd_datastore: false +{% endif %} k3s_registration_address: "{% raw %}{{ kube_vip_addr }}{% endraw %}" k3s_server_manifests_urls: + {% if bootstrap_kube_vip_enabled | default(true) %} # Kube-vip RBAC - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml filename: kube-vip-rbac.yaml + {% endif %} # Essential Prometheus Operator CRDs (the rest are installed with the kube-prometheus-stack helm release) - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml filename: custom-prometheus-podmonitors.yaml @@ -32,6 +37,8 @@ k3s_server_manifests_templates: - custom-cilium-l2.yaml.j2 {% endif %} - custom-coredns-helmchart.yaml.j2 +{% if bootstrap_kube_vip_enabled | default(true) %} # /var/lib/rancher/k3s/agent/pod-manifests k3s_server_pod_manifests_templates: - kube-vip-static-pod.yaml.j2 +{% endif %} diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index 73a595a9..597f9a57 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -39,6 +39,7 @@ bootstrap_cloudflare_tunnel_id: # CIDR your nodes are on (e.g. 192.168.1.0/24) bootstrap_node_cidr: # The IP address to use with kube-vip, choose an available IP in your nodes network that is not being used +# (Optional) Leave this blank if you are deploying a single master node, this will disable kube-vip and use the master node IP instead bootstrap_kube_vip_addr: # The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used bootstrap_k8s_gateway_addr: