From 88decb875fe38406674e907f5ea3a469c7cf79fd Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Fri, 9 Dec 2022 11:58:51 -0500 Subject: [PATCH] feat: add resource readiness check to cluster-installation playbook (#547) --- .../inventory/group_vars/kubernetes/k3s.yml | 18 +++++++ .../inventory/group_vars/kubernetes/os.yml | 4 ++ .../playbooks/cluster-installation.yml | 48 ++++++++++++++++--- provision/ansible/requirements.yml | 2 + 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/provision/ansible/inventory/group_vars/kubernetes/k3s.yml b/provision/ansible/inventory/group_vars/kubernetes/k3s.yml index 06e24e39..6a33136f 100644 --- a/provision/ansible/inventory/group_vars/kubernetes/k3s.yml +++ b/provision/ansible/inventory/group_vars/kubernetes/k3s.yml @@ -28,9 +28,27 @@ k3s_registration_address: "{{ kubevip_address }}" # (list) A list of URLs to deploy on the primary control plane. Read notes below. k3s_server_manifests_urls: + # Tigera Operator # https://github.com/projectcalico/calico/issues/7003 - url: https://raw.githubusercontent.com/projectcalico/calico/v3.24.2/manifests/tigera-operator.yaml filename: tigera-operator.yaml + # Prometheus Operator + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml + filename: monitoring.coreos.com_alertmanagerconfigs.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml + filename: monitoring.coreos.com_alertmanagers.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml + filename: monitoring.coreos.com_podmonitors.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml + filename: monitoring.coreos.com_probes.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml + filename: monitoring.coreos.com_prometheuses.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml + filename: monitoring.coreos.com_prometheusrules.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml + filename: monitoring.coreos.com_servicemonitors.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml + filename: monitoring.coreos.com_thanosrulers.yaml # (list) A flat list of templates to deploy on the primary control plane # /var/lib/rancher/k3s/server/manifests diff --git a/provision/ansible/inventory/group_vars/kubernetes/os.yml b/provision/ansible/inventory/group_vars/kubernetes/os.yml index 44262473..29bbf751 100644 --- a/provision/ansible/inventory/group_vars/kubernetes/os.yml +++ b/provision/ansible/inventory/group_vars/kubernetes/os.yml @@ -17,7 +17,9 @@ fedora: - nano - nvme-cli - socat + - python3-kubernetes - python3-libselinux + - python3-pyyaml ubuntu: # disable_ufw: true @@ -31,3 +33,5 @@ ubuntu: - nano - nvme-cli - socat + - python3-kubernetes + - python3-yaml diff --git a/provision/ansible/playbooks/cluster-installation.yml b/provision/ansible/playbooks/cluster-installation.yml index 51fecb71..bf1ddc96 100644 --- a/provision/ansible/playbooks/cluster-installation.yml +++ b/provision/ansible/playbooks/cluster-installation.yml @@ -57,6 +57,47 @@ regexp: "https://127.0.0.1:6443" replace: "https://{{ k3s_registration_address }}:6443" + - name: Resource Readiness Check + run_once: true + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + kind: "{{ item.kind }}" + name: "{{ item.name }}" + namespace: "{{ item.namespace | default('') }}" + wait: true + wait_sleep: 10 + wait_timeout: 360 + loop: + - kind: Deployment + name: tigera-operator + namespace: tigera-operator + - kind: DaemonSet + name: kube-vip + namespace: kube-system + - kind: Installation + name: default + - kind: CustomResourceDefinition + name: alertmanagerconfigs.monitoring.coreos.com + - kind: CustomResourceDefinition + name: alertmanagers.monitoring.coreos.com + - kind: CustomResourceDefinition + name: podmonitors.monitoring.coreos.com + - kind: CustomResourceDefinition + name: probes.monitoring.coreos.com + - kind: CustomResourceDefinition + name: prometheuses.monitoring.coreos.com + - kind: CustomResourceDefinition + name: prometheusrules.monitoring.coreos.com + - kind: CustomResourceDefinition + name: servicemonitors.monitoring.coreos.com + - kind: CustomResourceDefinition + name: thanosrulers.monitoring.coreos.com + when: + - k3s_server_manifests_templates | length > 0 + or k3s_server_manifests_urls | length > 0 + - k3s_control_node is defined + - k3s_control_node + # Cleaning up the manifests from the /var/lib/rancher/k3s/server/manifests # directory is needed because k3s has an awesome "feature" to always deploy # these on restarting the k3s systemd service. Removing them does NOT @@ -65,13 +106,6 @@ # Removing them means we can manage the lifecycle of these components # outside of the /var/lib/rancher/k3s/server/manifests directory - # FIXME(ansible): Check for deployments to be happy rather than waiting - - name: Wait for k3s to finish installing the deployed manifests - ansible.builtin.wait_for: - timeout: 15 - when: k3s_server_manifests_templates | length > 0 - or k3s_server_manifests_dir | length > 0 - - name: Remove deployed manifest templates ansible.builtin.file: path: "{{ k3s_server_manifests_dir }}/{{ item | basename | regex_replace('\\.j2$', '') }}" diff --git a/provision/ansible/requirements.yml b/provision/ansible/requirements.yml index 690fc869..5ebb2c40 100644 --- a/provision/ansible/requirements.yml +++ b/provision/ansible/requirements.yml @@ -8,6 +8,8 @@ collections: version: 1.4.0 - name: ansible.utils version: 2.8.0 + - name: kubernetes.core + version: 2.3.2 roles: - name: xanmanning.k3s src: https://github.com/PyratLabs/ansible-role-k3s.git