feat: add resource readiness check to cluster-installation playbook (#547)

provision/ansible/inventory/group_vars/kubernetes/k3s.yml

@@ -28,9 +28,27 @@ k3s_registration_address: "{{ kubevip_address }}"
 
 # (list) A list of URLs to deploy on the primary control plane. Read notes below.
 k3s_server_manifests_urls:
+  # Tigera Operator
   # https://github.com/projectcalico/calico/issues/7003
   - url: https://raw.githubusercontent.com/projectcalico/calico/v3.24.2/manifests/tigera-operator.yaml
     filename: tigera-operator.yaml
+  # Prometheus Operator
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+    filename: monitoring.coreos.com_alertmanagerconfigs.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+    filename: monitoring.coreos.com_alertmanagers.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+    filename: monitoring.coreos.com_podmonitors.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+    filename: monitoring.coreos.com_probes.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+    filename: monitoring.coreos.com_prometheuses.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+    filename: monitoring.coreos.com_prometheusrules.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+    filename: monitoring.coreos.com_servicemonitors.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+    filename: monitoring.coreos.com_thanosrulers.yaml
 
 # (list) A flat list of templates to deploy on the primary control plane
 # /var/lib/rancher/k3s/server/manifests

provision/ansible/inventory/group_vars/kubernetes/os.yml

@@ -17,7 +17,9 @@ fedora:
     - nano
     - nvme-cli
     - socat
+    - python3-kubernetes
     - python3-libselinux
+    - python3-pyyaml
 
 ubuntu:
   # disable_ufw: true
@@ -31,3 +33,5 @@ ubuntu:
     - nano
     - nvme-cli
     - socat
+    - python3-kubernetes
+    - python3-yaml

provision/ansible/playbooks/cluster-installation.yml

@@ -57,6 +57,47 @@
         regexp: "https://127.0.0.1:6443"
         replace: "https://{{ k3s_registration_address }}:6443"
 
+    - name: Resource Readiness Check
+      run_once: true
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        kind: "{{ item.kind }}"
+        name: "{{ item.name }}"
+        namespace: "{{ item.namespace | default('') }}"
+        wait: true
+        wait_sleep: 10
+        wait_timeout: 360
+      loop:
+        - kind: Deployment
+          name: tigera-operator
+          namespace: tigera-operator
+        - kind: DaemonSet
+          name: kube-vip
+          namespace: kube-system
+        - kind: Installation
+          name: default
+        - kind: CustomResourceDefinition
+          name: alertmanagerconfigs.monitoring.coreos.com
+        - kind: CustomResourceDefinition
+          name: alertmanagers.monitoring.coreos.com
+        - kind: CustomResourceDefinition
+          name: podmonitors.monitoring.coreos.com
+        - kind: CustomResourceDefinition
+          name: probes.monitoring.coreos.com
+        - kind: CustomResourceDefinition
+          name: prometheuses.monitoring.coreos.com
+        - kind: CustomResourceDefinition
+          name: prometheusrules.monitoring.coreos.com
+        - kind: CustomResourceDefinition
+          name: servicemonitors.monitoring.coreos.com
+        - kind: CustomResourceDefinition
+          name: thanosrulers.monitoring.coreos.com
+      when:
+        - k3s_server_manifests_templates | length > 0
+            or k3s_server_manifests_urls | length > 0
+        - k3s_control_node is defined
+        - k3s_control_node
+
     # Cleaning up the manifests from the /var/lib/rancher/k3s/server/manifests
     # directory is needed because k3s has an awesome "feature" to always deploy 
     # these on restarting the k3s systemd service. Removing them does NOT
@@ -65,13 +106,6 @@
     # Removing them means we can manage the lifecycle of these components 
     # outside of the /var/lib/rancher/k3s/server/manifests directory
 
-    # FIXME(ansible): Check for deployments to be happy rather than waiting
-    - name: Wait for k3s to finish installing the deployed manifests
-      ansible.builtin.wait_for:
-        timeout: 15
-      when: k3s_server_manifests_templates | length > 0
-        or k3s_server_manifests_dir | length > 0
-
     - name: Remove deployed manifest templates
       ansible.builtin.file:
         path: "{{ k3s_server_manifests_dir }}/{{ item | basename | regex_replace('\\.j2$', '') }}"

provision/ansible/requirements.yml

@@ -8,6 +8,8 @@ collections:
     version: 1.4.0
   - name: ansible.utils
     version: 2.8.0
+  - name: kubernetes.core
+    version: 2.3.2
 roles:
   - name: xanmanning.k3s
     src: https://github.com/PyratLabs/ansible-role-k3s.git