feat: cluster rollout update playbook & use new cmd for taskfiles (#829)

.taskfiles/AnsibleTasks.yaml

@@ -17,34 +17,44 @@ tasks:
       - pip3 install --user --requirement requirements.txt
       - ansible-galaxy install -r requirements.yaml --roles-path ~/.ansible/roles --force
       - ansible-galaxy collection install -r requirements.yaml --collections-path ~/.ansible/collections --force
+    preconditions:
+      - test -f "{{.ANSIBLE_DIR}}/requirements.txt"
+      - test -f "{{.ANSIBLE_DIR}}/requirements.yaml"
+    env:
+      ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"
 
   list:
     desc: List all the hosts
     dir: "{{.ANSIBLE_DIR}}"
-    cmds:
-      - ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --list-hosts
+    cmd: ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --list-hosts
+    env:
+      ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"
 
   prepare:
     desc: Prepare all the k8s nodes for running k3s
     dir: "{{.ANSIBLE_DIR}}"
-    cmds:
-      - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-prepare.yaml
+    cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-prepare.yaml
     env:
       ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"
 
   install:
     desc: Install Kubernetes on the nodes
     dir: "{{.ANSIBLE_DIR}}"
-    cmds:
-      - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-installation.yaml
+    cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-installation.yaml
     env:
       ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"
 
-  upgrade:
-    desc: Upgrade Kubernetes on the nodes (currently this only upgrades kube-vip)
+  rollout-update:
+    desc: Preform operating system updates and rollout restart the cluster
     dir: "{{.ANSIBLE_DIR}}"
-    cmds:
-      - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-upgrade.yaml
+    cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-rollout-update.yaml
+    env:
+      ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"
+
+  kube-vip:
+    desc: Apply new kube-vip configuration to the cluster
+    dir: "{{.ANSIBLE_DIR}}"
+    cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-kube-vip.yaml
     env:
       ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"
 
@@ -61,31 +71,27 @@ tasks:
   ping:
     desc: Ping all the hosts
     dir: "{{.ANSIBLE_DIR}}"
-    cmds:
-      - ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -m 'ping'
+    cmd: ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -m 'ping'
     env:
       ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"
 
   uptime:
     desc: Uptime of all the hosts
     dir: "{{.ANSIBLE_DIR}}"
-    cmds:
-      - ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -a 'uptime'
+    cmd: ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -a 'uptime'
     env:
       ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"
 
   force-reboot:
     desc: Reboot all the k8s nodes
     dir: "{{.ANSIBLE_DIR}}"
-    cmds:
-      - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-reboot.yaml
+    cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-reboot.yaml
     env:
       ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"
 
   force-poweroff:
     desc: Shutdown all the k8s nodes
     dir: "{{.ANSIBLE_DIR}}"
-    cmds:
-      - ansible kubernetes -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml -a '/usr/bin/systemctl poweroff' --become
+    cmd: ansible kubernetes -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml -a '/usr/bin/systemctl poweroff' --become
     env:
       ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg"

.taskfiles/ClusterTasks.yaml

@@ -5,8 +5,7 @@ tasks:
 
   verify:
     desc: Verify flux meets the prerequisites
-    cmds:
-      - flux check --pre
+    cmd: flux check --pre
 
   install:
     desc: Install Flux into your cluster
@@ -25,8 +24,7 @@ tasks:
 
   reconcile:
     desc: Force update Flux to pull in changes from your Git repository
-    cmds:
-      - flux reconcile -n flux-system kustomization cluster --with-source
+    cmd: flux reconcile -n flux-system kustomization cluster --with-source
 
   hr-restart:
     desc: Restart all failed Helm Releases
@@ -36,33 +34,27 @@ tasks:
 
   nodes:
     desc: List all the nodes in your cluster
-    cmds:
-      - kubectl get nodes {{.CLI_ARGS | default "-o wide"}}
+    cmd: kubectl get nodes {{.CLI_ARGS | default "-o wide"}}
 
   pods:
     desc: List all the pods in your cluster
-    cmds:
-      - kubectl get pods {{.CLI_ARGS | default "-A"}}
+    cmd: kubectl get pods {{.CLI_ARGS | default "-A"}}
 
   kustomizations:
     desc: List all the kustomizations in your cluster
-    cmds:
-      - kubectl get kustomizations {{.CLI_ARGS | default "-A"}}
+    cmd: kubectl get kustomizations {{.CLI_ARGS | default "-A"}}
 
   helmreleases:
     desc: List all the helmreleases in your cluster
-    cmds:
-      - kubectl get helmreleases {{.CLI_ARGS | default "-A"}}
+    cmd: kubectl get helmreleases {{.CLI_ARGS | default "-A"}}
 
   helmrepositories:
     desc: List all the helmrepositories in your cluster
-    cmds:
-      - kubectl get helmrepositories {{.CLI_ARGS | default "-A"}}
+    cmd: kubectl get helmrepositories {{.CLI_ARGS | default "-A"}}
 
   gitrepositories:
     desc: List all the gitrepositories in your cluster
-    cmds:
-      - kubectl get gitrepositories {{.CLI_ARGS | default "-A"}}
+    cmd: kubectl get gitrepositories {{.CLI_ARGS | default "-A"}}
 
   certificates:
     desc: List all the certificates in your cluster
@@ -72,8 +64,7 @@ tasks:
 
   ingresses:
     desc: List all the ingresses in your cluster
-    cmds:
-      - kubectl get ingress {{.CLI_ARGS | default "-A"}}
+    cmd: kubectl get ingress {{.CLI_ARGS | default "-A"}}
 
   resources:
     desc: Gather common resources in your cluster, useful when asking for support

ansible/playbooks/cluster-kube-vip.yaml

@@ -1,5 +1,5 @@
 ---
-- name: Cluster Upgrade
+- name: Cluster kube-vip
   hosts: master
   become: true
   gather_facts: true

ansible/playbooks/cluster-rollout-update.yaml

@@ -0,0 +1,76 @@
+---
+# https://github.com/kevincoakley/ansible-role-k8s-rolling-update
+- name: Cluster rollout update
+  hosts: all
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  serial: 1
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Get the node's details
+      ansible.builtin.command: |
+        kubectl get node {{ inventory_hostname }} -o json
+      register: kubectl_get_node
+      delegate_to: "{{ groups['master'][0] }}"
+      failed_when: false
+      changed_when: false
+
+    - name: Update node
+      when:
+        # When status.conditions[x].type == Ready then check stats.conditions[x].status for True|False
+        - kubectl_get_node['stdout'] | from_json | json_query("status.conditions[?type == 'Ready'].status")
+        # If spec.unschedulable is defined then the node is cordoned
+        - not (kubectl_get_node['stdout'] | from_json).spec.unschedulable is defined
+      block:
+        - name: Cordon node
+          ansible.builtin.command: |
+            kubectl cordon {{ inventory_hostname }}
+          delegate_to: "{{ groups['master'][0] }}"
+          changed_when: false
+
+        - name: Wait for node to cordon
+          ansible.builtin.command: |
+            kubectl get node {{ inventory_hostname }} -o json
+          register: wait_for_cordon
+          retries: 10
+          delay: 10
+          delegate_to: "{{ groups['master'][0] }}"
+          changed_when: false
+          until: (wait_for_cordon['stdout'] | from_json).spec.unschedulable
+
+        - name: Cordon & drain node
+          ansible.builtin.command: |
+            kubectl drain --ignore-daemonsets --delete-emptydir-data --force {{ inventory_hostname }}
+          delegate_to: "{{ groups['master'][0] }}"
+          changed_when: false
+
+        - name: Update all packages
+          ansible.builtin.apt:
+            upgrade: dist
+            update_cache: true
+
+        - name: Reboot
+          ansible.builtin.reboot:
+            msg: Rebooting node
+            post_reboot_delay: 60
+            reboot_timeout: 3600
+
+        - name: Uncordon node
+          ansible.builtin.command: |
+            kubectl uncordon {{ inventory_hostname }}
+          delegate_to: "{{ groups['master'][0] }}"
+          changed_when: false
+
+        - name: Wait for node to uncordon
+          ansible.builtin.command: |
+            kubectl get node {{ inventory_hostname }} -o json
+          register: wait_for_uncordon
+          retries: 10
+          delay: 10
+          delegate_to: "{{ groups['master'][0] }}"
+          changed_when: false
+          until: not (kubectl_get_node['stdout'] | from_json).spec.unschedulable is defined