feat: add cluster-rollout-reboot playbook

Signed-off-by: Devin Buhl <devin@buhl.casa>

.envrc

@@ -2,4 +2,5 @@
 export KUBECONFIG=$(expand_path ./kubeconfig)
 export ANSIBLE_CONFIG=$(expand_path ./ansible.cfg)
 export ANSIBLE_HOST_KEY_CHECKING="False"
+export K8S_AUTH_KUBECONFIG=$(expand_path ./kubeconfig)
 export SOPS_AGE_KEY_FILE=$(expand_path ~/.config/sops/age/keys.txt)

.taskfiles/AnsibleTasks.yml

@@ -3,6 +3,7 @@ version: "3"
 
 env:
   ANSIBLE_CONFIG: "{{.PROJECT_DIR}}/ansible.cfg"
+  K8S_AUTH_KUBECONFIG: "{{.PROJECT_DIR}}/kubeconfig"
 
 vars:
   ANSIBLE_PLAYBOOK_DIR: "{{.ANSIBLE_DIR}}/playbooks"
@@ -41,7 +42,7 @@ tasks:
     interactive: true
     cmds:
       - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-nuke.yml
-      - task: reboot
+      - task: force-reboot
 
   ping:
     desc: Ping all the hosts
@@ -55,13 +56,19 @@ tasks:
     cmds:
       - ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml --one-line -a 'uptime'
 
-  reboot:
+  rollout-reboot:
+    desc: Rollout a reboot across all the k8s nodes
+    dir: "{{.ANSIBLE_DIR}}"
+    cmds:
+      - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-rollout-reboot.yml
+
+  force-reboot:
     desc: Reboot all the k8s nodes
     dir: "{{.ANSIBLE_DIR}}"
     cmds:
       - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-reboot.yml
 
-  poweroff:
+  force-poweroff:
     desc: Shutdown all the k8s nodes
     dir: "{{.ANSIBLE_DIR}}"
     cmds:

ansible/playbooks/cluster-rollout-reboot.yml

@@ -0,0 +1,32 @@
+---
+- name: Cluster Rollout Reboot
+  hosts:
+    - master
+    - worker
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  serial: 1
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Draining node (this might take awhile)
+      kubernetes.core.k8s_drain:
+        state: drain
+        name: "{{ inventory_hostname }}"
+        delete_options:
+          delete_emptydir_data: true
+          ignore_daemonsets: true
+        force: true
+        terminate_grace_period: 900
+      notify: Reboot
+    - name: Uncordoning node
+      kubernetes.core.k8s_drain:
+        state: uncordon
+        name: "{{ inventory_hostname }}"
+
+  handlers:
+    - name: Reboot
+      ansible.builtin.reboot: