From 0e4bf5d49916eab95892b16e1cc23b66a0315e00 Mon Sep 17 00:00:00 2001
From: Devin Buhl <devin@buhl.casa>
Date: Mon, 9 Jan 2023 13:41:01 -0500
Subject: [PATCH] feat: add cluster-rollout-reboot playbook

Signed-off-by: Devin Buhl <devin@buhl.casa>
---
 .envrc                                       |  1 +
 .taskfiles/AnsibleTasks.yml                  | 13 ++++++--
 ansible/playbooks/cluster-rollout-reboot.yml | 32 ++++++++++++++++++++
 3 files changed, 43 insertions(+), 3 deletions(-)
 create mode 100644 ansible/playbooks/cluster-rollout-reboot.yml

diff --git a/.envrc b/.envrc
index 2b089196..76c51a41 100644
--- a/.envrc
+++ b/.envrc
@@ -2,4 +2,5 @@
 export KUBECONFIG=$(expand_path ./kubeconfig)
 export ANSIBLE_CONFIG=$(expand_path ./ansible.cfg)
 export ANSIBLE_HOST_KEY_CHECKING="False"
+export K8S_AUTH_KUBECONFIG=$(expand_path ./kubeconfig)
 export SOPS_AGE_KEY_FILE=$(expand_path ~/.config/sops/age/keys.txt)
diff --git a/.taskfiles/AnsibleTasks.yml b/.taskfiles/AnsibleTasks.yml
index e07ece62..9826259b 100644
--- a/.taskfiles/AnsibleTasks.yml
+++ b/.taskfiles/AnsibleTasks.yml
@@ -3,6 +3,7 @@ version: "3"
 
 env:
   ANSIBLE_CONFIG: "{{.PROJECT_DIR}}/ansible.cfg"
+  K8S_AUTH_KUBECONFIG: "{{.PROJECT_DIR}}/kubeconfig"
 
 vars:
   ANSIBLE_PLAYBOOK_DIR: "{{.ANSIBLE_DIR}}/playbooks"
@@ -41,7 +42,7 @@ tasks:
     interactive: true
     cmds:
       - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-nuke.yml
-      - task: reboot
+      - task: force-reboot
 
   ping:
     desc: Ping all the hosts
@@ -55,13 +56,19 @@ tasks:
     cmds:
       - ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml --one-line -a 'uptime'
 
-  reboot:
+  rollout-reboot:
+    desc: Rollout a reboot across all the k8s nodes
+    dir: "{{.ANSIBLE_DIR}}"
+    cmds:
+      - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-rollout-reboot.yml
+
+  force-reboot:
     desc: Reboot all the k8s nodes
     dir: "{{.ANSIBLE_DIR}}"
     cmds:
       - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-reboot.yml
 
-  poweroff:
+  force-poweroff:
     desc: Shutdown all the k8s nodes
     dir: "{{.ANSIBLE_DIR}}"
     cmds:
diff --git a/ansible/playbooks/cluster-rollout-reboot.yml b/ansible/playbooks/cluster-rollout-reboot.yml
new file mode 100644
index 00000000..46cbfba6
--- /dev/null
+++ b/ansible/playbooks/cluster-rollout-reboot.yml
@@ -0,0 +1,32 @@
+---
+- name: Cluster Rollout Reboot
+  hosts:
+    - master
+    - worker
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  serial: 1
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Draining node (this might take awhile)
+      kubernetes.core.k8s_drain:
+        state: drain
+        name: "{{ inventory_hostname }}"
+        delete_options:
+          delete_emptydir_data: true
+          ignore_daemonsets: true
+        force: true
+        terminate_grace_period: 900
+      notify: Reboot
+    - name: Uncordoning node
+      kubernetes.core.k8s_drain:
+        state: uncordon
+        name: "{{ inventory_hostname }}"
+
+  handlers:
+    - name: Reboot
+      ansible.builtin.reboot: