diff --git a/.taskfiles/AnsibleTasks.yaml b/.taskfiles/AnsibleTasks.yaml index 055776ad..b2bcac67 100644 --- a/.taskfiles/AnsibleTasks.yaml +++ b/.taskfiles/AnsibleTasks.yaml @@ -17,34 +17,44 @@ tasks: - pip3 install --user --requirement requirements.txt - ansible-galaxy install -r requirements.yaml --roles-path ~/.ansible/roles --force - ansible-galaxy collection install -r requirements.yaml --collections-path ~/.ansible/collections --force + preconditions: + - test -f "{{.ANSIBLE_DIR}}/requirements.txt" + - test -f "{{.ANSIBLE_DIR}}/requirements.yaml" + env: + ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" list: desc: List all the hosts dir: "{{.ANSIBLE_DIR}}" - cmds: - - ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --list-hosts + cmd: ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --list-hosts + env: + ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" prepare: desc: Prepare all the k8s nodes for running k3s dir: "{{.ANSIBLE_DIR}}" - cmds: - - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-prepare.yaml + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-prepare.yaml env: ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" install: desc: Install Kubernetes on the nodes dir: "{{.ANSIBLE_DIR}}" - cmds: - - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-installation.yaml + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-installation.yaml env: ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" - upgrade: - desc: Upgrade Kubernetes on the nodes (currently this only upgrades kube-vip) + rollout-update: + desc: Preform operating system updates and rollout restart the cluster dir: "{{.ANSIBLE_DIR}}" - cmds: - - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-upgrade.yaml + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-rollout-update.yaml + env: + ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" + + kube-vip: + desc: Apply new kube-vip configuration to the cluster + dir: "{{.ANSIBLE_DIR}}" + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-kube-vip.yaml env: ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" @@ -61,31 +71,27 @@ tasks: ping: desc: Ping all the hosts dir: "{{.ANSIBLE_DIR}}" - cmds: - - ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -m 'ping' + cmd: ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -m 'ping' env: ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" uptime: desc: Uptime of all the hosts dir: "{{.ANSIBLE_DIR}}" - cmds: - - ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -a 'uptime' + cmd: ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml --one-line -a 'uptime' env: ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" force-reboot: desc: Reboot all the k8s nodes dir: "{{.ANSIBLE_DIR}}" - cmds: - - ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-reboot.yaml + cmd: ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml {{.ANSIBLE_PLAYBOOK_DIR}}/cluster-reboot.yaml env: ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" force-poweroff: desc: Shutdown all the k8s nodes dir: "{{.ANSIBLE_DIR}}" - cmds: - - ansible kubernetes -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml -a '/usr/bin/systemctl poweroff' --become + cmd: ansible kubernetes -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yaml -a '/usr/bin/systemctl poweroff' --become env: ANSIBLE_CONFIG: "{{.ANSIBLE_DIR}}/ansible.cfg" diff --git a/.taskfiles/ClusterTasks.yaml b/.taskfiles/ClusterTasks.yaml index 91dd64af..50802c85 100644 --- a/.taskfiles/ClusterTasks.yaml +++ b/.taskfiles/ClusterTasks.yaml @@ -5,8 +5,7 @@ tasks: verify: desc: Verify flux meets the prerequisites - cmds: - - flux check --pre + cmd: flux check --pre install: desc: Install Flux into your cluster @@ -25,8 +24,7 @@ tasks: reconcile: desc: Force update Flux to pull in changes from your Git repository - cmds: - - flux reconcile -n flux-system kustomization cluster --with-source + cmd: flux reconcile -n flux-system kustomization cluster --with-source hr-restart: desc: Restart all failed Helm Releases @@ -36,33 +34,27 @@ tasks: nodes: desc: List all the nodes in your cluster - cmds: - - kubectl get nodes {{.CLI_ARGS | default "-o wide"}} + cmd: kubectl get nodes {{.CLI_ARGS | default "-o wide"}} pods: desc: List all the pods in your cluster - cmds: - - kubectl get pods {{.CLI_ARGS | default "-A"}} + cmd: kubectl get pods {{.CLI_ARGS | default "-A"}} kustomizations: desc: List all the kustomizations in your cluster - cmds: - - kubectl get kustomizations {{.CLI_ARGS | default "-A"}} + cmd: kubectl get kustomizations {{.CLI_ARGS | default "-A"}} helmreleases: desc: List all the helmreleases in your cluster - cmds: - - kubectl get helmreleases {{.CLI_ARGS | default "-A"}} + cmd: kubectl get helmreleases {{.CLI_ARGS | default "-A"}} helmrepositories: desc: List all the helmrepositories in your cluster - cmds: - - kubectl get helmrepositories {{.CLI_ARGS | default "-A"}} + cmd: kubectl get helmrepositories {{.CLI_ARGS | default "-A"}} gitrepositories: desc: List all the gitrepositories in your cluster - cmds: - - kubectl get gitrepositories {{.CLI_ARGS | default "-A"}} + cmd: kubectl get gitrepositories {{.CLI_ARGS | default "-A"}} certificates: desc: List all the certificates in your cluster @@ -72,8 +64,7 @@ tasks: ingresses: desc: List all the ingresses in your cluster - cmds: - - kubectl get ingress {{.CLI_ARGS | default "-A"}} + cmd: kubectl get ingress {{.CLI_ARGS | default "-A"}} resources: desc: Gather common resources in your cluster, useful when asking for support diff --git a/ansible/playbooks/cluster-upgrade.yaml b/ansible/playbooks/cluster-kube-vip.yaml similarity index 95% rename from ansible/playbooks/cluster-upgrade.yaml rename to ansible/playbooks/cluster-kube-vip.yaml index 4c7dfd13..d6fe862b 100644 --- a/ansible/playbooks/cluster-upgrade.yaml +++ b/ansible/playbooks/cluster-kube-vip.yaml @@ -1,5 +1,5 @@ --- -- name: Cluster Upgrade +- name: Cluster kube-vip hosts: master become: true gather_facts: true diff --git a/ansible/playbooks/cluster-rollout-update.yaml b/ansible/playbooks/cluster-rollout-update.yaml new file mode 100644 index 00000000..e307f10a --- /dev/null +++ b/ansible/playbooks/cluster-rollout-update.yaml @@ -0,0 +1,76 @@ +--- +# https://github.com/kevincoakley/ansible-role-k8s-rolling-update +- name: Cluster rollout update + hosts: all + become: true + gather_facts: true + any_errors_fatal: true + serial: 1 + pre_tasks: + - name: Pausing for 5 seconds... + ansible.builtin.pause: + seconds: 5 + tasks: + - name: Get the node's details + ansible.builtin.command: | + kubectl get node {{ inventory_hostname }} -o json + register: kubectl_get_node + delegate_to: "{{ groups['master'][0] }}" + failed_when: false + changed_when: false + + - name: Update node + when: + # When status.conditions[x].type == Ready then check stats.conditions[x].status for True|False + - kubectl_get_node['stdout'] | from_json | json_query("status.conditions[?type == 'Ready'].status") + # If spec.unschedulable is defined then the node is cordoned + - not (kubectl_get_node['stdout'] | from_json).spec.unschedulable is defined + block: + - name: Cordon node + ansible.builtin.command: | + kubectl cordon {{ inventory_hostname }} + delegate_to: "{{ groups['master'][0] }}" + changed_when: false + + - name: Wait for node to cordon + ansible.builtin.command: | + kubectl get node {{ inventory_hostname }} -o json + register: wait_for_cordon + retries: 10 + delay: 10 + delegate_to: "{{ groups['master'][0] }}" + changed_when: false + until: (wait_for_cordon['stdout'] | from_json).spec.unschedulable + + - name: Cordon & drain node + ansible.builtin.command: | + kubectl drain --ignore-daemonsets --delete-emptydir-data --force {{ inventory_hostname }} + delegate_to: "{{ groups['master'][0] }}" + changed_when: false + + - name: Update all packages + ansible.builtin.apt: + upgrade: dist + update_cache: true + + - name: Reboot + ansible.builtin.reboot: + msg: Rebooting node + post_reboot_delay: 60 + reboot_timeout: 3600 + + - name: Uncordon node + ansible.builtin.command: | + kubectl uncordon {{ inventory_hostname }} + delegate_to: "{{ groups['master'][0] }}" + changed_when: false + + - name: Wait for node to uncordon + ansible.builtin.command: | + kubectl get node {{ inventory_hostname }} -o json + register: wait_for_uncordon + retries: 10 + delay: 10 + delegate_to: "{{ groups['master'][0] }}" + changed_when: false + until: not (kubectl_get_node['stdout'] | from_json).spec.unschedulable is defined