From 7fc4250ef2d8e3351cf0ba1b8b0800fda2fa84e2 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Fri, 26 Jan 2024 20:34:04 -0500 Subject: [PATCH] fix: update taskfiles and validation logic (#1264) Signed-off-by: Devin Buhl --- .taskfiles/Ansible/Taskfile.yaml | 32 +++++++++++++--------------- .taskfiles/Flux/Taskfile.yaml | 4 ++-- .taskfiles/K0s/Taskfile.yaml | 15 +++++++------ .taskfiles/Sops/Taskfile.yaml | 6 +++--- .taskfiles/Talos/Taskfile.yaml | 2 +- .taskfiles/Workstation/Taskfile.yaml | 11 +++++----- Taskfile.yaml | 12 +++++++---- bootstrap/scripts/validation.py | 2 +- 8 files changed, 44 insertions(+), 40 deletions(-) diff --git a/.taskfiles/Ansible/Taskfile.yaml b/.taskfiles/Ansible/Taskfile.yaml index 00e539d7..02322eaf 100644 --- a/.taskfiles/Ansible/Taskfile.yaml +++ b/.taskfiles/Ansible/Taskfile.yaml @@ -2,29 +2,27 @@ # yaml-language-server: $schema=https://taskfile.dev/schema.json version: "3" -env: - PATH: "{{.ROOT_DIR}}/.venv/bin:$PATH" - VIRTUAL_ENV: "{{.ROOT_DIR}}/.venv" - ANSIBLE_COLLECTIONS_PATH: "{{.ROOT_DIR}}/.venv/galaxy" - ANSIBLE_ROLES_PATH: "{{.ROOT_DIR}}/.venv/galaxy/ansible_roles" - ANSIBLE_VARS_ENABLED: "host_group_vars" - ANSIBLE_LOCALHOST_WARNING: "False" - ANSIBLE_INVENTORY_UNPARSED_WARNING: "False" - vars: ANSIBLE_LINT_FILE: "{{.ANSIBLE_DIR}}/.ansible-lint" ANSIBLE_INVENTORY_FILE: "{{.ANSIBLE_DIR}}/inventory/hosts.yaml" ANSIBLE_REQUIREMENTS_FILE: "{{.ANSIBLE_DIR}}/requirements.yaml" ANSIBLE_PIP_REQUIREMENTS_FILE: "{{.ANSIBLE_DIR}}/requirements.txt" +env: + ANSIBLE_COLLECTIONS_PATH: "{{.VIRTUAL_ENV}}/galaxy" + ANSIBLE_ROLES_PATH: "{{.VIRTUAL_ENV}}/galaxy/ansible_roles" + ANSIBLE_VARS_ENABLED: "host_group_vars" + ANSIBLE_LOCALHOST_WARNING: "False" + ANSIBLE_INVENTORY_UNPARSED_WARNING: "False" + tasks: deps: desc: Set up Ansible dependencies deps: [":workstation:venv"] cmds: - - .venv/bin/python3 -m pip install --upgrade --requirement "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}" - - .venv/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}} + - '{{.VIRTUAL_ENV}}/bin/python3 -m pip install --upgrade --requirement "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}"' + - '{{.VIRTUAL_ENV}}/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}}' preconditions: - { msg: "Missing Ansible requirements file", sh: "test -f {{.ANSIBLE_REQUIREMENTS_FILE}}" } - { msg: "Missing Pip requirements file", sh: "test -f {{.ANSIBLE_PIP_REQUIREMENTS_FILE}}" } @@ -44,7 +42,7 @@ tasks: playbook: Playbook to run (required) prompt: Run Ansible playbook '{{.playbook}}'... continue? deps: ["deps"] - cmd: .venv/bin/ansible-playbook --inventory {{.ANSIBLE_INVENTORY_FILE}} {{.ANSIBLE_DIR}}/playbooks/{{.playbook}}.yaml {{.CLI_ARGS}} + cmd: "{{.VIRTUAL_ENV}}/bin/ansible-playbook --inventory {{.ANSIBLE_INVENTORY_FILE}} {{.ANSIBLE_DIR}}/playbooks/{{.playbook}}.yaml {{.CLI_ARGS}}" requires: vars: ["playbook"] preconditions: @@ -53,35 +51,35 @@ tasks: poweroff: desc: Shutdown all the k8s nodes deps: ["deps"] - cmd: .venv/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} -a '/usr/bin/systemctl poweroff' --become + cmd: "{{.VIRTUAL_ENV}}/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} -a '/usr/bin/systemctl poweroff' --become" preconditions: - { msg: "Missing Ansible inventory file", sh: "test -f {{.ANSIBLE_INVENTORY_FILE}}" } list: desc: List all the hosts deps: ["deps"] - cmd: .venv/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} --list-hosts + cmd: "{{.VIRTUAL_ENV}}/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} --list-hosts" preconditions: - { msg: "Missing Ansible inventory file", sh: "test -f {{.ANSIBLE_INVENTORY_FILE}}" } ping: desc: Ping all the hosts deps: ["deps"] - cmd: .venv/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} --one-line -m 'ping' + cmd: "{{.VIRTUAL_ENV}}/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} --one-line -m 'ping'" preconditions: - { msg: "Missing Ansible inventory file", sh: "test -f {{.ANSIBLE_INVENTORY_FILE}}" } uptime: desc: Uptime of all the hosts deps: ["deps"] - cmd: .venv/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} --one-line -a 'uptime' + cmd: "{{.VIRTUAL_ENV}}/bin/ansible kubernetes --inventory {{.ANSIBLE_INVENTORY_FILE}} --one-line -a 'uptime'" preconditions: - { msg: "Missing Ansible inventory file", sh: "test -f {{.ANSIBLE_INVENTORY_FILE}}" } lint: desc: Lint Ansible deps: ["deps"] - cmd: ./.venv/bin/ansible-lint --config-file {{.ANSIBLE_LINT_FILE}} {{.ANSIBLE_DIR}}/**/*.yaml + cmd: "{{.VIRTUAL_ENV}}/bin/ansible-lint --config-file {{.ANSIBLE_LINT_FILE}} {{.ANSIBLE_DIR}}/**/*.yaml" preconditions: - { msg: "Missing Ansible lint file", sh: "test -f {{.ANSIBLE_LINT_FILE}}" } diff --git a/.taskfiles/Flux/Taskfile.yaml b/.taskfiles/Flux/Taskfile.yaml index 99b2f3d7..28e79534 100644 --- a/.taskfiles/Flux/Taskfile.yaml +++ b/.taskfiles/Flux/Taskfile.yaml @@ -18,14 +18,14 @@ tasks: - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/{{.PROMETHEUS_OPERATOR_VERSION}}/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/{{.PROMETHEUS_OPERATOR_VERSION}}/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --kustomize {{.KUBERNETES_DIR}}/bootstrap - - cat {{.SOPS_AGE_KEY_FILE}} | kubectl -n flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin + - cat {{.AGE_FILE}} | kubectl -n flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin - sops --decrypt {{.CLUSTER_SECRET_SOPS_FILE}} | kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename - - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename {{.CLUSTER_SETTINGS_FILE}} - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config preconditions: - { msg: "Missing kubeconfig", sh: "test -f {{.KUBECONFIG_FILE}}" } - { msg: "Flux already installed", sh: "kubectl get namespace flux-system --kubeconfig {{.KUBECONFIG_FILE}} &>/dev/null && exit 1 || exit 0" } - - { msg: "Missing Sops Age key file", sh: "test -f {{.SOPS_AGE_KEY_FILE}}" } + - { msg: "Missing Sops Age key file", sh: "test -f {{.AGE_FILE}}" } apply: desc: Apply a Flux Kustomization resource for a cluster diff --git a/.taskfiles/K0s/Taskfile.yaml b/.taskfiles/K0s/Taskfile.yaml index fbf8fc88..c1f8eeff 100644 --- a/.taskfiles/K0s/Taskfile.yaml +++ b/.taskfiles/K0s/Taskfile.yaml @@ -2,18 +2,19 @@ # yaml-language-server: $schema=https://taskfile.dev/schema.json version: "3" +vars: + K0S_DIR: "{{.KUBERNETES_DIR}}/k0s" + K0S_CONFIG_FILE: "{{.K0S_DIR}}/k0sctl.yaml" + env: DISABLE_TELEMETRY: "true" DISABLE_UPGRADE_CHECK: "true" -vars: - K0S_CONFIG_FILE: "{{.KUBERNETES_DIR}}/k0s/k0sctl.yaml" - tasks: apply: desc: Apply k0s cluster config - dir: "{{.KUBERNETES_DIR}}/k0s" # Needed for uploading files in k0sctl.yaml + dir: "{{.K0S_DIR}}" cmds: - k0sctl apply --config {{.K0S_CONFIG_FILE}} - task: kubeconfig @@ -22,14 +23,14 @@ tasks: nuke: desc: Nuke the k0s cluster - deps: [":ansible:deps"] - cmds: - - k0sctl reset --config {{.K0S_CONFIG_FILE}} + dir: "{{.K0S_DIR}}" + cmd: k0sctl reset --config {{.K0S_CONFIG_FILE}} preconditions: - { msg: "Missing K0s config file", sh: "test -f {{.K0S_CONFIG_FILE}}" } kubeconfig: desc: Gets k0s kubeconfig + dir: "{{.K0S_DIR}}" prompt: This will overwrite the existing kubeconfig... continue? cmds: - k0sctl kubeconfig --config {{.K0S_CONFIG_FILE}} > {{.KUBECONFIG_FILE}} diff --git a/.taskfiles/Sops/Taskfile.yaml b/.taskfiles/Sops/Taskfile.yaml index e4506aae..ca93faaa 100644 --- a/.taskfiles/Sops/Taskfile.yaml +++ b/.taskfiles/Sops/Taskfile.yaml @@ -9,9 +9,9 @@ tasks: age-keygen: desc: Initialize Age Key for Sops - cmd: age-keygen --output {{.SOPS_AGE_KEY_FILE}} + cmd: age-keygen --output {{.AGE_FILE}} status: - - test -f "{{.SOPS_AGE_KEY_FILE}}" + - test -f "{{.AGE_FILE}}" encrypt: desc: Encrypt all Kubernetes SOPS secrets that are not already encrypted @@ -31,7 +31,7 @@ tasks: vars: ["file"] preconditions: - { msg: "Missing Sops config file", sh: "test -f {{.SOPS_CONFIG_FILE}}" } - - { msg: "Missing Sops Age key file", sh: "test -f {{.SOPS_AGE_KEY_FILE}}" } + - { msg: "Missing Sops Age key file", sh: "test -f {{.AGE_FILE}}" } .reset: internal: true diff --git a/.taskfiles/Talos/Taskfile.yaml b/.taskfiles/Talos/Taskfile.yaml index 4ae83ada..594b150b 100644 --- a/.taskfiles/Talos/Taskfile.yaml +++ b/.taskfiles/Talos/Taskfile.yaml @@ -3,7 +3,7 @@ version: "3" vars: - TALOS_DIR: "{{.ROOT_DIR}}/kubernetes/talos" + TALOS_DIR: "{{.KUBERNETES_DIR}}/talos" TALHELPER_SECRET_FILE: "{{.TALOS_DIR}}/talhelper.sops.yaml" TALHELPER_CONFIG_FILE: "{{.TALOS_DIR}}/talconfig.yaml" diff --git a/.taskfiles/Workstation/Taskfile.yaml b/.taskfiles/Workstation/Taskfile.yaml index b0e166f7..84c21e7c 100644 --- a/.taskfiles/Workstation/Taskfile.yaml +++ b/.taskfiles/Workstation/Taskfile.yaml @@ -5,6 +5,7 @@ version: "3" vars: ARCHFILE: "{{.ROOT_DIR}}/.taskfiles/Workstation/Archfile" BREWFILE: "{{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile" + GENERIC_BIN_DIR: "{{.ROOT_DIR}}/.bin" tasks: @@ -18,13 +19,13 @@ tasks: venv: desc: Set up virtual environment cmds: - - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv" - - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel - - .venv/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}" + - "{{.PYTHON_BIN}} -m venv {{.VIRTUAL_ENV}}" + - '{{.VIRTUAL_ENV}}/bin/python3 -m pip install --upgrade pip setuptools wheel' + - '{{.VIRTUAL_ENV}}/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}"' sources: - "{{.PIP_REQUIREMENTS_FILE}}" generates: - - "{{.ROOT_DIR}}/.venv/pyvenv.cfg" + - "{{.VIRTUAL_ENV}}/pyvenv.cfg" preconditions: - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" } @@ -46,7 +47,7 @@ tasks: generic-linux: desc: Install CLI tools into the projects .bin directory using curl - dir: "{{.ROOT_DIR}}/.bin" + dir: "{{.GENERIC_BIN_DIR}}" platforms: ["linux/amd64", "linux/arm64"] cmds: - for: diff --git a/Taskfile.yaml b/Taskfile.yaml index ec000164..16f4b4ac 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -9,15 +9,19 @@ vars: KUBERNETES_DIR: "{{.ROOT_DIR}}/kubernetes" PRIVATE_DIR: "{{.ROOT_DIR}}/.private" # Files + AGE_FILE: "{{.ROOT_DIR}}/age.key" BOOTSTRAP_CONFIG_FILE: "{{.BOOTSTRAP_DIR}}/vars/config.yaml" KUBECONFIG_FILE: "{{.ROOT_DIR}}/kubeconfig" MAKEJINJA_CONFIG_FILE: "{{.ROOT_DIR}}/makejinja.toml" PIP_REQUIREMENTS_FILE: "{{.ROOT_DIR}}/requirements.txt" - SOPS_AGE_KEY_FILE: "{{.ROOT_DIR}}/age.key" # Binaries PYTHON_BIN: python3 - # Python + +env: + KUBECONFIG: "{{.KUBECONFIG_FILE}}" PYTHONDONTWRITEBYTECODE: "1" + SOPS_AGE_KEY_FILE: "{{.AGE_FILE}}" + VIRTUAL_ENV: "{{.ROOT_DIR}}/.venv" includes: ansible: .taskfiles/Ansible/Taskfile.yaml @@ -62,9 +66,9 @@ tasks: .template: internal: true - cmd: ./.venv/bin/makejinja + cmd: "{{.VIRTUAL_ENV}}/bin/makejinja" preconditions: - - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" } + - { msg: "Missing virtual environment", sh: "test -d {{.VIRTUAL_ENV}}" } - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" } - { msg: "Missing Makejinja plugin file", sh: "test -f {{.BOOTSTRAP_DIR}}/scripts/plugin.py" } - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" } diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py index d4760079..7f51a06a 100644 --- a/bootstrap/scripts/validation.py +++ b/bootstrap/scripts/validation.py @@ -60,7 +60,7 @@ def _validate_node(node: dict, node_cidr: str, distribution: str) -> None: raise ValueError(f"A node is missing a name") if not node.get("ssh", {}).get("username") and distribution in ["k0s", "k3s"]: raise ValueError(f"Node {node.get('name')} is missing a SSH username") - if not node.get("disk_device") and distribution in ["talos"]: + if not node.get("talos", {}).get("disk_device") and distribution in ["talos"]: raise ValueError(f"Node {node.get('name')} is missing a disk device") if not re.match(r"^[a-z0-9-\.]+$", node.get('name')): raise ValueError(f"Node {node.get('name')} has an invalid name")