From 83118cfec32a4137664183e787c5072d75162d9b Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 22 Jan 2024 17:06:01 -0500 Subject: [PATCH] chore: improvements to e2e workflow (#1235) * chore: improvements to e2e workflow Signed-off-by: Devin Buhl * chore: improve taskfiles Signed-off-by: Devin Buhl * chore: update taskfiles Signed-off-by: Devin Buhl --------- Signed-off-by: Devin Buhl --- .envrc | 3 ++- .github/workflows/e2e.yaml | 24 +++++++++--------------- .taskfiles/Ansible/Taskfile.yaml | 2 +- .taskfiles/Kubernetes/Taskfile.yaml | 1 + .taskfiles/Workstation/Taskfile.yaml | 20 ++++++++++++++++++++ README.md | 12 +++++++----- Taskfile.yaml | 24 ++++++------------------ 7 files changed, 46 insertions(+), 40 deletions(-) diff --git a/.envrc b/.envrc index 1e4d7b7b..58b52711 100644 --- a/.envrc +++ b/.envrc @@ -1,9 +1,10 @@ #shellcheck disable=SC2148,SC2155 export KUBECONFIG="$(expand_path ./kubeconfig)" export SOPS_AGE_KEY_FILE="$(expand_path ./age.key)" -# ansible +# Venv PATH_add "$(expand_path ./.venv/bin)" export VIRTUAL_ENV="$(expand_path ./.venv)" +# Ann Seabelle export ANSIBLE_COLLECTIONS_PATH=$(expand_path ./.venv/galaxy) export ANSIBLE_ROLES_PATH=$(expand_path ./.venv/galaxy/ansible_roles) export ANSIBLE_VARS_ENABLED="host_group_vars" diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index bb8600fa..8f98801e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -76,23 +76,23 @@ jobs: shell: bash run: brew install go-task - - name: Run Workstation tasks + - name: Run Workstation Brew tasks if: ${{ github.event_name == 'pull_request' && steps.cache-homebrew-packages.outputs.cache-hit != 'true' }} shell: bash run: task workstation:brew - - name: Initialize Direnv + - name: Run Workstation venv tasks shell: bash - run: direnv allow . + run: task workstation:venv + + - name: Run Workstation direnv tasks + shell: bash + run: task workstation:direnv - name: Run Sops Age key task shell: bash run: task sops:age-keygen - - name: Run setup-virtual-env task - shell: bash - run: task setup-virtual-env - - name: Run init tasks shell: bash run: | @@ -102,25 +102,19 @@ jobs: export BOOTSTRAP_AGE_PUBLIC_KEY=$(sed -n 's/# public key: //gp' age.key) envsubst < ./bootstrap/vars/config.yaml | sponge ./bootstrap/vars/config.yaml - - name: Export Env from config.yaml - shell: bash - id: config-env - run: | - echo "distribution=$(yq '.bootstrap_distribution' ./bootstrap/vars/config.yaml)" >> $GITHUB_OUTPUT - - name: Run configure task shell: bash run: task configure --yes - name: Run Talos tasks - if: ${{ steps.config-env.outputs.distribution == 'talos' }} + if: ${{ startsWith(matrix.config-files, 'talos') }} shell: bash run: | task talos:gensecret --yes task talos:genconfig - name: Run Ansible tasks - if: ${{ steps.config-env.outputs.distribution == 'k3s' || steps.config-env.outputs.distribution == 'k0s' }} + if: ${{ startsWith(matrix.config-files, 'k0s') || startsWith(matrix.config-files, 'k3s') }} shell: bash run: | task ansible:deps force=false diff --git a/.taskfiles/Ansible/Taskfile.yaml b/.taskfiles/Ansible/Taskfile.yaml index a3ca0d64..00e539d7 100644 --- a/.taskfiles/Ansible/Taskfile.yaml +++ b/.taskfiles/Ansible/Taskfile.yaml @@ -21,7 +21,7 @@ tasks: deps: desc: Set up Ansible dependencies - deps: [":setup-virtual-env"] + deps: [":workstation:venv"] cmds: - .venv/bin/python3 -m pip install --upgrade --requirement "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}" - .venv/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}} diff --git a/.taskfiles/Kubernetes/Taskfile.yaml b/.taskfiles/Kubernetes/Taskfile.yaml index 8e69a614..8050f2a0 100644 --- a/.taskfiles/Kubernetes/Taskfile.yaml +++ b/.taskfiles/Kubernetes/Taskfile.yaml @@ -29,6 +29,7 @@ tasks: cmd: bash {{.KUBECONFORM_SCRIPT}} {{.KUBERNETES_DIR}} preconditions: - { msg: "Missing kubeconform script", sh: "test -f {{.KUBECONFORM_SCRIPT}}" } + - { msg: "Missing kubernetes directory", sh: "test -d {{.KUBERNETES_DIR}}" } .reset: internal: true diff --git a/.taskfiles/Workstation/Taskfile.yaml b/.taskfiles/Workstation/Taskfile.yaml index 5e2b7f3a..1a0eaa54 100644 --- a/.taskfiles/Workstation/Taskfile.yaml +++ b/.taskfiles/Workstation/Taskfile.yaml @@ -8,6 +8,26 @@ vars: tasks: + direnv: + desc: Run direnv hooks + cmd: direnv allow . + status: + - "[[ $(direnv status --json | jq '.state.foundRC.allowed') == 0 ]]" + - "[[ $(direnv status --json | jq '.state.loadedRC.allowed') == 0 ]]" + + venv: + desc: Set up virtual environment + cmds: + - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv" + - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel + - .venv/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}" + sources: + - "{{.PIP_REQUIREMENTS_FILE}}" + generates: + - "{{.ROOT_DIR}}/.venv/pyvenv.cfg" + preconditions: + - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" } + brew: desc: Install workstation dependencies with Brew cmd: brew bundle --file {{.BREWFILE}} diff --git a/README.md b/README.md index 234fd605..2bf17c26 100644 --- a/README.md +++ b/README.md @@ -173,8 +173,6 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g 2. Install the most recent version of [direnv](https://direnv.net/), see the [installation docs](https://direnv.net/docs/installation.html) for other supported platforms. - 📍 _After installing `direnv` be sure to **[hook it into your shell](https://direnv.net/docs/hook.html)** and after that is done run `direnv allow` while in your repos' directory._ - ```sh # Homebrew brew install direnv @@ -182,7 +180,11 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g yay -S direnv ``` -3. Install **required** CLI tools: [age](https://github.com/FiloSottile/age), [cloudflared](https://github.com/cloudflare/cloudflared), [flux](https://toolkit.fluxcd.io/), [kubeconform](https://github.com/yannh/kubeconform), [kubectl](https://kubectl.docs.kubernetes.io/installation/), [kustomize](https://kubectl.docs.kubernetes.io/installation/), [sops](https://github.com/getsops/sops). [k0sctl](https://github.com/k0sproject/k0sctl) is required for k0s. [talosctl](https://www.talos.dev/latest/learn-more/talosctl/) and [talhelper](https://github.com/budimanjojo/talhelper) is required for Talos. + 📍 _After `direnv` is installed be sure to **[hook it into your preferred shell](https://direnv.net/docs/hook.html)** and then run `task workstation:direnv`_ + +3. Install **required** CLI tools: [age](https://github.com/FiloSottile/age), [cloudflared](https://github.com/cloudflare/cloudflared), [flux](https://toolkit.fluxcd.io/), [kubeconform](https://github.com/yannh/kubeconform), [kubectl](https://kubectl.docs.kubernetes.io/installation/), [kustomize](https://kubectl.docs.kubernetes.io/installation/), [sops](https://github.com/getsops/sops). + + 📍 _[k0sctl](https://github.com/k0sproject/k0sctl) is required for k0s. [talosctl](https://www.talos.dev/latest/learn-more/talosctl/) and [talhelper](https://github.com/budimanjojo/talhelper) is required for Talos._ 📍 _Not using Homebrew or ArchLinux? Make sure to look up how to install the latest version of each of these CLI tools and install them._ @@ -193,12 +195,12 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g go-task workstation:yay ``` -4. Setup a Python virual env by running the following task command. +4. Setup a Python virual environment by running the following task command. 📍 _This commands requires Python 3.11+ to be installed._ ```sh - task setup-virtual-env + task workstation:venv ``` 5. Continue on to 🔧 [**Stage 3**](#-stage-3-do-bootstrap-configuration) diff --git a/Taskfile.yaml b/Taskfile.yaml index 286cb3b0..cba89569 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -37,8 +37,7 @@ tasks: default: task -l init: - desc: Initialize virtual env and configuration files - deps: ["setup-virtual-env"] + desc: Initialize configuration files cmds: - mkdir -p {{.PRIVATE_DIR}} - cp -n {{.BOOTSTRAP_ADDONS_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_ADDONS_FILE}} @@ -51,33 +50,22 @@ tasks: silent: true - cmd: echo {{.BOOTSTRAP_ADDONS_FILE}} silent: true + status: + - test -f "{{.BOOTSTRAP_ADDONS_FILE}}" + - test -f "{{.BOOTSTRAP_CONFIG_FILE}}" configure: desc: Configure repository from bootstrap vars prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue? - deps: ["setup-virtual-env"] + deps: ["workstation:direnv", "workstation:venv", "sops:age-keygen", "init"] cmds: - task: .template - task: sops:encrypt - task: .validate - setup-virtual-env: - desc: Set up virtual environment - cmds: - - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv" - - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel - - .venv/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}" - sources: - - "{{.PIP_REQUIREMENTS_FILE}}" - generates: - - "{{.ROOT_DIR}}/.venv/pyvenv.cfg" - preconditions: - - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" } - .template: internal: true - cmds: - - ./.venv/bin/makejinja + cmd: ./.venv/bin/makejinja preconditions: - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" } - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" }