diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml index 1136e99a..eb3372de 100644 --- a/.github/tests/config-talos.yaml +++ b/.github/tests/config-talos.yaml @@ -22,7 +22,7 @@ bootstrap_pod_network: 10.69.0.0/16 bootstrap_service_network: 10.96.0.0/16 bootstrap_controller_vip: 10.10.10.254 bootstrap_tls_sans: ["fake"] -bootstrap_sops_age_pubkey: $BOOTSTRAP_AGE_PUBLIC_KEY +bootstrap_age_pubkey: $BOOTSTRAP_AGE_PUBLIC_KEY bootstrap_bgp: enabled: false bootstrap_github_address: https://github.com/onedr0p/cluster-template diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 58b53e92..78cb5caf 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -70,7 +70,7 @@ jobs: - name: Run Sops Age key task shell: bash - run: task sops:age-keygen + run: task bootstrap:age-keygen - name: Run init tasks shell: bash @@ -84,8 +84,6 @@ jobs: shell: bash run: task configure --yes - - name: Run repo clean and reset tasks + - name: Run clean shell: bash - run: | - task repository:clean - task repository:reset --yes + run: task bootstrap:clean diff --git a/.taskfiles/Bootstrap/Taskfile.yaml b/.taskfiles/Bootstrap/Taskfile.yaml new file mode 100644 index 00000000..ffa726fb --- /dev/null +++ b/.taskfiles/Bootstrap/Taskfile.yaml @@ -0,0 +1,115 @@ +--- +# yaml-language-server: $schema=https://taskfile.dev/schema.json +version: '3' + +vars: + GITHUB_DEPLOY_KEY_FILE: '{{.KUBERNETES_DIR}}/bootstrap/flux/github-deploy-key.sops.yaml' + TALHELPER_CLUSTER_DIR: '{{.KUBERNETES_DIR}}/bootstrap/talos/clusterconfig' + TALHELPER_SECRET_FILE: '{{.KUBERNETES_DIR}}/bootstrap/talos/talsecret.sops.yaml' + TALHELPER_CONFIG_FILE: '{{.KUBERNETES_DIR}}/bootstrap/talos/talconfig.yaml' + +env: + TALOSCONFIG: '{{.TALHELPER_CLUSTER_DIR}}/talosconfig' + +tasks: + + talos: + desc: Bootstrap the Talos cluster + dir: '{{.KUBERNETES_DIR}}/bootstrap/talos' + cmds: + - '{{if eq .TALHELPER_SECRET_EXISTS "false"}}talhelper gensecret > {{.TALHELPER_SECRET_FILE}}{{end}}' + - '{{if eq .TALHELPER_SECRET_EXISTS "false"}}sops --encrypt --in-place {{.TALHELPER_SECRET_FILE}}{{end}}' + - talhelper genconfig --config-file {{.TALHELPER_CONFIG_FILE}} --secret-file {{.TALHELPER_SECRET_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} + - talhelper gencommand apply --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="--insecure" | bash + - until talhelper gencommand bootstrap --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} | bash; do sleep 10; done + - until talhelper gencommand kubeconfig --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="{{.ROOT_DIR}} --force" | bash; do sleep 10; done + - task: apps + - talosctl health --server=false + vars: + TALHELPER_SECRET_EXISTS: + sh: test -f {{.TALHELPER_SECRET_FILE}} && echo true || echo false + preconditions: + - msg: Missing talhelper config file + sh: test -f {{.TALHELPER_CONFIG_FILE}} + - msg: Missing Sops config file + sh: test -f {{.SOPS_CONFIG_FILE}} + - msg: Missing Sops Age key file + sh: test -f {{.SOPS_AGE_KEY_FILE}} + + flux: + desc: Bootstrap Flux into the Talos cluster + cmds: + - '{{if eq .GITHUB_DEPLOY_KEY_EXISTS "true"}}kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply --filename -{{end}}' + - '{{if eq .GITHUB_DEPLOY_KEY_EXISTS "true"}}sops exec-file {{.GITHUB_DEPLOY_KEY_FILE}} "kubectl apply --server-side --filename {}"{{end}}' + - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/bootstrap/flux + - '{{if eq .SOPS_SECRET_EXISTS "false"}}cat {{.SOPS_AGE_KEY_FILE}} | kubectl --namespace flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin{{end}}' + - sops exec-file {{.KUBERNETES_DIR}}/flux/vars/cluster-secrets.sops.yaml "kubectl apply --server-side --filename {}" + - kubectl apply --server-side --filename {{.KUBERNETES_DIR}}/flux/vars/cluster-settings.yaml + - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config + vars: + GITHUB_DEPLOY_KEY_EXISTS: + sh: test -f {{.GITHUB_DEPLOY_KEY_FILE}} && echo true || echo false + SOPS_SECRET_EXISTS: + sh: kubectl --namespace flux-system get secret sops-age &>/dev/null && echo true || echo false + preconditions: + - msg: Missing kubeconfig + sh: test -f {{.KUBECONFIG}} + - msg: Missing Sops Age key file + sh: test -f {{.SOPS_AGE_KEY_FILE}} + + apps: + internal: true + dir: '{{.KUBERNETES_DIR}}/bootstrap/talos' + cmds: + - until kubectl wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done + - helmfile --file {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml apply --skip-diff-on-install --suppress-diff + - until kubectl wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done + preconditions: + - msg: Missing kubeconfig + sh: test -f {{.KUBECONFIG}} + - msg: Missing helmfile + sh: test -f {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml + + age-keygen: + desc: Bootstrap the Sops Age key + cmd: age-keygen --output {{.SOPS_AGE_KEY_FILE}} + status: + - test -f {{.SOPS_AGE_KEY_FILE}} + + encrypt: + desc: Encrypt all Kubernetes SOPS secrets + cmd: | + find "{{.KUBERNETES_DIR}}" -type f -name "*.sops.*" | while read -r file; do + if sops filestatus "${file}" | jq --exit-status ".encrypted == false" &>/dev/null; then + sops --encrypt --in-place "${file}" + fi + done + preconditions: + - msg: Missing Sops config file + sh: test -f {{.SOPS_CONFIG_FILE}} + - msg: Missing Sops Age key file + sh: test -f {{.SOPS_AGE_KEY_FILE}} + + clean: + desc: Clean files and directories no longer needed after cluster bootstrap + cmds: + # Create backup directory + - mkdir -p {{.ROOT_DIR}}/.private + # Clean up CI + - rm -rf {{.ROOT_DIR}}/.github/tests + - rm -rf {{.ROOT_DIR}}/.github/workflows/e2e.yaml + # Clean up devcontainer + - rm -rf {{.ROOT_DIR}}/.devcontainer/ci + - rm -rf {{.ROOT_DIR}}/.github/workflows/devcontainer.yaml + # Move bootstrap directory to gitignored directory + - mv {{.BOOTSTRAP_DIR}} {{.ROOT_DIR}}/.private/bootstrap-{{.TS}} + - mv {{.MAKEJINJA_CONFIG_FILE}} {{.ROOT_DIR}}/.private/makejinja-{{.TS}}.toml + # Update renovate.json5 + - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5 + vars: + TS: '{{now | unixEpoch}}' + preconditions: + - msg: Missing bootstrap directory + sh: test -d {{.BOOTSTRAP_DIR}} + - msg: Missing Renovate config file + sh: test -f {{.ROOT_DIR}}/.github/renovate.json5 diff --git a/.taskfiles/Flux/Taskfile.yaml b/.taskfiles/Flux/Taskfile.yaml deleted file mode 100644 index 0e76700f..00000000 --- a/.taskfiles/Flux/Taskfile.yaml +++ /dev/null @@ -1,48 +0,0 @@ ---- -# yaml-language-server: $schema=https://taskfile.dev/schema.json -version: '3' - -vars: - CLUSTER_SECRET_SOPS_FILE: '{{.KUBERNETES_DIR}}/flux/vars/cluster-secrets.sops.yaml' - CLUSTER_SETTINGS_FILE: '{{.KUBERNETES_DIR}}/flux/vars/cluster-settings.yaml' - GITHUB_DEPLOY_KEY_FILE: '{{.KUBERNETES_DIR}}/bootstrap/flux/github-deploy-key.sops.yaml' - -tasks: - - bootstrap: - desc: Bootstrap Flux into a Kubernetes cluster - cmds: - - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/bootstrap/flux - - | - if ! kubectl --namespace flux-system get secret sops-age &>/dev/null; then - cat {{.SOPS_AGE_KEY_FILE}} | kubectl --namespace flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin - else - echo "sops-age secret already exists, skipping creation." - fi - - sops --decrypt {{.CLUSTER_SECRET_SOPS_FILE}} | kubectl apply --server-side --filename - - - kubectl apply --server-side --filename {{.CLUSTER_SETTINGS_FILE}} - - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config - preconditions: - - msg: Missing kubeconfig - sh: test -f {{.KUBECONFIG}} - - msg: Missing Sops Age key file - sh: test -f {{.SOPS_AGE_KEY_FILE}} - - reconcile: - desc: Force update Flux to pull in changes from your Git repository - cmd: flux reconcile --namespace flux-system kustomization cluster --with-source - preconditions: - - msg: Missing kubeconfig - sh: test -f {{.KUBECONFIG}} - - github-deploy-key: - cmds: - - kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply --filename - - - sops --decrypt {{.GITHUB_DEPLOY_KEY_FILE}} | kubectl apply --server-side --filename - - preconditions: - - msg: Missing kubeconfig - sh: test -f {{.KUBECONFIG}} - - msg: Missing Sops Age key file - sh: test -f {{.SOPS_AGE_KEY_FILE}} - - msg: Missing Github deploy key file - sh: test -f {{.GITHUB_DEPLOY_KEY_FILE}} diff --git a/.taskfiles/Kubernetes/Taskfile.yaml b/.taskfiles/Kubernetes/Taskfile.yaml index 585b6946..09cf5475 100644 --- a/.taskfiles/Kubernetes/Taskfile.yaml +++ b/.taskfiles/Kubernetes/Taskfile.yaml @@ -7,6 +7,41 @@ vars: tasks: + apply-ks: + desc: Apply a Flux Kustomization resource for a cluster + summary: |- + PATH: Path to the Flux Kustomization resource from the apps base dir (required, e.g. network/echo-server) + NS: Namespace the Flux Kustomization exists in (default: flux-system) + cmd: > + flux build --namespace {{.NS}} ks {{base .PATH}} + --kustomization-file {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml + --path {{.KUBERNETES_DIR}}/apps/{{.PATH}} + {{- if contains "not found" .KS }}--dry-run \{{ end }} + | yq 'with(select(.apiVersion == "kustomize.toolkit.fluxcd.io/v1" and .kind == "Kustomization"); .metadata.namespace = "{{.NS}}")' - + | kubectl apply --server-side --field-manager=kustomize-controller -f - + requires: + vars: [PATH] + vars: + NS: '{{.NS | default "flux-system"}}' + KS: + sh: flux --namespace {{.NS}} get kustomizations {{base .PATH}} 2>&1 + preconditions: + - test -f {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml + + reconcile: + desc: Force update Flux to pull in changes from your Git repository + cmd: flux reconcile --namespace flux-system kustomization cluster --with-source + preconditions: + - msg: Missing kubeconfig + sh: test -f {{.KUBECONFIG}} + + kubeconform: + desc: Validate Kubernetes manifests with kubeconform + cmd: bash {{.KUBECONFORM_SCRIPT}} {{.KUBERNETES_DIR}} + preconditions: + - msg: Missing kubeconform script + sh: test -f {{.KUBECONFORM_SCRIPT}} + resources: desc: Gather common resources in your cluster, useful when asking for support cmds: @@ -23,14 +58,6 @@ tasks: certificaterequests ingresses pods - - kubeconform: - desc: Validate Kubernetes manifests with kubeconform - cmd: bash {{.KUBECONFORM_SCRIPT}} {{.KUBERNETES_DIR}} preconditions: - - msg: Missing kubeconform script - sh: test -f {{.KUBECONFORM_SCRIPT}} - - reset: - internal: true - cmd: rm -rf {{.KUBERNETES_DIR}} + - msg: Missing kubeconfig + sh: test -f {{.KUBECONFIG}} diff --git a/.taskfiles/Repository/Taskfile.yaml b/.taskfiles/Repository/Taskfile.yaml deleted file mode 100644 index cbf29d35..00000000 --- a/.taskfiles/Repository/Taskfile.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- -# yaml-language-server: $schema=https://taskfile.dev/schema.json -version: '3' - -vars: - PRIVATE_DIR: '{{.ROOT_DIR}}/.private' - -tasks: - - clean: - desc: Clean files and directories no longer needed after cluster bootstrap - cmds: - - mkdir -p {{.PRIVATE_DIR}} - # Clean up CI - - rm -rf {{.ROOT_DIR}}/.github/tests - - rm -rf {{.ROOT_DIR}}/.github/workflows/e2e.yaml - # Clean up devcontainer - - rm -rf {{.ROOT_DIR}}/.devcontainer/ci - - rm -rf {{.ROOT_DIR}}/.github/workflows/devcontainer.yaml - # Move bootstrap directory to gitignored directory - - mv {{.BOOTSTRAP_DIR}} {{.PRIVATE_DIR}}/bootstrap-{{now | date "150405"}} - - mv {{.MAKEJINJA_CONFIG_FILE}} {{.PRIVATE_DIR}}/makejinja-{{now | date "150405"}}.toml - # Update renovate.json5 - - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5 - preconditions: - - msg: Missing bootstrap directory - sh: test -d {{.BOOTSTRAP_DIR}} - - msg: Missing Renovate config file - sh: test -f {{.ROOT_DIR}}/.github/renovate.json5 - - reset: - desc: Reset templated configuration files (use --force to reset repo back to HEAD) - prompt: Reset templated configuration files... continue? - cmds: - - task: :kubernetes:reset - - task: :sops:reset - - task: :talos:reset - - cmd: | - {{- if eq .CLI_FORCE true }} - git reset --hard HEAD && git clean -f -d && git pull origin main - {{- end }} diff --git a/.taskfiles/Sops/Taskfile.yaml b/.taskfiles/Sops/Taskfile.yaml deleted file mode 100644 index b2b92d08..00000000 --- a/.taskfiles/Sops/Taskfile.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -# yaml-language-server: $schema=https://taskfile.dev/schema.json -version: '3' - -tasks: - - age-keygen: - desc: Initialize Age Key for Sops - cmd: age-keygen --output {{.SOPS_AGE_KEY_FILE}} - status: - - test -f {{.SOPS_AGE_KEY_FILE}} - - encrypt: - desc: Encrypt all Kubernetes SOPS secrets - cmds: - - cmd: | - find "{{.KUBERNETES_DIR}}" -type f -name "*.sops.*" | while read -r file; do - if sops filestatus "$file" | jq --exit-status ".encrypted == false" &>/dev/null; then - sops --encrypt --in-place "$file" - echo "Encrypted ${file}" - fi - done - preconditions: - - msg: Missing Sops config file - sh: test -f {{.SOPS_CONFIG_FILE}} - - msg: Missing Sops Age key file - sh: test -f {{.SOPS_AGE_KEY_FILE}} - - reset: - internal: true - cmd: rm -rf {{.SOPS_CONFIG_FILE}} diff --git a/.taskfiles/Talos/Taskfile.yaml b/.taskfiles/Talos/Taskfile.yaml index ff4d81d9..38c51fc3 100644 --- a/.taskfiles/Talos/Taskfile.yaml +++ b/.taskfiles/Talos/Taskfile.yaml @@ -4,107 +4,71 @@ version: '3' vars: TALHELPER_CLUSTER_DIR: '{{.KUBERNETES_DIR}}/bootstrap/talos/clusterconfig' - TALHELPER_SECRET_FILE: '{{.KUBERNETES_DIR}}/bootstrap/talos/talsecret.sops.yaml' TALHELPER_CONFIG_FILE: '{{.KUBERNETES_DIR}}/bootstrap/talos/talconfig.yaml' - HELMFILE_FILE: '{{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml' env: TALOSCONFIG: '{{.TALHELPER_CLUSTER_DIR}}/talosconfig' tasks: - bootstrap: - desc: Bootstrap the Talos cluster - dir: '{{.KUBERNETES_DIR}}/bootstrap/talos' + upgrade-node: + desc: Upgrade Talos on a single node + dir: "{{.KUBERNETES_DIR}}/bootstrap/talos" cmds: - - | - if [ ! -f "{{.TALHELPER_SECRET_FILE}}" ]; then - talhelper gensecret > {{.TALHELPER_SECRET_FILE}} - sops --encrypt --in-place {{.TALHELPER_SECRET_FILE}} - fi - - talhelper genconfig --config-file {{.TALHELPER_CONFIG_FILE}} --secret-file {{.TALHELPER_SECRET_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} - - talhelper gencommand apply --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="--insecure" | bash - - until talhelper gencommand bootstrap --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} | bash; do sleep 10; done - - task: conf - - task: apps - - talosctl health --server=false - preconditions: - - msg: Missing talhelper config file - sh: test -f {{.TALHELPER_CONFIG_FILE}} - - msg: Missing Sops config file - sh: test -f {{.SOPS_CONFIG_FILE}} - - msg: Missing Sops Age key file - sh: test -f {{.SOPS_AGE_KEY_FILE}} - - upgrade: - desc: Upgrade Talos on a node - dir: '{{.KUBERNETES_DIR}}/bootstrap/talos' - cmds: - - talosctl --nodes {{.NODE}} upgrade --image="factory.talos.dev/installer/{{.TALOS_SCHEMATIC_ID}}:{{.VERSION}}" --timeout=10m --reboot-mode={{.MODE}} - - talosctl --nodes {{.NODE}} health --wait-timeout=10m --server=false + - talosctl --nodes {{.HOSTNAME}} upgrade --image="factory.talos.dev/installer/{{.SCHEMATIC_ID}}:{{.VERSION}}" --timeout=10m + - talosctl --nodes {{.HOSTNAME}} health --wait-timeout=10m --server=false vars: - MODE: '{{.MODE | default "default"}}' - TALOS_SCHEMATIC_ID: - sh: talosctl get nodeannotationspecs --nodes {{.NODE}} -o jsonpath='{.spec.value}' + SCHEMATIC_ID: + sh: kubectl get node {{.HOSTNAME}} --output=jsonpath='{.metadata.annotations.extensions\.talos\.dev/schematic}' requires: - vars: ['NODE', 'VERSION'] + vars: [HOSTNAME, VERSION] preconditions: - msg: Missing talosconfig sh: test -f {{.TALOSCONFIG}} - msg: Unable to retrieve Talos config sh: talosctl config info &>/dev/null - msg: Node not found - sh: talosctl --nodes {{.NODE}} get machineconfig &>/dev/null + sh: talosctl --nodes {{.HOSTNAME}} get machineconfig &>/dev/null - msg: Upstream Talos version not found - sh: | - curl -fsSL -H "Authorization: Bearer $(curl -fsSL 'https://ghcr.io/token?scope=repository:siderolabs/talos:pull' | jq -r '.token')" https://ghcr.io/v2/siderolabs/talos/manifests/{{.VERSION}} &>/dev/null + sh: curl -fsSL -o /dev/null --fail https://github.com/siderolabs/talos/releases/tag/{{.VERSION}} + + upgrade-cluster: + desc: Upgrade Talos on the whole cluster + dir: "{{.KUBERNETES_DIR}}/bootstrap/talos" + cmds: + - for: { var: HOSTNAMES } + task: upgrade-node + vars: + HOSTNAME: '{{.ITEM}}' + VERSION: '{{.VERSION}}' + vars: + HOSTNAMES: + sh: kubectl get nodes --output=jsonpath='{.items[*].metadata.name}' + requires: + vars: [VERSION] upgrade-k8s: - desc: Upgrade Kubernetes across the cluster - dir: '{{.KUBERNETES_DIR}}/bootstrap/talos' - cmd: talosctl --nodes {{.NODE}} upgrade-k8s --to {{.VERSION}} + desc: Upgrade Kubernetes + dir: "{{.KUBERNETES_DIR}}/bootstrap/talos" + cmds: + - talosctl --nodes {{.CONTROLLER}} upgrade-k8s --to {{.VERSION}} + vars: + CONTROLLER: + sh: talosctl config info --output json | jq --raw-output '.endpoints[]' | shuf -n 1 requires: - vars: ['NODE', 'VERSION'] + vars: [VERSION] preconditions: - msg: Missing talosconfig sh: test -f {{.TALOSCONFIG}} - msg: Unable to retrieve Talos config sh: talosctl config info &>/dev/null - msg: Node not found - sh: talosctl --nodes {{.NODE}} get machineconfig &>/dev/null - - msg: Node not controller - sh: talosctl config info --output json | jq --exit-status '.endpoints | index("{{.NODE}}") != null' &>/dev/null + sh: talosctl --nodes {{.CONTROLLER}} get machineconfig &>/dev/null - msg: Upstream Kubernetes version not found - sh: | - curl -fsSL -H "Authorization: Bearer $(curl -fsSL 'https://ghcr.io/token?scope=repository:siderolabs/kubelet:pull' | jq -r '.token')" https://ghcr.io/v2/siderolabs/kubelet/manifests/{{.VERSION}} &>/dev/null + sh: curl -fsSL -o /dev/null --fail https://github.com/siderolabs/kubelet/releases/tag/{{.VERSION}} - nuke: + reset: desc: Resets nodes back to maintenance mode dir: "{{.KUBERNETES_DIR}}/bootstrap/talos" prompt: This will destroy your cluster and reset the nodes back to maintenance mode... continue? cmd: talhelper gencommand reset --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="--reboot {{- if eq .CLI_FORCE false }} --system-labels-to-wipe STATE --system-labels-to-wipe EPHEMERAL{{ end }} --graceful=false --wait=false" | bash - - conf: - internal: true - dir: '{{.KUBERNETES_DIR}}/bootstrap/talos' - cmd: until talhelper gencommand kubeconfig --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="{{.ROOT_DIR}} --force" | bash; do sleep 10; done - preconditions: - - msg: Missing talhelper config file - sh: test -f {{.TALHELPER_CONFIG_FILE}} - - apps: - internal: true - dir: '{{.KUBERNETES_DIR}}/bootstrap/talos' - cmds: - - until kubectl wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done - - helmfile --file {{.HELMFILE_FILE}} apply --skip-diff-on-install --suppress-diff - - until kubectl wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done - preconditions: - - msg: Missing kubeconfig - sh: test -f {{.KUBECONFIG}} - - msg: Missing helmfile - sh: test -f {{.HELMFILE_FILE}} - - reset: - internal: true - cmd: rm -rf {{.TALHELPER_CLUSTER_DIR}} {{.TALHELPER_SECRET_FILE}} {{.TALHELPER_CONFIG_FILE}} diff --git a/.taskfiles/Workstation/Taskfile.yaml b/.taskfiles/Workstation/Taskfile.yaml index 894cab41..fe38064b 100644 --- a/.taskfiles/Workstation/Taskfile.yaml +++ b/.taskfiles/Workstation/Taskfile.yaml @@ -2,33 +2,33 @@ # yaml-language-server: $schema=https://taskfile.dev/schema.json version: '3' -vars: - ARCHFILE: '{{.ROOT_DIR}}/.taskfiles/Workstation/Archfile' - BREWFILE: '{{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile' - GENERIC_BIN_DIR: '{{.ROOT_DIR}}/.bin' - PIP_REQUIREMENTS_FILE: '{{.ROOT_DIR}}/requirements.txt' - PYTHON_BIN: python3 - tasks: arch: - desc: Install Arch workstation dependencies with Paru Or Yay - cmd: '{{.PKGMGR}} -Syu --needed --noconfirm --noprogressbar $(cat {{.ARCHFILE}} | xargs)' + desc: Set up Arch Linux tools + cmd: '{{.PKGMGR}} -Syu --needed --noconfirm --noprogressbar $(cat {{.ROOT_DIR}}/.taskfiles/Workstation/Archfile | xargs)' vars: PKGMGR: sh: command -v yay || command -v paru preconditions: + - msg: Missing yay or paru + sh: command -v brew &>/dev/null || command -v paru &>/dev/null - msg: Missing Archfile - sh: test -f {{.ARCHFILE}} + sh: test -f {{.ROOT_DIR}}/.taskfiles/Workstation/Archfile brew: - desc: Install workstation dependencies with Brew - cmd: brew bundle --file {{.BREWFILE}} + desc: Set up Homebrew tools + cmds: + - brew bundle --file {{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile + sources: + - '{{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile' + generates: + - '{{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile.lock.json' preconditions: - - msg: Missing Homebrew - sh: command -v brew + - msg: Missing brew + sh: command -v brew &>/dev/null - msg: Missing Brewfile - sh: test -f {{.BREWFILE}} + sh: test -f {{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile direnv: desc: Run direnv hooks @@ -36,10 +36,13 @@ tasks: status: - '[[ $(direnv status --json | jq ".state.foundRC.allowed") == 0 ]]' - '[[ $(direnv status --json | jq ".state.loadedRC.allowed") == 0 ]]' + preconditions: + - msg: Missing direnv + sh: command -v direnv &>/dev/null generic-linux: - desc: Try to install CLI tools into the projects .bin directory - dir: '{{.GENERIC_BIN_DIR}}' + desc: Setup CLI tools into the projects .bin directory + dir: '{{.ROOT_DIR}}/.bin' platforms: ['linux/amd64', 'linux/arm64'] cmds: - for: @@ -65,13 +68,15 @@ tasks: venv: desc: Set up virtual environment cmds: - - '{{.PYTHON_BIN}} -m venv {{.VIRTUAL_ENV}}' + - python3 -m venv {{.VIRTUAL_ENV}} - '{{.VIRTUAL_ENV}}/bin/python3 -m pip install --upgrade pip setuptools wheel' - - '{{.VIRTUAL_ENV}}/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}"' + - '{{.VIRTUAL_ENV}}/bin/python3 -m pip install --upgrade --requirement "{{.ROOT_DIR}}/requirements.txt"' sources: - - '{{.PIP_REQUIREMENTS_FILE}}' + - '{{.ROOT_DIR}}/requirements.txt' generates: - '{{.VIRTUAL_ENV}}/pyvenv.cfg' preconditions: + - msg: Missing python3 + sh: command -v python3 &>/dev/null - msg: Missing Pip requirements file - sh: test -f {{.PIP_REQUIREMENTS_FILE}} + sh: test -f {{.ROOT_DIR}}/requirements.txt diff --git a/README.md b/README.md index e341a7b4..cefa2b9d 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,8 @@ You have two different options for setting up your local workstation. # Homebrew brew install direnv go-task # or, Arch - pacman -S --noconfirm direnv go-task && ln -sf /usr/bin/go-task /usr/local/bin/task + pacman -S --noconfirm direnv go-task \ + && ln -sf /usr/bin/go-task /usr/local/bin/task ``` 2. [Hook `direnv` into your preferred shell](https://direnv.net/docs/hook.html), then run: @@ -154,7 +155,7 @@ You have two different options for setting up your local workstation. 1. Deploy your cluster and bootstrap it. This generates secrets, generates the config files for your nodes and applies them. It bootstraps the cluster afterwards, fetches the kubeconfig file and installs Cilium and kubelet-csr-approver. It finishes with some health checks. ```sh - task talos:bootstrap + task bootstrap:talos ``` 2. ⚠️ It might take a while for the cluster to be setup (10+ minutes is normal), during which time you will see a variety of error messages like: "couldn't get current server API group list," "error: no matching resources found", etc. This is a normal. If this step gets interrupted, e.g. by pressing Ctrl + C, you likely will need to [nuke the cluster](#-Nuke) before trying again. @@ -190,16 +191,14 @@ You have two different options for setting up your local workstation. 2. Install Flux and sync the cluster to the Git repository - 📍 _Run `task flux:github-deploy-key` first if using a private repository._ - ```sh - task flux:bootstrap + task bootstrap:flux # namespace/flux-system configured # customresourcedefinition.apiextensions.k8s.io/alerts.notification.toolkit.fluxcd.io created # ... ``` -1. Verify Flux components are running in the cluster +3. Verify Flux components are running in the cluster ```sh kubectl -n flux-system get pods -o wide @@ -281,12 +280,12 @@ By default Flux will periodically check your git repository for changes. In orde 3. Navigate to the settings of your repository on Github, under "Settings/Webhooks" press the "Add webhook" button. Fill in the webhook URL and your `bootstrap_github_webhook_token` secret in `config.yaml`, Content type: `application/json`, Events: Choose Just the push event, and save. -## 💥 Nuke +## 💥 Reset There might be a situation where you want to destroy your Kubernetes cluster. The following command will reset your nodes back to maintenance mode, append `--force` to completely format your the Talos installation. Either way the nodes should reboot after the command has run. ```sh -task talos:nuke +task talos:reset # --force ``` ## 🤖 Renovate @@ -345,22 +344,22 @@ Resolving problems that you have could take some tweaking of your YAML manifests ## ⬆️ Upgrading Talos and Kubernetes -### Manual - ```sh -# Upgrade Talos to a newer version -# NOTE: This needs to be run once on every node -task talos:upgrade NODE=? VERSION=? -# e.g. -# task talos:upgrade NODE=192.168.42.10 VERSION=v1.8.1 +# Upgrade cluster to a newer Talos version +task talos:upgrade-cluster VERSION=? +# e.g. task talos:upgrade-cluster VERSION=v1.8.1 ``` ```sh -# Upgrade Kubernetes to a newer version -# NOTE: This only needs to be run once against a controller node -task talos:upgrade-k8s NODE=? VERSION=? -# e.g. -# task talos:upgrade-k8s NODE=192.168.42.10 VERSION=1.30.1 +# Upgrade node to a newer Talos version +task talos:upgrade-node HOSTNAME=? VERSION=? +# e.g. task talos:upgrade HOSTNAME=k8s-0 VERSION=v1.8.1 +``` + +```sh +# Upgrade cluster to a newer Kubernetes version +task talos:upgrade-k8s VERSION=? +# e.g. task talos:upgrade-k8s VERSION=1.30.1 ``` ## 👉 Help diff --git a/Taskfile.yaml b/Taskfile.yaml index 61775a26..a090cc96 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -17,11 +17,9 @@ env: VIRTUAL_ENV: '{{.ROOT_DIR}}/.venv' includes: + bootstrap: .taskfiles/Bootstrap kubernetes: .taskfiles/Kubernetes - flux: .taskfiles/Flux - repository: .taskfiles/Repository talos: .taskfiles/Talos - sops: .taskfiles/Sops workstation: .taskfiles/Workstation user: taskfile: .taskfiles/User @@ -45,10 +43,10 @@ tasks: configure: desc: Configure repository from bootstrap vars prompt: Any conflicting config in the kubernetes directory will be overwritten... continue? - deps: ['workstation:direnv', 'workstation:venv', 'sops:age-keygen', 'init'] + deps: ['workstation:direnv', 'workstation:venv', 'bootstrap:age-keygen', 'init'] cmds: - task: template - - task: sops:encrypt + - task: bootstrap:encrypt - task: validate template: diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py index b3a75a07..456f9c13 100644 --- a/bootstrap/scripts/validation.py +++ b/bootstrap/scripts/validation.py @@ -81,7 +81,7 @@ def validate_cli_tools(cloudflare: dict, **_) -> None: raise ValueError(f"Missing required CLI tool {tool}") -@required("bootstrap_sops_age_pubkey") +@required("bootstrap_age_pubkey") def validate_age(key: str, **_) -> None: if not re.match(r"^age1[a-z0-9]{0,58}$", key): raise ValueError(f"Invalid Age public key {key}") diff --git a/bootstrap/templates/.sops.yaml.j2 b/bootstrap/templates/.sops.yaml.j2 index cb7aa764..aa3b06fa 100644 --- a/bootstrap/templates/.sops.yaml.j2 +++ b/bootstrap/templates/.sops.yaml.j2 @@ -4,9 +4,9 @@ creation_rules: path_regex: talos/.*\.sops\.ya?ml key_groups: - age: - - "#{ bootstrap_sops_age_pubkey }#" + - "#{ bootstrap_age_pubkey }#" - path_regex: kubernetes/.*\.sops\.ya?ml encrypted_regex: "^(data|stringData)$" key_groups: - age: - - "#{ bootstrap_sops_age_pubkey }#" + - "#{ bootstrap_age_pubkey }#" diff --git a/bootstrap/templates/kubernetes/apps/network/ingress-nginx/external/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/network/ingress-nginx/external/helmrelease.yaml.j2 index fe74b145..1a5c10ab 100644 --- a/bootstrap/templates/kubernetes/apps/network/ingress-nginx/external/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/network/ingress-nginx/external/helmrelease.yaml.j2 @@ -30,10 +30,10 @@ spec: annotations: external-dns.alpha.kubernetes.io/hostname: "external.${SECRET_DOMAIN}" lbipam.cilium.io/ips: "#{ bootstrap_cloudflare.tunnel.ingress_vip }#" - #% if bootstrap_loadbalancer_mode in ['dsr'] %# - externalTrafficPolicy: Cluster - #% else %# + #% if bgp.enabled %# externalTrafficPolicy: Local + #% else %# + externalTrafficPolicy: Cluster #% endif %# ingressClassResource: name: external diff --git a/bootstrap/templates/kubernetes/apps/network/ingress-nginx/internal/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/network/ingress-nginx/internal/helmrelease.yaml.j2 index b69d2311..f924f402 100644 --- a/bootstrap/templates/kubernetes/apps/network/ingress-nginx/internal/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/network/ingress-nginx/internal/helmrelease.yaml.j2 @@ -27,10 +27,10 @@ spec: service: annotations: lbipam.cilium.io/ips: "#{ bootstrap_cloudflare.ingress_vip }#" - #% if bootstrap_loadbalancer_mode in ['dsr'] %# - externalTrafficPolicy: Cluster - #% else %# + #% if bgp.enabled %# externalTrafficPolicy: Local + #% else %# + externalTrafficPolicy: Cluster #% endif %# ingressClassResource: name: internal diff --git a/bootstrap/templates/kubernetes/apps/network/k8s-gateway/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/network/k8s-gateway/app/helmrelease.yaml.j2 index 8d65c19b..982dfd93 100644 --- a/bootstrap/templates/kubernetes/apps/network/k8s-gateway/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/network/k8s-gateway/app/helmrelease.yaml.j2 @@ -29,9 +29,9 @@ spec: port: 53 annotations: lbipam.cilium.io/ips: "#{ bootstrap_cloudflare.gateway_vip }#" - #% if bootstrap_loadbalancer_mode in ['dsr'] %# - externalTrafficPolicy: Cluster - #% else %# + #% if bgp.enabled %# externalTrafficPolicy: Local + #% else %# + externalTrafficPolicy: Cluster #% endif %# watchedResources: ["Ingress", "Service"] diff --git a/config.sample.yaml b/config.sample.yaml index 052f2234..05ef430a 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -80,9 +80,9 @@ bootstrap_vlan: "" # (Required) Age Public Key (e.g. age1...) # 1. Generate a new key with the following command: -# > task sops:age-keygen +# > task bootstrap:age-keygen # 2. Copy the PUBLIC key and paste it below -bootstrap_sops_age_pubkey: "" +bootstrap_age_pubkey: "" # (Optional) Use cilium BGP control plane when L2 announcements won't traverse VLAN network segments. # Needs a BGP capable router setup with the node IPs as peers.