diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml
index 1136e99a..eb3372de 100644
--- a/.github/tests/config-talos.yaml
+++ b/.github/tests/config-talos.yaml
@@ -22,7 +22,7 @@ bootstrap_pod_network: 10.69.0.0/16
bootstrap_service_network: 10.96.0.0/16
bootstrap_controller_vip: 10.10.10.254
bootstrap_tls_sans: ["fake"]
-bootstrap_sops_age_pubkey: $BOOTSTRAP_AGE_PUBLIC_KEY
+bootstrap_age_pubkey: $BOOTSTRAP_AGE_PUBLIC_KEY
bootstrap_bgp:
enabled: false
bootstrap_github_address: https://github.com/onedr0p/cluster-template
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 58b53e92..78cb5caf 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -70,7 +70,7 @@ jobs:
- name: Run Sops Age key task
shell: bash
- run: task sops:age-keygen
+ run: task bootstrap:age-keygen
- name: Run init tasks
shell: bash
@@ -84,8 +84,6 @@ jobs:
shell: bash
run: task configure --yes
- - name: Run repo clean and reset tasks
+ - name: Run clean
shell: bash
- run: |
- task repository:clean
- task repository:reset --yes
+ run: task bootstrap:clean
diff --git a/.taskfiles/Bootstrap/Taskfile.yaml b/.taskfiles/Bootstrap/Taskfile.yaml
new file mode 100644
index 00000000..ffa726fb
--- /dev/null
+++ b/.taskfiles/Bootstrap/Taskfile.yaml
@@ -0,0 +1,115 @@
+---
+# yaml-language-server: $schema=https://taskfile.dev/schema.json
+version: '3'
+
+vars:
+ GITHUB_DEPLOY_KEY_FILE: '{{.KUBERNETES_DIR}}/bootstrap/flux/github-deploy-key.sops.yaml'
+ TALHELPER_CLUSTER_DIR: '{{.KUBERNETES_DIR}}/bootstrap/talos/clusterconfig'
+ TALHELPER_SECRET_FILE: '{{.KUBERNETES_DIR}}/bootstrap/talos/talsecret.sops.yaml'
+ TALHELPER_CONFIG_FILE: '{{.KUBERNETES_DIR}}/bootstrap/talos/talconfig.yaml'
+
+env:
+ TALOSCONFIG: '{{.TALHELPER_CLUSTER_DIR}}/talosconfig'
+
+tasks:
+
+ talos:
+ desc: Bootstrap the Talos cluster
+ dir: '{{.KUBERNETES_DIR}}/bootstrap/talos'
+ cmds:
+ - '{{if eq .TALHELPER_SECRET_EXISTS "false"}}talhelper gensecret > {{.TALHELPER_SECRET_FILE}}{{end}}'
+ - '{{if eq .TALHELPER_SECRET_EXISTS "false"}}sops --encrypt --in-place {{.TALHELPER_SECRET_FILE}}{{end}}'
+ - talhelper genconfig --config-file {{.TALHELPER_CONFIG_FILE}} --secret-file {{.TALHELPER_SECRET_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}}
+ - talhelper gencommand apply --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="--insecure" | bash
+ - until talhelper gencommand bootstrap --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} | bash; do sleep 10; done
+ - until talhelper gencommand kubeconfig --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="{{.ROOT_DIR}} --force" | bash; do sleep 10; done
+ - task: apps
+ - talosctl health --server=false
+ vars:
+ TALHELPER_SECRET_EXISTS:
+ sh: test -f {{.TALHELPER_SECRET_FILE}} && echo true || echo false
+ preconditions:
+ - msg: Missing talhelper config file
+ sh: test -f {{.TALHELPER_CONFIG_FILE}}
+ - msg: Missing Sops config file
+ sh: test -f {{.SOPS_CONFIG_FILE}}
+ - msg: Missing Sops Age key file
+ sh: test -f {{.SOPS_AGE_KEY_FILE}}
+
+ flux:
+ desc: Bootstrap Flux into the Talos cluster
+ cmds:
+ - '{{if eq .GITHUB_DEPLOY_KEY_EXISTS "true"}}kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply --filename -{{end}}'
+ - '{{if eq .GITHUB_DEPLOY_KEY_EXISTS "true"}}sops exec-file {{.GITHUB_DEPLOY_KEY_FILE}} "kubectl apply --server-side --filename {}"{{end}}'
+ - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/bootstrap/flux
+ - '{{if eq .SOPS_SECRET_EXISTS "false"}}cat {{.SOPS_AGE_KEY_FILE}} | kubectl --namespace flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin{{end}}'
+ - sops exec-file {{.KUBERNETES_DIR}}/flux/vars/cluster-secrets.sops.yaml "kubectl apply --server-side --filename {}"
+ - kubectl apply --server-side --filename {{.KUBERNETES_DIR}}/flux/vars/cluster-settings.yaml
+ - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config
+ vars:
+ GITHUB_DEPLOY_KEY_EXISTS:
+ sh: test -f {{.GITHUB_DEPLOY_KEY_FILE}} && echo true || echo false
+ SOPS_SECRET_EXISTS:
+ sh: kubectl --namespace flux-system get secret sops-age &>/dev/null && echo true || echo false
+ preconditions:
+ - msg: Missing kubeconfig
+ sh: test -f {{.KUBECONFIG}}
+ - msg: Missing Sops Age key file
+ sh: test -f {{.SOPS_AGE_KEY_FILE}}
+
+ apps:
+ internal: true
+ dir: '{{.KUBERNETES_DIR}}/bootstrap/talos'
+ cmds:
+ - until kubectl wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done
+ - helmfile --file {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml apply --skip-diff-on-install --suppress-diff
+ - until kubectl wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done
+ preconditions:
+ - msg: Missing kubeconfig
+ sh: test -f {{.KUBECONFIG}}
+ - msg: Missing helmfile
+ sh: test -f {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml
+
+ age-keygen:
+ desc: Bootstrap the Sops Age key
+ cmd: age-keygen --output {{.SOPS_AGE_KEY_FILE}}
+ status:
+ - test -f {{.SOPS_AGE_KEY_FILE}}
+
+ encrypt:
+ desc: Encrypt all Kubernetes SOPS secrets
+ cmd: |
+ find "{{.KUBERNETES_DIR}}" -type f -name "*.sops.*" | while read -r file; do
+ if sops filestatus "${file}" | jq --exit-status ".encrypted == false" &>/dev/null; then
+ sops --encrypt --in-place "${file}"
+ fi
+ done
+ preconditions:
+ - msg: Missing Sops config file
+ sh: test -f {{.SOPS_CONFIG_FILE}}
+ - msg: Missing Sops Age key file
+ sh: test -f {{.SOPS_AGE_KEY_FILE}}
+
+ clean:
+ desc: Clean files and directories no longer needed after cluster bootstrap
+ cmds:
+ # Create backup directory
+ - mkdir -p {{.ROOT_DIR}}/.private
+ # Clean up CI
+ - rm -rf {{.ROOT_DIR}}/.github/tests
+ - rm -rf {{.ROOT_DIR}}/.github/workflows/e2e.yaml
+ # Clean up devcontainer
+ - rm -rf {{.ROOT_DIR}}/.devcontainer/ci
+ - rm -rf {{.ROOT_DIR}}/.github/workflows/devcontainer.yaml
+ # Move bootstrap directory to gitignored directory
+ - mv {{.BOOTSTRAP_DIR}} {{.ROOT_DIR}}/.private/bootstrap-{{.TS}}
+ - mv {{.MAKEJINJA_CONFIG_FILE}} {{.ROOT_DIR}}/.private/makejinja-{{.TS}}.toml
+ # Update renovate.json5
+ - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
+ vars:
+ TS: '{{now | unixEpoch}}'
+ preconditions:
+ - msg: Missing bootstrap directory
+ sh: test -d {{.BOOTSTRAP_DIR}}
+ - msg: Missing Renovate config file
+ sh: test -f {{.ROOT_DIR}}/.github/renovate.json5
diff --git a/.taskfiles/Flux/Taskfile.yaml b/.taskfiles/Flux/Taskfile.yaml
deleted file mode 100644
index 0e76700f..00000000
--- a/.taskfiles/Flux/Taskfile.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
----
-# yaml-language-server: $schema=https://taskfile.dev/schema.json
-version: '3'
-
-vars:
- CLUSTER_SECRET_SOPS_FILE: '{{.KUBERNETES_DIR}}/flux/vars/cluster-secrets.sops.yaml'
- CLUSTER_SETTINGS_FILE: '{{.KUBERNETES_DIR}}/flux/vars/cluster-settings.yaml'
- GITHUB_DEPLOY_KEY_FILE: '{{.KUBERNETES_DIR}}/bootstrap/flux/github-deploy-key.sops.yaml'
-
-tasks:
-
- bootstrap:
- desc: Bootstrap Flux into a Kubernetes cluster
- cmds:
- - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/bootstrap/flux
- - |
- if ! kubectl --namespace flux-system get secret sops-age &>/dev/null; then
- cat {{.SOPS_AGE_KEY_FILE}} | kubectl --namespace flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin
- else
- echo "sops-age secret already exists, skipping creation."
- fi
- - sops --decrypt {{.CLUSTER_SECRET_SOPS_FILE}} | kubectl apply --server-side --filename -
- - kubectl apply --server-side --filename {{.CLUSTER_SETTINGS_FILE}}
- - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config
- preconditions:
- - msg: Missing kubeconfig
- sh: test -f {{.KUBECONFIG}}
- - msg: Missing Sops Age key file
- sh: test -f {{.SOPS_AGE_KEY_FILE}}
-
- reconcile:
- desc: Force update Flux to pull in changes from your Git repository
- cmd: flux reconcile --namespace flux-system kustomization cluster --with-source
- preconditions:
- - msg: Missing kubeconfig
- sh: test -f {{.KUBECONFIG}}
-
- github-deploy-key:
- cmds:
- - kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply --filename -
- - sops --decrypt {{.GITHUB_DEPLOY_KEY_FILE}} | kubectl apply --server-side --filename -
- preconditions:
- - msg: Missing kubeconfig
- sh: test -f {{.KUBECONFIG}}
- - msg: Missing Sops Age key file
- sh: test -f {{.SOPS_AGE_KEY_FILE}}
- - msg: Missing Github deploy key file
- sh: test -f {{.GITHUB_DEPLOY_KEY_FILE}}
diff --git a/.taskfiles/Kubernetes/Taskfile.yaml b/.taskfiles/Kubernetes/Taskfile.yaml
index 585b6946..09cf5475 100644
--- a/.taskfiles/Kubernetes/Taskfile.yaml
+++ b/.taskfiles/Kubernetes/Taskfile.yaml
@@ -7,6 +7,41 @@ vars:
tasks:
+ apply-ks:
+ desc: Apply a Flux Kustomization resource for a cluster
+ summary: |-
+ PATH: Path to the Flux Kustomization resource from the apps base dir (required, e.g. network/echo-server)
+ NS: Namespace the Flux Kustomization exists in (default: flux-system)
+ cmd: >
+ flux build --namespace {{.NS}} ks {{base .PATH}}
+ --kustomization-file {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml
+ --path {{.KUBERNETES_DIR}}/apps/{{.PATH}}
+ {{- if contains "not found" .KS }}--dry-run \{{ end }}
+ | yq 'with(select(.apiVersion == "kustomize.toolkit.fluxcd.io/v1" and .kind == "Kustomization"); .metadata.namespace = "{{.NS}}")' -
+ | kubectl apply --server-side --field-manager=kustomize-controller -f -
+ requires:
+ vars: [PATH]
+ vars:
+ NS: '{{.NS | default "flux-system"}}'
+ KS:
+ sh: flux --namespace {{.NS}} get kustomizations {{base .PATH}} 2>&1
+ preconditions:
+ - test -f {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml
+
+ reconcile:
+ desc: Force update Flux to pull in changes from your Git repository
+ cmd: flux reconcile --namespace flux-system kustomization cluster --with-source
+ preconditions:
+ - msg: Missing kubeconfig
+ sh: test -f {{.KUBECONFIG}}
+
+ kubeconform:
+ desc: Validate Kubernetes manifests with kubeconform
+ cmd: bash {{.KUBECONFORM_SCRIPT}} {{.KUBERNETES_DIR}}
+ preconditions:
+ - msg: Missing kubeconform script
+ sh: test -f {{.KUBECONFORM_SCRIPT}}
+
resources:
desc: Gather common resources in your cluster, useful when asking for support
cmds:
@@ -23,14 +58,6 @@ tasks:
certificaterequests
ingresses
pods
-
- kubeconform:
- desc: Validate Kubernetes manifests with kubeconform
- cmd: bash {{.KUBECONFORM_SCRIPT}} {{.KUBERNETES_DIR}}
preconditions:
- - msg: Missing kubeconform script
- sh: test -f {{.KUBECONFORM_SCRIPT}}
-
- reset:
- internal: true
- cmd: rm -rf {{.KUBERNETES_DIR}}
+ - msg: Missing kubeconfig
+ sh: test -f {{.KUBECONFIG}}
diff --git a/.taskfiles/Repository/Taskfile.yaml b/.taskfiles/Repository/Taskfile.yaml
deleted file mode 100644
index cbf29d35..00000000
--- a/.taskfiles/Repository/Taskfile.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
----
-# yaml-language-server: $schema=https://taskfile.dev/schema.json
-version: '3'
-
-vars:
- PRIVATE_DIR: '{{.ROOT_DIR}}/.private'
-
-tasks:
-
- clean:
- desc: Clean files and directories no longer needed after cluster bootstrap
- cmds:
- - mkdir -p {{.PRIVATE_DIR}}
- # Clean up CI
- - rm -rf {{.ROOT_DIR}}/.github/tests
- - rm -rf {{.ROOT_DIR}}/.github/workflows/e2e.yaml
- # Clean up devcontainer
- - rm -rf {{.ROOT_DIR}}/.devcontainer/ci
- - rm -rf {{.ROOT_DIR}}/.github/workflows/devcontainer.yaml
- # Move bootstrap directory to gitignored directory
- - mv {{.BOOTSTRAP_DIR}} {{.PRIVATE_DIR}}/bootstrap-{{now | date "150405"}}
- - mv {{.MAKEJINJA_CONFIG_FILE}} {{.PRIVATE_DIR}}/makejinja-{{now | date "150405"}}.toml
- # Update renovate.json5
- - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
- preconditions:
- - msg: Missing bootstrap directory
- sh: test -d {{.BOOTSTRAP_DIR}}
- - msg: Missing Renovate config file
- sh: test -f {{.ROOT_DIR}}/.github/renovate.json5
-
- reset:
- desc: Reset templated configuration files (use --force to reset repo back to HEAD)
- prompt: Reset templated configuration files... continue?
- cmds:
- - task: :kubernetes:reset
- - task: :sops:reset
- - task: :talos:reset
- - cmd: |
- {{- if eq .CLI_FORCE true }}
- git reset --hard HEAD && git clean -f -d && git pull origin main
- {{- end }}
diff --git a/.taskfiles/Sops/Taskfile.yaml b/.taskfiles/Sops/Taskfile.yaml
deleted file mode 100644
index b2b92d08..00000000
--- a/.taskfiles/Sops/Taskfile.yaml
+++ /dev/null
@@ -1,31 +0,0 @@
----
-# yaml-language-server: $schema=https://taskfile.dev/schema.json
-version: '3'
-
-tasks:
-
- age-keygen:
- desc: Initialize Age Key for Sops
- cmd: age-keygen --output {{.SOPS_AGE_KEY_FILE}}
- status:
- - test -f {{.SOPS_AGE_KEY_FILE}}
-
- encrypt:
- desc: Encrypt all Kubernetes SOPS secrets
- cmds:
- - cmd: |
- find "{{.KUBERNETES_DIR}}" -type f -name "*.sops.*" | while read -r file; do
- if sops filestatus "$file" | jq --exit-status ".encrypted == false" &>/dev/null; then
- sops --encrypt --in-place "$file"
- echo "Encrypted ${file}"
- fi
- done
- preconditions:
- - msg: Missing Sops config file
- sh: test -f {{.SOPS_CONFIG_FILE}}
- - msg: Missing Sops Age key file
- sh: test -f {{.SOPS_AGE_KEY_FILE}}
-
- reset:
- internal: true
- cmd: rm -rf {{.SOPS_CONFIG_FILE}}
diff --git a/.taskfiles/Talos/Taskfile.yaml b/.taskfiles/Talos/Taskfile.yaml
index ff4d81d9..38c51fc3 100644
--- a/.taskfiles/Talos/Taskfile.yaml
+++ b/.taskfiles/Talos/Taskfile.yaml
@@ -4,107 +4,71 @@ version: '3'
vars:
TALHELPER_CLUSTER_DIR: '{{.KUBERNETES_DIR}}/bootstrap/talos/clusterconfig'
- TALHELPER_SECRET_FILE: '{{.KUBERNETES_DIR}}/bootstrap/talos/talsecret.sops.yaml'
TALHELPER_CONFIG_FILE: '{{.KUBERNETES_DIR}}/bootstrap/talos/talconfig.yaml'
- HELMFILE_FILE: '{{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml'
env:
TALOSCONFIG: '{{.TALHELPER_CLUSTER_DIR}}/talosconfig'
tasks:
- bootstrap:
- desc: Bootstrap the Talos cluster
- dir: '{{.KUBERNETES_DIR}}/bootstrap/talos'
+ upgrade-node:
+ desc: Upgrade Talos on a single node
+ dir: "{{.KUBERNETES_DIR}}/bootstrap/talos"
cmds:
- - |
- if [ ! -f "{{.TALHELPER_SECRET_FILE}}" ]; then
- talhelper gensecret > {{.TALHELPER_SECRET_FILE}}
- sops --encrypt --in-place {{.TALHELPER_SECRET_FILE}}
- fi
- - talhelper genconfig --config-file {{.TALHELPER_CONFIG_FILE}} --secret-file {{.TALHELPER_SECRET_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}}
- - talhelper gencommand apply --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="--insecure" | bash
- - until talhelper gencommand bootstrap --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} | bash; do sleep 10; done
- - task: conf
- - task: apps
- - talosctl health --server=false
- preconditions:
- - msg: Missing talhelper config file
- sh: test -f {{.TALHELPER_CONFIG_FILE}}
- - msg: Missing Sops config file
- sh: test -f {{.SOPS_CONFIG_FILE}}
- - msg: Missing Sops Age key file
- sh: test -f {{.SOPS_AGE_KEY_FILE}}
-
- upgrade:
- desc: Upgrade Talos on a node
- dir: '{{.KUBERNETES_DIR}}/bootstrap/talos'
- cmds:
- - talosctl --nodes {{.NODE}} upgrade --image="factory.talos.dev/installer/{{.TALOS_SCHEMATIC_ID}}:{{.VERSION}}" --timeout=10m --reboot-mode={{.MODE}}
- - talosctl --nodes {{.NODE}} health --wait-timeout=10m --server=false
+ - talosctl --nodes {{.HOSTNAME}} upgrade --image="factory.talos.dev/installer/{{.SCHEMATIC_ID}}:{{.VERSION}}" --timeout=10m
+ - talosctl --nodes {{.HOSTNAME}} health --wait-timeout=10m --server=false
vars:
- MODE: '{{.MODE | default "default"}}'
- TALOS_SCHEMATIC_ID:
- sh: talosctl get nodeannotationspecs --nodes {{.NODE}} -o jsonpath='{.spec.value}'
+ SCHEMATIC_ID:
+ sh: kubectl get node {{.HOSTNAME}} --output=jsonpath='{.metadata.annotations.extensions\.talos\.dev/schematic}'
requires:
- vars: ['NODE', 'VERSION']
+ vars: [HOSTNAME, VERSION]
preconditions:
- msg: Missing talosconfig
sh: test -f {{.TALOSCONFIG}}
- msg: Unable to retrieve Talos config
sh: talosctl config info &>/dev/null
- msg: Node not found
- sh: talosctl --nodes {{.NODE}} get machineconfig &>/dev/null
+ sh: talosctl --nodes {{.HOSTNAME}} get machineconfig &>/dev/null
- msg: Upstream Talos version not found
- sh: |
- curl -fsSL -H "Authorization: Bearer $(curl -fsSL 'https://ghcr.io/token?scope=repository:siderolabs/talos:pull' | jq -r '.token')" https://ghcr.io/v2/siderolabs/talos/manifests/{{.VERSION}} &>/dev/null
+ sh: curl -fsSL -o /dev/null --fail https://github.com/siderolabs/talos/releases/tag/{{.VERSION}}
+
+ upgrade-cluster:
+ desc: Upgrade Talos on the whole cluster
+ dir: "{{.KUBERNETES_DIR}}/bootstrap/talos"
+ cmds:
+ - for: { var: HOSTNAMES }
+ task: upgrade-node
+ vars:
+ HOSTNAME: '{{.ITEM}}'
+ VERSION: '{{.VERSION}}'
+ vars:
+ HOSTNAMES:
+ sh: kubectl get nodes --output=jsonpath='{.items[*].metadata.name}'
+ requires:
+ vars: [VERSION]
upgrade-k8s:
- desc: Upgrade Kubernetes across the cluster
- dir: '{{.KUBERNETES_DIR}}/bootstrap/talos'
- cmd: talosctl --nodes {{.NODE}} upgrade-k8s --to {{.VERSION}}
+ desc: Upgrade Kubernetes
+ dir: "{{.KUBERNETES_DIR}}/bootstrap/talos"
+ cmds:
+ - talosctl --nodes {{.CONTROLLER}} upgrade-k8s --to {{.VERSION}}
+ vars:
+ CONTROLLER:
+ sh: talosctl config info --output json | jq --raw-output '.endpoints[]' | shuf -n 1
requires:
- vars: ['NODE', 'VERSION']
+ vars: [VERSION]
preconditions:
- msg: Missing talosconfig
sh: test -f {{.TALOSCONFIG}}
- msg: Unable to retrieve Talos config
sh: talosctl config info &>/dev/null
- msg: Node not found
- sh: talosctl --nodes {{.NODE}} get machineconfig &>/dev/null
- - msg: Node not controller
- sh: talosctl config info --output json | jq --exit-status '.endpoints | index("{{.NODE}}") != null' &>/dev/null
+ sh: talosctl --nodes {{.CONTROLLER}} get machineconfig &>/dev/null
- msg: Upstream Kubernetes version not found
- sh: |
- curl -fsSL -H "Authorization: Bearer $(curl -fsSL 'https://ghcr.io/token?scope=repository:siderolabs/kubelet:pull' | jq -r '.token')" https://ghcr.io/v2/siderolabs/kubelet/manifests/{{.VERSION}} &>/dev/null
+ sh: curl -fsSL -o /dev/null --fail https://github.com/siderolabs/kubelet/releases/tag/{{.VERSION}}
- nuke:
+ reset:
desc: Resets nodes back to maintenance mode
dir: "{{.KUBERNETES_DIR}}/bootstrap/talos"
prompt: This will destroy your cluster and reset the nodes back to maintenance mode... continue?
cmd: talhelper gencommand reset --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="--reboot {{- if eq .CLI_FORCE false }} --system-labels-to-wipe STATE --system-labels-to-wipe EPHEMERAL{{ end }} --graceful=false --wait=false" | bash
-
- conf:
- internal: true
- dir: '{{.KUBERNETES_DIR}}/bootstrap/talos'
- cmd: until talhelper gencommand kubeconfig --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="{{.ROOT_DIR}} --force" | bash; do sleep 10; done
- preconditions:
- - msg: Missing talhelper config file
- sh: test -f {{.TALHELPER_CONFIG_FILE}}
-
- apps:
- internal: true
- dir: '{{.KUBERNETES_DIR}}/bootstrap/talos'
- cmds:
- - until kubectl wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done
- - helmfile --file {{.HELMFILE_FILE}} apply --skip-diff-on-install --suppress-diff
- - until kubectl wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done
- preconditions:
- - msg: Missing kubeconfig
- sh: test -f {{.KUBECONFIG}}
- - msg: Missing helmfile
- sh: test -f {{.HELMFILE_FILE}}
-
- reset:
- internal: true
- cmd: rm -rf {{.TALHELPER_CLUSTER_DIR}} {{.TALHELPER_SECRET_FILE}} {{.TALHELPER_CONFIG_FILE}}
diff --git a/.taskfiles/Workstation/Taskfile.yaml b/.taskfiles/Workstation/Taskfile.yaml
index 894cab41..fe38064b 100644
--- a/.taskfiles/Workstation/Taskfile.yaml
+++ b/.taskfiles/Workstation/Taskfile.yaml
@@ -2,33 +2,33 @@
# yaml-language-server: $schema=https://taskfile.dev/schema.json
version: '3'
-vars:
- ARCHFILE: '{{.ROOT_DIR}}/.taskfiles/Workstation/Archfile'
- BREWFILE: '{{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile'
- GENERIC_BIN_DIR: '{{.ROOT_DIR}}/.bin'
- PIP_REQUIREMENTS_FILE: '{{.ROOT_DIR}}/requirements.txt'
- PYTHON_BIN: python3
-
tasks:
arch:
- desc: Install Arch workstation dependencies with Paru Or Yay
- cmd: '{{.PKGMGR}} -Syu --needed --noconfirm --noprogressbar $(cat {{.ARCHFILE}} | xargs)'
+ desc: Set up Arch Linux tools
+ cmd: '{{.PKGMGR}} -Syu --needed --noconfirm --noprogressbar $(cat {{.ROOT_DIR}}/.taskfiles/Workstation/Archfile | xargs)'
vars:
PKGMGR:
sh: command -v yay || command -v paru
preconditions:
+ - msg: Missing yay or paru
+ sh: command -v brew &>/dev/null || command -v paru &>/dev/null
- msg: Missing Archfile
- sh: test -f {{.ARCHFILE}}
+ sh: test -f {{.ROOT_DIR}}/.taskfiles/Workstation/Archfile
brew:
- desc: Install workstation dependencies with Brew
- cmd: brew bundle --file {{.BREWFILE}}
+ desc: Set up Homebrew tools
+ cmds:
+ - brew bundle --file {{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile
+ sources:
+ - '{{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile'
+ generates:
+ - '{{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile.lock.json'
preconditions:
- - msg: Missing Homebrew
- sh: command -v brew
+ - msg: Missing brew
+ sh: command -v brew &>/dev/null
- msg: Missing Brewfile
- sh: test -f {{.BREWFILE}}
+ sh: test -f {{.ROOT_DIR}}/.taskfiles/Workstation/Brewfile
direnv:
desc: Run direnv hooks
@@ -36,10 +36,13 @@ tasks:
status:
- '[[ $(direnv status --json | jq ".state.foundRC.allowed") == 0 ]]'
- '[[ $(direnv status --json | jq ".state.loadedRC.allowed") == 0 ]]'
+ preconditions:
+ - msg: Missing direnv
+ sh: command -v direnv &>/dev/null
generic-linux:
- desc: Try to install CLI tools into the projects .bin directory
- dir: '{{.GENERIC_BIN_DIR}}'
+ desc: Setup CLI tools into the projects .bin directory
+ dir: '{{.ROOT_DIR}}/.bin'
platforms: ['linux/amd64', 'linux/arm64']
cmds:
- for:
@@ -65,13 +68,15 @@ tasks:
venv:
desc: Set up virtual environment
cmds:
- - '{{.PYTHON_BIN}} -m venv {{.VIRTUAL_ENV}}'
+ - python3 -m venv {{.VIRTUAL_ENV}}
- '{{.VIRTUAL_ENV}}/bin/python3 -m pip install --upgrade pip setuptools wheel'
- - '{{.VIRTUAL_ENV}}/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}"'
+ - '{{.VIRTUAL_ENV}}/bin/python3 -m pip install --upgrade --requirement "{{.ROOT_DIR}}/requirements.txt"'
sources:
- - '{{.PIP_REQUIREMENTS_FILE}}'
+ - '{{.ROOT_DIR}}/requirements.txt'
generates:
- '{{.VIRTUAL_ENV}}/pyvenv.cfg'
preconditions:
+ - msg: Missing python3
+ sh: command -v python3 &>/dev/null
- msg: Missing Pip requirements file
- sh: test -f {{.PIP_REQUIREMENTS_FILE}}
+ sh: test -f {{.ROOT_DIR}}/requirements.txt
diff --git a/README.md b/README.md
index e341a7b4..cefa2b9d 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,8 @@ You have two different options for setting up your local workstation.
# Homebrew
brew install direnv go-task
# or, Arch
- pacman -S --noconfirm direnv go-task && ln -sf /usr/bin/go-task /usr/local/bin/task
+ pacman -S --noconfirm direnv go-task \
+ && ln -sf /usr/bin/go-task /usr/local/bin/task
```
2. [Hook `direnv` into your preferred shell](https://direnv.net/docs/hook.html), then run:
@@ -154,7 +155,7 @@ You have two different options for setting up your local workstation.
1. Deploy your cluster and bootstrap it. This generates secrets, generates the config files for your nodes and applies them. It bootstraps the cluster afterwards, fetches the kubeconfig file and installs Cilium and kubelet-csr-approver. It finishes with some health checks.
```sh
- task talos:bootstrap
+ task bootstrap:talos
```
2. ⚠️ It might take a while for the cluster to be setup (10+ minutes is normal), during which time you will see a variety of error messages like: "couldn't get current server API group list," "error: no matching resources found", etc. This is a normal. If this step gets interrupted, e.g. by pressing Ctrl + C, you likely will need to [nuke the cluster](#-Nuke) before trying again.
@@ -190,16 +191,14 @@ You have two different options for setting up your local workstation.
2. Install Flux and sync the cluster to the Git repository
- 📍 _Run `task flux:github-deploy-key` first if using a private repository._
-
```sh
- task flux:bootstrap
+ task bootstrap:flux
# namespace/flux-system configured
# customresourcedefinition.apiextensions.k8s.io/alerts.notification.toolkit.fluxcd.io created
# ...
```
-1. Verify Flux components are running in the cluster
+3. Verify Flux components are running in the cluster
```sh
kubectl -n flux-system get pods -o wide
@@ -281,12 +280,12 @@ By default Flux will periodically check your git repository for changes. In orde
3. Navigate to the settings of your repository on Github, under "Settings/Webhooks" press the "Add webhook" button. Fill in the webhook URL and your `bootstrap_github_webhook_token` secret in `config.yaml`, Content type: `application/json`, Events: Choose Just the push event, and save.
-## 💥 Nuke
+## 💥 Reset
There might be a situation where you want to destroy your Kubernetes cluster. The following command will reset your nodes back to maintenance mode, append `--force` to completely format your the Talos installation. Either way the nodes should reboot after the command has run.
```sh
-task talos:nuke
+task talos:reset # --force
```
## 🤖 Renovate
@@ -345,22 +344,22 @@ Resolving problems that you have could take some tweaking of your YAML manifests
## ⬆️ Upgrading Talos and Kubernetes
-### Manual
-
```sh
-# Upgrade Talos to a newer version
-# NOTE: This needs to be run once on every node
-task talos:upgrade NODE=? VERSION=?
-# e.g.
-# task talos:upgrade NODE=192.168.42.10 VERSION=v1.8.1
+# Upgrade cluster to a newer Talos version
+task talos:upgrade-cluster VERSION=?
+# e.g. task talos:upgrade-cluster VERSION=v1.8.1
```
```sh
-# Upgrade Kubernetes to a newer version
-# NOTE: This only needs to be run once against a controller node
-task talos:upgrade-k8s NODE=? VERSION=?
-# e.g.
-# task talos:upgrade-k8s NODE=192.168.42.10 VERSION=1.30.1
+# Upgrade node to a newer Talos version
+task talos:upgrade-node HOSTNAME=? VERSION=?
+# e.g. task talos:upgrade HOSTNAME=k8s-0 VERSION=v1.8.1
+```
+
+```sh
+# Upgrade cluster to a newer Kubernetes version
+task talos:upgrade-k8s VERSION=?
+# e.g. task talos:upgrade-k8s VERSION=1.30.1
```
## 👉 Help
diff --git a/Taskfile.yaml b/Taskfile.yaml
index 61775a26..a090cc96 100644
--- a/Taskfile.yaml
+++ b/Taskfile.yaml
@@ -17,11 +17,9 @@ env:
VIRTUAL_ENV: '{{.ROOT_DIR}}/.venv'
includes:
+ bootstrap: .taskfiles/Bootstrap
kubernetes: .taskfiles/Kubernetes
- flux: .taskfiles/Flux
- repository: .taskfiles/Repository
talos: .taskfiles/Talos
- sops: .taskfiles/Sops
workstation: .taskfiles/Workstation
user:
taskfile: .taskfiles/User
@@ -45,10 +43,10 @@ tasks:
configure:
desc: Configure repository from bootstrap vars
prompt: Any conflicting config in the kubernetes directory will be overwritten... continue?
- deps: ['workstation:direnv', 'workstation:venv', 'sops:age-keygen', 'init']
+ deps: ['workstation:direnv', 'workstation:venv', 'bootstrap:age-keygen', 'init']
cmds:
- task: template
- - task: sops:encrypt
+ - task: bootstrap:encrypt
- task: validate
template:
diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py
index b3a75a07..456f9c13 100644
--- a/bootstrap/scripts/validation.py
+++ b/bootstrap/scripts/validation.py
@@ -81,7 +81,7 @@ def validate_cli_tools(cloudflare: dict, **_) -> None:
raise ValueError(f"Missing required CLI tool {tool}")
-@required("bootstrap_sops_age_pubkey")
+@required("bootstrap_age_pubkey")
def validate_age(key: str, **_) -> None:
if not re.match(r"^age1[a-z0-9]{0,58}$", key):
raise ValueError(f"Invalid Age public key {key}")
diff --git a/bootstrap/templates/.sops.yaml.j2 b/bootstrap/templates/.sops.yaml.j2
index cb7aa764..aa3b06fa 100644
--- a/bootstrap/templates/.sops.yaml.j2
+++ b/bootstrap/templates/.sops.yaml.j2
@@ -4,9 +4,9 @@ creation_rules:
path_regex: talos/.*\.sops\.ya?ml
key_groups:
- age:
- - "#{ bootstrap_sops_age_pubkey }#"
+ - "#{ bootstrap_age_pubkey }#"
- path_regex: kubernetes/.*\.sops\.ya?ml
encrypted_regex: "^(data|stringData)$"
key_groups:
- age:
- - "#{ bootstrap_sops_age_pubkey }#"
+ - "#{ bootstrap_age_pubkey }#"
diff --git a/bootstrap/templates/kubernetes/apps/network/ingress-nginx/external/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/network/ingress-nginx/external/helmrelease.yaml.j2
index fe74b145..1a5c10ab 100644
--- a/bootstrap/templates/kubernetes/apps/network/ingress-nginx/external/helmrelease.yaml.j2
+++ b/bootstrap/templates/kubernetes/apps/network/ingress-nginx/external/helmrelease.yaml.j2
@@ -30,10 +30,10 @@ spec:
annotations:
external-dns.alpha.kubernetes.io/hostname: "external.${SECRET_DOMAIN}"
lbipam.cilium.io/ips: "#{ bootstrap_cloudflare.tunnel.ingress_vip }#"
- #% if bootstrap_loadbalancer_mode in ['dsr'] %#
- externalTrafficPolicy: Cluster
- #% else %#
+ #% if bgp.enabled %#
externalTrafficPolicy: Local
+ #% else %#
+ externalTrafficPolicy: Cluster
#% endif %#
ingressClassResource:
name: external
diff --git a/bootstrap/templates/kubernetes/apps/network/ingress-nginx/internal/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/network/ingress-nginx/internal/helmrelease.yaml.j2
index b69d2311..f924f402 100644
--- a/bootstrap/templates/kubernetes/apps/network/ingress-nginx/internal/helmrelease.yaml.j2
+++ b/bootstrap/templates/kubernetes/apps/network/ingress-nginx/internal/helmrelease.yaml.j2
@@ -27,10 +27,10 @@ spec:
service:
annotations:
lbipam.cilium.io/ips: "#{ bootstrap_cloudflare.ingress_vip }#"
- #% if bootstrap_loadbalancer_mode in ['dsr'] %#
- externalTrafficPolicy: Cluster
- #% else %#
+ #% if bgp.enabled %#
externalTrafficPolicy: Local
+ #% else %#
+ externalTrafficPolicy: Cluster
#% endif %#
ingressClassResource:
name: internal
diff --git a/bootstrap/templates/kubernetes/apps/network/k8s-gateway/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/network/k8s-gateway/app/helmrelease.yaml.j2
index 8d65c19b..982dfd93 100644
--- a/bootstrap/templates/kubernetes/apps/network/k8s-gateway/app/helmrelease.yaml.j2
+++ b/bootstrap/templates/kubernetes/apps/network/k8s-gateway/app/helmrelease.yaml.j2
@@ -29,9 +29,9 @@ spec:
port: 53
annotations:
lbipam.cilium.io/ips: "#{ bootstrap_cloudflare.gateway_vip }#"
- #% if bootstrap_loadbalancer_mode in ['dsr'] %#
- externalTrafficPolicy: Cluster
- #% else %#
+ #% if bgp.enabled %#
externalTrafficPolicy: Local
+ #% else %#
+ externalTrafficPolicy: Cluster
#% endif %#
watchedResources: ["Ingress", "Service"]
diff --git a/config.sample.yaml b/config.sample.yaml
index 052f2234..05ef430a 100644
--- a/config.sample.yaml
+++ b/config.sample.yaml
@@ -80,9 +80,9 @@ bootstrap_vlan: ""
# (Required) Age Public Key (e.g. age1...)
# 1. Generate a new key with the following command:
-# > task sops:age-keygen
+# > task bootstrap:age-keygen
# 2. Copy the PUBLIC key and paste it below
-bootstrap_sops_age_pubkey: ""
+bootstrap_age_pubkey: ""
# (Optional) Use cilium BGP control plane when L2 announcements won't traverse VLAN network segments.
# Needs a BGP capable router setup with the node IPs as peers.