feat: Do not rely on k0s file uploads - instead use curl (#1229)

* feat: Do not rely on k0s file uploads - instead use curl

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: add bootstrap_advanced_flags to validation

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: only skip tests for non-static config

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: only skip tests for non-static config

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: issues in k0s scripts

Signed-off-by: Devin Buhl <devin@buhl.casa>

* update: readme

Signed-off-by: Devin Buhl <devin@buhl.casa>

* update: taskfile

Signed-off-by: Devin Buhl <devin@buhl.casa>

* update: taskfile

Signed-off-by: Devin Buhl <devin@buhl.casa>

* update: k0s config and remove ansible hook

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>

.github/tests/config-k0s.yaml

@@ -4,10 +4,9 @@ skip_tests: true
 bootstrap_distribution: k0s
 bootstrap_github_username: onedr0p
 bootstrap_github_repository_name: flux-cluster-template
-bootstrap_github_repository_branch: main
 bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
 bootstrap_timezone: Etc/UTC
-bootstrap_acme_email: fake
+bootstrap_acme_email: fake@example.com
 bootstrap_acme_production_enabled: false
 bootstrap_flux_github_webhook_token: fake
 bootstrap_cloudflare_domain: fake

.github/tests/config-k3s-ipv4.yaml

@@ -4,10 +4,9 @@ skip_tests: true
 bootstrap_distribution: k3s
 bootstrap_github_username: onedr0p
 bootstrap_github_repository_name: flux-cluster-template
-bootstrap_github_repository_branch: main
 bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
 bootstrap_timezone: Etc/UTC
-bootstrap_acme_email: fake
+bootstrap_acme_email: fake@example.com
 bootstrap_acme_production_enabled: false
 bootstrap_flux_github_webhook_token: fake
 bootstrap_cloudflare_domain: fake

.github/tests/config-k3s-ipv6.yaml

@@ -4,10 +4,9 @@ skip_tests: true
 bootstrap_distribution: k3s
 bootstrap_github_username: onedr0p
 bootstrap_github_repository_name: flux-cluster-template
-bootstrap_github_repository_branch: main
 bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
 bootstrap_timezone: Etc/UTC
-bootstrap_acme_email: fake
+bootstrap_acme_email: fake@example.com
 bootstrap_acme_production_enabled: false
 bootstrap_flux_github_webhook_token: fake
 bootstrap_cloudflare_domain: fake

.github/tests/config-talos.yaml

@@ -4,10 +4,9 @@ skip_tests: true
 bootstrap_distribution: talos
 bootstrap_github_username: onedr0p
 bootstrap_github_repository_name: flux-cluster-template
-bootstrap_github_repository_branch: main
 bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
 bootstrap_timezone: Etc/UTC
-bootstrap_acme_email: fake
+bootstrap_acme_email: fake@example.com
 bootstrap_acme_production_enabled: false
 bootstrap_flux_github_webhook_token: fake
 bootstrap_cloudflare_domain: fake

.taskfiles/Ansible/Taskfile.yaml

@@ -21,8 +21,8 @@ tasks:
 
   deps:
     desc: Set up Ansible dependencies
+    deps: [":setup-virtual-env"]
     cmds:
-      - task: :setup-virtual-env
       - .venv/bin/python3 -m pip install --upgrade --requirement "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}"
       - .venv/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}}
     preconditions:

README.md

@@ -284,8 +284,17 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
     task configure
     ```
 
-8. Continue on to ⚡ [**Stage 4**](#-stage-4-prepare-your-nodes-for-kubernetes)
+8. Push you changes to git
 
+   📍 **Verify** all the `*.sops.yaml` and `*.sops.yaml` files under the `./ansible`, and `./kubernetes` directories are **encrypted** with SOPS
+
+    ```sh
+    git add -A
+    git commit -m "Initial commit :rocket:"
+    git push
+    ```
+
+9. Continue on to ⚡ [**Stage 4**](#-stage-4-prepare-your-nodes-for-kubernetes)
 
 ### ⚡ Stage 4: Prepare your nodes for Kubernetes
 
@@ -396,17 +405,7 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
     # ✔ prerequisites checks passed
     ```
 
-2. Push you changes to git
+2. Install Flux and sync the cluster to the Git repository
-
-   📍 **Verify** all the `*.sops.yaml` and `*.sops.yaml` files under the `./ansible`, and `./kubernetes` directories are **encrypted** with SOPS
-
-    ```sh
-    git add -A
-    git commit -m "Initial commit :rocket:"
-    git push
-    ```
-
-3. Install Flux and sync the cluster to the Git repository
 
     ```sh
     task flux:bootstrap
@@ -415,7 +414,7 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
     # ...
     ```
 
-4. Verify Flux components are running in the cluster
+3. Verify Flux components are running in the cluster
 
     ```sh
     kubectl -n flux-system get pods -o wide

Taskfile.yaml

@@ -38,8 +38,8 @@ tasks:
 
   init:
     desc: Initialize virtual env and configuration files
+    deps: ["setup-virtual-env"]
     cmds:
-      - task: setup-virtual-env
       - mkdir -p {{.PRIVATE_DIR}}
       - cp -n {{.BOOTSTRAP_ADDONS_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_ADDONS_FILE}}
       - cp -n {{.BOOTSTRAP_CONFIG_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_CONFIG_FILE}}
@@ -53,28 +53,13 @@ tasks:
         silent: true
 
   configure:
-    desc: Configure repository from Ansible vars
+    desc: Configure repository from bootstrap vars
     prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue?
+    deps: ["setup-virtual-env"]
     cmds:
       - task: .template
-      - task: .post-validate
-
-  .template:
-    internal: true
-    cmds:
-      - ./.venv/bin/makejinja
       - task: sops:encrypt
-    preconditions:
+      - task: .validate
-      - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" }
-      - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" }
-      - { msg: "Missing Makejinja plugin file", sh: "test -f {{.BOOTSTRAP_DIR}}/scripts/plugin.py" }
-      - { msg: "Missing bootstrap addons file", sh: "test -f {{.BOOTSTRAP_ADDONS_FILE}}" }
-      - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" }
-
-  .post-validate:
-    internal: true
-    cmds:
-      - task: kubernetes:kubeconform
 
   setup-virtual-env:
     desc: Set up virtual environment
@@ -88,3 +73,19 @@ tasks:
       - "{{.ROOT_DIR}}/.venv/pyvenv.cfg"
     preconditions:
       - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" }
+
+  .template:
+    internal: true
+    cmds:
+      - ./.venv/bin/makejinja
+    preconditions:
+      - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" }
+      - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" }
+      - { msg: "Missing Makejinja plugin file", sh: "test -f {{.BOOTSTRAP_DIR}}/scripts/plugin.py" }
+      - { msg: "Missing bootstrap addons file", sh: "test -f {{.BOOTSTRAP_ADDONS_FILE}}" }
+      - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" }
+
+  .validate:
+    internal: true
+    cmds:
+      - task: kubernetes:kubeconform

bootstrap/scripts/plugin.py

@@ -20,9 +20,6 @@ def encrypt(value: str) -> str:
 
 class Plugin(makejinja.plugin.Plugin):
     def __init__(self, data: dict[str, Any]):
-        if data.get("skip_tests", False):
-            return
-
         validation.validate(data)
 
     def filters(self) -> makejinja.plugin.Filters:

bootstrap/scripts/validation.py

@@ -100,14 +100,17 @@ def validate_distribution(distribution: str, **_) -> None:
     _validate_distribution(distribution)
 
 
-@required("bootstrap_github_username", "bootstrap_github_repository_name", "bootstrap_github_repository_branch")
+@required("bootstrap_github_username", "bootstrap_github_repository_name", "bootstrap_advanced_flags")
-def validate_github(username: str, repository: str, branch: str, **_) -> None:
+def validate_github(username: str, repository: str, advanced_flags: dict, **_) -> None:
     try:
-        request = requests.get(f"https://api.github.com/repos/{username}/{repository}/branches/{branch}")
+        request = requests.get(
+            f"https://api.github.com/repos/{username}/{repository}/branches/{advanced_flags.get('github_repository_branch', 'main')}")
         if request.status_code != 200:
-            raise ValueError(f"GitHub repository {username}/{repository} branch {branch} not found")
+            raise ValueError(
+                f"GitHub repository {username}/{repository} branch {advanced_flags.get('github_repository_branch', 'main')} not found")
     except requests.exceptions.RequestException as e:
-        raise ValueError(f"GitHub repository {username}/{repository} branch {branch} not found") from e
+        raise ValueError(
+            f"GitHub repository {username}/{repository} branch {advanced_flags.get('github_repository_branch', 'main')} not found") from e
 
 
 @required("bootstrap_age_public_key")
@@ -240,18 +243,27 @@ def validate_nodes(node_cidr: str, nodes: dict[list], distribution: str, **_) ->
         _validate_node(node, node_cidr, distribution)
 
 
+def massage(data: dict) -> dict:
+    data["bootstrap_advanced_flags"] = data.get("bootstrap_advanced_flags", {})
+    return data
+
 def validate(data: dict) -> None:
+    user_data = massage(data)
+
     validate_python_version()
-    validate_cli_tools(data)
+    validate_cli_tools(user_data)
-    validate_distribution(data)
+    validate_distribution(user_data)
-    if not data.get("bootstrap_private_github_repo"):
+    validate_age(user_data)
-        validate_github(data)
+    validate_timezone(user_data)
-    validate_age(data)
+    validate_bootstrap_dns_server(user_data)
-    validate_timezone(data)
+    validate_cluster_cidrs(user_data)
-    validate_acme_email(data)
+    validate_flux_github_webhook_token(user_data)
-    validate_flux_github_webhook_token(data)
+    validate_host_network(user_data)
-    validate_cloudflare(data)
+    validate_acme_email(user_data)
-    validate_host_network(data)
+
-    validate_bootstrap_dns_server(data)
+    if not user_data.get("bootstrap_private_github_repo"):
-    validate_cluster_cidrs(data)
+        validate_github(user_data)
-    validate_nodes(data)
+
+    if not user_data.get("skip_tests", False):
+        validate_cloudflare(user_data)
+        validate_nodes(user_data)

bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2

@@ -62,7 +62,7 @@
               ff02::2 ip6-allrouters
             dest: /etc/hosts
             mode: preserve
-        #% if bootstrap_advanced_flags.update_resolv_conf %#
+        #% if bootstrap_advanced_flags.update_resolv_conf|default(true) %#
         # https://github.com/onedr0p/flux-cluster-template/discussions/635
         - name: Network Configuration | Remove immutable flag from /etc/resolv.conf
           ansible.builtin.file:

bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2

@@ -7,7 +7,7 @@ metadata:
 spec:
   interval: 30m
   ref:
-    branch: "#{ bootstrap_github_repository_branch }#"
+    branch: "#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#"
   #% if bootstrap_private_github_repo %#
   secretRef:
     name: github-deploy-key

bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2

@@ -14,37 +14,15 @@ spec:
       installFlags:
         - --disable-components=metrics-server
         - --no-taints
-      #% if loop.first %#
+      hooks:
-      files: &files
-        - name: system-config
-          src: resources/hooks/
-          dstDir: /home/#{ item.username }#/k0s/hooks/
-          perm: 0644
-          user: root
-          group: root
-        - name: containerd-config
-          src: resources/containerd/
-          dstDir: /home/#{ item.username }#/k0s/containerd/
-          perm: 0644
-          user: root
-          group: root
-      #% else %#
-      files: *files
-      #% endif %#
-      #% if loop.first %#
-      hooks: &hooks
         apply:
           before:
-            - sudo bash /home/#{ item.username }#/k0s/hooks/apply-system.sh
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-system.sh | INSTALL_K0S_HOSTNAME="#{ item.name }#" bash -s -
-            - sudo mv /home/#{ item.username }#/k0s/containerd/default.toml /etc/k0s/containerd.d/default.toml
         reset:
           before:
-            - sudo bash /home/#{ item.username }#/k0s/hooks/reset-cilium.sh
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-cilium.sh | bash -s -
           after:
-            - sudo bash /home/#{ item.username }#/k0s/hooks/reset-system.sh
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s -
-      #% else %#
-      hooks: *hooks
-      #% endif %#
     #% endfor %#
     #% if bootstrap_nodes.worker | length > 0 %#
     #% for item in bootstrap_nodes.worker %#
@@ -52,8 +30,15 @@ spec:
       ssh:
         address: "#{ item.address }#"
         user: "#{ item.username }#"
-      files: *files
+      hooks:
-      hooks: *hooks
+        apply:
+          before:
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-system.sh | INSTALL_K0S_HOSTNAME="#{ item.name }#" bash -s -
+        reset:
+          before:
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-cilium.sh | bash -s -
+          after:
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s -
     #% endfor %#
     #% endif %#
   k0s:

bootstrap/templates/kubernetes/k0s/resources/apply-system.sh.j2

@@ -1,11 +1,25 @@
 #% if bootstrap_distribution == 'k0s' %#
 #!/usr/bin/env bash
+set -e
+set -o noglob
 
 [ $(id -u) -eq 0 ] || exec sudo $0 $@
 
 # Prepare
 sudo apt-get update -y
 
+# Hostname
+if [ -n "${INSTALL_K0S_HOSTNAME}" ] && [ "$(hostnamectl hostname)" != "${INSTALL_K0S_HOSTNAME}" ]; then
+    hostnamectl set-hostname "${INSTALL_K0S_HOSTNAME}"
+tee /etc/hosts > /dev/null <<EOF
+127.0.0.1 localhost
+127.0.1.1 $INSTALL_K0S_HOSTNAME
+::1       localhost ip6-localhost ip6-loopback
+ff02::1   ip6-allnodes
+ff02::2   ip6-allrouters
+EOF
+fi
+
 # Timezone
 timedatectl set-timezone "#{ bootstrap_timezone }#"
 
@@ -16,7 +30,7 @@ apt-get install -y --no-install-recommends \
     nvme-cli open-iscsi parted psmisc python3 python3-apt python3-kubernetes python3-yaml \
     smartmontools socat software-properties-common unzip util-linux
 
-#% if bootstrap_advanced_flags.update_resolv_conf %#
+#% if bootstrap_advanced_flags.update_resolv_conf|default(true) %#
 # Update DNS
 chattr -i /etc/resolv.conf
 rm -f /etc/resolv.conf
@@ -37,9 +51,9 @@ systemctl restart systemd-modules-load.service
 
 # Sysctls
 cat <<EOF > /etc/sysctl.d/99-kubernetes.conf
-fs.inotify.max_queued_events: 65536
+fs.inotify.max_queued_events = 65536
-fs.inotify.max_user_watches: 524288
+fs.inotify.max_user_watches = 524288
-fs.inotify.max_user_instances: 8192
+fs.inotify.max_user_instances = 8192
 EOF
 sysctl -p /etc/sysctl.d/99-kubernetes.conf
 
@@ -57,6 +71,17 @@ neofetch --config none
 EOF
 chmod 755 /etc/profile.d/neofetch.sh
 
-# Ensure k0s containerd directory exists
+# Create containerd config
 mkdir -p /etc/k0s/containerd.d
+cat <<EOF > /etc/k0s/containerd.d/default.toml
+[plugins."io.containerd.grpc.v1.cri"]
+  enable_unprivileged_ports = true
+  enable_unprivileged_icmp = true
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "/var/lib/k0s/containerd/certs.d"
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  discard_unpacked_layers = false
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  discard_unpacked_layers = false
+EOF
 #% endif %#

bootstrap/templates/kubernetes/k0s/resources/containerd/default.toml.j2

@@ -1,11 +0,0 @@
-#% if bootstrap_distribution == 'k0s' %#
-[plugins."io.containerd.grpc.v1.cri"]
-  enable_unprivileged_ports = true
-  enable_unprivileged_icmp = true
-[plugins."io.containerd.grpc.v1.cri".registry]
-  config_path = "/var/lib/k0s/containerd/certs.d"
-[plugins."io.containerd.grpc.v1.cri".containerd]
-  discard_unpacked_layers = false
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-  discard_unpacked_layers = false
-#% endif %#

bootstrap/templates/kubernetes/k0s/resources/reset-cilium.sh.j2

@@ -1,5 +1,7 @@
 #% if bootstrap_distribution == 'k0s' %#
 #!/usr/bin/env bash
+set -e
+set -o noglob
 
 [ $(id -u) -eq 0 ] || exec sudo $0 $@
 

bootstrap/templates/kubernetes/k0s/resources/reset-system.sh.j2

@@ -1,8 +1,13 @@
 #% if bootstrap_distribution == 'k0s' %#
 #!/usr/bin/env bash
+set -e
+set -o noglob
 
 [ $(id -u) -eq 0 ] || exec sudo $0 $@
 
+# Remove containerd config
+rm -rf /etc/k0s/containerd.d
+
 # Remove local storage data
 rm -rf "#{ bootstrap_advanced_flags.local_storage_path|default('/var/openebs/local', true) }#"
 

bootstrap/vars/config.sample.yaml

@@ -10,8 +10,6 @@ bootstrap_distribution: k3s
 bootstrap_github_username:
 # Github repository (e.g. flux-cluster-template)
 bootstrap_github_repository_name:
-# Github repository branch (e.g. main)
-bootstrap_github_repository_branch: main
 # Age Public Key (e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta)
 bootstrap_age_public_key:
 # Choose your timezone (e.g. America/New_York)
@@ -90,6 +88,8 @@ bootstrap_nodes:
 #
 
 # bootstrap_advanced_flags:
+#   # Github repository branch
+#   github_repository_branch: main
 #   # Disable to prevent changes to nodes /etc/resolv.conf
 #   update_resolv_conf: true
 #   # Cilium load balancer mode, choose either 'dsr' or 'snat'