feat: k0s remove isolated controllers support and ansible prepare playbook (#1205)

Signed-off-by: Devin Buhl <devin@buhl.casa>

README.md

@@ -297,9 +297,9 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g
 ### ⚡ Stage 4: Prepare your nodes for Kubernetes
 
 > [!NOTE]
-> For **Talos** skip ahead to ⛵ [**Stage 5**](#-stage-5-install-kubernetes)
+> For **Talos** or **k0s** skip ahead to ⛵ [**Stage 5**](#-stage-5-install-kubernetes)
 
-#### k3s or k0s
+#### k3s
 
 📍 _Here we will be running an Ansible playbook to prepare your nodes for running a Kubernetes cluster._
 

bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k3s' or bootstrap_distribution == 'k0s' %#
+#% if bootstrap_distribution == 'k3s' %#
 ---
 - name: Prepare System
   hosts: kubernetes

bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2

@@ -3,67 +3,50 @@
 apiVersion: k0sctl.k0sproject.io/v1beta1
 kind: Cluster
 metadata:
-  name: k0s-cluster
+  name: home-kubernetes
 spec:
   hosts:
     #% for item in bootstrap_nodes.master %#
-    - role: "#{ item.role | default('controller+worker') }#"
+    - role: controller+worker
       ssh:
         address: "#{ item.address }#"
         user: "#{ item.username }#"
       installFlags:
         - --disable-components=metrics-server
-        #% if item.role | default('controller+worker') == 'controller+worker' %#
         - --no-taints
-        #% endif %#
-      #% if item.role | default('') != 'controller' %#
-      files:
+      #% if loop.first %#
+      files: &files
+        - name: system-config
+          src: resources/hooks/
+          dstDir: /home/#{ item.username }#/k0s/hooks/
+          perm: 0644
+          user: root
+          group: root
         - name: containerd-config
           src: resources/containerd/
-          dstDir: /home/#{ item.username }#/
+          dstDir: /home/#{ item.username }#/k0s/containerd/
           perm: 0644
           user: root
           group: root
-      #% endif %#
-      hooks:
-        #% if loop.first %#
-        #% if item.role | default('') != 'controller' %#
-        apply:
-          before:
-            - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
-            #% if spegel.enabled | default(false) %#
-            - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml
-            #% endif %#
-        #% endif %#
-        reset: &reset
-          before:
-            - sudo ip link delete cilium_host > /dev/null 2>&1 || true
-            - sudo ip link delete cilium_net > /dev/null 2>&1 || true
-            - sudo ip link delete cilium_vxlan > /dev/null 2>&1 || true
-            - sudo iptables -t nat -F
-            - sudo iptables -t mangle -F
-            - sudo iptables -t filter -F
-            - sudo iptables -t raw -F
-            - sudo iptables -X
-            - sudo ip6tables -t nat -F
-            - sudo ip6tables -t mangle -F
-            - sudo ip6tables -t filter -F
-            - sudo ip6tables -t raw -F
-            - sudo ip6tables -X
-            - sudo rm -rf /etc/cni/net.d
-          after:
-            - sudo rm -rf #{ bootstrap_local_storage_path }#
-            - nohup sudo -b bash -c 'sleep 5; reboot' &>/dev/null
       #% else %#
-        #% if item.role | default('') != 'controller' %#
+      files: *files
+      #% endif %#
+      #% if loop.first %#
+      hooks: &hooks
         apply:
           before:
-            - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
+            - sudo bash /home/#{ item.username }#/k0s/hooks/apply-system.sh
+            - sudo mv /home/#{ item.username }#/k0s/containerd/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
             #% if spegel.enabled | default(false) %#
-            - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml
+            - sudo mv /home/#{ item.username }#/k0s/containerd/spegel.toml /etc/k0s/containerd.d/spegel.toml
             #% endif %#
-        #% endif %#
-        reset: *reset
+        reset:
+          before:
+            - sudo bash /home/#{ item.username }#/k0s/hooks/reset-cilium.sh
+          after:
+            - sudo bash /home/#{ item.username }#/k0s/hooks/reset-system.sh
+      #% else %#
+      hooks: *hooks
       #% endif %#
     #% endfor %#
     #% if bootstrap_nodes.worker | default([]) | length > 0 %#
@@ -72,30 +55,8 @@ spec:
       ssh:
         address: "#{ item.address }#"
         user: "#{ item.username }#"
-      #% if loop.first %#
-      files: &files
-        - name: containerd-config
-          src: resources/containerd/
-          dstDir: /home/#{ item.username }#/
-          perm: 0644
-          user: root
-          group: root
-      #% else %#
       files: *files
-      #% endif %#
-      hooks:
-        #% if loop.first %#
-        apply: &apply
-          before:
-            - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
-            #% if spegel.enabled | default(false) %#
-            - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml
-            #% endif %#
-        reset: *reset
-        #% else %#
-        apply: *apply
-        reset: *reset
-        #% endif %#
+      hooks: *hooks
     #% endfor %#
     #% endif %#
   k0s:

bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2

@@ -1,6 +1,8 @@
 #% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %#
-[plugins."io.containerd.grpc.v1.cri".containerd]
-  discard_unpacked_layers = false
 [plugins."io.containerd.grpc.v1.cri".registry]
   config_path = "/var/lib/k0s/containerd/certs.d"
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  discard_unpacked_layers = false
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  discard_unpacked_layers = false
 #% endif %#

bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2

@@ -1,4 +1,4 @@
-#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %#
+#% if bootstrap_distribution == 'k0s' %#
 [plugins."io.containerd.grpc.v1.cri"]
   enable_unprivileged_ports = true
   enable_unprivileged_icmp = true

bootstrap/templates/kubernetes/k0s/resources/hooks/apply-system.sh.j2

@@ -0,0 +1,60 @@
+#% if bootstrap_distribution == 'k0s' %#
+#!/usr/bin/env bash
+
+[ $(id -u) -eq 0 ] || exec sudo $0 $@
+
+# Prepare
+sudo apt-get update -y
+
+# Timezone
+timedatectl set-timezone "#{ bootstrap_timezone }#"
+
+# Install Packages
+apt-get install -y --no-install-recommends \
+    apt-transport-https ca-certificates conntrack curl dirmngr gdisk gnupg hdparm htop \
+    iptables iputils-ping ipvsadm libseccomp2 lm-sensors neofetch net-tools nfs-common \
+    nvme-cli open-iscsi parted psmisc python3 python3-apt python3-kubernetes python3-yaml \
+    smartmontools socat software-properties-common unzip util-linux
+
+# Update DNS
+chattr -i /etc/resolv.conf
+rm -f /etc/resolv.conf
+cat <<EOF > /etc/resolv.conf
+search .
+nameserver #{ bootstrap_dns_server | default('1.1.1.1', true) }#
+EOF
+chattr +i /etc/resolv.conf
+
+# Kernel Modules
+for module in br_netfilter ceph ip_vs ip_vs_rr nbd overlay rbd; do
+    if ! test -f /etc/modules-load.d/$module.conf; then
+        echo $module > /etc/modules-load.d/$module.conf
+    fi
+done
+systemctl restart systemd-modules-load.service
+
+# Sysctls
+cat <<EOF > /etc/sysctl.d/99-kubernetes.conf
+fs.inotify.max_queued_events: 65536
+fs.inotify.max_user_watches: 524288
+fs.inotify.max_user_instances: 8192
+EOF
+sysctl -p /etc/sysctl.d/99-kubernetes.conf
+
+# Disable swap
+swapoff -a
+
+# Disable AppArmor
+if systemctl is-enabled apparmor.service; then
+    systemctl mask apparmor.service
+fi
+
+# Neofetch
+cat <<EOF > /etc/profile.d/neofetch.sh
+neofetch --config none
+EOF
+chmod 755 /etc/profile.d/neofetch.sh
+
+# Ensure k0s containerd directory exists
+mkdir -p /etc/k0s/containerd.d
+#% endif %#

bootstrap/templates/kubernetes/k0s/resources/hooks/reset-cilium.sh.j2

@@ -0,0 +1,40 @@
+#% if bootstrap_distribution == 'k0s' %#
+#!/usr/bin/env bash
+
+[ $(id -u) -eq 0 ] || exec sudo $0 $@
+
+remove_interfaces() {
+  ip link show 2>/dev/null | grep 'cilium' | while read ignore iface ignore; do
+      iface=${iface%%@*}
+      [ -z "$iface" ] || ip link delete $iface
+  done
+}
+
+reset_iptables() {
+    iptables -t nat -F
+    iptables -t mangle -F
+    iptables -t filter -F
+    iptables -t raw -F
+    iptables -X
+    ip6tables -t nat -F
+    ip6tables -t mangle -F
+    ip6tables -t filter -F
+    ip6tables -t raw -F
+    ip6tables -X
+}
+
+do_unmount_and_remove() {
+    set +x
+    while read -r _ path _; do
+        case "$path" in $1*) echo "$path" ;; esac
+    done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount -f "$0" && rm -rf "$0"'
+    set -x
+}
+
+do_unmount_and_remove '/run/netns/cni-'
+ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete
+remove_interfaces
+reset_iptables
+rm -rf /var/lib/cni
+rm -rf /etc/cni/net.d
+#% endif %#

bootstrap/templates/kubernetes/k0s/resources/hooks/reset-system.sh.j2

@@ -0,0 +1,11 @@
+#% if bootstrap_distribution == 'k0s' %#
+#!/usr/bin/env bash
+
+[ $(id -u) -eq 0 ] || exec sudo $0 $@
+
+# Remove local storage data
+rm -rf "#{ bootstrap_local_storage_path }#"
+
+# Reboot
+(sleep 10 && systemctl reboot)&
+#% endif %#

bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

@@ -4,7 +4,7 @@ bpf:
 bgp:
   enabled: false
 cluster:
-  name: home-cluster
+  name: home-kubernetes
   id: 1
 containerRuntime:
   integration: containerd

bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

@@ -4,7 +4,7 @@ bpf:
 bgp:
   enabled: false
 cluster:
-  name: home-cluster
+  name: home-kubernetes
   id: 1
 containerRuntime:
   integration: containerd

bootstrap/vars/config.sample.yaml

@@ -87,7 +87,6 @@ bootstrap_nodes:
     # - name:       # name or hostname of the master node (must match [a-z0-9-\.]+)
     #   address:    # ip address of the master node
     #   username:   # ssh username of the master node
-    #   role:       # (Optional: k0s) controller+worker (default) or controller
     #   diskSerial: # (Required: Talos) Serial number of Disk for this master node
     # ...
   worker: # set to [] or omit if no workers are needed