From 965bfcdcb983f2b2836b68c11dd30b41565e54ec Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Fri, 19 Jan 2024 17:35:33 -0500 Subject: [PATCH] feat: k0s remove isolated controllers support and ansible prepare playbook (#1205) Signed-off-by: Devin Buhl --- README.md | 4 +- .../ansible/playbooks/cluster-prepare.yaml.j2 | 2 +- .../templates/kubernetes/k0s/k0sctl.yaml.j2 | 91 ++++++------------- .../k0s/resources/containerd/spegel.toml.j2 | 6 +- .../containerd/unprivileged-ports.toml.j2 | 2 +- .../k0s/resources/hooks/apply-system.sh.j2 | 60 ++++++++++++ .../k0s/resources/hooks/reset-cilium.sh.j2 | 40 ++++++++ .../k0s/resources/hooks/reset-system.sh.j2 | 11 +++ .../cilium-values-full.partial.yaml.j2 | 2 +- .../cilium-values-init.partial.yaml.j2 | 2 +- bootstrap/vars/config.sample.yaml | 1 - 11 files changed, 147 insertions(+), 74 deletions(-) create mode 100755 bootstrap/templates/kubernetes/k0s/resources/hooks/apply-system.sh.j2 create mode 100755 bootstrap/templates/kubernetes/k0s/resources/hooks/reset-cilium.sh.j2 create mode 100755 bootstrap/templates/kubernetes/k0s/resources/hooks/reset-system.sh.j2 diff --git a/README.md b/README.md index ad29eacd..b767fd01 100644 --- a/README.md +++ b/README.md @@ -297,9 +297,9 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g ### ⚡ Stage 4: Prepare your nodes for Kubernetes > [!NOTE] -> For **Talos** skip ahead to ⛵ [**Stage 5**](#-stage-5-install-kubernetes) +> For **Talos** or **k0s** skip ahead to ⛵ [**Stage 5**](#-stage-5-install-kubernetes) -#### k3s or k0s +#### k3s 📍 _Here we will be running an Ansible playbook to prepare your nodes for running a Kubernetes cluster._ diff --git a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 index 88b21217..0ac865aa 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k3s' or bootstrap_distribution == 'k0s' %# +#% if bootstrap_distribution == 'k3s' %# --- - name: Prepare System hosts: kubernetes diff --git a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 index 02e7a148..62dcd602 100644 --- a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 +++ b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 @@ -3,67 +3,50 @@ apiVersion: k0sctl.k0sproject.io/v1beta1 kind: Cluster metadata: - name: k0s-cluster + name: home-kubernetes spec: hosts: #% for item in bootstrap_nodes.master %# - - role: "#{ item.role | default('controller+worker') }#" + - role: controller+worker ssh: address: "#{ item.address }#" user: "#{ item.username }#" installFlags: - --disable-components=metrics-server - #% if item.role | default('controller+worker') == 'controller+worker' %# - --no-taints - #% endif %# - #% if item.role | default('') != 'controller' %# - files: - - name: containerd-config - src: resources/containerd/ - dstDir: /home/#{ item.username }#/ + #% if loop.first %# + files: &files + - name: system-config + src: resources/hooks/ + dstDir: /home/#{ item.username }#/k0s/hooks/ perm: 0644 user: root group: root + - name: containerd-config + src: resources/containerd/ + dstDir: /home/#{ item.username }#/k0s/containerd/ + perm: 0644 + user: root + group: root + #% else %# + files: *files #% endif %# - hooks: - #% if loop.first %# - #% if item.role | default('') != 'controller' %# + #% if loop.first %# + hooks: &hooks apply: before: - - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml + - sudo bash /home/#{ item.username }#/k0s/hooks/apply-system.sh + - sudo mv /home/#{ item.username }#/k0s/containerd/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml #% if spegel.enabled | default(false) %# - - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml + - sudo mv /home/#{ item.username }#/k0s/containerd/spegel.toml /etc/k0s/containerd.d/spegel.toml #% endif %# - #% endif %# - reset: &reset + reset: before: - - sudo ip link delete cilium_host > /dev/null 2>&1 || true - - sudo ip link delete cilium_net > /dev/null 2>&1 || true - - sudo ip link delete cilium_vxlan > /dev/null 2>&1 || true - - sudo iptables -t nat -F - - sudo iptables -t mangle -F - - sudo iptables -t filter -F - - sudo iptables -t raw -F - - sudo iptables -X - - sudo ip6tables -t nat -F - - sudo ip6tables -t mangle -F - - sudo ip6tables -t filter -F - - sudo ip6tables -t raw -F - - sudo ip6tables -X - - sudo rm -rf /etc/cni/net.d + - sudo bash /home/#{ item.username }#/k0s/hooks/reset-cilium.sh after: - - sudo rm -rf #{ bootstrap_local_storage_path }# - - nohup sudo -b bash -c 'sleep 5; reboot' &>/dev/null - #% else %# - #% if item.role | default('') != 'controller' %# - apply: - before: - - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml - #% if spegel.enabled | default(false) %# - - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml - #% endif %# - #% endif %# - reset: *reset + - sudo bash /home/#{ item.username }#/k0s/hooks/reset-system.sh + #% else %# + hooks: *hooks #% endif %# #% endfor %# #% if bootstrap_nodes.worker | default([]) | length > 0 %# @@ -72,30 +55,8 @@ spec: ssh: address: "#{ item.address }#" user: "#{ item.username }#" - #% if loop.first %# - files: &files - - name: containerd-config - src: resources/containerd/ - dstDir: /home/#{ item.username }#/ - perm: 0644 - user: root - group: root - #% else %# files: *files - #% endif %# - hooks: - #% if loop.first %# - apply: &apply - before: - - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml - #% if spegel.enabled | default(false) %# - - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml - #% endif %# - reset: *reset - #% else %# - apply: *apply - reset: *reset - #% endif %# + hooks: *hooks #% endfor %# #% endif %# k0s: diff --git a/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 b/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 index 887ff9a7..0d8b2b5c 100644 --- a/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 @@ -1,6 +1,8 @@ #% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %# -[plugins."io.containerd.grpc.v1.cri".containerd] - discard_unpacked_layers = false [plugins."io.containerd.grpc.v1.cri".registry] config_path = "/var/lib/k0s/containerd/certs.d" +[plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + discard_unpacked_layers = false #% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2 b/bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2 index 66583b86..ef02dc94 100644 --- a/bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %# +#% if bootstrap_distribution == 'k0s' %# [plugins."io.containerd.grpc.v1.cri"] enable_unprivileged_ports = true enable_unprivileged_icmp = true diff --git a/bootstrap/templates/kubernetes/k0s/resources/hooks/apply-system.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/hooks/apply-system.sh.j2 new file mode 100755 index 00000000..4fc7d518 --- /dev/null +++ b/bootstrap/templates/kubernetes/k0s/resources/hooks/apply-system.sh.j2 @@ -0,0 +1,60 @@ +#% if bootstrap_distribution == 'k0s' %# +#!/usr/bin/env bash + +[ $(id -u) -eq 0 ] || exec sudo $0 $@ + +# Prepare +sudo apt-get update -y + +# Timezone +timedatectl set-timezone "#{ bootstrap_timezone }#" + +# Install Packages +apt-get install -y --no-install-recommends \ + apt-transport-https ca-certificates conntrack curl dirmngr gdisk gnupg hdparm htop \ + iptables iputils-ping ipvsadm libseccomp2 lm-sensors neofetch net-tools nfs-common \ + nvme-cli open-iscsi parted psmisc python3 python3-apt python3-kubernetes python3-yaml \ + smartmontools socat software-properties-common unzip util-linux + +# Update DNS +chattr -i /etc/resolv.conf +rm -f /etc/resolv.conf +cat < /etc/resolv.conf +search . +nameserver #{ bootstrap_dns_server | default('1.1.1.1', true) }# +EOF +chattr +i /etc/resolv.conf + +# Kernel Modules +for module in br_netfilter ceph ip_vs ip_vs_rr nbd overlay rbd; do + if ! test -f /etc/modules-load.d/$module.conf; then + echo $module > /etc/modules-load.d/$module.conf + fi +done +systemctl restart systemd-modules-load.service + +# Sysctls +cat < /etc/sysctl.d/99-kubernetes.conf +fs.inotify.max_queued_events: 65536 +fs.inotify.max_user_watches: 524288 +fs.inotify.max_user_instances: 8192 +EOF +sysctl -p /etc/sysctl.d/99-kubernetes.conf + +# Disable swap +swapoff -a + +# Disable AppArmor +if systemctl is-enabled apparmor.service; then + systemctl mask apparmor.service +fi + +# Neofetch +cat < /etc/profile.d/neofetch.sh +neofetch --config none +EOF +chmod 755 /etc/profile.d/neofetch.sh + +# Ensure k0s containerd directory exists +mkdir -p /etc/k0s/containerd.d +#% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-cilium.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-cilium.sh.j2 new file mode 100755 index 00000000..ef69479d --- /dev/null +++ b/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-cilium.sh.j2 @@ -0,0 +1,40 @@ +#% if bootstrap_distribution == 'k0s' %# +#!/usr/bin/env bash + +[ $(id -u) -eq 0 ] || exec sudo $0 $@ + +remove_interfaces() { + ip link show 2>/dev/null | grep 'cilium' | while read ignore iface ignore; do + iface=${iface%%@*} + [ -z "$iface" ] || ip link delete $iface + done +} + +reset_iptables() { + iptables -t nat -F + iptables -t mangle -F + iptables -t filter -F + iptables -t raw -F + iptables -X + ip6tables -t nat -F + ip6tables -t mangle -F + ip6tables -t filter -F + ip6tables -t raw -F + ip6tables -X +} + +do_unmount_and_remove() { + set +x + while read -r _ path _; do + case "$path" in $1*) echo "$path" ;; esac + done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount -f "$0" && rm -rf "$0"' + set -x +} + +do_unmount_and_remove '/run/netns/cni-' +ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete +remove_interfaces +reset_iptables +rm -rf /var/lib/cni +rm -rf /etc/cni/net.d +#% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-system.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-system.sh.j2 new file mode 100755 index 00000000..48695e77 --- /dev/null +++ b/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-system.sh.j2 @@ -0,0 +1,11 @@ +#% if bootstrap_distribution == 'k0s' %# +#!/usr/bin/env bash + +[ $(id -u) -eq 0 ] || exec sudo $0 $@ + +# Remove local storage data +rm -rf "#{ bootstrap_local_storage_path }#" + +# Reboot +(sleep 10 && systemctl reboot)& +#% endif %# diff --git a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 index 9a173f5e..64ecf698 100644 --- a/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-full.partial.yaml.j2 @@ -4,7 +4,7 @@ bpf: bgp: enabled: false cluster: - name: home-cluster + name: home-kubernetes id: 1 containerRuntime: integration: containerd diff --git a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 index a0d96904..4a4843e5 100644 --- a/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 +++ b/bootstrap/templates/partials/cilium-values-init.partial.yaml.j2 @@ -4,7 +4,7 @@ bpf: bgp: enabled: false cluster: - name: home-cluster + name: home-kubernetes id: 1 containerRuntime: integration: containerd diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index 03ae3309..4fd4e424 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -87,7 +87,6 @@ bootstrap_nodes: # - name: # name or hostname of the master node (must match [a-z0-9-\.]+) # address: # ip address of the master node # username: # ssh username of the master node - # role: # (Optional: k0s) controller+worker (default) or controller # diskSerial: # (Required: Talos) Serial number of Disk for this master node # ... worker: # set to [] or omit if no workers are needed