feat: add spegel support for k0s (#1178)

* feat: add spegel support for k0s

Signed-off-by: Devin Buhl <devin@buhl.casa>

* chore: add spegel to e2e tests

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: update containerd file paths and run apply from k0s-config dir

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: move containerd files after upload

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>

bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2

@@ -8,3 +8,6 @@ resources:
   - ./coredns/ks.yaml
   #% endif %#
   - ./metrics-server/ks.yaml
+  #% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+  - ./spegel/ks.yaml
+  #% endif %#

bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2

@@ -0,0 +1,37 @@
+#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: spegel
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: spegel
+      version: v0.0.17
+      sourceRef:
+        kind: HelmRepository
+        name: xenitab
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    spegel:
+      #% if bootstrap_distribution == "k0s" %#
+      containerdSock: /run/k0s/containerd.sock
+      containerdRegistryConfigPath: /var/lib/k0s/containerd/certs.d
+      #% endif %#
+    service:
+      registry:
+        hostPort: 29999
+    serviceMonitor:
+      enabled: true
+#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2

@@ -0,0 +1,7 @@
+#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2

@@ -0,0 +1,22 @@
+#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app spegel
+  namespace: flux-system
+spec:
+  targetNamespace: kube-system
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/kube-system/spegel/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+#% endif %#

bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2

@@ -21,3 +21,4 @@ resources:
   - ./prometheus-community.yaml
   - ./stakater.yaml
   - ./weave-gitops.yaml
+  - ./xenitab.yaml

bootstrap/templates/kubernetes/flux/repositories/helm/xenitab.yaml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: xenitab
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 5m
+  url: oci://ghcr.io/xenitab/helm-charts

bootstrap/templates/kubernetes/k0s/k0s-config.yaml.j2

@@ -16,6 +16,19 @@ spec:
         #% if item.role | default('') == 'controller+worker' %#
         - --no-taints
         #% endif %#
+      files:
+        - name: containerd-config
+          src: resources/containerd/
+          perm: 0644
+          user: root
+          group: root
+      hooks:
+        apply:
+          before:
+            #% if spegel.enabled | default(false) %#
+            - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml
+            #% endif %#
+            - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
     #% endfor %#
     #% if bootstrap_nodes.worker | default([]) | length > 0 %#
     #% for item in bootstrap_nodes.worker %#
@@ -23,6 +36,19 @@ spec:
       ssh:
         address: "#{ item.address }#"
         user: "#{ item.username }#"
+      files:
+        - name: containerd-config
+          src: resources/containerd/
+          perm: 0644
+          user: root
+          group: root
+      hooks:
+        apply:
+          before:
+            #% if spegel.enabled | default(false) %#
+            - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml
+            #% endif %#
+            - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
     #% endfor %#
     #% endif %#
   k0s:
@@ -55,6 +81,15 @@ spec:
             - "#{ item.name }#"
             #% endif %#
             #% endfor %#
+        network:
+          kubeProxy:
+            disabled: true
+          #%  if bootstrap_nodes.master | length > 1 %#
+          nodeLocalLoadBalancing:
+            enabled: true
+            type: EnvoyProxy
+          #% endif %#
+          provider: custom
         extensions:
           helm:
             repositories:
@@ -119,13 +154,4 @@ spec:
                   routingMode: native
                   securityContext:
                     privileged: true
-        network:
-          kubeProxy:
-            disabled: true
-          #%  if bootstrap_nodes.master | length > 1 %#
-          nodeLocalLoadBalancing:
-            enabled: true
-            type: EnvoyProxy
-          #% endif %#
-          provider: custom
 #% endif %#

bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2

@@ -0,0 +1,6 @@
+#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %#
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  discard_unpacked_layers = false
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "/var/lib/k0s/containerd/certs.d"
+#% endif %#

bootstrap/templates/kubernetes/k0s/resources/containerd/unprivileged-ports.toml.j2

@@ -0,0 +1,5 @@
+#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %#
+[plugins."io.containerd.grpc.v1.cri"]
+  enable_unprivileged_ports = true
+  enable_unprivileged_icmp = true
+#% endif %#