feat: add spegel support for k0s (#1178)

* feat: add spegel support for k0s Signed-off-by: Devin Buhl <devin@buhl.casa> * chore: add spegel to e2e tests Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update containerd file paths and run apply from k0s-config dir Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: move containerd files after upload Signed-off-by: Devin Buhl <devin@buhl.casa> --------- Signed-off-by: Devin Buhl <devin@buhl.casa>
2024-01-17 16:00:28 -05:00
1 parent edbcf28139
commit ca9d1818fc
@@ -223,8 +223,7 @@
      "fileMatch": [
        "(^|/).taskfiles/.+\\.ya?ml$",
        "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
-        "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
-        "(^|/)k0s-config.ya?ml(\\.j2)?(\\.j2)?$"
+        "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
      ],
      "matchStrings": [
        // Example:
@@ -32,3 +32,6 @@ discord_template_notifier:

 volsync:
  enabled: true
+
+spegel:
+  enabled: true
@@ -6,12 +6,13 @@ env:
  DISABLE_UPGRADE_CHECK: "true"

 vars:
-  K0S_CONFIG_FILE: "{{.ROOT_DIR}}/k0s-config.yaml"
+  K0S_CONFIG_FILE: "{{.KUBERNETES_DIR}}/k0s/k0s-config.yaml"

 tasks:

  apply:
    desc: Apply k0s cluster config
+    dir: "{{.KUBERNETES_DIR}}/k0s" # Needed for uploading files in k0s-config.yaml
    cmds:
      - k0sctl apply --config {{.K0S_CONFIG_FILE}}
      - task: kubeconfig
@@ -8,3 +8,6 @@ resources:
  - ./coredns/ks.yaml
  #% endif %#
  - ./metrics-server/ks.yaml
+  #% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+  - ./spegel/ks.yaml
+  #% endif %#
@@ -0,0 +1,37 @@
+#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: spegel
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: spegel
+      version: v0.0.17
+      sourceRef:
+        kind: HelmRepository
+        name: xenitab
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    spegel:
+      #% if bootstrap_distribution == "k0s" %#
+      containerdSock: /run/k0s/containerd.sock
+      containerdRegistryConfigPath: /var/lib/k0s/containerd/certs.d
+      #% endif %#
+    service:
+      registry:
+        hostPort: 29999
+    serviceMonitor:
+      enabled: true
+#% endif %#
@@ -0,0 +1,7 @@
+#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+#% endif %#
@@ -0,0 +1,22 @@
+#% if bootstrap_distribution == "k0s" and spegel.enabled | default(false) %#
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app spegel
+  namespace: flux-system
+spec:
+  targetNamespace: kube-system
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/kube-system/spegel/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+#% endif %#
@@ -21,3 +21,4 @@ resources:
  - ./prometheus-community.yaml
  - ./stakater.yaml
  - ./weave-gitops.yaml
+  - ./xenitab.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: xenitab
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 5m
+  url: oci://ghcr.io/xenitab/helm-charts
@@ -16,6 +16,19 @@ spec:
        #% if item.role | default('') == 'controller+worker' %#
        - --no-taints
        #% endif %#
+      files:
+        - name: containerd-config
+          src: resources/containerd/
+          perm: 0644
+          user: root
+          group: root
+      hooks:
+        apply:
+          before:
+            #% if spegel.enabled | default(false) %#
+            - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml
+            #% endif %#
+            - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
    #% endfor %#
    #% if bootstrap_nodes.worker | default([]) | length > 0 %#
    #% for item in bootstrap_nodes.worker %#
@@ -23,6 +36,19 @@ spec:
      ssh:
        address: "#{ item.address }#"
        user: "#{ item.username }#"
+      files:
+        - name: containerd-config
+          src: resources/containerd/
+          perm: 0644
+          user: root
+          group: root
+      hooks:
+        apply:
+          before:
+            #% if spegel.enabled | default(false) %#
+            - sudo mv ~/spegel.toml /etc/k0s/containerd.d/spegel.toml
+            #% endif %#
+            - sudo mv ~/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml
    #% endfor %#
    #% endif %#
  k0s:
@@ -55,6 +81,15 @@ spec:
            - "#{ item.name }#"
            #% endif %#
            #% endfor %#
+        network:
+          kubeProxy:
+            disabled: true
+          #%  if bootstrap_nodes.master | length > 1 %#
+          nodeLocalLoadBalancing:
+            enabled: true
+            type: EnvoyProxy
+          #% endif %#
+          provider: custom
        extensions:
          helm:
            repositories:
@@ -119,13 +154,4 @@ spec:
                  routingMode: native
                  securityContext:
                    privileged: true
-        network:
-          kubeProxy:
-            disabled: true
-          #%  if bootstrap_nodes.master | length > 1 %#
-          nodeLocalLoadBalancing:
-            enabled: true
-            type: EnvoyProxy
-          #% endif %#
-          provider: custom
 #% endif %#
@@ -0,0 +1,6 @@
+#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %#
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  discard_unpacked_layers = false
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "/var/lib/k0s/containerd/certs.d"
+#% endif %#
@@ -0,0 +1,5 @@
+#% if bootstrap_distribution == 'k0s' and spegel.enabled | default(false) %#
+[plugins."io.containerd.grpc.v1.cri"]
+  enable_unprivileged_ports = true
+  enable_unprivileged_icmp = true
+#% endif %#
@@ -41,7 +41,7 @@ system_upgrade_controller:
  # WARNING: Only enable this if you also track the version of k3s in the
  # ansible configuration files. Running ansible against an already provisioned
  # cluster with this enabled might cause your cluster to be downgraded.
-  # Note that if bootstrap_distribution is set to k0s enable: true will be ignored.
+  # Note: If bootstrap_distribution is set to k0s this will be ignored.
  enabled: false

 # https://github.com/morphy2k/rss-forwarder
@@ -54,3 +54,8 @@ discord_template_notifier:
 # https://github.com/backube/volsync
 volsync:
  enabled: false
+
+# https://github.com/XenitAB/spegel
+spegel:
+  # Note: This only applies to k0s at the moment
+  enabled: false