chore: add ci tests for k0s (#1152)

.envrc

@@ -12,3 +12,4 @@ export ANSIBLE_INVENTORY_UNPARSED_WARNING="False"
 export K8S_AUTH_KUBECONFIG="$(expand_path ./kubeconfig)"
 # k0s
 export DISABLE_TELEMETRY="true"
+export DISABLE_UPGRADE_CHECK="true"

.github/tests/config-k0s.yaml

@@ -0,0 +1,36 @@
+---
+ci_test: true
+
+bootstrap_distribution: k0s
+bootstrap_github_username: onedr0p
+bootstrap_github_repository_name: flux-cluster-template
+bootstrap_github_repository_branch: main
+bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY
+bootstrap_timezone: Etc/UTC
+bootstrap_acme_email: fake
+bootstrap_acme_production_enabled: false
+bootstrap_flux_github_webhook_token: fake
+bootstrap_cloudflare_domain: fake
+bootstrap_cloudflare_token: fake
+bootstrap_cloudflare_account_tag: fake
+bootstrap_cloudflare_tunnel_secret: fake
+bootstrap_cloudflare_tunnel_id: fake
+bootstrap_node_cidr: 10.10.10.0/24
+bootstrap_kubeapi_addr: 10.10.10.254
+bootstrap_k8s_gateway_addr: 10.10.10.253
+bootstrap_external_ingress_addr: 10.10.10.252
+bootstrap_internal_ingress_addr: 10.10.10.251
+bootstrap_cilium_loadbalancer_mode: dsr
+bootstrap_ipv6_enabled: false
+bootstrap_cluster_cidr: 10.42.0.0/16
+bootstrap_service_cidr: 10.43.0.0/16
+bootstrap_local_storage_path: /var/openebs/local
+bootstrap_nodes:
+  master:
+    - name: k8s-0
+      address: 10.10.10.100
+      username: fake
+  worker:
+    - name: k8s-1
+      address: 10.10.10.101
+      username: fake

.github/workflows/e2e.yaml

@@ -34,13 +34,13 @@ jobs:
     name: e2e-configure
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         config-files:
-          - config-ipv4
-          - config-ipv6
-        addon-files:
-          - addons
-      fail-fast: false
+          - k0s
+          - k3s-ipv4
+          - k3s-ipv6
+        addon-files: ["addons"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -96,11 +96,17 @@ jobs:
         shell: bash
         run: |
           task init
-          cp ./.github/tests/${{ matrix.config-files }}.yaml ./bootstrap/vars/config.yaml
+          cp ./.github/tests/config-${{ matrix.config-files }}.yaml ./bootstrap/vars/config.yaml
           cp ./.github/tests/${{ matrix.addon-files }}.yaml ./bootstrap/vars/addons.yaml
           export BOOTSTRAP_AGE_PUBLIC_KEY=$(sed -n 's/# public key: //gp' age.key)
           envsubst < ./bootstrap/vars/config.yaml | sponge ./bootstrap/vars/config.yaml
 
+      - name: Export Env from config.yaml
+        shell: bash
+        id: config-env
+        run: |
+          echo "distribution=$(yq '.bootstrap_distribution' ./bootstrap/vars/config.yaml)" >> $GITHUB_OUTPUT
+
       - name: Run configure
         shell: bash
         run: task --yes configure
@@ -110,6 +116,7 @@ jobs:
         run: ./scripts/kubeconform.sh ./kubernetes
 
       - name: List Hosts with Ansible
+        if: ${{ steps.config-env.outputs.distribution == 'k3s' || steps.config-env.outputs.distribution == 'k0s' }}
         shell: bash
         run: task ansible:list
 

.taskfiles/K0s/Taskfile.yaml

@@ -3,6 +3,7 @@ version: "3"
 
 env:
   DISABLE_TELEMETRY: "true"
+  DISABLE_UPGRADE_CHECK: "true"
 
 tasks:
 

bootstrap/tasks/addons/coredns.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
   block:
@@ -32,3 +36,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/addons/csi_driver_nfs.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
   block:
@@ -32,3 +36,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/addons/discord_template_notifier.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
   block:
@@ -32,3 +36,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/addons/grafana.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
   block:
@@ -32,3 +36,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/addons/homepage.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 # https://github.com/ansible-collections/community.sops/issues/153
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
@@ -33,3 +37,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/addons/kube_prometheus_stack.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
   block:
@@ -32,3 +36,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/addons/kubernetes_dashboard.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
   block:
@@ -32,3 +36,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/addons/system_upgrade_controller.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
   block:
@@ -32,3 +36,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/addons/volsync.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
   block:
@@ -32,3 +36,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/addons/weave_gitops.yaml

@@ -11,6 +11,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
   when: item.state == 'file' and 'sops' not in item.path
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
   block:
@@ -32,3 +36,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/ansible/main.yaml

@@ -8,6 +8,8 @@
   with_community.general.filetree:
     - "../templates/ansible/shared/"
     - "../templates/ansible/{{ bootstrap_distribution }}/"
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template Ansible unencrypted files
   when:
@@ -21,6 +23,8 @@
   with_community.general.filetree:
     - "../templates/ansible/shared/"
     - "../templates/ansible/{{ bootstrap_distribution }}/"
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template Ansible encrypted files
   block:
@@ -38,3 +42,5 @@
       with_community.general.filetree:
         - "../templates/ansible/shared/"
         - "../templates/ansible/{{ bootstrap_distribution }}/"
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/tasks/kubernetes/main.yaml

@@ -6,6 +6,8 @@
     state: directory
     mode: "0755"
   with_community.general.filetree: ["../templates/kubernetes/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template Kubernetes unencrypted files
   when:
@@ -19,6 +21,8 @@
     dest: "{{ repository_path }}/kubernetes/{{ item.path | regex_replace('.j2$', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/kubernetes/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Check if the cluster user settings file already exists
   stat:
@@ -35,6 +39,8 @@
     dest: "{{ repository_path }}/kubernetes/{{ item.path | regex_replace('.j2$', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/kubernetes/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Check if the cluster user secrets file already exists
   stat:
@@ -51,6 +57,8 @@
     dest: "{{ repository_path }}/kubernetes/{{ item.path | regex_replace('.j2$', '') }}"
     mode: "0644"
   with_community.general.filetree: ["../templates/kubernetes/"]
+  loop_control:
+    label: "{{ item.path }}"
 
 - name: Template Kubernetes encrypted files
   block:
@@ -66,3 +74,5 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/kubernetes/"]
+      loop_control:
+        label: "{{ item.path }}"

bootstrap/templates/kubernetes/apps/tools/kustomization.yaml.j2

@@ -6,6 +6,6 @@ resources:
   - ./namespace.yaml
   - ./descheduler/ks.yaml
   - ./reloader/ks.yaml
-  {% if system_upgrade_controller.enabled | default(false) %}
+  {% if bootstrap_distribution == "k3s" and system_upgrade_controller.enabled | default(false) %}
   - ./system-upgrade-controller/ks.yaml
   {% endif %}