add k0s support (#1123)

* ignore idea

* Initial work on support for k0s

* *  Changed config param to bootstrap_distribution
*  Replaced democratic-csi with openebs
*  Added notes about system-upgrade-controller being ignored with k0s
*  Removed password for nodes

* *  Switched to single storage path for openebs
*  Removed democratic-csi helm repo
*  Added missing openebs helm repo
*  Fixed template formatting
*  Made bootstrap_private_github_repo hidden/undocumented
*  Updated flux cluster config to accommodate bootstrap_private_github_repo

* *  added the storageclass name to be declarative

* *  added newline
*  set chart wait false

* *  fixed spacing in K0sTasks
*  fixed openebs path
*  properly added openebs helm repo

* *  updated configure task to use venv
*  renamed bootstrap_kube_vip_addr to bootstrap_kubeapi_addr
*  renamed bootstrap_kube_vip_addr derived variables accordingly

* *  tweaked configure task cmd
*  started work on k0s-config.yaml template

* *  tweaked k0s task cmd
*  moved coredns under addons rather than optional
*  fixed typo in coredns task
*  normalized formatting in k0s-config
*  disabled telemetry in k0s-config
*  added renovate to k0s-config
*  removed tunnel and added routing mode in k0s-config cilium chart values
*  updated docs in config example
*  fixed issue with k0s-config controller role so it didn't default to worker+controller if more than one master

* *  fixed whitespace

* add k0sctl to brew tasks

* Update bootstrap/vars/config.sample.yaml

* Update bootstrap/vars/config.sample.yaml

* Update k0s-config.yaml.j2

* Update config.sample.yaml

* controllerManager and scheduler bind to `0.0.0.0` for metrics

* Update custom-cilium-helmchart.yaml.j2.j2

* Update helmvalues.yaml.j2

* Update bootstrap/tasks/addons/main.yaml

* first pass at readme updates

* update placement of k0sconfig options

* disable telemetry on k0sctl too

* disable k0s telemetry in env too

* update: jinja spacing in k0sconfig

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: ensure bootstrap_local_storage_path is delete on nuke

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: update renovate regex for k3s/k0s

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: update nuke playbook and only include k3s task when k3s

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: update nuke playbook and only include k3s task when k3s and include jinja comments

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>
Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com>
Co-authored-by: Devin Buhl <devin@buhl.casa>

bootstrap/configure.yaml

@@ -17,11 +17,11 @@
       ansible.builtin.set_fact:
         repository_path: "{{ repository.stdout }}"
 
-    - name: Override kube-vip address when there is a single master node and no address is defined
-      when: bootstrap_nodes.master | length == 1 and not bootstrap_kube_vip_addr
+    - name: Override kubeapi address when there is a single master node and no address is defined
+      when: bootstrap_nodes.master | length == 1 and not bootstrap_kubeapi_addr
       ansible.builtin.set_fact:
         bootstrap_kube_vip_enabled: false
-        bootstrap_kube_vip_addr: "{{ bootstrap_nodes.master[0].address }}"
+        bootstrap_kubeapi_addr: "{{ bootstrap_nodes.master[0].address }}"
 
     - name: Verify configuration
       ansible.builtin.include_tasks: tasks/validation/main.yaml
@@ -29,6 +29,10 @@
     - name: Template Sops configuration
       ansible.builtin.include_tasks: tasks/sops/main.yaml
 
+    - name: Template k0s configuration
+      when: bootstrap_distribution == "k0s"
+      ansible.builtin.include_tasks: tasks/k0s/main.yaml
+
     - name: Template Ansible configuration
       ansible.builtin.include_tasks: tasks/ansible/main.yaml
 

bootstrap/tasks/addons/coredns.yaml

@@ -0,0 +1,34 @@
+---
+- name: Set addon facts
+  ansible.builtin.set_fact:
+    addon_name: coredns
+    addon_namespace: kube-system
+
+- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }}
+  when: item.state == 'directory'
+  ansible.builtin.file:
+    path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}"
+    state: directory
+    mode: "0755"
+  with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+
+- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }}
+  when: item.state == 'file' and 'sops' not in item.path
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
+    mode: "0644"
+  with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]
+
+- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }}
+  block:
+    - name: Template encrypted files
+      when: item.state == 'file' and 'sops' in item.path
+      community.sops.sops_encrypt:
+        path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}"
+        encrypted_regex: ^(data|stringData)$
+        age: ["{{ bootstrap_age_public_key }}"]
+        content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}"
+        mode: "0644"
+        force: true
+      with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"]

bootstrap/tasks/addons/main.yaml

@@ -1,5 +1,9 @@
 ---
 
+- name: Process optional coredns
+  when: bootstrap_distribution == "k3s"
+  ansible.builtin.include_tasks: coredns.yaml
+
 - name: Process addon csi-driver-nfs
   when: csi_driver_nfs.enabled | default(false)
   ansible.builtin.include_tasks: csi_driver_nfs.yaml
@@ -21,7 +25,9 @@
   ansible.builtin.include_tasks: kube_prometheus_stack.yaml
 
 - name: Process addon system-upgrade-controller
-  when: system_upgrade_controller.enabled | default(false)
+  when:
+    - bootstrap_distribution == "k3s"
+    - system_upgrade_controller.enabled | default(false)
   ansible.builtin.include_tasks: system_upgrade_controller.yaml
 
 - name: Process addon weave-gitops

bootstrap/tasks/ansible/main.yaml

@@ -27,13 +27,4 @@
         mode: "0644"
         force: true
       with_community.general.filetree: ["../templates/ansible/"]
-    - name: Template encrypted node secrets
-      community.sops.sops_encrypt:
-        path: "{{ repository_path }}/ansible/inventory/host_vars/{{ item.name }}.sops.yaml"
-        age: ["{{ bootstrap_age_public_key }}"]
-        content_yaml: "{{ lookup('ansible.builtin.template', 'templates/node.sops.yaml.j2', template_vars=dict(password=item.password)) | from_yaml }}"
-        mode: "0644"
-        force: true
-      loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
-      loop_control:
-        label: "{{ item.address }}"
+

bootstrap/tasks/k0s/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: Template k0s configuration file
+  ansible.builtin.template:
+    src: "templates/k0s-config.yaml.j2"
+    dest: "{{ repository_path }}/k0s-config.yaml"
+    mode: "0644"

bootstrap/tasks/validation/github.yaml

@@ -14,6 +14,7 @@
     fail_msg: Github user {{ bootstrap_github_username }} does not exist
 
 - name: Query Github repo
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.uri:
     url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}
     timeout: 5
@@ -22,12 +23,14 @@
   register: result
 
 - name: Check if repo exists
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.assert:
     that: result.json.full_name == bootstrap_github_username + '/' + bootstrap_github_repository_name
     success_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} exists
     fail_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} does not exist
 
 - name: Query Github repo branch
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.uri:
     url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}/branches/{{ bootstrap_github_repository_branch | default('main', true) }}
     timeout: 5
@@ -36,6 +39,7 @@
   register: result
 
 - name: Check if repo branch exists
+  when: not bootstrap_private_github_repo | default(false)
   ansible.builtin.assert:
     that: result.json.name == bootstrap_github_repository_branch | default('main', true)
     success_msg: Github repo branch {{ bootstrap_github_repository_branch | default('main', true) }} exists

bootstrap/tasks/validation/net.yaml

@@ -98,17 +98,17 @@
     success_msg: external ingress address {{ bootstrap_external_ingress_addr }} is within {{ bootstrap_node_cidr }}.
     fail_msg: external ingress address {{ bootstrap_external_ingress_addr }} is not within {{ bootstrap_node_cidr }}.
 
-- name: Verify kube-vip
+- name: Verify kubeapi address
   ansible.builtin.assert:
-    that: bootstrap_kube_vip_addr is ansible.utils.ipv4
-    success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is valid.
-    fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is invalid.
+    that: bootstrap_kubeapi_addr is ansible.utils.ipv4
+    success_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is valid.
+    fail_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is invalid.
 
-- name: Verify kube-vip in node CIDR
+- name: Verify kubeapi address in node CIDR
   ansible.builtin.assert:
-    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kube_vip_addr)
-    success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}.
-    fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}.
+    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kubeapi_addr)
+    success_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is within {{ bootstrap_node_cidr }}.
+    fail_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is not within {{ bootstrap_node_cidr }}.
 
 - name: Verify all IP addresses are unique
   ansible.builtin.assert:
@@ -117,7 +117,7 @@
         bootstrap_k8s_gateway_addr,
         bootstrap_external_ingress_addr,
         bootstrap_internal_ingress_addr,
-        bootstrap_kube_vip_addr
+        bootstrap_kubeapi_addr
       ] | unique | length == 4
     success_msg: All IP addresses are unique.
     fail_msg: All IP addresses are not unique.
@@ -133,12 +133,12 @@
   loop_control:
     label: "{{ item.address }}"
 
-- name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kube-vip
-  when: bootstrap_kube_vip_enabled | default(true)
+- name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kubeapi address
+  when: (bootstrap_distribution == "k3s") and (bootstrap_kube_vip_enabled | default(true))
   ansible.builtin.assert:
-    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kube_vip_addr)
-    success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip.
-    fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip.
+    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kubeapi_addr)
+    success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kubeapi.
+    fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kubeapi.
     quiet: true
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
   loop_control:

bootstrap/tasks/validation/vars.yaml

@@ -16,23 +16,24 @@
     - bootstrap_cloudflare_tunnel_id
     - bootstrap_cloudflare_tunnel_secret
     - bootstrap_cluster_cidr
-    - bootstrap_flux_github_webhook_token
-    - bootstrap_github_repository_name
-    - bootstrap_github_repository_branch
-    - bootstrap_github_username
+    - bootstrap_distribution
     - bootstrap_external_ingress_addr
+    - bootstrap_flux_github_webhook_token
+    - bootstrap_github_repository_branch
+    - bootstrap_github_repository_name
+    - bootstrap_github_username
     - bootstrap_internal_ingress_addr
     - bootstrap_ipv6_enabled
     - bootstrap_k8s_gateway_addr
-    - bootstrap_kube_vip_addr
-    - bootstrap_local_path_provisioner_path
+    - bootstrap_kubeapi_addr
+    - bootstrap_local_storage_path
     - bootstrap_node_cidr
     - bootstrap_service_cidr
     - bootstrap_timezone
 
 - name: Verify bootstrap node names are valid
   ansible.builtin.assert:
-    that: item.name is match('^[a-z0-9-]+$')
+    that: item.name is match('^[a-z0-9-\.]+$')
     success_msg: Node name {{ item.name }} is valid
     fail_msg: Node name {{ item.name }} is not valid
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"

bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2

@@ -24,7 +24,7 @@ spec:
   uninstall:
     keepHistory: false
   dependsOn:
-    - name: local-path-provisioner
+    - name: openebs
       namespace: storage
   values:
     deploymentStrategy:
@@ -168,6 +168,6 @@ spec:
             - *host
     persistence:
       enabled: true
-      storageClassName: local-hostpath
+      storageClassName: openebs-hostpath
     testFramework:
       enabled: false

bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2

@@ -27,7 +27,7 @@ spec:
   uninstall:
     keepHistory: false
   dependsOn:
-    - name: local-path-provisioner
+    - name: openebs
       namespace: storage
   valuesFrom:
     - name: kube-prometheus-stack-values

bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2

@@ -114,7 +114,7 @@ data:
         storageSpec:
           volumeClaimTemplate:
             spec:
-              storageClassName: local-hostpath
+              storageClassName: openebs-hostpath
               resources:
                 requests:
                   storage: 10Gi

bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2

@@ -14,7 +14,7 @@ k3s_etcd_datastore: true
 {% else %}
 k3s_etcd_datastore: false
 {% endif %}
-k3s_registration_address: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+k3s_registration_address: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
   - custom-cilium-helmchart.yaml.j2

bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2

@@ -3,7 +3,7 @@
 timezone: "{{ bootstrap_timezone }}"
 github_username: "{{ bootstrap_github_username }}"
 coredns_addr: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}"
-kube_vip_addr: "{{ bootstrap_kube_vip_addr }}"
+kubeapi_addr: "{{ bootstrap_kubeapi_addr }}"
 cluster_cidr: "{{ bootstrap_cluster_cidr.split(',')[0] }}"
 service_cidr: "{{ bootstrap_service_cidr.split(',')[0] }}"
 node_cidr: "{{ bootstrap_node_cidr }}"

bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2

@@ -11,7 +11,7 @@ k3s_server:
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
 {% endif %}
   tls-san:
-    - "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+    - "{% raw %}{{ kubeapi_addr }}{% endraw %}"
   docker: false
   flannel-backend: "none"             # This needs to be in quotes
   disable:

bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2

@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 ---
 - name: Cluster Nuke
   hosts: kubernetes
@@ -21,6 +22,7 @@
       ansible.builtin.pause:
         seconds: 5
   tasks:
+    {% if bootstrap_distribution == "k3s" %}
     - name: Stop Kubernetes # noqa: ignore-errors
       ignore_errors: true
       block:
@@ -30,6 +32,7 @@
             public: true
           vars:
             k3s_state: stopped
+    {% endif %}
 
     # https://github.com/k3s-io/docs/blob/main/docs/installation/network-options.md
     - name: Networking
@@ -55,6 +58,7 @@
             path: /etc/cni/net.d
             state: absent
 
+    {% if bootstrap_distribution == "k3s" %}
     - name: Check to see if k3s-killall.sh exits
       ansible.builtin.stat:
         path: /usr/local/bin/k3s-killall.sh
@@ -89,6 +93,12 @@
         path: "{% raw %}{{ k3s_install_dir }}/{{ item }}{% endraw %}"
         state: absent
       loop: ["kubectl", "crictl", "ctr"]
+    {% endif %}
+
+    - name: Remove local storage path
+      ansible.builtin.file:
+        path: "{% raw %}{{ bootstrap_local_storage_path }}{% endraw %}"
+        state: absent
 
     - name: Reboot
       ansible.builtin.reboot:

bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2

@@ -37,7 +37,7 @@ spec:
     ipv6:
       enabled: true
     {% endif %}
-    k8sServiceHost: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+    k8sServiceHost: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
     k8sServicePort: 6443
     kubeProxyReplacement: true
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
@@ -59,6 +59,6 @@ spec:
       replicas: 1
       rollOutPods: true
     rollOutCiliumPods: true
+    routingMode: native
     securityContext:
       privileged: true
-    tunnel: disabled

bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2

@@ -15,7 +15,7 @@ spec:
       args: ["manager"]
       env:
         - name: address
-          value: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+          value: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
         - name: vip_arp
           value: "true"
         - name: lb_enable

bootstrap/templates/k0s-config.yaml.j2

@@ -0,0 +1,126 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: k0sctl.k0sproject.io/v1beta1
+kind: Cluster
+metadata:
+  name: k0s-cluster
+spec:
+  hosts:
+    {% for item in bootstrap_nodes.master %}
+    - role: {{ item.role | default('controller+worker') }}
+      ssh:
+        address: {{ item.address }}
+        user: {{ item.username }}
+      installFlags:
+        - --disable-components=metrics-server
+        {% if item.role | default('') == 'controller+worker' %}
+        - --no-taints
+        {% endif %}
+    {% endfor %}
+    {% if bootstrap_nodes.worker | default([]) | length > 0 %}
+    {% for item in bootstrap_nodes.worker %}
+    - role: worker
+      ssh:
+        address: {{ item.address }}
+        user: {{ item.username }}
+    {% endfor %}
+    {% endif %}
+  k0s:
+    # renovate: datasource=github-releases depName=k0sproject/k0s
+    version: "v1.28.4+k0s.0"
+    dynamicConfig: false
+    config:
+      spec:
+        telemetry:
+          enabled: false
+        controllerManager:
+          extraArgs:
+            # Required to monitor kube-controller-manager with kube-prometheus-stack
+            bind-address: "0.0.0.0"
+        scheduler:
+          extraArgs:
+            # Required to monitor kube-scheduler with kube-prometheus-stack
+            bind-address: "0.0.0.0"
+        api:
+          sans:
+            - {{ bootstrap_kubeapi_addr }}
+            {% if bootstrap_kubeapi_hostname is defined %}
+            - {{ bootstrap_kubeapi_hostname }}
+            {% endif %}
+            {% for item in bootstrap_nodes.master %}
+            {% if item.address != bootstrap_kubeapi_addr %}
+            - {{ item.address }}
+            {% endif %}
+            {% if (bootstrap_kubeapi_hostname is not defined) or (item.name != bootstrap_kubeapi_hostname) %}
+            - {{ item.name }}
+            {% endif %}
+            {% endfor %}
+        extensions:
+          helm:
+            repositories:
+              - name: cilium
+                url: https://helm.cilium.io
+            charts:
+              - name: cilium
+                chartname: cilium/cilium
+                # renovate: datasource=github-releases depName=cilium/cilium
+                version: "1.14.5"
+                namespace: kube-system
+                values: |2
+                  autoDirectNodeRoutes: true
+                  bpf:
+                    masquerade: true
+                  bgp:
+                    enabled: false
+                  cluster:
+                    name: home-cluster
+                    id: 1
+                  containerRuntime:
+                    integration: containerd
+                    socketPath: /var/run/k0s/containerd.sock
+                  endpointRoutes:
+                    enabled: true
+                  hubble:
+                    enabled: false
+                  ipam:
+                    mode: kubernetes
+                  ipv4NativeRoutingCIDR: "{{ bootstrap_cluster_cidr }}"
+                  {% if bootstrap_ipv6_enabled | default(false) %}
+                  ipv6NativeRoutingCIDR: "{{ bootstrap_cluster_cidr_v6 }}"
+                  ipv6:
+                    enabled: true
+                  {% endif %}
+                  k8sServiceHost: "{{ bootstrap_kubeapi_addr }}"
+                  k8sServicePort: 6443
+                  kubeProxyReplacement: true
+                  kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+                  l2announcements:
+                    {% if bootstrap_ipv6_enabled | default(false) %}
+                    enabled: false
+                    {% else %}
+                    enabled: true
+                    # https://github.com/cilium/cilium/issues/26586
+                    leaseDuration: 120s
+                    leaseRenewDeadline: 60s
+                    leaseRetryPeriod: 1s
+                    {% endif %}
+                  loadBalancer:
+                    algorithm: maglev
+                    mode: {{ bootstrap_cilium_loadbalancer_mode | default('dsr', true) }}
+                  localRedirectPolicy: true
+                  operator:
+                    replicas: 1
+                    rollOutPods: true
+                  rollOutCiliumPods: true
+                  routingMode: native
+                  securityContext:
+                    privileged: true
+        network:
+          kubeProxy:
+            disabled: true
+          {%  if bootstrap_nodes.master | length > 1 %}
+          nodeLocalLoadBalancing:
+            enabled: true
+            type: EnvoyProxy
+          {% endif %}
+          provider: custom

bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2

@@ -16,7 +16,11 @@ data:
       id: 1
     containerRuntime:
       integration: containerd
+      {% if bootstrap_distribution == "k3s" %}
       socketPath: /var/run/k3s/containerd/containerd.sock
+      {% else %}
+      socketPath: /var/run/k0s/containerd.sock
+      {% endif %}
     endpointRoutes:
       enabled: true
     hubble:
@@ -63,7 +67,7 @@ data:
     ipv6:
       enabled: true
     {% endif %}
-    k8sServiceHost: "${KUBE_VIP_ADDR}"
+    k8sServiceHost: "${KUBEAPI_ADDR}"
     k8sServicePort: 6443
     kubeProxyReplacement: true
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
@@ -102,6 +106,6 @@ data:
       annotations:
         grafana_folder: Cilium
     rollOutCiliumPods: true
+    routingMode: native
     securityContext:
       privileged: true
-    tunnel: disabled

bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2

@@ -5,5 +5,7 @@ kind: Kustomization
 resources:
   - ./namespace.yaml
   - ./cilium/ks.yaml
+  {% if bootstrap_distribution == "k3s" %}
   - ./coredns/ks.yaml
+  {% endif %}
   - ./metrics-server/ks.yaml

bootstrap/templates/kubernetes/apps/storage/kustomization.yaml.j2

@@ -4,7 +4,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
-  - ./local-path-provisioner/ks.yaml
+  - ./openebs/ks.yaml
   - ./snapshot-controller/ks.yaml
   - ./volsync/ks.yaml
   {% if csi_driver_nfs.enabled | default(false) %}

bootstrap/templates/kubernetes/apps/storage/local-path-provisioner/app/helmrelease.yaml.j2

@@ -1,76 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2beta2
-kind: HelmRelease
-metadata:
-  name: local-path-provisioner
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: democratic-csi
-      version: 0.14.3
-      sourceRef:
-        name: democratic-csi
-        kind: HelmRepository
-        namespace: flux-system
-  install:
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  uninstall:
-    keepHistory: false
-  values:
-    fullnameOverride: local-path-provisioner
-    controller:
-      strategy: node
-      externalProvisioner:
-        image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.3
-        extraArgs:
-          - --leader-election=false
-          - --node-deployment=true
-          - --node-deployment-immediate-binding=false
-          - --feature-gates=Topology=true
-          - --strict-topology=true
-          - --enable-capacity=true
-          - --capacity-ownerref-level=1
-      externalResizer:
-        enabled: false
-      externalAttacher:
-        enabled: false
-      externalSnapshotter:
-        enabled: false
-    csiDriver:
-      name: local-hostpath.cluster.local
-      storageCapacity: true
-      attachRequired: false
-      fsGroupPolicy: File
-    storageClasses:
-      - name: local-hostpath
-        defaultClass: false
-        reclaimPolicy: Delete
-        volumeBindingMode: WaitForFirstConsumer
-        allowVolumeExpansion: true
-    driver:
-      config:
-        driver: local-hostpath
-        local-hostpath:
-          shareBasePath: &storagePath "{{ bootstrap_local_path_provisioner_path | default('/var/lib/rancher/k3s/local-hostpath', true) }}"
-          controllerBasePath: *storagePath
-          dirPermissionsMode: "0770"
-          dirPermissionsUser: 0
-          dirPermissionsGroup: 0
-    node:
-      driver:
-        image: ghcr.io/democratic-csi/democratic-csi:v1.8.4
-        extraVolumeMounts:
-          - name: local-hostpath
-            mountPath: *storagePath
-            mountPropagation: Bidirectional
-      extraVolumes:
-        - name: local-hostpath
-          hostPath:
-            path: *storagePath
-            type: DirectoryOrCreate

bootstrap/templates/kubernetes/apps/storage/openebs/app/helmrelease.yaml.j2

@@ -0,0 +1,33 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: openebs
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: openebs
+      version: 3.3.0
+      sourceRef:
+        kind: HelmRepository
+        name: openebs
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    localprovisioner:
+      hostpathClass:
+        enabled: true
+        name: openebs-hostpath
+        isDefaultClass: false
+        basePath: "{{ bootstrap_local_storage_path }}"
+

bootstrap/templates/kubernetes/apps/storage/openebs/ks.yaml.j2

@@ -2,14 +2,14 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: &app local-path-provisioner
+  name: &app openebs
   namespace: flux-system
 spec:
   targetNamespace: storage
   commonMetadata:
     labels:
       app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/storage/local-path-provisioner/app
+  path: ./kubernetes/apps/storage/openebs/app
   prune: true
   sourceRef:
     kind: GitRepository

bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2

@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 ---
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: GitRepository
@@ -8,7 +9,13 @@ spec:
   interval: 30m
   ref:
     branch: {{ bootstrap_github_repository_branch | default('main', true) }}
+  {% if bootstrap_private_github_repo | default(false) %}
+  secretRef:
+    name: github-deploy-key
+  url: "ssh://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}"
+  {% else %}
   url: "https://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}.git"
+  {% endif %}
   ignore: |
     # exclude all
     /*

bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2

@@ -8,7 +8,6 @@ resources:
   - ./cilium.yaml
   - ./coredns.yaml
   - ./csi-driver-nfs.yaml
-  - ./democratic-csi.yaml
   - ./descheduler.yaml
   - ./external-dns.yaml
   - ./grafana.yaml
@@ -18,6 +17,7 @@ resources:
   - ./k8s-gateway.yaml
   - ./kubernetes-dashboard.yaml
   - ./metrics-server.yaml
+  - ./openebs.yaml
   - ./piraeus.yaml
   - ./prometheus-community.yaml
   - ./stakater.yaml

bootstrap/templates/kubernetes/flux/repositories/helm/openebs.yaml.j2

@@ -2,8 +2,8 @@
 apiVersion: source.toolkit.fluxcd.io/v1beta2
 kind: HelmRepository
 metadata:
-  name: democratic-csi
+  name: openebs
   namespace: flux-system
 spec:
   interval: 1h
-  url: https://democratic-csi.github.io/charts/
+  url: https://openebs.github.io/charts

bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2

@@ -8,7 +8,7 @@ metadata:
 data:
   TIMEZONE: "{{ bootstrap_timezone }}"
   COREDNS_ADDR: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}"
-  KUBE_VIP_ADDR: "{{ bootstrap_kube_vip_addr }}"
+  KUBEAPI_ADDR: "{{ bootstrap_kubeapi_addr }}"
   CLUSTER_CIDR: "{{ bootstrap_cluster_cidr.split(',')[0] }}"
   SERVICE_CIDR: "{{ bootstrap_service_cidr.split(',')[0] }}"
   NODE_CIDR: "{{ bootstrap_node_cidr }}"

bootstrap/templates/node.sops.yaml.j2

@@ -1,2 +0,0 @@
----
-ansible_become_pass: "{{ password }}"

bootstrap/vars/addons.sample.yaml

@@ -41,6 +41,7 @@ system_upgrade_controller:
   # WARNING: Only enable this if you also track the version of k3s in the
   # ansible configuration files. Running ansible against an already provisioned
   # cluster with this enabled might cause your cluster to be downgraded.
+  # Note that if bootstrap_distribution is set to k0s enable: true will be ignored.
   enabled: false
 
 # https://github.com/morphy2k/rss-forwarder

bootstrap/vars/config.sample.yaml

@@ -3,6 +3,10 @@
 # Bootstrap configuration - config.yaml is gitignored
 #
 
+# Distribution can either be 'k3s' or 'k0s'
+# Note that changing this to k0s will ignore `system_upgrade_controller` in the addons.
+bootstrap_distribution: k3s
+
 # Github username (e.g. onedr0p)
 bootstrap_github_username:
 # Github repository (e.g. flux-cluster-template)
@@ -38,9 +42,13 @@ bootstrap_cloudflare_tunnel_id:
 
 # CIDR your nodes are on (e.g. 192.168.1.0/24)
 bootstrap_node_cidr:
-# The IP address to use with kube-vip, choose an available IP in your nodes network that is not being used
-# (Optional) Leave this blank if you are deploying a single master node, this will disable kube-vip and use the master node IP instead
-bootstrap_kube_vip_addr:
+# The IP address of the kubeapi, choose an available IP in your nodes network that is not being used
+# (Optional) Leave this blank if you are deploying a single master node, this will disable kube-vip in k3s
+# or keepalived in k0s and use the master node IP instead
+bootstrap_kubeapi_addr:
+# The hostname of the kubeapi, set this if you intend to call the kubeapi by hostname rather than IP
+# (Optional) This is currently only utilized in k0s and is added to the kubeapi cert sans
+#bootstrap_kubeapi_hostname:
 # The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used
 bootstrap_k8s_gateway_addr:
 # The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used
@@ -63,23 +71,23 @@ bootstrap_cluster_cidr: 10.42.0.0/16
 # (Advanced) For ipv6 use format 10.43.0.0/16,fd78:c889:47fb:e0::/112
 bootstrap_service_cidr: 10.43.0.0/16
 
-# (Advanced) Path for democratic-csi local-hostpath to use for storage on ALL nodes
-bootstrap_local_path_provisioner_path: /var/lib/rancher/k3s/local-hostpath
+# (Advanced) Path for openebs openebs-hostpath to use for storage on ALL nodes
+bootstrap_local_storage_path: /var/openebs/local
 
 # Node information
 bootstrap_nodes:
   # Use only 1, 3 or more odd master nodes, recommended is 3
   master:
-    # - name:     # name of the master node (must match [a-z0-9-]+)
+    # - name:     # name or hostname of the master node (must match [a-z0-9-\.]+)
     #   address:  # ip address of the master node
     #   username: # ssh username of the master node
-    #   password: # password of ssh username for the master node
-    #   # external_address: # Only use when SSH is not reachable on the IP specified in 'address' field
+    #   role:     # (Optional) k0s only: can be controller+worker (default) or controller 
+    #   # external_address: # (Optional) k3s only: Only use when SSH is not reachable on the IP specified in 'address' field
     # ...
   worker: # set to [] or omit if no workers are needed
-    # - name:     # name of the worker node (must match [a-z0-9-]+)
+    # - name:     # name or hostname of the worker node (must match [a-z0-9-\.]+)
     #   address:  # ip address of the worker node
     #   username: # ssh username of the worker node
-    #   password: # password of ssh username for the worker node
-    #   # external_address: # Only use when SSH is not reachable on the IP specified in 'address' field
+    #   # external_address: # (Optional) k3s only: Only use when SSH is not reachable on the IP specified in 'address' field
     # ...
+