add k0s support (#1123)

* ignore idea * Initial work on support for k0s * * Changed config param to bootstrap_distribution * Replaced democratic-csi with openebs * Added notes about system-upgrade-controller being ignored with k0s * Removed password for nodes * * Switched to single storage path for openebs * Removed democratic-csi helm repo * Added missing openebs helm repo * Fixed template formatting * Made bootstrap_private_github_repo hidden/undocumented * Updated flux cluster config to accommodate bootstrap_private_github_repo * * added the storageclass name to be declarative * * added newline * set chart wait false * * fixed spacing in K0sTasks * fixed openebs path * properly added openebs helm repo * * updated configure task to use venv * renamed bootstrap_kube_vip_addr to bootstrap_kubeapi_addr * renamed bootstrap_kube_vip_addr derived variables accordingly * * tweaked configure task cmd * started work on k0s-config.yaml template * * tweaked k0s task cmd * moved coredns under addons rather than optional * fixed typo in coredns task * normalized formatting in k0s-config * disabled telemetry in k0s-config * added renovate to k0s-config * removed tunnel and added routing mode in k0s-config cilium chart values * updated docs in config example * fixed issue with k0s-config controller role so it didn't default to worker+controller if more than one master * * fixed whitespace * add k0sctl to brew tasks * Update bootstrap/vars/config.sample.yaml * Update bootstrap/vars/config.sample.yaml * Update k0s-config.yaml.j2 * Update config.sample.yaml * controllerManager and scheduler bind to `0.0.0.0` for metrics * Update custom-cilium-helmchart.yaml.j2.j2 * Update helmvalues.yaml.j2 * Update bootstrap/tasks/addons/main.yaml * first pass at readme updates * update placement of k0sconfig options * disable telemetry on k0sctl too * disable k0s telemetry in env too * update: jinja spacing in k0sconfig Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: ensure bootstrap_local_storage_path is delete on nuke Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update renovate regex for k3s/k0s Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update nuke playbook and only include k3s task when k3s Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update nuke playbook and only include k3s task when k3s and include jinja comments Signed-off-by: Devin Buhl <devin@buhl.casa> --------- Signed-off-by: Devin Buhl <devin@buhl.casa> Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com> Co-authored-by: Devin Buhl <devin@buhl.casa>
2024-01-11 15:03:36 -05:00
1 parent 1b73543d34
commit d093d0a4e1
@@ -16,7 +16,11 @@ data:
      id: 1
    containerRuntime:
      integration: containerd
+      {% if bootstrap_distribution == "k3s" %}
      socketPath: /var/run/k3s/containerd/containerd.sock
+      {% else %}
+      socketPath: /var/run/k0s/containerd.sock
+      {% endif %}
    endpointRoutes:
      enabled: true
    hubble:
@@ -63,7 +67,7 @@ data:
    ipv6:
      enabled: true
    {% endif %}
-    k8sServiceHost: "${KUBE_VIP_ADDR}"
+    k8sServiceHost: "${KUBEAPI_ADDR}"
    k8sServicePort: 6443
    kubeProxyReplacement: true
    kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
@@ -102,6 +106,6 @@ data:
      annotations:
        grafana_folder: Cilium
    rollOutCiliumPods: true
+    routingMode: native
    securityContext:
      privileged: true
-    tunnel: disabled
@@ -1,89 +0,0 @@
-#jinja2: trim_blocks: True, lstrip_blocks: True
---
-apiVersion: helm.toolkit.fluxcd.io/v2beta2
-kind: HelmRelease
-metadata:
-  name: coredns
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: coredns
-      version: 1.29.0
-      sourceRef:
-        kind: HelmRepository
-        name: coredns
-        namespace: flux-system
-  maxHistory: 2
-  install:
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  uninstall:
-    keepHistory: false
-  values:
-    fullnameOverride: coredns
-    replicaCount: 1
-    k8sAppLabelOverride: kube-dns
-    service:
-      name: kube-dns
-      clusterIP: "${COREDNS_ADDR}"
-    serviceAccount:
-      create: true
-    deployment:
-      annotations:
-        reloader.stakater.com/auto: "true"
-    servers:
-      - zones:
-          - zone: .
-            scheme: dns://
-            use_tcp: true
-        port: 53
-        plugins:
-          - name: log
-          - name: errors
-          - name: health
-            configBlock: |-
-              lameduck 5s
-          - name: ready
-          - name: kubernetes
-            parameters: cluster.local in-addr.arpa ip6.arpa
-            configBlock: |-
-              pods insecure
-              fallthrough in-addr.arpa ip6.arpa
-              ttl 30
-          - name: prometheus
-            parameters: 0.0.0.0:9153
-          - name: forward
-            parameters: . /etc/resolv.conf
-          - name: cache
-            parameters: 30
-          - name: loop
-          - name: reload
-          - name: loadbalance
-    affinity:
-      nodeAffinity:
-        requiredDuringSchedulingIgnoredDuringExecution:
-          nodeSelectorTerms:
-            - matchExpressions:
-                - key: node-role.kubernetes.io/control-plane
-                  operator: Exists
-    tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-        effect: NoSchedule
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-    topologySpreadConstraints:
-      - maxSkew: 1
-        topologyKey: kubernetes.io/hostname
-        whenUnsatisfiable: DoNotSchedule
-        labelSelector:
-          matchLabels:
-            app.kubernetes.io/instance: coredns
@@ -1,20 +0,0 @@
---
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: &app coredns
-  namespace: flux-system
-spec:
-  targetNamespace: kube-system
-  commonMetadata:
-    labels:
-      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/kube-system/coredns/app
-  prune: false # never should be deleted
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: false
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
@@ -5,5 +5,7 @@ kind: Kustomization
 resources:
  - ./namespace.yaml
  - ./cilium/ks.yaml
+  {% if bootstrap_distribution == "k3s" %}
  - ./coredns/ks.yaml
+  {% endif %}
  - ./metrics-server/ks.yaml
@@ -4,7 +4,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - ./namespace.yaml
-  - ./local-path-provisioner/ks.yaml
+  - ./openebs/ks.yaml
  - ./snapshot-controller/ks.yaml
  - ./volsync/ks.yaml
  {% if csi_driver_nfs.enabled | default(false) %}
@@ -1,76 +0,0 @@
---
-apiVersion: helm.toolkit.fluxcd.io/v2beta2
-kind: HelmRelease
-metadata:
-  name: local-path-provisioner
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: democratic-csi
-      version: 0.14.3
-      sourceRef:
-        name: democratic-csi
-        kind: HelmRepository
-        namespace: flux-system
-  install:
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  uninstall:
-    keepHistory: false
-  values:
-    fullnameOverride: local-path-provisioner
-    controller:
-      strategy: node
-      externalProvisioner:
-        image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.3
-        extraArgs:
-          - --leader-election=false
-          - --node-deployment=true
-          - --node-deployment-immediate-binding=false
-          - --feature-gates=Topology=true
-          - --strict-topology=true
-          - --enable-capacity=true
-          - --capacity-ownerref-level=1
-      externalResizer:
-        enabled: false
-      externalAttacher:
-        enabled: false
-      externalSnapshotter:
-        enabled: false
-    csiDriver:
-      name: local-hostpath.cluster.local
-      storageCapacity: true
-      attachRequired: false
-      fsGroupPolicy: File
-    storageClasses:
-      - name: local-hostpath
-        defaultClass: false
-        reclaimPolicy: Delete
-        volumeBindingMode: WaitForFirstConsumer
-        allowVolumeExpansion: true
-    driver:
-      config:
-        driver: local-hostpath
-        local-hostpath:
-          shareBasePath: &storagePath "{{ bootstrap_local_path_provisioner_path | default('/var/lib/rancher/k3s/local-hostpath', true) }}"
-          controllerBasePath: *storagePath
-          dirPermissionsMode: "0770"
-          dirPermissionsUser: 0
-          dirPermissionsGroup: 0
-    node:
-      driver:
-        image: ghcr.io/democratic-csi/democratic-csi:v1.8.4
-        extraVolumeMounts:
-          - name: local-hostpath
-            mountPath: *storagePath
-            mountPropagation: Bidirectional
-      extraVolumes:
-        - name: local-hostpath
-          hostPath:
-            path: *storagePath
-            type: DirectoryOrCreate
@@ -1,5 +0,0 @@
---
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ./helmrelease.yaml
@@ -0,0 +1,33 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: openebs
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: openebs
+      version: 3.3.0
+      sourceRef:
+        kind: HelmRepository
+        name: openebs
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    localprovisioner:
+      hostpathClass:
+        enabled: true
+        name: openebs-hostpath
+        isDefaultClass: false
+        basePath: "{{ bootstrap_local_storage_path }}"
+
@@ -2,14 +2,14 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: &app local-path-provisioner
+  name: &app openebs
  namespace: flux-system
 spec:
  targetNamespace: storage
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/storage/local-path-provisioner/app
+  path: ./kubernetes/apps/storage/openebs/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 ---
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: GitRepository
@@ -8,7 +9,13 @@ spec:
  interval: 30m
  ref:
    branch: {{ bootstrap_github_repository_branch | default('main', true) }}
+  {% if bootstrap_private_github_repo | default(false) %}
+  secretRef:
+    name: github-deploy-key
+  url: "ssh://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}"
+  {% else %}
  url: "https://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}.git"
+  {% endif %}
  ignore: |
    # exclude all
    /*
@@ -8,7 +8,6 @@ resources:
  - ./cilium.yaml
  - ./coredns.yaml
  - ./csi-driver-nfs.yaml
-  - ./democratic-csi.yaml
  - ./descheduler.yaml
  - ./external-dns.yaml
  - ./grafana.yaml
@@ -18,6 +17,7 @@ resources:
  - ./k8s-gateway.yaml
  - ./kubernetes-dashboard.yaml
  - ./metrics-server.yaml
+  - ./openebs.yaml
  - ./piraeus.yaml
  - ./prometheus-community.yaml
  - ./stakater.yaml
@@ -2,8 +2,8 @@
 apiVersion: source.toolkit.fluxcd.io/v1beta2
 kind: HelmRepository
 metadata:
-  name: democratic-csi
+  name: openebs
  namespace: flux-system
 spec:
  interval: 1h
-  url: https://democratic-csi.github.io/charts/
+  url: https://openebs.github.io/charts
@@ -8,7 +8,7 @@ metadata:
 data:
  TIMEZONE: "{{ bootstrap_timezone }}"
  COREDNS_ADDR: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}"
-  KUBE_VIP_ADDR: "{{ bootstrap_kube_vip_addr }}"
+  KUBEAPI_ADDR: "{{ bootstrap_kubeapi_addr }}"
  CLUSTER_CIDR: "{{ bootstrap_cluster_cidr.split(',')[0] }}"
  SERVICE_CIDR: "{{ bootstrap_service_cidr.split(',')[0] }}"
  NODE_CIDR: "{{ bootstrap_node_cidr }}"