add k0s support (#1123)
* ignore idea * Initial work on support for k0s * * Changed config param to bootstrap_distribution * Replaced democratic-csi with openebs * Added notes about system-upgrade-controller being ignored with k0s * Removed password for nodes * * Switched to single storage path for openebs * Removed democratic-csi helm repo * Added missing openebs helm repo * Fixed template formatting * Made bootstrap_private_github_repo hidden/undocumented * Updated flux cluster config to accommodate bootstrap_private_github_repo * * added the storageclass name to be declarative * * added newline * set chart wait false * * fixed spacing in K0sTasks * fixed openebs path * properly added openebs helm repo * * updated configure task to use venv * renamed bootstrap_kube_vip_addr to bootstrap_kubeapi_addr * renamed bootstrap_kube_vip_addr derived variables accordingly * * tweaked configure task cmd * started work on k0s-config.yaml template * * tweaked k0s task cmd * moved coredns under addons rather than optional * fixed typo in coredns task * normalized formatting in k0s-config * disabled telemetry in k0s-config * added renovate to k0s-config * removed tunnel and added routing mode in k0s-config cilium chart values * updated docs in config example * fixed issue with k0s-config controller role so it didn't default to worker+controller if more than one master * * fixed whitespace * add k0sctl to brew tasks * Update bootstrap/vars/config.sample.yaml * Update bootstrap/vars/config.sample.yaml * Update k0s-config.yaml.j2 * Update config.sample.yaml * controllerManager and scheduler bind to `0.0.0.0` for metrics * Update custom-cilium-helmchart.yaml.j2.j2 * Update helmvalues.yaml.j2 * Update bootstrap/tasks/addons/main.yaml * first pass at readme updates * update placement of k0sconfig options * disable telemetry on k0sctl too * disable k0s telemetry in env too * update: jinja spacing in k0sconfig Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: ensure bootstrap_local_storage_path is delete on nuke Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update renovate regex for k3s/k0s Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update nuke playbook and only include k3s task when k3s Signed-off-by: Devin Buhl <devin@buhl.casa> * fix: update nuke playbook and only include k3s task when k3s and include jinja comments Signed-off-by: Devin Buhl <devin@buhl.casa> --------- Signed-off-by: Devin Buhl <devin@buhl.casa> Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com> Co-authored-by: Devin Buhl <devin@buhl.casa>
This commit is contained in:
43 files changed
+357
-155
No files matched your search
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -24,7 +24,7 @@ spec:
|
||||
uninstall:
|
||||
keepHistory: false
|
||||
dependsOn:
|
||||
- name: local-path-provisioner
|
||||
- name: openebs
|
||||
namespace: storage
|
||||
values:
|
||||
deploymentStrategy:
|
||||
@@ -168,6 +168,6 @@ spec:
|
||||
- *host
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClassName: local-hostpath
|
||||
storageClassName: openebs-hostpath
|
||||
testFramework:
|
||||
enabled: false
|
||||
@@ -27,7 +27,7 @@ spec:
|
||||
uninstall:
|
||||
keepHistory: false
|
||||
dependsOn:
|
||||
- name: local-path-provisioner
|
||||
- name: openebs
|
||||
namespace: storage
|
||||
valuesFrom:
|
||||
- name: kube-prometheus-stack-values
|
||||
|
||||
@@ -114,7 +114,7 @@ data:
|
||||
storageSpec:
|
||||
volumeClaimTemplate:
|
||||
spec:
|
||||
storageClassName: local-hostpath
|
||||
storageClassName: openebs-hostpath
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
|
||||
@@ -14,7 +14,7 @@ k3s_etcd_datastore: true
|
||||
{% else %}
|
||||
k3s_etcd_datastore: false
|
||||
{% endif %}
|
||||
k3s_registration_address: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
|
||||
k3s_registration_address: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
|
||||
# /var/lib/rancher/k3s/server/manifests
|
||||
k3s_server_manifests_templates:
|
||||
- custom-cilium-helmchart.yaml.j2
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
timezone: "{{ bootstrap_timezone }}"
|
||||
github_username: "{{ bootstrap_github_username }}"
|
||||
coredns_addr: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}"
|
||||
kube_vip_addr: "{{ bootstrap_kube_vip_addr }}"
|
||||
kubeapi_addr: "{{ bootstrap_kubeapi_addr }}"
|
||||
cluster_cidr: "{{ bootstrap_cluster_cidr.split(',')[0] }}"
|
||||
service_cidr: "{{ bootstrap_service_cidr.split(',')[0] }}"
|
||||
node_cidr: "{{ bootstrap_node_cidr }}"
|
||||
|
||||
@@ -11,7 +11,7 @@ k3s_server:
|
||||
node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
|
||||
{% endif %}
|
||||
tls-san:
|
||||
- "{% raw %}{{ kube_vip_addr }}{% endraw %}"
|
||||
- "{% raw %}{{ kubeapi_addr }}{% endraw %}"
|
||||
docker: false
|
||||
flannel-backend: "none" # This needs to be in quotes
|
||||
disable:
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#jinja2: trim_blocks: True, lstrip_blocks: True
|
||||
---
|
||||
- name: Cluster Nuke
|
||||
hosts: kubernetes
|
||||
@@ -21,6 +22,7 @@
|
||||
ansible.builtin.pause:
|
||||
seconds: 5
|
||||
tasks:
|
||||
{% if bootstrap_distribution == "k3s" %}
|
||||
- name: Stop Kubernetes # noqa: ignore-errors
|
||||
ignore_errors: true
|
||||
block:
|
||||
@@ -30,6 +32,7 @@
|
||||
public: true
|
||||
vars:
|
||||
k3s_state: stopped
|
||||
{% endif %}
|
||||
|
||||
# https://github.com/k3s-io/docs/blob/main/docs/installation/network-options.md
|
||||
- name: Networking
|
||||
@@ -55,6 +58,7 @@
|
||||
path: /etc/cni/net.d
|
||||
state: absent
|
||||
|
||||
{% if bootstrap_distribution == "k3s" %}
|
||||
- name: Check to see if k3s-killall.sh exits
|
||||
ansible.builtin.stat:
|
||||
path: /usr/local/bin/k3s-killall.sh
|
||||
@@ -89,6 +93,12 @@
|
||||
path: "{% raw %}{{ k3s_install_dir }}/{{ item }}{% endraw %}"
|
||||
state: absent
|
||||
loop: ["kubectl", "crictl", "ctr"]
|
||||
{% endif %}
|
||||
|
||||
- name: Remove local storage path
|
||||
ansible.builtin.file:
|
||||
path: "{% raw %}{{ bootstrap_local_storage_path }}{% endraw %}"
|
||||
state: absent
|
||||
|
||||
- name: Reboot
|
||||
ansible.builtin.reboot:
|
||||
|
||||
@@ -37,7 +37,7 @@ spec:
|
||||
ipv6:
|
||||
enabled: true
|
||||
{% endif %}
|
||||
k8sServiceHost: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
|
||||
k8sServiceHost: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
|
||||
k8sServicePort: 6443
|
||||
kubeProxyReplacement: true
|
||||
kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
|
||||
@@ -59,6 +59,6 @@ spec:
|
||||
replicas: 1
|
||||
rollOutPods: true
|
||||
rollOutCiliumPods: true
|
||||
routingMode: native
|
||||
securityContext:
|
||||
privileged: true
|
||||
tunnel: disabled
|
||||
@@ -15,7 +15,7 @@ spec:
|
||||
args: ["manager"]
|
||||
env:
|
||||
- name: address
|
||||
value: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
|
||||
value: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
|
||||
- name: vip_arp
|
||||
value: "true"
|
||||
- name: lb_enable
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
#jinja2: trim_blocks: True, lstrip_blocks: True
|
||||
---
|
||||
apiVersion: k0sctl.k0sproject.io/v1beta1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: k0s-cluster
|
||||
spec:
|
||||
hosts:
|
||||
{% for item in bootstrap_nodes.master %}
|
||||
- role: {{ item.role | default('controller+worker') }}
|
||||
ssh:
|
||||
address: {{ item.address }}
|
||||
user: {{ item.username }}
|
||||
installFlags:
|
||||
- --disable-components=metrics-server
|
||||
{% if item.role | default('') == 'controller+worker' %}
|
||||
- --no-taints
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if bootstrap_nodes.worker | default([]) | length > 0 %}
|
||||
{% for item in bootstrap_nodes.worker %}
|
||||
- role: worker
|
||||
ssh:
|
||||
address: {{ item.address }}
|
||||
user: {{ item.username }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
k0s:
|
||||
# renovate: datasource=github-releases depName=k0sproject/k0s
|
||||
version: "v1.28.4+k0s.0"
|
||||
dynamicConfig: false
|
||||
config:
|
||||
spec:
|
||||
telemetry:
|
||||
enabled: false
|
||||
controllerManager:
|
||||
extraArgs:
|
||||
# Required to monitor kube-controller-manager with kube-prometheus-stack
|
||||
bind-address: "0.0.0.0"
|
||||
scheduler:
|
||||
extraArgs:
|
||||
# Required to monitor kube-scheduler with kube-prometheus-stack
|
||||
bind-address: "0.0.0.0"
|
||||
api:
|
||||
sans:
|
||||
- {{ bootstrap_kubeapi_addr }}
|
||||
{% if bootstrap_kubeapi_hostname is defined %}
|
||||
- {{ bootstrap_kubeapi_hostname }}
|
||||
{% endif %}
|
||||
{% for item in bootstrap_nodes.master %}
|
||||
{% if item.address != bootstrap_kubeapi_addr %}
|
||||
- {{ item.address }}
|
||||
{% endif %}
|
||||
{% if (bootstrap_kubeapi_hostname is not defined) or (item.name != bootstrap_kubeapi_hostname) %}
|
||||
- {{ item.name }}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
extensions:
|
||||
helm:
|
||||
repositories:
|
||||
- name: cilium
|
||||
url: https://helm.cilium.io
|
||||
charts:
|
||||
- name: cilium
|
||||
chartname: cilium/cilium
|
||||
# renovate: datasource=github-releases depName=cilium/cilium
|
||||
version: "1.14.5"
|
||||
namespace: kube-system
|
||||
values: |2
|
||||
autoDirectNodeRoutes: true
|
||||
bpf:
|
||||
masquerade: true
|
||||
bgp:
|
||||
enabled: false
|
||||
cluster:
|
||||
name: home-cluster
|
||||
id: 1
|
||||
containerRuntime:
|
||||
integration: containerd
|
||||
socketPath: /var/run/k0s/containerd.sock
|
||||
endpointRoutes:
|
||||
enabled: true
|
||||
hubble:
|
||||
enabled: false
|
||||
ipam:
|
||||
mode: kubernetes
|
||||
ipv4NativeRoutingCIDR: "{{ bootstrap_cluster_cidr }}"
|
||||
{% if bootstrap_ipv6_enabled | default(false) %}
|
||||
ipv6NativeRoutingCIDR: "{{ bootstrap_cluster_cidr_v6 }}"
|
||||
ipv6:
|
||||
enabled: true
|
||||
{% endif %}
|
||||
k8sServiceHost: "{{ bootstrap_kubeapi_addr }}"
|
||||
k8sServicePort: 6443
|
||||
kubeProxyReplacement: true
|
||||
kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
|
||||
l2announcements:
|
||||
{% if bootstrap_ipv6_enabled | default(false) %}
|
||||
enabled: false
|
||||
{% else %}
|
||||
enabled: true
|
||||
# https://github.com/cilium/cilium/issues/26586
|
||||
leaseDuration: 120s
|
||||
leaseRenewDeadline: 60s
|
||||
leaseRetryPeriod: 1s
|
||||
{% endif %}
|
||||
loadBalancer:
|
||||
algorithm: maglev
|
||||
mode: {{ bootstrap_cilium_loadbalancer_mode | default('dsr', true) }}
|
||||
localRedirectPolicy: true
|
||||
operator:
|
||||
replicas: 1
|
||||
rollOutPods: true
|
||||
rollOutCiliumPods: true
|
||||
routingMode: native
|
||||
securityContext:
|
||||
privileged: true
|
||||
network:
|
||||
kubeProxy:
|
||||
disabled: true
|
||||
{% if bootstrap_nodes.master | length > 1 %}
|
||||
nodeLocalLoadBalancing:
|
||||
enabled: true
|
||||
type: EnvoyProxy
|
||||
{% endif %}
|
||||
provider: custom
|
||||
@@ -16,7 +16,11 @@ data:
|
||||
id: 1
|
||||
containerRuntime:
|
||||
integration: containerd
|
||||
{% if bootstrap_distribution == "k3s" %}
|
||||
socketPath: /var/run/k3s/containerd/containerd.sock
|
||||
{% else %}
|
||||
socketPath: /var/run/k0s/containerd.sock
|
||||
{% endif %}
|
||||
endpointRoutes:
|
||||
enabled: true
|
||||
hubble:
|
||||
@@ -63,7 +67,7 @@ data:
|
||||
ipv6:
|
||||
enabled: true
|
||||
{% endif %}
|
||||
k8sServiceHost: "${KUBE_VIP_ADDR}"
|
||||
k8sServiceHost: "${KUBEAPI_ADDR}"
|
||||
k8sServicePort: 6443
|
||||
kubeProxyReplacement: true
|
||||
kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
|
||||
@@ -102,6 +106,6 @@ data:
|
||||
annotations:
|
||||
grafana_folder: Cilium
|
||||
rollOutCiliumPods: true
|
||||
routingMode: native
|
||||
securityContext:
|
||||
privileged: true
|
||||
tunnel: disabled
|
||||
@@ -5,5 +5,7 @@ kind: Kustomization
|
||||
resources:
|
||||
- ./namespace.yaml
|
||||
- ./cilium/ks.yaml
|
||||
{% if bootstrap_distribution == "k3s" %}
|
||||
- ./coredns/ks.yaml
|
||||
{% endif %}
|
||||
- ./metrics-server/ks.yaml
|
||||
@@ -4,7 +4,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ./namespace.yaml
|
||||
- ./local-path-provisioner/ks.yaml
|
||||
- ./openebs/ks.yaml
|
||||
- ./snapshot-controller/ks.yaml
|
||||
- ./volsync/ks.yaml
|
||||
{% if csi_driver_nfs.enabled | default(false) %}
|
||||
|
||||
-76
@@ -1,76 +0,0 @@
|
||||
---
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2beta2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: local-path-provisioner
|
||||
spec:
|
||||
interval: 30m
|
||||
chart:
|
||||
spec:
|
||||
chart: democratic-csi
|
||||
version: 0.14.3
|
||||
sourceRef:
|
||||
name: democratic-csi
|
||||
kind: HelmRepository
|
||||
namespace: flux-system
|
||||
install:
|
||||
remediation:
|
||||
retries: 3
|
||||
upgrade:
|
||||
cleanupOnFail: true
|
||||
remediation:
|
||||
retries: 3
|
||||
uninstall:
|
||||
keepHistory: false
|
||||
values:
|
||||
fullnameOverride: local-path-provisioner
|
||||
controller:
|
||||
strategy: node
|
||||
externalProvisioner:
|
||||
image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.3
|
||||
extraArgs:
|
||||
- --leader-election=false
|
||||
- --node-deployment=true
|
||||
- --node-deployment-immediate-binding=false
|
||||
- --feature-gates=Topology=true
|
||||
- --strict-topology=true
|
||||
- --enable-capacity=true
|
||||
- --capacity-ownerref-level=1
|
||||
externalResizer:
|
||||
enabled: false
|
||||
externalAttacher:
|
||||
enabled: false
|
||||
externalSnapshotter:
|
||||
enabled: false
|
||||
csiDriver:
|
||||
name: local-hostpath.cluster.local
|
||||
storageCapacity: true
|
||||
attachRequired: false
|
||||
fsGroupPolicy: File
|
||||
storageClasses:
|
||||
- name: local-hostpath
|
||||
defaultClass: false
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: WaitForFirstConsumer
|
||||
allowVolumeExpansion: true
|
||||
driver:
|
||||
config:
|
||||
driver: local-hostpath
|
||||
local-hostpath:
|
||||
shareBasePath: &storagePath "{{ bootstrap_local_path_provisioner_path | default('/var/lib/rancher/k3s/local-hostpath', true) }}"
|
||||
controllerBasePath: *storagePath
|
||||
dirPermissionsMode: "0770"
|
||||
dirPermissionsUser: 0
|
||||
dirPermissionsGroup: 0
|
||||
node:
|
||||
driver:
|
||||
image: ghcr.io/democratic-csi/democratic-csi:v1.8.4
|
||||
extraVolumeMounts:
|
||||
- name: local-hostpath
|
||||
mountPath: *storagePath
|
||||
mountPropagation: Bidirectional
|
||||
extraVolumes:
|
||||
- name: local-hostpath
|
||||
hostPath:
|
||||
path: *storagePath
|
||||
type: DirectoryOrCreate
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2beta2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: openebs
|
||||
spec:
|
||||
interval: 30m
|
||||
chart:
|
||||
spec:
|
||||
chart: openebs
|
||||
version: 3.3.0
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: openebs
|
||||
namespace: flux-system
|
||||
maxHistory: 2
|
||||
install:
|
||||
remediation:
|
||||
retries: 3
|
||||
upgrade:
|
||||
cleanupOnFail: true
|
||||
remediation:
|
||||
retries: 3
|
||||
uninstall:
|
||||
keepHistory: false
|
||||
values:
|
||||
localprovisioner:
|
||||
hostpathClass:
|
||||
enabled: true
|
||||
name: openebs-hostpath
|
||||
isDefaultClass: false
|
||||
basePath: "{{ bootstrap_local_storage_path }}"
|
||||
|
||||
File renamed without changes.
+2
-2
@@ -2,14 +2,14 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: &app local-path-provisioner
|
||||
name: &app openebs
|
||||
namespace: flux-system
|
||||
spec:
|
||||
targetNamespace: storage
|
||||
commonMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: *app
|
||||
path: ./kubernetes/apps/storage/local-path-provisioner/app
|
||||
path: ./kubernetes/apps/storage/openebs/app
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
@@ -1,3 +1,4 @@
|
||||
#jinja2: trim_blocks: True, lstrip_blocks: True
|
||||
---
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: GitRepository
|
||||
@@ -8,7 +9,13 @@ spec:
|
||||
interval: 30m
|
||||
ref:
|
||||
branch: {{ bootstrap_github_repository_branch | default('main', true) }}
|
||||
{% if bootstrap_private_github_repo | default(false) %}
|
||||
secretRef:
|
||||
name: github-deploy-key
|
||||
url: "ssh://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}"
|
||||
{% else %}
|
||||
url: "https://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}.git"
|
||||
{% endif %}
|
||||
ignore: |
|
||||
# exclude all
|
||||
/*
|
||||
|
||||
@@ -8,7 +8,6 @@ resources:
|
||||
- ./cilium.yaml
|
||||
- ./coredns.yaml
|
||||
- ./csi-driver-nfs.yaml
|
||||
- ./democratic-csi.yaml
|
||||
- ./descheduler.yaml
|
||||
- ./external-dns.yaml
|
||||
- ./grafana.yaml
|
||||
@@ -18,6 +17,7 @@ resources:
|
||||
- ./k8s-gateway.yaml
|
||||
- ./kubernetes-dashboard.yaml
|
||||
- ./metrics-server.yaml
|
||||
- ./openebs.yaml
|
||||
- ./piraeus.yaml
|
||||
- ./prometheus-community.yaml
|
||||
- ./stakater.yaml
|
||||
|
||||
+2
-2
@@ -2,8 +2,8 @@
|
||||
apiVersion: source.toolkit.fluxcd.io/v1beta2
|
||||
kind: HelmRepository
|
||||
metadata:
|
||||
name: democratic-csi
|
||||
name: openebs
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 1h
|
||||
url: https://democratic-csi.github.io/charts/
|
||||
url: https://openebs.github.io/charts
|
||||
@@ -8,7 +8,7 @@ metadata:
|
||||
data:
|
||||
TIMEZONE: "{{ bootstrap_timezone }}"
|
||||
COREDNS_ADDR: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}"
|
||||
KUBE_VIP_ADDR: "{{ bootstrap_kube_vip_addr }}"
|
||||
KUBEAPI_ADDR: "{{ bootstrap_kubeapi_addr }}"
|
||||
CLUSTER_CIDR: "{{ bootstrap_cluster_cidr.split(',')[0] }}"
|
||||
SERVICE_CIDR: "{{ bootstrap_service_cidr.split(',')[0] }}"
|
||||
NODE_CIDR: "{{ bootstrap_node_cidr }}"
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
---
|
||||
ansible_become_pass: "{{ password }}"
|
||||
Reference in new issue
Block a user