infra/home-cluster
2
0 Fork 0
Code Issues Pull Requests 40 Actions Packages Projects Releases Wiki Activity

feat(k0s): unsupport it (#1291)

Browse Source 
Signed-off-by: Devin Buhl <devin@buhl.casa>
This commit is contained in:
Devin Buhl authored and GitHub committed 2024-02-02 18:02:14 -05:00
1 parent 3e0fb233b6
commit eab63476b7
48 files changed
+22 -505
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

bootstrap/scripts/validation.py
+1 -5
View File
@@ -5,10 +5,9 @@ from zoneinfo import available_timezones
import netaddr
import sys
DISTRIBUTIONS = ["k0s", "k3s", "talos"]
DISTRIBUTIONS = ["k3s", "talos"]
GLOBAL_CLI_TOOLS = ["age", "cloudflared", "flux", "sops", "jq", "kubeconform", "kustomize"]
TALOS_CLI_TOOLS = ["talosctl", "talhelper"]
K0S_CLI_TOOLS = ["k0sctl"]
def required(*keys: str):
def wrapper_outter(func: Callable):
@@ -48,9 +47,6 @@ def validate_cli_tools(distribution: dict, **_) -> None:
for tool in TALOS_CLI_TOOLS if distribution in ["talos"] else []:
if not which(tool):
raise ValueError(f"Missing required CLI tool {tool}")
for tool in K0S_CLI_TOOLS if distribution in ["k0s"] else []:
if not which(tool):
raise ValueError(f"Missing required CLI tool {tool}")
@required("distribution")
bootstrap/templates/.sops.yaml.j2
+1 -1
View File
@@ -13,7 +13,7 @@ creation_rules:
key_groups:
- age:
- "{{ flux.sops_age_public_key }}"
{% if distribution.type in ["k0s", "k3s"] %}
{% if distribution.type in ["k3s"] %}
- path_regex: ansible/.*\.sops\.ya?ml
key_groups:
- age:
bootstrap/templates/ansible/.mjfilter.py
+1 -1
View File
@@ -1 +1 @@
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k0s", "k3s"]
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k3s"]
bootstrap/templates/ansible/inventory/.mjfilter.py
-1
View File
@@ -1 +0,0 @@
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k0s", "k3s"]
bootstrap/templates/ansible/inventory/group_vars/kubernetes/.mjfilter.py
-1
View File
@@ -1 +0,0 @@
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k3s"]
bootstrap/templates/ansible/inventory/group_vars/master/.mjfilter.py
-1
View File
@@ -1 +0,0 @@
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k3s"]
bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2
-2
View File
@@ -1,4 +1,3 @@
{% if distribution.type in ["k3s"] %}
---
- name: Cluster Installation
hosts: kubernetes
@@ -59,4 +58,3 @@
- name: Cruft
when: k3s_primary_control_node
ansible.builtin.include_tasks: tasks/cruft.yaml
{% endif %}
bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2
+1 -6
View File
@@ -10,20 +10,15 @@
ansible.builtin.pause:
seconds: 5
tasks:
{% if distribution.type in ["k3s"] %}
- name: Ensure Kubernetes is running
ansible.builtin.include_role:
name: xanmanning.k3s
public: true
vars:
k3s_state: started
{% endif %}
- name: Upgrade kube-vip
ansible.builtin.template:
src: templates/kube-vip-ds.yaml.j2
{% if distribution.type in ["k3s"] %}
dest: "{% raw %}{{ k3s_server_manifests_dir }}{% endraw %}/kube-vip-ds.yaml"
{% elif distribution.type in ["k0s"] %}
dest: "/var/lib/k0s/manifests/kube-vip/kube-vip-ds.yaml"
{% endif %}
mode: preserve
bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2
-2
View File
@@ -1,4 +1,3 @@
{% if distribution.type in ["k3s"] %}
---
- name: Cluster Nuke
hosts: kubernetes
@@ -100,4 +99,3 @@
ansible.builtin.reboot:
msg: Rebooting hosts
reboot_timeout: 3600
{% endif %}
bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2
-2
View File
@@ -1,4 +1,3 @@
{% if distribution.type in ["k3s"] %}
---
- name: Prepare System
hosts: kubernetes
@@ -112,4 +111,3 @@
ansible.builtin.reboot:
msg: Rebooting hosts
reboot_timeout: 3600
{% endif %}
bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2
-17
View File
@@ -1,5 +1,4 @@
---
# https://github.com/kevincoakley/ansible-role-k8s-rolling-update
- name: Cluster rollout update
hosts: kubernetes
become: true
@@ -12,11 +11,7 @@
seconds: 5
tasks:
- name: Details
{% if distribution.type in ["k3s"] %}
ansible.builtin.command: "k3s kubectl get node {% raw %}{{ inventory_hostname }}{% endraw %} -o json"
{% elif distribution.type in ["k0s"] %}
ansible.builtin.command: "k0s kubectl get node {% raw %}{{ inventory_hostname }}{% endraw %} -o json"
{% endif %}
register: kubectl_get_node
delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}"
failed_when: false
@@ -32,22 +27,14 @@
- name: Cordon
kubernetes.core.k8s_drain:
name: "{% raw %}{{ inventory_hostname }}{% endraw %}"
{% if distribution.type in ["k3s"] %}
kubeconfig: /etc/rancher/k3s/k3s.yaml
{% elif distribution.type in ["k0s"] %}
kubeconfig: /var/lib/k0s/pki/admin.conf
{% endif %}
state: cordon
delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}"
- name: Drain
kubernetes.core.k8s_drain:
name: "{% raw %}{{ inventory_hostname }}{% endraw %}"
{% if distribution.type in ["k3s"] %}
kubeconfig: /etc/rancher/k3s/k3s.yaml
{% elif distribution.type in ["k0s"] %}
kubeconfig: /var/lib/k0s/pki/admin.conf
{% endif %}
state: drain
delete_options:
delete_emptydir_data: true
@@ -78,10 +65,6 @@
- name: Uncordon
kubernetes.core.k8s_drain:
name: "{% raw %}{{ inventory_hostname }}{% endraw %}"
{% if distribution.type in ["k3s"] %}
kubeconfig: /etc/rancher/k3s/k3s.yaml
{% elif distribution.type in ["k0s"] %}
kubeconfig: /var/lib/k0s/pki/admin.conf
{% endif %}
state: uncordon
delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}"
bootstrap/templates/ansible/playbooks/tasks/.mjfilter.py
-1
View File
@@ -1 +0,0 @@
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k3s"]
bootstrap/templates/ansible/playbooks/templates/.mjfilter.py
-1
View File
@@ -1 +0,0 @@
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k3s"]
bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2
-1
View File
@@ -1,5 +1,4 @@
---
# https://docs.k3s.io/helm
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
bootstrap/templates/ansible/requirements.txt.j2
-1
View File
@@ -1,4 +1,3 @@
# Ansible
ansible-lint==6.22.2
ansible==9.2.0
jmespath==1.0.1
bootstrap/templates/ansible/requirements.yaml.j2
-2
View File
@@ -8,9 +8,7 @@ collections:
version: 8.3.0
- name: kubernetes.core
version: 3.0.0
{% if distribution.type in ["k3s"] %}
roles:
- name: xanmanning.k3s
src: https://github.com/PyratLabs/ansible-role-k3s
version: v3.4.4
{% endif %}
bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2
+1 -1
View File
@@ -8,7 +8,7 @@ resources:
- ./kubelet-csr-approver/ks.yaml
{% endif %}
- ./metrics-server/ks.yaml
{% if distribution.type in ["k0s", "talos"] %}
{% if distribution.type in ["talos"] %}
- ./spegel/ks.yaml
{% endif %}
- ./reloader/ks.yaml
bootstrap/templates/kubernetes/apps/kube-system/spegel/.mjfilter.py
+1 -1
View File
@@ -1 +1 @@
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k0s", "talos"]
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["talos"]
bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2
-5
View File
@@ -24,13 +24,8 @@ spec:
keepHistory: false
values:
spegel:
{% if distribution.type in ["k0s"] %}
containerdSock: /run/k0s/containerd.sock
containerdRegistryConfigPath: /var/lib/k0s/containerd/certs.d
{% elif distribution.type in ["talos"] %}
containerdSock: /run/containerd/containerd.sock
containerdRegistryConfigPath: /etc/cri/conf.d/hosts
{% endif %}
service:
registry:
hostPort: 29999
bootstrap/templates/kubernetes/apps/system-upgrade/k0s/.mjfilter.py
-1
View File
@@ -1 +0,0 @@
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k0s"]
bootstrap/templates/kubernetes/apps/system-upgrade/k0s/app/kustomization.yaml.j2
-5
View File
@@ -1,5 +0,0 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./plan.yaml
bootstrap/templates/kubernetes/apps/system-upgrade/k0s/app/plan.yaml.j2
-25
View File
@@ -1,25 +0,0 @@
---
apiVersion: autopilot.k0sproject.io/v1beta2
kind: Plan
metadata:
name: autopilot
spec:
id: not-used
timestamp: not-used
commands:
- k0supdate:
version: "${KUBE_VERSION}"
platforms:
linux-amd64:
url: "https://get.k0sproject.io/${KUBE_VERSION}/k0s-${KUBE_VERSION}-amd64"
linux-arm64:
url: "https://get.k0sproject.io/${KUBE_VERSION}/k0s-${KUBE_VERSION}-arm64"
targets:
controllers:
discovery:
selector: {}
{% if nodes.inventory | selectattr('controller', 'equalto', False) | list | length %}
workers:
discovery:
selector: {}
{% endif %}
bootstrap/templates/kubernetes/apps/system-upgrade/k0s/ks.yaml.j2
-24
View File
@@ -1,24 +0,0 @@
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: &app system-upgrade-k0s
namespace: flux-system
spec:
targetNamespace: system-upgrade
commonMetadata:
labels:
app.kubernetes.io/name: *app
path: ./kubernetes/apps/system-upgrade/k0s/app
prune: true
sourceRef:
kind: GitRepository
name: home-kubernetes
wait: false
interval: 30m
retryInterval: 1m
timeout: 5m
postBuild:
substitute:
# renovate: datasource=github-releases depName=k0sproject/k0s
KUBE_VERSION: v1.29.1+k0s.0
bootstrap/templates/kubernetes/apps/system-upgrade/kustomization.yaml.j2
-3
View File
@@ -6,9 +6,6 @@ resources:
{% if distribution.type in ['k3s', 'talos'] %}
- ./system-upgrade-controller/ks.yaml
{% endif %}
{% if distribution.type in ["k0s"] %}
- ./k0s/ks.yaml
{% endif %}
{% if distribution.type in ["k3s"] %}
- ./k3s/ks.yaml
{% endif %}
bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2
+1 -1
View File
@@ -16,6 +16,6 @@ resources:
- ./postfinance.yaml
{% endif %}
- ./stakater.yaml
{% if distribution.type in ["k0s", "talos"] %}
{% if distribution.type in ["talos"] %}
- ./xenitab.yaml
{% endif %}
bootstrap/templates/kubernetes/flux/repositories/helm/xenitab.yaml.j2
+1 -1
View File
@@ -1,4 +1,4 @@
{% if distribution.type in ["k0s", "talos"] %}
{% if distribution.type in ["talos"] %}
---
apiVersion: source.toolkit.fluxcd.io/v1beta2
kind: HelmRepository
bootstrap/templates/kubernetes/k0s/.mjfilter.py
-1
View File
@@ -1 +0,0 @@
main = lambda data: data.get("distribution", {}).get("type", "k3s") in ["k0s"]
bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2
-97
View File
@@ -1,97 +0,0 @@
---
apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
name: home-kubernetes
spec:
hosts:
{% for item in nodes.inventory %}
{% if item.controller %}
- role: controller+worker
{% else %}
- role: worker
{% endif %}
ssh:
address: "{{ item.address }}"
user: "{{ item.ssh_username }}"
installFlags:
{% if item.controller %}
- --disable-components=metrics-server
- --no-taints
{% endif %}
- --kubelet-extra-args="--image-gc-low-threshold=50 --image-gc-high-threshold=60"
files:
- name: bootstrap-scripts
src: scripts
hooks:
apply:
before:
- bash ~/apply-system.sh "{{ item.name }}"
{% if item.controller %}
- bash ~/apply-kube-vip.sh
{% endif %}
reset:
before:
- bash ~/reset-cilium.sh
after:
- bash ~/reset-system.sh
{% endfor %}
k0s:
# renovate: datasource=github-releases depName=k0sproject/k0s
version: v1.29.1+k0s.0
config:
apiVersion: k0s.k0sproject.io/v1beta1
kind: ClusterConfig
metadata:
name: home-kubernetes
spec:
telemetry:
enabled: false
controllerManager:
extraArgs:
bind-address: "0.0.0.0"
scheduler:
extraArgs:
bind-address: "0.0.0.0"
storage:
etcd:
extraArgs:
listen-metrics-urls: http://0.0.0.0:2381
api:
sans:
{% if cluster.endpoint_vip %}
- "{{ cluster.endpoint_vip }}"
{% endif %}
{% for item in cluster.tls_sans %}
- "{{ item }}"
{% endfor %}
{% for item in controllers %}
{% if item.controller %}
- "{{ item.address }}"
- "{{ item.name }}"
{% endif %}
{% endfor %}
network:
podCIDR: "{{ cluster.pod_network.split(',')[0] }}"
serviceCIDR: "{{ cluster.service_network.split(',')[0] }}"
provider: custom
kubeProxy:
disabled: true
nodeLocalLoadBalancing:
enabled: true
type: EnvoyProxy
extensions:
helm:
repositories:
- name: cilium
url: https://helm.cilium.io
charts:
- name: cilium
chartname: cilium/cilium
# renovate: datasource=helm depName=cilium repository=https://helm.cilium.io
version: 1.15.0
namespace: kube-system
values: |2
{% filter indent(width=18, first=True) %}
{% include 'partials/cilium-values-init.partial.yaml.j2' %}
{% endfilter %}
bootstrap/templates/kubernetes/k0s/scripts/apply-kube-vip.sh.j2
-18
View File
@@ -1,18 +0,0 @@
#!/usr/bin/env bash
set -e
set -o noglob
[ $(id -u) -eq 0 ] || exec sudo $0 $@
# Create kube-vip directory
mkdir -p /var/lib/k0s/manifests/kube-vip
# Create kube-vip rbac
cat <<EOF > /var/lib/k0s/manifests/kube-vip/rbac.yaml
{% include 'partials/kube-vip-rbac.partial.yaml.j2' %}
EOF
# Create kube-vip daemonset
cat <<EOF > /var/lib/k0s/manifests/kube-vip/ds.yaml
{% include 'partials/kube-vip-ds.partial.yaml.j2' %}
EOF
bootstrap/templates/kubernetes/k0s/scripts/apply-system.sh.j2
-81
View File
@@ -1,81 +0,0 @@
#!/usr/bin/env bash
set -e
set -o noglob
[ $(id -u) -eq 0 ] || exec sudo $0 $@
K0S_HOSTNAME="$1"
# Prepare
sudo apt-get update -y
# Hostname
if [ -n "${K0S_HOSTNAME}" ] && [ "$(hostnamectl hostname)" != "${K0S_HOSTNAME}" ]; then
hostnamectl set-hostname "${K0S_HOSTNAME}"
cat >/etc/hosts <<EOF
127.0.0.1 localhost
127.0.1.1 $K0S_HOSTNAME
::1 localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
EOF
fi
# Timezone
timedatectl set-timezone "{{ timezone }}"
# Install Packages
apt-get install -y --no-install-recommends \
apt-transport-https ca-certificates conntrack curl dirmngr gdisk gnupg hdparm htop \
iptables iputils-ping ipvsadm libseccomp2 lm-sensors net-tools nfs-common \
nvme-cli open-iscsi parted psmisc python3 python3-apt python3-kubernetes python3-yaml \
smartmontools socat software-properties-common unzip util-linux
# Update DNS
chattr -i /etc/resolv.conf
rm -f /etc/resolv.conf
cat <<EOF > /etc/resolv.conf
search {{ nodes.search_domain|default('.', true) }}
{% for item in nodes.dns_servers | default(['1.1.1.1', '1.0.0.1']) %}
nameserver {{ item }}
{% endfor %}
EOF
chattr +i /etc/resolv.conf
# Kernel Modules
for module in br_netfilter ceph ip_vs ip_vs_rr nbd overlay rbd; do
if ! test -f /etc/modules-load.d/$module.conf; then
echo $module > /etc/modules-load.d/$module.conf
fi
done
systemctl restart systemd-modules-load.service
# Sysctls
cat <<EOF > /etc/sysctl.d/99-kubernetes.conf
fs.inotify.max_queued_events = 65536
fs.inotify.max_user_watches = 524288
fs.inotify.max_user_instances = 8192
EOF
sysctl -p /etc/sysctl.d/99-kubernetes.conf
# Disable swap
swapoff -a
# Disable AppArmor
if systemctl is-enabled apparmor.service; then
systemctl mask apparmor.service
fi
# Create containerd config
mkdir -p /etc/k0s/containerd.d
cat <<EOF > /etc/k0s/containerd.d/default.toml
[plugins."io.containerd.grpc.v1.cri"]
enable_unprivileged_ports = true
enable_unprivileged_icmp = true
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/var/lib/k0s/containerd/certs.d"
[plugins."io.containerd.grpc.v1.cri".containerd]
discard_unpacked_layers = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
discard_unpacked_layers = false
EOF
bootstrap/templates/kubernetes/k0s/scripts/reset-cilium.sh.j2
-40
View File
@@ -1,40 +0,0 @@
#!/usr/bin/env bash
set -e
set -o noglob
[ $(id -u) -eq 0 ] || exec sudo $0 $@
remove_interfaces() {
ip link show 2>/dev/null | grep 'cilium' | while read ignore iface ignore; do
iface=${iface%%@*}
[ -z "$iface" ] || (ip link delete $iface || true)
done
}
reset_iptables() {
iptables -t nat -F
iptables -t mangle -F
iptables -t filter -F
iptables -t raw -F
iptables -X
ip6tables -t nat -F
ip6tables -t mangle -F
ip6tables -t filter -F
ip6tables -t raw -F
ip6tables -X
}
do_unmount_and_remove() {
set +x
while read -r _ path _; do
case "$path" in $1*) echo "$path" ;; esac
done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount -f "$0" && rm -rf "$0"'
set -x
}
do_unmount_and_remove '/run/netns/cni-'
ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete
remove_interfaces
reset_iptables
rm -rf /var/lib/cni
rm -rf /etc/cni/net.d
bootstrap/templates/kubernetes/k0s/scripts/reset-system.sh.j2
-14
View File
@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -e
set -o noglob
[ $(id -u) -eq 0 ] || exec sudo $0 $@
# Remove containerd config
rm -rf /etc/k0s/containerd.d
# Remove local storage data
rm -rf /var/openebs/local
# Reboot
(sleep 30 && systemctl reboot)&
bootstrap/templates/partials/cilium-values-full.partial.yaml.j2
-5
View File
@@ -8,8 +8,6 @@ containerRuntime:
integration: containerd
{% if distribution.type in ["k3s"] %}
socketPath: /var/run/k3s/containerd/containerd.sock
{% elif distribution.type in ["k0s"] %}
socketPath: /var/run/k0s/containerd.sock
{% endif %}
endpointRoutes:
enabled: true
@@ -63,9 +61,6 @@ ipv6:
{% if distribution.type in ["k3s"] %}
k8sServiceHost: 127.0.0.1
k8sServicePort: 6444
{% elif distribution.type in ["k0s"] %}
k8sServiceHost: localhost
k8sServicePort: 7443
{% elif distribution.type in ["talos"] %}
k8sServiceHost: 127.0.0.1
k8sServicePort: 7445
bootstrap/templates/partials/cilium-values-init.partial.yaml.j2
-5
View File
@@ -8,8 +8,6 @@ containerRuntime:
integration: containerd
{% if distribution.type in ["k3s"] %}
socketPath: /var/run/k3s/containerd/containerd.sock
{% elif distribution.type in ["k0s"] %}
socketPath: /var/run/k0s/containerd.sock
{% endif %}
endpointRoutes:
enabled: true
@@ -26,9 +24,6 @@ ipv6:
{% if distribution.type in ["k3s"] %}
k8sServiceHost: 127.0.0.1
k8sServicePort: 6444
{% elif distribution.type in ["k0s"] %}
k8sServiceHost: localhost
k8sServicePort: 7443
{% elif distribution.type in ["talos"] %}
k8sServiceHost: 127.0.0.1
k8sServicePort: 7445