feat: implement kube-vip for k0s & k3s in daemonset mode (#1230)

* feat: implement kube-vip for k0s

Signed-off-by: Devin Buhl <devin@buhl.casa>

* feat: implement kube-vip for k0s

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: k3s to ds as well

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: no j2j2 nonsense

Signed-off-by: Devin Buhl <devin@buhl.casa>

* upadte: readme and taskfile

Signed-off-by: Devin Buhl <devin@buhl.casa>

* update: renovate config

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>

.github/renovate.json5

@@ -15,35 +15,35 @@
   "schedule": ["on saturday"],
   "flux": {
     "fileMatch": [
-      "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
+      "(^|/)ansible/.+\\.ya?ml(\\.j2)?$",
-      "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
+      "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$"
     ]
   },
   "helm-values": {
     "fileMatch": [
-      "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
+      "(^|/)ansible/.+\\.ya?ml(\\.j2)?$",
-      "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
+      "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$"
     ]
   },
   "kubernetes": {
     "fileMatch": [
-      "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
+      "(^|/)ansible/.+\\.ya?ml(\\.j2)?$",
-      "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
+      "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$"
     ]
   },
   "kustomize": {
     "fileMatch": [
-      "(^|/)kustomization\\.ya?ml(\\.j2)?(\\.j2)?$"
+      "(^|/)kustomization\\.ya?ml(\\.j2)?$"
     ]
   },
   "pip_requirements": {
     "fileMatch": [
-      "(^|/)[\\w-]*requirements(-\\w+)?\\.(txt|pip)(\\.j2)?(\\.j2)?$"
+      "(^|/)[\\w-]*requirements(-\\w+)?\\.(txt|pip)(\\.j2)?$"
     ]
   },
   "ansible-galaxy": {
     "fileMatch": [
-      "(^|/)(galaxy|requirements)(\\.ansible)?\\.ya?ml(\\.j2)?(\\.j2)?$"
+      "(^|/)(galaxy|requirements)(\\.ansible)?\\.ya?ml(\\.j2)?$"
     ]
   },
   // commit message topics
@@ -232,8 +232,8 @@
       "description": "Process various other dependencies",
       "fileMatch": [
         "(^|/).taskfiles/.+\\.ya?ml$",
-        "(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
+        "(^|/)ansible/.+\\.ya?ml(\\.j2)?$",
-        "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
+        "(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$"
       ],
       "matchStrings": [
         // Example:
@@ -242,11 +242,6 @@
         // Example:
         //   - https://github.com/rancher/system-upgrade-controller/releases/download/v0.11.0/crd.yaml
         "datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)( versioning=(?<versioning>\\S+))?\n.*?-\\s(.*?)\/(?<currentValue>[^/]+)\/[^/]+\n",
-        // Example:
-        //   repo: https://helm.cilium.io
-        //   chart: cilium
-        //   version: 1.14.5
-        "datasource=(?<datasource>\\S+)\n.*?repo: (?<registryUrl>\\S+)\n.*?chart: (?<depName>\\S+)\n.*?version: (?<currentValue>\\S+)\n"
       ],
       "datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}",
       "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}"

.taskfiles/Repository/Taskfile.yaml

@@ -14,7 +14,7 @@ tasks:
       - mv {{.BOOTSTRAP_DIR}} {{.PRIVATE_DIR}}/bootstrap-{{now | date "150405"}}
       - mv {{.MAKEJINJA_CONFIG_FILE}} {{.PRIVATE_DIR}}/makejinja-{{now | date "150405"}}.toml
       # Update renovate.json5
-      - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
+      - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
     preconditions:
       - { msg: "Missing bootstrap directory", sh: "test -d {{.BOOTSTRAP_DIR}}" }
       - { msg: "Missing private directory", sh: "test -d {{.PRIVATE_DIR}}" }

bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2

@@ -7,15 +7,13 @@ k3s_install_hard_links: true
 k3s_become: true
 k3s_etcd_datastore: true
 k3s_use_unsupported_config: true
-k3s_registration_address: "{{ kube_api_addr }}"
+k3s_registration_address: "#{ bootstrap_kube_api_addr }#"
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
   - custom-cilium-helmchart.yaml.j2
+  - kube-vip-ds.yaml.j2
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_urls:
-  - url: https://raw.githubusercontent.com/kube-vip/website/main/content/manifests/rbac.yaml
+  - url: https://kube-vip.io/manifests/rbac.yaml
     filename: kube-vip-rbac.yaml
-# /var/lib/rancher/k3s/agent/pod-manifests
-k3s_server_pod_manifests_templates:
-  - kube-vip-static-pod.yaml.j2
 #% endif %#

bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2

@@ -1,14 +0,0 @@
-#% if bootstrap_distribution in ['k3s', 'k0s'] %#
----
-timezone: "#{ bootstrap_timezone }#"
-github_username: "#{ bootstrap_github_username }#"
-coredns_addr: "#{ bootstrap_service_cidr.split(',')[0] | nthhost(10) }#"
-kube_api_addr: "#{ bootstrap_kube_api_addr }#"
-cluster_cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#"
-service_cidr: "#{ bootstrap_service_cidr.split(',')[0] }#"
-node_cidr: "#{ bootstrap_node_cidr }#"
-#% if bootstrap_ipv6_enabled %#
-cluster_cidr_v6: "#{ bootstrap_cluster_cidr.split(',')[1] }#"
-service_cidr_v6: "#{ bootstrap_service_cidr.split(',')[1] }#"
-#% endif %#
-#% endif %#

bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2

@@ -10,7 +10,7 @@ k3s_server:
   node-ip: "{{ ansible_host }}"
   #% endif %#
   tls-san:
-    - "{{ kube_api_addr }}"
+    - "#{ bootstrap_kube_api_addr }#"
     #% for item in bootstrap_kube_additional_tls_sans %#
     - "{{ item }}"
     #% endfor %#
@@ -24,11 +24,11 @@ k3s_server:
   pause-image: registry.k8s.io/pause:3.9
   secrets-encryption: true
   #% if bootstrap_ipv6_enabled %#
-  cluster-cidr: "{{ cluster_cidr }},{{ cluster_cidr_v6 }}"
+  cluster-cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#,#{ bootstrap_cluster_cidr.split(',')[1] }#"
-  service-cidr: "{{ service_cidr }},{{ service_cidr_v6 }}"
+  service-cidr: "#{ bootstrap_service_cidr.split(',')[0] }#,#{ bootstrap_service_cidr.split(',')[1] }#"
   #% else %#
-  cluster-cidr: "{{ cluster_cidr }}"
+  cluster-cidr: "#{ bootstrap_cluster_cidr }#"
-  service-cidr: "{{ service_cidr }}"
+  service-cidr: "#{ bootstrap_service_cidr }#"
   #% endif %#
   etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack
   kube-controller-manager-arg:

bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2

@@ -16,7 +16,7 @@
       block:
         - name: Locale | Set timezone
           community.general.timezone:
-            name: "{{ timezone | default('Etc/UTC') }}"
+            name: "#{ bootstrap_timezone }#"
 
     - name: Packages
       block:
@@ -33,7 +33,7 @@
         - name: User Configuration | SSH keys
           ansible.posix.authorized_key:
             user: "{{ ansible_user }}"
-            key: "https://github.com/{{ github_username }}.keys"
+            key: "https://github.com/#{ bootstrap_github_username }#.keys"
         - name: User Configuration | Silence login
           ansible.builtin.file:
             dest: "{{ '/home/' + ansible_user if ansible_user != 'root' else '/root' }}/.hushlogin"

bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2

@@ -7,10 +7,10 @@ metadata:
   name: cilium
   namespace: kube-system
 spec:
-  # renovate: datasource=helm
   repo: https://helm.cilium.io/
   chart: cilium
-  version: 1.14.6
+  # renovate: datasource=github-releases depName=cilium/cilium
+  version: "1.14.5"
   targetNamespace: kube-system
   bootstrap: true
   valuesContent: |-

bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2

@@ -0,0 +1,77 @@
+#% if bootstrap_distribution == 'k3s' %#
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-vip
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kube-vip
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-vip
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kube-vip
+    spec:
+      containers:
+        - name: kube-vip
+          image: ghcr.io/kube-vip/kube-vip:v0.6.4
+          imagePullPolicy: IfNotPresent
+          args: ["manager"]
+          env:
+            - name: address
+              value: "#{ bootstrap_kube_api_addr }#"
+            - name: vip_arp
+              value: "true"
+            - name: lb_enable
+              value: "true"
+            - name: port
+              value: "6443"
+            - name: vip_cidr
+              value: "32"
+            - name: cp_enable
+              value: "true"
+            - name: cp_namespace
+              value: kube-system
+            - name: vip_ddns
+              value: "false"
+            - name: svc_enable
+              value: "false"
+            - name: vip_leaderelection
+              value: "true"
+            - name: vip_leaseduration
+              value: "15"
+            - name: vip_renewdeadline
+              value: "10"
+            - name: vip_retryperiod
+              value: "2"
+            - name: prometheus_server
+              value: :2112
+          securityContext:
+            capabilities:
+              add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"]
+      hostAliases:
+        - hostnames:
+            - kubernetes
+          ip: 127.0.0.1
+      hostNetwork: true
+      serviceAccountName: kube-vip
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/master
+                    operator: Exists
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+#% endif %#

bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2

@@ -1,61 +0,0 @@
-#% if bootstrap_distribution == 'k3s' %#
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-vip
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/instance: kube-vip
-    app.kubernetes.io/name: kube-vip
-spec:
-  containers:
-    - name: kube-vip
-      image: ghcr.io/kube-vip/kube-vip:v0.6.4
-      imagePullPolicy: IfNotPresent
-      args: ["manager"]
-      env:
-        - name: address
-          value: "{{ kube_api_addr }}"
-        - name: vip_arp
-          value: "true"
-        - name: lb_enable
-          value: "true"
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "false"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "15"
-        - name: vip_renewdeadline
-          value: "10"
-        - name: vip_retryperiod
-          value: "2"
-        - name: prometheus_server
-          value: :2112
-      securityContext:
-        capabilities:
-          add: ["NET_ADMIN", "NET_RAW"]
-      volumeMounts:
-        - mountPath: /etc/kubernetes/admin.conf
-          name: kubeconfig
-  hostAliases:
-    - hostnames:
-        - kubernetes
-      ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-    - name: kubeconfig
-      hostPath:
-        path: /etc/rancher/k3s/k3s.yaml
-#% endif %#

bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2

@@ -18,6 +18,7 @@ spec:
         apply:
           before:
             - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-system.sh | INSTALL_K0S_HOSTNAME="#{ item.name }#" bash -s -
+            - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-kube-vip.sh | bash -s -
         reset:
           before:
             - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-cilium.sh | bash -s -

bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -e
+set -o noglob
+
+[ $(id -u) -eq 0 ] || exec sudo $0 $@
+
+# Create directory
+mkdir -p /var/lib/k0s/manifests/kube-vip
+
+# Create kube-vip rbac
+cat <<EOF > /var/lib/k0s/manifests/kube-vip/rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-vip
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: system:kube-vip-role
+rules:
+  - apiGroups: [""]
+    resources: ["services/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["services", "endpoints"]
+    verbs: ["list","get","watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list","get","watch", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["list", "get", "watch", "update", "create"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["list","get","watch", "update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:kube-vip-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-vip-role
+subjects:
+- kind: ServiceAccount
+  name: kube-vip
+  namespace: kube-system
+EOF
+
+cat <<EOF > /var/lib/k0s/manifests/kube-vip/ds.yaml
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-vip
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kube-vip
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-vip
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kube-vip
+    spec:
+      containers:
+        - name: kube-vip
+          image: ghcr.io/kube-vip/kube-vip:v0.6.4
+          imagePullPolicy: IfNotPresent
+          args: ["manager"]
+          env:
+            - name: address
+              value: "#{ bootstrap_kube_api_addr }#"
+            - name: vip_arp
+              value: "true"
+            - name: lb_enable
+              value: "true"
+            - name: port
+              value: "6443"
+            - name: vip_cidr
+              value: "32"
+            - name: cp_enable
+              value: "true"
+            - name: cp_namespace
+              value: kube-system
+            - name: vip_ddns
+              value: "false"
+            - name: svc_enable
+              value: "false"
+            - name: vip_leaderelection
+              value: "true"
+            - name: vip_leaseduration
+              value: "15"
+            - name: vip_renewdeadline
+              value: "10"
+            - name: vip_retryperiod
+              value: "2"
+            - name: prometheus_server
+              value: :2112
+          securityContext:
+            capabilities:
+              add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"]
+      hostAliases:
+        - hostnames:
+            - kubernetes
+          ip: 127.0.0.1
+      hostNetwork: true
+      serviceAccountName: kube-vip
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/master
+                    operator: Exists
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+EOF

bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2

@@ -5,7 +5,7 @@ kind: Kustomization
 helmCharts:
   - name: cilium
     repo: https://helm.cilium.io
-    version: 1.14.6
+    version: 1.14.5
     releaseName: cilium
     namespace: kube-system
     valuesFile: values.yaml

bootstrap/vars/config.sample.yaml

@@ -39,18 +39,18 @@ bootstrap_cloudflare_tunnel_id:
 
 # CIDR your nodes are on (e.g. 192.168.1.0/24)
 bootstrap_node_cidr:
-# The IP address of the Kube API, choose an available IP in your nodes network that is not being used
+
-# NOTE: For k0s set this to a IP you can then later use to manually setup a load balancer for the Kube API (e.g. Haproxy or Keepalived)
+# The IP address of the Kube API, choose an available IP in your nodes network that is not being used. This is announced over L2.
-# Ref: https://github.com/k0sproject/k0s/issues/1150
 bootstrap_kube_api_addr:
+# The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used. This is announced over L2.
+bootstrap_k8s_gateway_addr:
+# The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used. This is announced over L2.
+bootstrap_external_ingress_addr:
+# The Load balancer IP for internal ingress, choose an available IP in your nodes network that is not being used. This is announced over L2.
+bootstrap_internal_ingress_addr:
+
 # Add additional SANs to the Kube API cert, this is useful if you want to call the Kube API by hostname rather than IP
 bootstrap_kube_additional_tls_sans: []
-# The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used
-bootstrap_k8s_gateway_addr:
-# The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used
-bootstrap_external_ingress_addr:
-# The Load balancer IP for internal ingress, choose an available IP in your nodes network that is not being used
-bootstrap_internal_ingress_addr:
 
 # The DNS server to use for the cluster, this can be an existing local DNS server or a public one
 # If using a local DNS server make sure it meets the following requirements: