infra/home-cluster
2
0 Fork 0
Code Issues Pull Requests 40 Actions Packages Projects Releases Wiki Activity

feat: implement kube-vip for k0s & k3s in daemonset mode (#1230)

Browse Source 
* feat: implement kube-vip for k0s

Signed-off-by: Devin Buhl <devin@buhl.casa>

* feat: implement kube-vip for k0s

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: k3s to ds as well

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: no j2j2 nonsense

Signed-off-by: Devin Buhl <devin@buhl.casa>

* upadte: readme and taskfile

Signed-off-by: Devin Buhl <devin@buhl.casa>

* update: renovate config

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>
This commit is contained in:
Devin Buhl authored and GitHub committed 2024-01-22 12:45:11 -05:00
1 parent d9c7fd44ba
commit 9373065338
13 files changed
+243 -116
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/renovate.json5
+11 -16
View File
@@ -15,35 +15,35 @@
"schedule": ["on saturday"],
"flux": {
"fileMatch": [
"(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
"(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
"(^|/)ansible/.+\\.ya?ml(\\.j2)?$",
"(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$"
]
},
"helm-values": {
"fileMatch": [
"(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
"(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
"(^|/)ansible/.+\\.ya?ml(\\.j2)?$",
"(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$"
]
},
"kubernetes": {
"fileMatch": [
"(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
"(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
"(^|/)ansible/.+\\.ya?ml(\\.j2)?$",
"(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$"
]
},
"kustomize": {
"fileMatch": [
"(^|/)kustomization\\.ya?ml(\\.j2)?(\\.j2)?$"
"(^|/)kustomization\\.ya?ml(\\.j2)?$"
]
},
"pip_requirements": {
"fileMatch": [
"(^|/)[\\w-]*requirements(-\\w+)?\\.(txt|pip)(\\.j2)?(\\.j2)?$"
"(^|/)[\\w-]*requirements(-\\w+)?\\.(txt|pip)(\\.j2)?$"
]
},
"ansible-galaxy": {
"fileMatch": [
"(^|/)(galaxy|requirements)(\\.ansible)?\\.ya?ml(\\.j2)?(\\.j2)?$"
"(^|/)(galaxy|requirements)(\\.ansible)?\\.ya?ml(\\.j2)?$"
]
},
// commit message topics
@@ -232,8 +232,8 @@
"description": "Process various other dependencies",
"fileMatch": [
"(^|/).taskfiles/.+\\.ya?ml$",
"(^|/)ansible/.+\\.ya?ml(\\.j2)?(\\.j2)?$",
"(^|/)kubernetes/.+\\.ya?ml(\\.j2)?(\\.j2)?$"
"(^|/)ansible/.+\\.ya?ml(\\.j2)?$",
"(^|/)kubernetes/.+\\.ya?ml(\\.j2)?$"
],
"matchStrings": [
// Example:
@@ -242,11 +242,6 @@
// Example:
// - https://github.com/rancher/system-upgrade-controller/releases/download/v0.11.0/crd.yaml
"datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)( versioning=(?<versioning>\\S+))?\n.*?-\\s(.*?)\/(?<currentValue>[^/]+)\/[^/]+\n",
// Example:
// repo: https://helm.cilium.io
// chart: cilium
// version: 1.14.5
"datasource=(?<datasource>\\S+)\n.*?repo: (?<registryUrl>\\S+)\n.*?chart: (?<depName>\\S+)\n.*?version: (?<currentValue>\\S+)\n"
],
"datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}",
"versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}"
.taskfiles/Repository/Taskfile.yaml
+1 -1
View File
@@ -14,7 +14,7 @@ tasks:
- mv {{.BOOTSTRAP_DIR}} {{.PRIVATE_DIR}}/bootstrap-{{now | date "150405"}}
- mv {{.MAKEJINJA_CONFIG_FILE}} {{.PRIVATE_DIR}}/makejinja-{{now | date "150405"}}.toml
# Update renovate.json5
- sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
- sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
preconditions:
- { msg: "Missing bootstrap directory", sh: "test -d {{.BOOTSTRAP_DIR}}" }
- { msg: "Missing private directory", sh: "test -d {{.PRIVATE_DIR}}" }
bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2
+3 -5
View File
@@ -7,15 +7,13 @@ k3s_install_hard_links: true
k3s_become: true
k3s_etcd_datastore: true
k3s_use_unsupported_config: true
k3s_registration_address: "{{ kube_api_addr }}"
k3s_registration_address: "#{ bootstrap_kube_api_addr }#"
# /var/lib/rancher/k3s/server/manifests
k3s_server_manifests_templates:
- custom-cilium-helmchart.yaml.j2
- kube-vip-ds.yaml.j2
# /var/lib/rancher/k3s/server/manifests
k3s_server_manifests_urls:
- url: https://raw.githubusercontent.com/kube-vip/website/main/content/manifests/rbac.yaml
- url: https://kube-vip.io/manifests/rbac.yaml
filename: kube-vip-rbac.yaml
# /var/lib/rancher/k3s/agent/pod-manifests
k3s_server_pod_manifests_templates:
- kube-vip-static-pod.yaml.j2
#% endif %#
bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2
-14
View File
@@ -1,14 +0,0 @@
#% if bootstrap_distribution in ['k3s', 'k0s'] %#
---
timezone: "#{ bootstrap_timezone }#"
github_username: "#{ bootstrap_github_username }#"
coredns_addr: "#{ bootstrap_service_cidr.split(',')[0] | nthhost(10) }#"
kube_api_addr: "#{ bootstrap_kube_api_addr }#"
cluster_cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#"
service_cidr: "#{ bootstrap_service_cidr.split(',')[0] }#"
node_cidr: "#{ bootstrap_node_cidr }#"
#% if bootstrap_ipv6_enabled %#
cluster_cidr_v6: "#{ bootstrap_cluster_cidr.split(',')[1] }#"
service_cidr_v6: "#{ bootstrap_service_cidr.split(',')[1] }#"
#% endif %#
#% endif %#
bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
+5 -5
View File
@@ -10,7 +10,7 @@ k3s_server:
node-ip: "{{ ansible_host }}"
#% endif %#
tls-san:
- "{{ kube_api_addr }}"
- "#{ bootstrap_kube_api_addr }#"
#% for item in bootstrap_kube_additional_tls_sans %#
- "{{ item }}"
#% endfor %#
@@ -24,11 +24,11 @@ k3s_server:
pause-image: registry.k8s.io/pause:3.9
secrets-encryption: true
#% if bootstrap_ipv6_enabled %#
cluster-cidr: "{{ cluster_cidr }},{{ cluster_cidr_v6 }}"
service-cidr: "{{ service_cidr }},{{ service_cidr_v6 }}"
cluster-cidr: "#{ bootstrap_cluster_cidr.split(',')[0] }#,#{ bootstrap_cluster_cidr.split(',')[1] }#"
service-cidr: "#{ bootstrap_service_cidr.split(',')[0] }#,#{ bootstrap_service_cidr.split(',')[1] }#"
#% else %#
cluster-cidr: "{{ cluster_cidr }}"
service-cidr: "{{ service_cidr }}"
cluster-cidr: "#{ bootstrap_cluster_cidr }#"
service-cidr: "#{ bootstrap_service_cidr }#"
#% endif %#
etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack
kube-controller-manager-arg:
bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2
+2 -2
View File
@@ -16,7 +16,7 @@
block:
- name: Locale | Set timezone
community.general.timezone:
name: "{{ timezone | default('Etc/UTC') }}"
name: "#{ bootstrap_timezone }#"
- name: Packages
block:
@@ -33,7 +33,7 @@
- name: User Configuration | SSH keys
ansible.posix.authorized_key:
user: "{{ ansible_user }}"
key: "https://github.com/{{ github_username }}.keys"
key: "https://github.com/#{ bootstrap_github_username }#.keys"
- name: User Configuration | Silence login
ansible.builtin.file:
dest: "{{ '/home/' + ansible_user if ansible_user != 'root' else '/root' }}/.hushlogin"
bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 → bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2
+2 -2
View File
@@ -7,10 +7,10 @@ metadata:
name: cilium
namespace: kube-system
spec:
# renovate: datasource=helm
repo: https://helm.cilium.io/
chart: cilium
version: 1.14.6
# renovate: datasource=github-releases depName=cilium/cilium
version: "1.14.5"
targetNamespace: kube-system
bootstrap: true
valuesContent: |-
bootstrap/templates/ansible/playbooks/templates/kube-vip-ds.yaml.j2
+77
View File
@@ -0,0 +1,77 @@
#% if bootstrap_distribution == 'k3s' %#
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-vip
namespace: kube-system
labels:
app.kubernetes.io/name: kube-vip
spec:
selector:
matchLabels:
app.kubernetes.io/name: kube-vip
template:
metadata:
labels:
app.kubernetes.io/name: kube-vip
spec:
containers:
- name: kube-vip
image: ghcr.io/kube-vip/kube-vip:v0.6.4
imagePullPolicy: IfNotPresent
args: ["manager"]
env:
- name: address
value: "#{ bootstrap_kube_api_addr }#"
- name: vip_arp
value: "true"
- name: lb_enable
value: "true"
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_ddns
value: "false"
- name: svc_enable
value: "false"
- name: vip_leaderelection
value: "true"
- name: vip_leaseduration
value: "15"
- name: vip_renewdeadline
value: "10"
- name: vip_retryperiod
value: "2"
- name: prometheus_server
value: :2112
securityContext:
capabilities:
add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"]
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
serviceAccountName: kube-vip
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/master
operator: Exists
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
#% endif %#
bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2
-61
View File
@@ -1,61 +0,0 @@
#% if bootstrap_distribution == 'k3s' %#
---
apiVersion: v1
kind: Pod
metadata:
name: kube-vip
namespace: kube-system
labels:
app.kubernetes.io/instance: kube-vip
app.kubernetes.io/name: kube-vip
spec:
containers:
- name: kube-vip
image: ghcr.io/kube-vip/kube-vip:v0.6.4
imagePullPolicy: IfNotPresent
args: ["manager"]
env:
- name: address
value: "{{ kube_api_addr }}"
- name: vip_arp
value: "true"
- name: lb_enable
value: "true"
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_ddns
value: "false"
- name: svc_enable
value: "false"
- name: vip_leaderelection
value: "true"
- name: vip_leaseduration
value: "15"
- name: vip_renewdeadline
value: "10"
- name: vip_retryperiod
value: "2"
- name: prometheus_server
value: :2112
securityContext:
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- name: kubeconfig
hostPath:
path: /etc/rancher/k3s/k3s.yaml
#% endif %#
bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2
+1
View File
@@ -18,6 +18,7 @@ spec:
apply:
before:
- curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-system.sh | INSTALL_K0S_HOSTNAME="#{ item.name }#" bash -s -
- curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-kube-vip.sh | bash -s -
reset:
before:
- curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-cilium.sh | bash -s -
bootstrap/templates/kubernetes/k0s/resources/apply-kube-vip.sh.j2
Executable
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env bash
set -e
set -o noglob
[ $(id -u) -eq 0 ] || exec sudo $0 $@
# Create directory
mkdir -p /var/lib/k0s/manifests/kube-vip
# Create kube-vip rbac
cat <<EOF > /var/lib/k0s/manifests/kube-vip/rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-vip
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
name: system:kube-vip-role
rules:
- apiGroups: [""]
resources: ["services/status"]
verbs: ["update"]
- apiGroups: [""]
resources: ["services", "endpoints"]
verbs: ["list","get","watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["list","get","watch", "update", "patch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["list", "get", "watch", "update", "create"]
- apiGroups: ["discovery.k8s.io"]
resources: ["endpointslices"]
verbs: ["list","get","watch", "update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: system:kube-vip-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kube-vip-role
subjects:
- kind: ServiceAccount
name: kube-vip
namespace: kube-system
EOF
cat <<EOF > /var/lib/k0s/manifests/kube-vip/ds.yaml
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-vip
namespace: kube-system
labels:
app.kubernetes.io/name: kube-vip
spec:
selector:
matchLabels:
app.kubernetes.io/name: kube-vip
template:
metadata:
labels:
app.kubernetes.io/name: kube-vip
spec:
containers:
- name: kube-vip
image: ghcr.io/kube-vip/kube-vip:v0.6.4
imagePullPolicy: IfNotPresent
args: ["manager"]
env:
- name: address
value: "#{ bootstrap_kube_api_addr }#"
- name: vip_arp
value: "true"
- name: lb_enable
value: "true"
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_ddns
value: "false"
- name: svc_enable
value: "false"
- name: vip_leaderelection
value: "true"
- name: vip_leaseduration
value: "15"
- name: vip_renewdeadline
value: "10"
- name: vip_retryperiod
value: "2"
- name: prometheus_server
value: :2112
securityContext:
capabilities:
add: ["NET_ADMIN", "NET_RAW", "SYS_TIME"]
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
serviceAccountName: kube-vip
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/master
operator: Exists
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
EOF
bootstrap/templates/kubernetes/talos/cilium/kustomization.yaml.j2
+1 -1
View File
@@ -5,7 +5,7 @@ kind: Kustomization
helmCharts:
- name: cilium
repo: https://helm.cilium.io
version: 1.14.6
version: 1.14.5
releaseName: cilium
namespace: kube-system
valuesFile: values.yaml
bootstrap/vars/config.sample.yaml
+9 -9
View File
@@ -39,18 +39,18 @@ bootstrap_cloudflare_tunnel_id:
# CIDR your nodes are on (e.g. 192.168.1.0/24)
bootstrap_node_cidr:
# The IP address of the Kube API, choose an available IP in your nodes network that is not being used
# NOTE: For k0s set this to a IP you can then later use to manually setup a load balancer for the Kube API (e.g. Haproxy or Keepalived)
# Ref: https://github.com/k0sproject/k0s/issues/1150
# The IP address of the Kube API, choose an available IP in your nodes network that is not being used. This is announced over L2.
bootstrap_kube_api_addr:
# The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used. This is announced over L2.
bootstrap_k8s_gateway_addr:
# The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used. This is announced over L2.
bootstrap_external_ingress_addr:
# The Load balancer IP for internal ingress, choose an available IP in your nodes network that is not being used. This is announced over L2.
bootstrap_internal_ingress_addr:
# Add additional SANs to the Kube API cert, this is useful if you want to call the Kube API by hostname rather than IP
bootstrap_kube_additional_tls_sans: []
# The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used
bootstrap_k8s_gateway_addr:
# The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used
bootstrap_external_ingress_addr:
# The Load balancer IP for internal ingress, choose an available IP in your nodes network that is not being used
bootstrap_internal_ingress_addr:
# The DNS server to use for the cluster, this can be an existing local DNS server or a public one
# If using a local DNS server make sure it meets the following requirements: