feat: do not template out k3s ansible when k0s is choosen

Signed-off-by: Devin Buhl <devin@buhl.casa>
This commit is contained in:
Devin Buhl committed 2024-01-11 16:46:36 -05:00
1 parent 8e7d36c0c5
commit c8ebab7508
29 files changed
+33 -15

No files matched your search

@@ -0,0 +1,33 @@
#jinja2: trim_blocks: True, lstrip_blocks: True
---
#
# Below vars are for the xanmanning.k3s role
# ...see https://github.com/PyratLabs/ansible-role-k3s
#
# renovate: datasource=github-releases depName=k3s-io/k3s
k3s_release_version: "v1.29.0+k3s1"
k3s_install_hard_links: true
k3s_become: true
{% if bootstrap_nodes.master | length > 1 %}
k3s_etcd_datastore: true
{% else %}
k3s_etcd_datastore: false
{% endif %}
k3s_registration_address: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
# /var/lib/rancher/k3s/server/manifests
k3s_server_manifests_templates:
- custom-cilium-helmchart.yaml.j2
{% if not bootstrap_ipv6_enabled | default(false) %}
- custom-cilium-l2.yaml.j2
{% endif %}
- custom-coredns-helmchart.yaml.j2
{% if bootstrap_kube_vip_enabled | default(true) %}
# /var/lib/rancher/k3s/server/manifests
k3s_server_manifests_urls:
- url: https://raw.githubusercontent.com/kube-vip/website/main/content/manifests/rbac.yaml
filename: kube-vip-rbac.yaml
# /var/lib/rancher/k3s/agent/pod-manifests
k3s_server_pod_manifests_templates:
- kube-vip-static-pod.yaml.j2
{% endif %}
@@ -0,0 +1,47 @@
#jinja2: trim_blocks: True, lstrip_blocks: True
---
# https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/
# https://github.com/PyratLabs/ansible-role-k3s
k3s_control_node: true
k3s_server:
{% if bootstrap_ipv6_enabled | default(false) %}
node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
{% else %}
node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
{% endif %}
tls-san:
- "{% raw %}{{ kubeapi_addr }}{% endraw %}"
docker: false
flannel-backend: "none" # This needs to be in quotes
disable:
- coredns # Disable coredns - replaced with Coredns Helm Chart
- flannel # Disable flannel - replaced with Cilium Helm Chart
- local-storage # Disable local-path-provisioner - replaced with democratic-csi
- metrics-server # Disable metrics-server - installed with Flux
- servicelb # Disable servicelb - replaced with Cilium Helm Chart
- traefik # Disable traefik - replaced with ingress-nginx and installed with Flux
disable-network-policy: true
disable-cloud-controller: true
disable-kube-proxy: true # Cilium uses eBPF
write-kubeconfig-mode: "644"
pause-image: registry.k8s.io/pause:3.9
secrets-encryption: true
{% if bootstrap_ipv6_enabled | default(false) %}
cluster-cidr: "{% raw %}{{ cluster_cidr }},{{ cluster_cidr_v6 }}{% endraw %}"
service-cidr: "{% raw %}{{ service_cidr }},{{ service_cidr_v6 }}{% endraw %}"
{% else %}
cluster-cidr: "{% raw %}{{ cluster_cidr }}{% endraw %}"
service-cidr: "{% raw %}{{ service_cidr }}{% endraw %}"
{% endif %}
etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack
kube-controller-manager-arg:
- "bind-address=0.0.0.0" # Required to monitor kube-controller-manager with kube-prometheus-stack
kube-scheduler-arg:
- "bind-address=0.0.0.0" # Required to monitor kube-scheduler with kube-prometheus-stack
kube-apiserver-arg:
- "anonymous-auth=true" # Required for HAProxy health-checks
# TODO: Move these options to a kubelet config file
kubelet-arg:
- "image-gc-high-threshold=55"
- "image-gc-low-threshold=50"
@@ -0,0 +1,17 @@
#jinja2: trim_blocks: True, lstrip_blocks: True
---
# https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/
# https://github.com/PyratLabs/ansible-role-k3s
k3s_control_node: false
k3s_agent:
{% if bootstrap_ipv6_enabled | default(false) %}
node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
{% else %}
node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
{% endif %}
pause-image: registry.k8s.io/pause:3.9
# TODO: Move these options to a kubelet config file
kubelet-arg:
- "image-gc-high-threshold=55"
- "image-gc-low-threshold=50"
@@ -0,0 +1,66 @@
#jinja2: trim_blocks: True, lstrip_blocks: True
---
- name: Cluster Installation
hosts: kubernetes
become: true
gather_facts: true
any_errors_fatal: true
pre_tasks:
- name: Pausing for 5 seconds...
ansible.builtin.pause:
seconds: 5
tasks:
- name: Check if cluster is installed
check_mode: false
ansible.builtin.stat:
path: /etc/rancher/k3s/config.yaml
register: k3s_installed
- name: Ignore manifests templates and urls if the cluster is already installed
when: k3s_installed.stat.exists
ansible.builtin.set_fact:
k3s_server_manifests_templates: []
k3s_server_manifests_urls: []
- name: Install Kubernetes
ansible.builtin.include_role:
name: xanmanning.k3s
public: true
vars:
k3s_state: installed
- name: Kubeconfig
ansible.builtin.include_tasks: tasks/kubeconfig.yaml
- name: Wait for custom manifests to rollout
when:
- k3s_primary_control_node
- (k3s_server_manifests_templates | length > 0
or k3s_server_manifests_urls | length > 0)
kubernetes.core.k8s_info:
kubeconfig: /etc/rancher/k3s/k3s.yaml
kind: "{% raw %}{{ item.kind }}{% endraw %}"
name: "{% raw %}{{ item.name }}{% endraw %}"
namespace: "{% raw %}{{ item.namespace | default('') }}{% endraw %}"
wait: true
wait_sleep: 10
wait_timeout: 360
loop:
- { name: cilium, kind: HelmChart, namespace: kube-system }
- { name: coredns, kind: HelmChart, namespace: kube-system }
{% if not bootstrap_ipv6_enabled | default(false) %}
- { name: policy, kind: CiliumL2AnnouncementPolicy }
- { name: pool, kind: CiliumLoadBalancerIPPool }
{% endif %}
- name: Coredns
when: k3s_primary_control_node
ansible.builtin.include_tasks: tasks/coredns.yaml
- name: Cilium
when: k3s_primary_control_node
ansible.builtin.include_tasks: tasks/cilium.yaml
- name: Cruft
when: k3s_primary_control_node
ansible.builtin.include_tasks: tasks/cruft.yaml
@@ -0,0 +1,24 @@
---
- name: Cluster kube-vip
hosts: master
serial: 1
become: true
gather_facts: true
any_errors_fatal: true
pre_tasks:
- name: Pausing for 5 seconds...
ansible.builtin.pause:
seconds: 5
tasks:
- name: Ensure Kubernetes is running
ansible.builtin.include_role:
name: xanmanning.k3s
public: true
vars:
k3s_state: started
- name: Upgrade kube-vip
ansible.builtin.template:
src: templates/kube-vip-static-pod.yaml.j2
dest: "{% raw %}{{ k3s_server_pod_manifests_dir }}{% endraw %}/kube-vip-static-pod.yaml"
mode: preserve
@@ -0,0 +1,56 @@
---
- name: Cilium
block:
- name: Cilium | Check if Cilium HelmChart exists
kubernetes.core.k8s_info:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: cilium
kind: HelmChart
namespace: kube-system
register: cilium_helmchart
- name: Cilium | Wait for Cilium to rollout
when: cilium_helmchart.resources | count > 0
kubernetes.core.k8s_info:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: helm-install-cilium
kind: Job
namespace: kube-system
wait: true
wait_condition:
type: Complete
status: true
wait_timeout: 360
- name: Cilium | Patch the Cilium HelmChart to unmanage it
when: cilium_helmchart.resources | count > 0
kubernetes.core.k8s_json_patch:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: cilium
kind: HelmChart
namespace: kube-system
patch:
- op: add
path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged
value: "true"
- name: Cilium | Delete the Cilium HelmChart CR
when: cilium_helmchart.resources | count > 0
kubernetes.core.k8s:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: cilium
kind: HelmChart
namespace: kube-system
state: absent
- name: Cilium | Force delete the Cilium HelmChart
when: cilium_helmchart.resources | count > 0
kubernetes.core.k8s:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: cilium
kind: HelmChart
namespace: kube-system
state: patched
definition:
metadata:
finalizers: []
@@ -0,0 +1,56 @@
---
- name: Coredns
block:
- name: Coredns | Check if Coredns HelmChart exists
kubernetes.core.k8s_info:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: coredns
kind: HelmChart
namespace: kube-system
register: coredns_helmchart
- name: Coredns | Wait for Coredns to rollout
when: coredns_helmchart.resources | count > 0
kubernetes.core.k8s_info:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: helm-install-coredns
kind: Job
namespace: kube-system
wait: true
wait_condition:
type: Complete
status: true
wait_timeout: 360
- name: Coredns | Patch the Coredns HelmChart to unmanage it
when: coredns_helmchart.resources | count > 0
kubernetes.core.k8s_json_patch:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: coredns
kind: HelmChart
namespace: kube-system
patch:
- op: add
path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged
value: "true"
- name: Coredns | Delete the Coredns HelmChart CR
when: coredns_helmchart.resources | count > 0
kubernetes.core.k8s:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: coredns
kind: HelmChart
namespace: kube-system
state: absent
- name: Coredns | Force delete the Coredns HelmChart
when: coredns_helmchart.resources | count > 0
kubernetes.core.k8s:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: coredns
kind: HelmChart
namespace: kube-system
state: patched
definition:
metadata:
finalizers: []
@@ -0,0 +1,32 @@
---
# https://github.com/k3s-io/k3s/issues/1971
- name: Cruft
block:
- name: Cruft | Get list of custom mantifests
ansible.builtin.find:
paths: "{% raw %}{{ k3s_server_manifests_dir }}{% endraw %}"
file_type: file
use_regex: true
patterns: ["^custom-.*"]
register: custom_manifest
- name: Cruft | Delete custom mantifests
ansible.builtin.file:
path: "{% raw %}{{ item.path }}{% endraw %}"
state: absent
loop: "{% raw %}{{ custom_manifest.files }}{% endraw %}"
- name: Cruft | Get list of custom addons
kubernetes.core.k8s_info:
kubeconfig: /etc/rancher/k3s/k3s.yaml
kind: Addon
register: addons_list
- name: Cruft | Delete addons
kubernetes.core.k8s:
kubeconfig: /etc/rancher/k3s/k3s.yaml
name: "{% raw %}{{ item.metadata.name }}{% endraw %}"
kind: Addon
namespace: kube-system
state: absent
loop: "{% raw %}{{ addons_list.resources | selectattr('metadata.name', 'match', '^custom-.*') | list }}{% endraw %}"
@@ -0,0 +1,26 @@
---
- name: Get absolute path to this Git repository # noqa: command-instead-of-module
ansible.builtin.command: git rev-parse --show-toplevel
delegate_to: localhost
become: false
run_once: true
register: repository_path
changed_when: false
check_mode: false
failed_when: repository_path.rc != 0
- name: Copy kubeconfig to the project directory
when: k3s_primary_control_node
ansible.builtin.fetch:
src: /etc/rancher/k3s/k3s.yaml
dest: "{% raw %}{{ repository_path.stdout }}{% endraw %}/kubeconfig"
flat: true
- name: Update kubeconfig with the correct load balancer address
delegate_to: localhost
become: false
run_once: true
ansible.builtin.replace:
path: "{% raw %}{{ repository_path.stdout }}{% endraw %}/kubeconfig"
regexp: https://127.0.0.1:6443
replace: "https://{% raw %}{{ k3s_registration_address }}{% endraw %}:6443"
@@ -0,0 +1,64 @@
#jinja2: trim_blocks: True, lstrip_blocks: True
---
# https://docs.k3s.io/helm
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: cilium
namespace: kube-system
spec:
# renovate: datasource=helm
repo: https://helm.cilium.io/
chart: cilium
version: 1.14.5
targetNamespace: kube-system
bootstrap: true
valuesContent: |-
autoDirectNodeRoutes: true
bpf:
masquerade: true
bgp:
enabled: false
cluster:
name: home-cluster
id: 1
containerRuntime:
integration: containerd
socketPath: /var/run/k3s/containerd/containerd.sock
endpointRoutes:
enabled: true
hubble:
enabled: false
ipam:
mode: kubernetes
ipv4NativeRoutingCIDR: "{% raw %}{{ cluster_cidr }}{% endraw %}"
{% if bootstrap_ipv6_enabled | default(false) %}
ipv6NativeRoutingCIDR: "{% raw %}{{ cluster_cidr_v6 }}{% endraw %}"
ipv6:
enabled: true
{% endif %}
k8sServiceHost: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
k8sServicePort: 6443
kubeProxyReplacement: true
kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
l2announcements:
{% if bootstrap_ipv6_enabled | default(false) %}
enabled: false
{% else %}
enabled: true
# https://github.com/cilium/cilium/issues/26586
leaseDuration: 120s
leaseRenewDeadline: 60s
leaseRetryPeriod: 1s
{% endif %}
loadBalancer:
algorithm: maglev
mode: {{ bootstrap_cilium_loadbalancer_mode | default('dsr', true) }}
localRedirectPolicy: true
operator:
replicas: 1
rollOutPods: true
rollOutCiliumPods: true
routingMode: native
securityContext:
privileged: true
@@ -0,0 +1,22 @@
---
# https://docs.cilium.io/en/latest/network/l2-announcements
apiVersion: cilium.io/v2alpha1
kind: CiliumL2AnnouncementPolicy
metadata:
name: policy
spec:
loadBalancerIPs: true
# NOTE: This might need to be set if you have more than one active NIC on your nodes
# interfaces:
# - ^eno[0-9]+
nodeSelector:
matchLabels:
kubernetes.io/os: linux
---
apiVersion: cilium.io/v2alpha1
kind: CiliumLoadBalancerIPPool
metadata:
name: pool
spec:
cidrs:
- cidr: "{% raw %}{{ node_cidr }}{% endraw %}"
@@ -0,0 +1,78 @@
#jinja2: trim_blocks: True, lstrip_blocks: True
---
# https://docs.k3s.io/helm
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: coredns
namespace: kube-system
spec:
# renovate: datasource=helm
repo: https://coredns.github.io/helm
chart: coredns
version: 1.29.0
targetNamespace: kube-system
bootstrap: true
valuesContent: |-
fullnameOverride: coredns
replicaCount: 1
k8sAppLabelOverride: kube-dns
service:
name: kube-dns
clusterIP: "{% raw %}{{ coredns_addr }}{% endraw %}"
serviceAccount:
create: true
deployment:
annotations:
reloader.stakater.com/auto: "true"
servers:
- zones:
- zone: .
scheme: dns://
use_tcp: true
port: 53
plugins:
- name: log
- name: errors
- name: health
configBlock: |-
lameduck 5s
- name: ready
- name: kubernetes
parameters: cluster.local in-addr.arpa ip6.arpa
configBlock: |-
pods insecure
fallthrough in-addr.arpa ip6.arpa
ttl 30
- name: prometheus
parameters: 0.0.0.0:9153
- name: forward
parameters: . /etc/resolv.conf
- name: cache
parameters: 30
- name: loop
- name: reload
- name: loadbalance
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/instance: coredns
@@ -0,0 +1,59 @@
---
apiVersion: v1
kind: Pod
metadata:
name: kube-vip
namespace: kube-system
labels:
app.kubernetes.io/instance: kube-vip
app.kubernetes.io/name: kube-vip
spec:
containers:
- name: kube-vip
image: ghcr.io/kube-vip/kube-vip:v0.6.4
imagePullPolicy: IfNotPresent
args: ["manager"]
env:
- name: address
value: "{% raw %}{{ kubeapi_addr }}{% endraw %}"
- name: vip_arp
value: "true"
- name: lb_enable
value: "true"
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_ddns
value: "false"
- name: svc_enable
value: "false"
- name: vip_leaderelection
value: "true"
- name: vip_leaseduration
value: "15"
- name: vip_renewdeadline
value: "10"
- name: vip_retryperiod
value: "2"
- name: prometheus_server
value: :2112
securityContext:
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- name: kubeconfig
hostPath:
path: /etc/rancher/k3s/k3s.yaml