feat!: template all ansible dir and rename template to bootstrap (#847)

bootstrap/templates/ansible/ansible.cfg.j2

@@ -0,0 +1,35 @@
+[defaults]
+# General settings
+nocows                      = True
+executable                  = /bin/bash
+stdout_callback             = yaml
+force_valid_group_names     = ignore
+# File/Directory settings
+log_path                    = ~/.ansible/ansible.log
+inventory                   = ./inventory
+roles_path                  = ~/.ansible/roles:./roles
+collections_path            = ~/.ansible/collections
+remote_tmp                  = ~/.ansible/tmp
+local_tmp                   = ~/.ansible/tmp
+# Fact Caching settings
+fact_caching                = jsonfile
+fact_caching_connection     = ~/.ansible/facts_cache
+# SSH settings
+remote_port                 = 22
+timeout                     = 60
+host_key_checking           = False
+# Plugin settings
+vars_plugins_enabled        = host_group_vars,community.sops.sops
+
+[inventory]
+unparsed_is_failed          = True
+
+[privilege_escalation]
+become                      = True
+
+[ssh_connection]
+scp_if_ssh                  = smart
+retries                     = 3
+ssh_args                    = -o ControlMaster=auto -o ControlPersist=30m -o Compression=yes -o ServerAliveInterval=15s
+pipelining                  = True
+control_path                = %(directory)s/%%h-%%r

bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2

@@ -0,0 +1,32 @@
+---
+#
+# Below vars are for the xanmanning.k3s role
+# ...see https://github.com/PyratLabs/ansible-role-k3s
+#
+
+# renovate: datasource=github-releases depName=k3s-io/k3s
+k3s_release_version: "v1.27.3+k3s1"
+k3s_install_hard_links: true
+k3s_become: true
+k3s_etcd_datastore: true
+k3s_use_unsupported_config: true
+k3s_registration_address: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+k3s_server_manifests_urls:
+  # Kube-vip RBAC
+  - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml
+    filename: kube-vip-rbac.yaml
+  # Essential Prometheus Operator CRDs (the rest are installed with the kube-prometheus-stack helm release)
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+    filename: custom-prometheus-podmonitors.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+    filename: custom-prometheus-prometheusrules.yaml
+  - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+    filename: custom-prometheus-servicemonitors.yaml
+# /var/lib/rancher/k3s/server/manifests
+k3s_server_manifests_templates:
+  - custom-cilium-helmchart.yaml.j2
+  - custom-cilium-l2.yaml.j2
+  - custom-coredns-helmchart.yaml.j2
+# /var/lib/rancher/k3s/agent/pod-manifests
+k3s_server_pod_manifests_templates:
+  - kube-vip-static-pod.yaml.j2

bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2

@@ -0,0 +1,8 @@
+---
+timezone: "{{ bootstrap_timezone }}"
+github_username: "{{ bootstrap_github_username }}"
+coredns_addr: "{{ bootstrap_coredns_addr }}"
+kube_vip_addr: "{{ bootstrap_kube_vip_addr }}"
+cluster_cidr: "{{ bootstrap_cluster_cidr }}"
+service_cidr: "{{ bootstrap_service_cidr }}"
+node_cidr: "{{ bootstrap_node_cidr }}"

bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2

@@ -0,0 +1,31 @@
+---
+# https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/
+# https://github.com/PyratLabs/ansible-role-k3s
+
+k3s_control_node: true
+k3s_server:
+  node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
+  tls-san:
+    - "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+  docker: false
+  flannel-backend: "none"             # This needs to be in quotes
+  disable:
+    - coredns                         # Disable coredns                 - replaced with Coredns Helm Chart
+    - flannel                         # Disable flannel                 - replaced with Cilium Helm Chart
+    - local-storage                   # Disable local-path-provisioner  - installed with Flux
+    - metrics-server                  # Disable metrics-server          - installed with Flux
+    - servicelb                       # Disable servicelb               - replaced with Cilium Helm Chart
+    - traefik                         # Disable traefik                 - replaced with ingress-nginx and installed with Flux
+  disable-network-policy: true
+  disable-cloud-controller: true
+  disable-kube-proxy: true            # Cilium uses eBPF
+  write-kubeconfig-mode: "644"
+  cluster-cidr: "{% raw %}{{ cluster_cidr }}{% endraw %}"
+  service-cidr: "{% raw %}{{ service_cidr }}{% endraw %}"
+  etcd-expose-metrics: true           # Required to monitor etcd with kube-prometheus-stack
+  kube-controller-manager-arg:
+    - "bind-address=0.0.0.0"          # Required to monitor kube-controller-manager with kube-prometheus-stack
+  kube-scheduler-arg:
+    - "bind-address=0.0.0.0"          # Required to monitor kube-scheduler with kube-prometheus-stack
+  kube-apiserver-arg:
+    - "anonymous-auth=true"           # Required for HAProxy health-checks

bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2

@@ -0,0 +1,7 @@
+---
+# https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/
+# https://github.com/PyratLabs/ansible-role-k3s
+
+k3s_control_node: false
+k3s_agent:
+  node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"

bootstrap/templates/ansible/inventory/hosts.yaml.j2

@@ -0,0 +1,20 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+kubernetes:
+  children:
+    master:
+      hosts:
+      {% for item in bootstrap_nodes.master %}
+        {{ item.name }}:
+          ansible_user: {{ item.username }}
+          ansible_host: {{ item.address }}
+      {% endfor %}
+    {% if bootstrap_nodes.worker | default([]) | length > 0 %}
+    worker:
+      hosts:
+      {% for item in bootstrap_nodes.worker %}
+        {{ item.name }}:
+          ansible_user: {{ item.username }}
+          ansible_host: {{ item.address }}
+      {% endfor %}
+    {% endif %}

bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2

@@ -0,0 +1,71 @@
+---
+- name: Cluster Installation
+  hosts: all
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Check if cluster is installed
+      check_mode: false
+      ansible.builtin.stat:
+        path: /etc/rancher/k3s/config.yaml
+      register: k3s_installed
+
+    - name: Ignore manifests templates and urls if the cluster is already installed
+      when: k3s_installed.stat.exists
+      ansible.builtin.set_fact:
+        k3s_server_manifests_templates: []
+        k3s_server_manifests_urls: []
+
+    - name: Install Kubernetes
+      ansible.builtin.include_role:
+        name: xanmanning.k3s
+        public: true
+      vars:
+        k3s_state: installed
+
+    - name: Kubeconfig
+      ansible.builtin.include_tasks: tasks/kubeconfig.yaml
+
+    - name: Wait for custom manifests to rollout
+      when:
+        - k3s_primary_control_node
+        - (k3s_server_manifests_templates | length > 0
+            or k3s_server_manifests_urls | length > 0)
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        kind: "{% raw %}{{ item.kind }}{% endraw %}"
+        name: "{% raw %}{{ item.name }}{% endraw %}"
+        namespace: "{% raw %}{{ item.namespace | default('') }}{% endraw %}"
+        wait: true
+        wait_sleep: 10
+        wait_timeout: 360
+      loop:
+        - { name: cilium, kind: HelmChart, namespace: kube-system }
+        - { name: coredns, kind: HelmChart, namespace: kube-system }
+        - { name: policy, kind: CiliumL2AnnouncementPolicy }
+        - { name: pool, kind: CiliumLoadBalancerIPPool }
+        - { name: podmonitors.monitoring.coreos.com, kind: CustomResourceDefinition }
+        - { name: prometheusrules.monitoring.coreos.com, kind: CustomResourceDefinition }
+        - { name: servicemonitors.monitoring.coreos.com, kind: CustomResourceDefinition }
+
+    - name: Coredns
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/coredns.yaml
+
+    - name: Cilium
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/cilium.yaml
+
+    - name: Cruft
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/cruft.yaml
+
+    - name: Stale Containers
+      ansible.builtin.include_tasks: tasks/stale_containers.yaml
+      vars:
+        stale_containers_state: enabled

bootstrap/templates/ansible/playbooks/cluster-kube-vip.yaml.j2

@@ -0,0 +1,23 @@
+---
+- name: Cluster kube-vip
+  hosts: master
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Ensure Kubernetes is running
+      ansible.builtin.include_role:
+        name: xanmanning.k3s
+        public: true
+      vars:
+        k3s_state: started
+
+    - name: Upgrade kube-vip
+      ansible.builtin.template:
+        src: templates/kube-vip-static-pod.yaml.j2
+        dest: "{% raw %}{{ k3s_server_pod_manifests_dir }}{% endraw %}/kube-vip-static-pod.yaml"
+        mode: preserve

bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2

@@ -0,0 +1,73 @@
+---
+- name: Cluster Nuke
+  hosts: all
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  vars_prompt:
+    - name: nuke
+      prompt: |-
+        Are you sure you want to nuke this cluster?
+        Type 'YES I WANT TO DESTROY THIS CLUSTER' to proceed
+      default: "n"
+      private: false
+  pre_tasks:
+    - name: Check for confirmation
+      ansible.builtin.fail:
+        msg: Aborted nuking the cluster
+      when: nuke != 'YES I WANT TO DESTROY THIS CLUSTER'
+
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Stop Kubernetes # noqa: ignore-errors
+      ignore_errors: true
+      block:
+        - name: Stop Kubernetes
+          ansible.builtin.include_role:
+            name: xanmanning.k3s
+            public: true
+          vars:
+            k3s_state: stopped
+
+    # https://github.com/k3s-io/docs/blob/main/docs/installation/network-options.md
+    - name: Networking
+      block:
+        - name: Networking | Delete Cilium links
+          ansible.builtin.command:
+            cmd: "ip link delete {% raw %}{{ item }}{% endraw %}"
+            removes: "/sys/class/net/{% raw %}{{ item }}{% endraw %}"
+          loop: ["cilium_host", "cilium_net", "cilium_vxlan"]
+        - name: Networking | Flush iptables
+          ansible.builtin.iptables:
+            table: "{% raw %}{{ item }}{% endraw %}"
+            flush: true
+          loop: ["filter", "nat", "mangle", "raw"]
+        - name: Networking | Flush ip6tables
+          ansible.builtin.iptables:
+            table: "{% raw %}{{ item }}{% endraw %}"
+            flush: true
+            ip_version: ipv6
+          loop: ["filter", "nat", "mangle", "raw"]
+        - name: Networking | Delete CNI directory
+          ansible.builtin.file:
+            path: /etc/cni/net.d
+            state: absent
+
+    - name: Uninstall Kubernetes
+      ansible.builtin.include_role:
+        name: xanmanning.k3s
+        public: true
+      vars:
+        k3s_state: uninstalled
+
+    - name: Stale Containers
+      ansible.builtin.include_tasks: tasks/stale_containers.yaml
+      vars:
+        stale_containers_state: disabled
+
+    - name: Reboot
+      ansible.builtin.reboot:
+        msg: Rebooting nodes
+        reboot_timeout: 3600

bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2

@@ -0,0 +1,120 @@
+---
+- name: Prepare System
+  hosts: all
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Locale
+      block:
+        - name: Locale | Set timezone
+          community.general.timezone:
+            name: "{% raw %}{{ timezone | default('Etc/UTC') }}{% endraw %}"
+
+    - name: Packages
+      block:
+        - name: Packages | Install
+          ansible.builtin.apt:
+            name: apt-transport-https,ca-certificates,conntrack,curl,dirmngr,gdisk,gnupg,htop,
+              iptables,iputils-ping,ipvsadm,libseccomp2,lm-sensors,neofetch,net-tools,nfs-common,
+              nvme-cli,open-iscsi,parted,psmisc,python3,python3-apt,python3-kubernetes,python3-yaml,
+              smartmontools,socat,software-properties-common,unzip,util-linux
+            install_recommends: false
+
+    - name: User Configuration
+      block:
+        - name: User Configuration | SSH keys
+          ansible.posix.authorized_key:
+            user: "{% raw %}{{ ansible_user }}{% endraw %}"
+            key: "https://github.com/{% raw %}{{ github_username }}{% endraw %}.keys"
+        - name: User Configuration | Silence login
+          ansible.builtin.file:
+            dest: /root/.hushlogin
+            state: touch
+            owner: "{% raw %}{{ ansible_user }}{% endraw %}"
+            mode: "0644"
+            modification_time: preserve
+            access_time: preserve
+
+    - name: Network Configuration
+      notify: Reboot
+      block:
+        - name: Network Configuration | Set hostname
+          ansible.builtin.hostname:
+            name: "{% raw %}{{ inventory_hostname }}{% endraw %}"
+        - name: Network Configuration | Update hosts
+          ansible.builtin.copy:
+            content: |
+              127.0.0.1 localhost
+              127.0.1.1 {% raw %}{{ inventory_hostname }}{% endraw %}
+
+              # The following lines are desirable for IPv6 capable hosts
+              ::1     localhost ip6-localhost ip6-loopback
+              ff02::1 ip6-allnodes
+              ff02::2 ip6-allrouters
+            dest: /etc/hosts
+            mode: preserve
+        # https://github.com/cilium/cilium/issues/18706
+        - name: Network Configuration | Cilium (1)
+          ansible.builtin.lineinfile:
+            dest: /etc/systemd/networkd.conf
+            regexp: ManageForeignRoutingPolicyRules
+            line: ManageForeignRoutingPolicyRules=no
+        - name: Network Configuration | Cilium (2)
+          ansible.builtin.lineinfile:
+            dest: /etc/systemd/networkd.conf
+            regexp: ManageForeignRoutes
+            line: ManageForeignRoutes=no
+
+    - name: System Configuration
+      notify: Reboot
+      block:
+        - name: System Configuration | Neofetch
+          ansible.builtin.copy:
+            dest: /etc/profile.d/neofetch.sh
+            mode: "0755"
+            content: neofetch --config none
+        - name: System Configuration | Disable apparmor
+          ansible.builtin.systemd:
+            name: apparmor
+            state: stopped
+            masked: true
+        - name: System Configuration | Disable swap
+          ansible.posix.mount:
+            name: "{% raw %}{{ item }}{% endraw %}"
+            fstype: swap
+            state: absent
+          loop: ["none", "swap"]
+        - name: System Configuration | Kernel modules (1)
+          community.general.modprobe:
+            name: "{% raw %}{{ item }}{% endraw %}"
+            state: present
+          loop: ["br_netfilter", "ceph", "ip_vs", "ip_vs_rr", "nbd", "overlay", "rbd"]
+        - name: System Configuration | Kernel modules (2)
+          ansible.builtin.copy:
+            dest: "/etc/modules-load.d/{% raw %}{{ item }}{% endraw %}.conf"
+            mode: "0644"
+            content: "{% raw %}{{ item }}{% endraw %}"
+          loop: ["br_netfilter", "ceph", "ip_vs", "ip_vs_rr", "nbd", "overlay", "rbd"]
+        - name: System Configuration | Sysctl
+          ansible.posix.sysctl:
+            name: "{% raw %}{{ item.key }}{% endraw %}"
+            value: "{% raw %}{{ item.value }}{% endraw %}"
+            sysctl_file: /etc/sysctl.d/99-kubernetes.conf
+            reload: true
+          with_dict: "{% raw %}{{ sysctl_config }}{% endraw %}"
+          vars:
+            sysctl_config:
+              fs.inotify.max_queued_events: 65536
+              fs.inotify.max_user_watches: 524288
+              fs.inotify.max_user_instances: 8192
+
+  handlers:
+    - name: Reboot
+      ansible.builtin.reboot:
+        msg: Rebooting nodes
+        reboot_timeout: 3600

bootstrap/templates/ansible/playbooks/cluster-reboot.yaml.j2

@@ -0,0 +1,15 @@
+---
+- name: Reboot
+  hosts: all
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Reboot
+      ansible.builtin.reboot:
+        msg: Rebooting nodes
+        reboot_timeout: 3600

bootstrap/templates/ansible/playbooks/cluster-rollout-update.yaml.j2

@@ -0,0 +1,70 @@
+---
+# https://github.com/kevincoakley/ansible-role-k8s-rolling-update
+- name: Cluster rollout update
+  hosts: all
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  serial: 1
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Get the node's details
+      ansible.builtin.command: "kubectl get node {% raw %}{{ inventory_hostname }}{% endraw %} -o json"
+      register: kubectl_get_node
+      delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}"
+      failed_when: false
+      changed_when: false
+
+    - name: Update node
+      when:
+        # When status.conditions[x].type == Ready then check stats.conditions[x].status for True|False
+        - kubectl_get_node['stdout'] | from_json | json_query("status.conditions[?type == 'Ready'].status")
+        # If spec.unschedulable is defined then the node is cordoned
+        - not (kubectl_get_node['stdout'] | from_json).spec.unschedulable is defined
+      block:
+        - name: Cordon node
+          ansible.builtin.command: "kubectl cordon {% raw %}{{ inventory_hostname }}{% endraw %}"
+          delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}"
+          changed_when: false
+
+        - name: Wait for node to cordon
+          ansible.builtin.command: "kubectl get node {% raw %}{{ inventory_hostname }}{% endraw %} -o json"
+          register: wait_for_cordon
+          retries: 10
+          delay: 10
+          delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}"
+          changed_when: false
+          until: (wait_for_cordon['stdout'] | from_json).spec.unschedulable
+
+        - name: Cordon & drain node
+          ansible.builtin.command: "kubectl drain --ignore-daemonsets --delete-emptydir-data --force {% raw %}{{ inventory_hostname }}{% endraw %}"
+          delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}"
+          changed_when: false
+
+        - name: Update all packages
+          ansible.builtin.apt:
+            upgrade: dist
+            update_cache: true
+
+        - name: Reboot
+          ansible.builtin.reboot:
+            msg: Rebooting node
+            post_reboot_delay: 60
+            reboot_timeout: 3600
+
+        - name: Uncordon node
+          ansible.builtin.command: "kubectl uncordon {% raw %}{{ inventory_hostname }}{% endraw %}"
+          delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}"
+          changed_when: false
+
+        - name: Wait for node to uncordon
+          ansible.builtin.command: "kubectl get node {% raw %}{{ inventory_hostname }}{% endraw %} -o json"
+          register: wait_for_uncordon
+          retries: 10
+          delay: 10
+          delegate_to: "{% raw %}{{ groups['master'][0] }}{% endraw %}"
+          changed_when: false
+          until: not (kubectl_get_node['stdout'] | from_json).spec.unschedulable is defined

bootstrap/templates/ansible/playbooks/files/stale-containers.service.j2

@@ -0,0 +1,6 @@
+[Unit]
+Description=Stale containers
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/k3s crictl rmi --prune

bootstrap/templates/ansible/playbooks/files/stale-containers.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Stale containers
+
+[Timer]
+OnCalendar=weekly
+AccuracySec=1h
+Persistent=true
+RandomizedDelaySec=6000
+
+[Install]
+WantedBy=timers.target

bootstrap/templates/ansible/playbooks/tasks/cilium.yaml.j2

@@ -0,0 +1,56 @@
+---
+- name: Cilium
+  block:
+    - name: Cilium | Check if Cilium HelmChart exists
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: cilium
+        kind: HelmChart
+        namespace: kube-system
+      register: cilium_helmchart
+
+    - name: Cilium | Wait for Cilium to rollout
+      when: cilium_helmchart.resources | count > 0
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: helm-install-cilium
+        kind: Job
+        namespace: kube-system
+        wait: true
+        wait_condition:
+          type: Complete
+          status: true
+        wait_timeout: 360
+
+    - name: Cilium | Patch the Cilium HelmChart to unmanage it
+      when: cilium_helmchart.resources | count > 0
+      kubernetes.core.k8s_json_patch:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: cilium
+        kind: HelmChart
+        namespace: kube-system
+        patch:
+          - op: add
+            path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged
+            value: "true"
+
+    - name: Cilium | Delete the Cilium HelmChart CR
+      when: cilium_helmchart.resources | count > 0
+      kubernetes.core.k8s:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: cilium
+        kind: HelmChart
+        namespace: kube-system
+        state: absent
+
+    - name: Cilium | Force delete the Cilium HelmChart
+      when: cilium_helmchart.resources | count > 0
+      kubernetes.core.k8s:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: cilium
+        kind: HelmChart
+        namespace: kube-system
+        state: patched
+        definition:
+          metadata:
+            finalizers: []

bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2

@@ -0,0 +1,56 @@
+---
+- name: Coredns
+  block:
+    - name: Coredns | Check if Coredns HelmChart exists
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: coredns
+        kind: HelmChart
+        namespace: kube-system
+      register: coredns_helmchart
+
+    - name: Coredns | Wait for Coredns to rollout
+      when: coredns_helmchart.resources | count > 0
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: helm-install-coredns
+        kind: Job
+        namespace: kube-system
+        wait: true
+        wait_condition:
+          type: Complete
+          status: true
+        wait_timeout: 360
+
+    - name: Coredns | Patch the Coredns HelmChart to unmanage it
+      when: coredns_helmchart.resources | count > 0
+      kubernetes.core.k8s_json_patch:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: coredns
+        kind: HelmChart
+        namespace: kube-system
+        patch:
+          - op: add
+            path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged
+            value: "true"
+
+    - name: Coredns | Delete the Coredns HelmChart CR
+      when: coredns_helmchart.resources | count > 0
+      kubernetes.core.k8s:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: coredns
+        kind: HelmChart
+        namespace: kube-system
+        state: absent
+
+    - name: Coredns | Force delete the Coredns HelmChart
+      when: coredns_helmchart.resources | count > 0
+      kubernetes.core.k8s:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: coredns
+        kind: HelmChart
+        namespace: kube-system
+        state: patched
+        definition:
+          metadata:
+            finalizers: []

bootstrap/templates/ansible/playbooks/tasks/cruft.yaml.j2

@@ -0,0 +1,32 @@
+---
+# https://github.com/k3s-io/k3s/issues/1971
+- name: Cruft
+  block:
+    - name: Cruft | Get list of custom mantifests
+      ansible.builtin.find:
+        paths: "{% raw %}{{ k3s_server_manifests_dir }}{% endraw %}"
+        file_type: file
+        use_regex: true
+        patterns: ["^custom-.*"]
+      register: custom_manifest
+
+    - name: Cruft | Delete custom mantifests
+      ansible.builtin.file:
+        path: "{% raw %}{{ item.path }}{% endraw %}"
+        state: absent
+      loop: "{% raw %}{{ custom_manifest.files }}{% endraw %}"
+
+    - name: Cruft | Get list of custom addons
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        kind: Addon
+      register: addons_list
+
+    - name: Cruft | Delete addons
+      kubernetes.core.k8s:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        name: "{% raw %}{{ item.metadata.name }}{% endraw %}"
+        kind: Addon
+        namespace: kube-system
+        state: absent
+      loop: "{% raw %}{{ addons_list.resources | selectattr('metadata.name', 'match', '^custom-.*') | list }}{% endraw %}"

bootstrap/templates/ansible/playbooks/tasks/kubeconfig.yaml.j2

@@ -0,0 +1,26 @@
+---
+- name: Get absolute path to this Git repository # noqa: command-instead-of-module
+  ansible.builtin.command: git rev-parse --show-toplevel
+  delegate_to: localhost
+  become: false
+  run_once: true
+  register: repository_path
+  changed_when: false
+  check_mode: false
+  failed_when: repository_path.rc != 0
+
+- name: Copy kubeconfig to the project directory
+  when: k3s_primary_control_node
+  ansible.builtin.fetch:
+    src: /etc/rancher/k3s/k3s.yaml
+    dest: "{% raw %}{{ repository_path.stdout }}{% endraw %}/kubeconfig"
+    flat: true
+
+- name: Update kubeconfig with the correct load balancer address
+  delegate_to: localhost
+  become: false
+  run_once: true
+  ansible.builtin.replace:
+    path: "{% raw %}{{ repository_path.stdout }}{% endraw %}/kubeconfig"
+    regexp: https://127.0.0.1:6443
+    replace: "https://{% raw %}{{ k3s_registration_address }}{% endraw %}:6443"

bootstrap/templates/ansible/playbooks/tasks/stale_containers.yaml.j2

@@ -0,0 +1,36 @@
+---
+# https://github.com/k3s-io/k3s/issues/1900
+- name: Enabled Stale containers
+  when: stale_containers_state == "enabled"
+  block:
+    - name: Stale containers | Create systemd unit
+      ansible.builtin.copy:
+        src: files/stale-containers.service
+        dest: /etc/systemd/system/stale-containers.service
+        owner: root
+        group: root
+        mode: "0644"
+
+    - name: Stale containers | Create systemd timer
+      ansible.builtin.copy:
+        src: files/stale-containers.timer
+        dest: /etc/systemd/system/stale-containers.timer
+        owner: root
+        group: root
+        mode: "0644"
+
+    - name: Stale containers | Start the systemd timer
+      ansible.builtin.systemd:
+        name: stale-containers.timer
+        enabled: true
+        daemon_reload: true
+        masked: false
+        state: started
+
+- name: Disable Stale containers
+  when: stale_containers_state == "disabled"
+  block:
+    - name: Stale containers | Mask the systemd timer
+      ansible.builtin.systemd:
+        name: stale-containers.timer
+        masked: true

bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2

@@ -0,0 +1,54 @@
+---
+# https://docs.k3s.io/helm
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: cilium
+  namespace: kube-system
+spec:
+  repo: https://helm.cilium.io/
+  chart: cilium
+  # renovate: datasource=helm depName=cilium registryUrl=https://helm.cilium.io
+  version: "1.14.0-rc.0"
+  targetNamespace: kube-system
+  bootstrap: true
+  valuesContent: |-
+    autoDirectNodeRoutes: true
+    bpf:
+      masquerade: true
+    bgp:
+      enabled: false
+    cluster:
+      name: home-cluster
+      id: 1
+    containerRuntime:
+      integration: containerd
+      socketPath: /var/run/k3s/containerd/containerd.sock
+    endpointRoutes:
+      enabled: true
+    hubble:
+      enabled: false
+    ipam:
+      mode: kubernetes
+    ipv4NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'] }}{% endraw %}"
+    k8sServiceHost: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+    k8sServicePort: 6443
+    kubeProxyReplacement: strict
+    kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+    # https://github.com/cilium/cilium/issues/26586
+    l2announcements:
+      enabled: true
+      leaseDuration: 120s
+      leaseRenewDeadline: 60s
+      leaseRetryPeriod: 1s
+    loadBalancer:
+      algorithm: maglev
+      mode: dsr
+    localRedirectPolicy: true
+    operator:
+      replicas: 1
+      rollOutPods: true
+    rollOutCiliumPods: true
+    securityContext:
+      privileged: true
+    tunnel: disabled

bootstrap/templates/ansible/playbooks/templates/custom-cilium-l2.yaml.j2.j2

@@ -0,0 +1,22 @@
+---
+# https://docs.cilium.io/en/latest/network/l2-announcements
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: policy
+spec:
+  loadBalancerIPs: true
+  # NOTE: This might need to be set if you have more than one active NIC on your nodes
+  # interfaces:
+  #   - ^eno[0-9]+
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/os: linux
+---
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: pool
+spec:
+  cidrs:
+    - cidr: "{% raw %}{{ node_cidr }}{% endraw %}"

bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2

@@ -0,0 +1,78 @@
+---
+# https://docs.k3s.io/helm
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: coredns
+  namespace: kube-system
+spec:
+  repo: https://coredns.github.io/helm
+  chart: coredns
+  # renovate: datasource=helm depName=coredns registryUrl=https://coredns.github.io/helm
+  version: "1.24.1"
+  targetNamespace: kube-system
+  bootstrap: true
+  valuesContent: |-
+    fullnameOverride: coredns
+    replicaCount: 1
+    k8sAppLabelOverride: kube-dns
+    service:
+      name: kube-dns
+      # Choose the 10th IP address from the start of the service-cidr
+      clusterIP: "{% raw %}{{ k3s_server['service-cidr'] | ansible.utils.nthhost(10) }}{% endraw %}"
+    serviceAccount:
+      create: true
+    deployment:
+      annotations:
+        reloader.stakater.com/auto: "true"
+    servers:
+      - zones:
+          - zone: .
+            scheme: dns://
+            use_tcp: true
+        port: 53
+        plugins:
+          - name: log
+          - name: errors
+          - name: health
+            configBlock: |-
+              lameduck 5s
+          - name: ready
+          - name: kubernetes
+            parameters: cluster.local in-addr.arpa ip6.arpa
+            configBlock: |-
+              pods insecure
+              fallthrough in-addr.arpa ip6.arpa
+              ttl 30
+          - name: prometheus
+            parameters: 0.0.0.0:9153
+          - name: forward
+            parameters: . /etc/resolv.conf
+          - name: cache
+            parameters: 30
+          - name: loop
+          - name: reload
+          - name: loadbalance
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: node-role.kubernetes.io/control-plane
+                  operator: Exists
+    tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+    topologySpreadConstraints:
+      - maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+        labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: coredns

bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2

@@ -0,0 +1,57 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-vip
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/instance: kube-vip
+    app.kubernetes.io/name: kube-vip
+spec:
+  containers:
+    - name: kube-vip
+      image: ghcr.io/kube-vip/kube-vip:v0.6.0
+      imagePullPolicy: IfNotPresent
+      args: ["manager"]
+      env:
+        - name: address
+          value: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
+        - name: vip_arp
+          value: "true"
+        - name: port
+          value: "6443"
+        - name: vip_cidr
+          value: "32"
+        - name: cp_enable
+          value: "true"
+        - name: cp_namespace
+          value: kube-system
+        - name: vip_ddns
+          value: "false"
+        - name: svc_enable
+          value: "false"
+        - name: vip_leaderelection
+          value: "true"
+        - name: vip_leaseduration
+          value: "15"
+        - name: vip_renewdeadline
+          value: "10"
+        - name: vip_retryperiod
+          value: "2"
+        - name: prometheus_server
+          value: :2112
+      securityContext:
+        capabilities:
+          add: ["NET_ADMIN", "NET_RAW"]
+      volumeMounts:
+        - mountPath: /etc/kubernetes/admin.conf
+          name: kubeconfig
+  hostAliases:
+    - hostnames:
+        - kubernetes
+      ip: 127.0.0.1
+  hostNetwork: true
+  volumes:
+    - name: kubeconfig
+      hostPath:
+        path: /etc/rancher/k3s/k3s.yaml

bootstrap/templates/ansible/requirements.txt.j2

@@ -0,0 +1 @@
+openshift==0.13.1

bootstrap/templates/ansible/requirements.yaml.j2

@@ -0,0 +1,16 @@
+---
+collections:
+  - name: ansible.posix
+    version: 1.5.4
+  - name: ansible.utils
+    version: 2.10.3
+  - name: community.general
+    version: 7.1.0
+  - name: community.sops
+    version: 1.6.4
+  - name: kubernetes.core
+    version: 2.4.0
+roles:
+  - name: xanmanning.k3s
+    src: https://github.com/PyratLabs/ansible-role-k3s.git
+    version: v3.4.2