feat: move kubernetes dir into templates to support jinja2 across all files (#831)

.github/renovate.json5

@@ -21,15 +21,22 @@
   "rebaseWhen": "conflicted",
   "schedule": ["every saturday"],
   "flux": {
-    "fileMatch": ["kubernetes/.+\\.ya?ml$"]
+    "fileMatch": [
+      "kubernetes/.+\\.ya?ml$",
+      "template/templates/kubernetes/.+\\.ya?ml.j2$"
+    ]
   },
   "helm-values": {
-    "fileMatch": ["kubernetes/.+\\.ya?ml$"]
+    "fileMatch": [
+      "kubernetes/.+\\.ya?ml$",
+      "template/templates/kubernetes/.+\\.ya?ml.j2$"
+    ]
   },
   "kubernetes": {
     "fileMatch": [
       "ansible/.+\\.ya?ml(\\.j2)?$",
-      "kubernetes/.+\\.ya?ml$"
+      "kubernetes/.+\\.ya?ml$",
+      "template/templates/kubernetes/.+\\.ya?ml.j2$"
     ]
   },
   "regexManagers": [
@@ -37,7 +44,8 @@
       "description": "Process various other dependencies",
       "fileMatch": [
         "ansible/.+\\.ya?ml(\\.j2)?$",
-        "kubernetes/.+\\.ya?ml$"
+        "kubernetes/.+\\.ya?ml$",
+        "template/templates/kubernetes/.+\\.ya?ml.j2$"
       ],
       "matchStrings": [
         "datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)( registryUrl=(?<registryUrl>\\S+))?\n.*?\"(?<currentValue>.*)\"\n"

README.md

@@ -138,7 +138,7 @@ Clone **your new repo** to you local workstation and `cd` into it.
 
 1. Install the following CLI tools on your workstation, if you are **NOT** using [Homebrew](https://brew.sh/) on MacOS or Linux **ignore** steps 4 and 5.
 
-   - Required: [age](https://github.com/FiloSottile/age), [ansible](https://www.ansible.com), [flux](https://toolkit.fluxcd.io/), [cloudflared](https://github.com/cloudflare/cloudflared), [go-task](https://github.com/go-task/task), [direnv](https://github.com/direnv/direnv), [jq](https://stedolan.github.io/jq/), [kubectl](https://kubernetes.io/docs/tasks/tools/), [python-pip3](https://pypi.org/project/pip/), [sops v3](https://github.com/getsops/sops)
+   - Required: [age](https://github.com/FiloSottile/age), [ansible](https://www.ansible.com), [flux](https://toolkit.fluxcd.io/), [cloudflared](https://github.com/cloudflare/cloudflared), [go-task](https://github.com/go-task/task), [direnv](https://github.com/direnv/direnv), [kubectl](https://kubernetes.io/docs/tasks/tools/), [python-pip3](https://pypi.org/project/pip/), [sops](https://github.com/getsops/sops)
 
    - Recommended: [helm](https://helm.sh/), [kustomize](https://github.com/kubernetes-sigs/kustomize), [stern](https://github.com/stern/stern)
 
@@ -239,7 +239,7 @@ In order to expose services to the internet you will need to create a [Cloudflar
    cp template/vars/config.sample.yaml template/vars/config.yaml
    ```
 
-2. Once done run the following command which will verify and generate the files needed to continue
+2. Once done run the following command which will verify and generate all the files needed to continue.
 
    ```sh
    task configure
@@ -314,8 +314,10 @@ In order to expose services to the internet you will need to create a [Cloudflar
    ```sh
    task cluster:nodes
    # NAME           STATUS   ROLES                       AGE     VERSION
-   # k8s-0          Ready    control-plane,master      4d20h   v1.21.5+k3s1
+   # k8s-0          Ready    control-plane,etcd,master   1h      v1.27.3+k3s1
-   # k8s-1          Ready    worker                    4d20h   v1.21.5+k3s1
+   # k8s-1          Ready    control-plane,etcd,master   1h      v1.27.3+k3s1
+   # k8s-2          Ready    control-plane,etcd,master   1h      v1.27.3+k3s1
+   # k8s-3          Ready    worker                      1h      v1.27.3+k3s1
    ```
 
 ### 🔹 GitOps with Flux

Taskfile.yaml

@@ -33,7 +33,6 @@ tasks:
         fluxcd/tap/flux
         go-task/tap/go-task
         helm
-        jq
         kubernetes-cli
         kustomize
         sops

ansible/playbooks/tasks/kubeconfig.yaml

@@ -1,11 +1,10 @@
 ---
-- name: Get absolute path to this Git repository
+- name: Get absolute path to this Git repository # noqa: command-instead-of-module
-  delegate_to: localhost
-  become: false
-  run_once: true
-  check_mode: false
   ansible.builtin.command: git rev-parse --show-toplevel
+  changed_when: false
+  check_mode: false
   register: repository_path
+  failed_when: repository_path.rc != 0
 
 - name: Copy kubeconfig to the project directory
   when: k3s_primary_control_node

template/configure.yaml

@@ -3,82 +3,17 @@
   hosts: localhost
   connection: local
   vars_files:
+    - defaults/main.yaml
     - vars/config.yaml
-    - defaults/templates.yaml
-    - defaults/extras.yaml
   tasks:
-    - name: Get absolute path to this Git repository # noqa: command-instead-of-module
-      ansible.builtin.command: git rev-parse --show-toplevel
-      changed_when: false
-      check_mode: false
-      register: repository_path
-      failed_when: repository_path.rc != 0
-
     - name: Verify configuration
-      ansible.builtin.include_tasks: tasks/verify_{{ task }}.yaml
+      ansible.builtin.include_tasks: tasks/validation/main.yaml
-      loop: [age, cli, net, cloudflare, github]
-      loop_control:
-        loop_var: task
 
-    - name: Ensure template directories exist
+    - name: Template Sops configuration
-      ansible.builtin.file:
+      ansible.builtin.include_tasks: tasks/sops/main.yaml
-        path: "{{ repository_path.stdout }}/{{ item.dest | dirname }}"
-        state: directory
-        mode: "0755"
-      loop: "{{ templates }}"
 
-    - name: Template out non-encrypted files
+    - name: Template Ansible configuration
-      ansible.builtin.template:
+      ansible.builtin.include_tasks: tasks/ansible/main.yaml
-        src: "templates/{{ item.src }}"
-        dest: "{{ repository_path.stdout }}/{{ item.dest }}"
-        mode: "0755"
-      loop: "{{ templates | selectattr('encrypt', 'equalto', false) | list }}"
 
-    # https://github.com/ansible-collections/community.sops/issues/153
+    - name: Template Kubernetes configuration
-    - name: Backup sops configuration file
+      ansible.builtin.include_tasks: tasks/kubernetes/main.yaml
-      ansible.builtin.copy:
-        src: "{{ repository_path.stdout }}/.sops.yaml"
-        dest: "{{ repository_path.stdout }}/.sops.yaml.orig"
-        mode: "0755"
-    - name: Remove sops configuration file
-      ansible.builtin.file:
-        path: "{{ repository_path.stdout }}/.sops.yaml"
-        state: absent
-
-    - name: Template out encrypted files
-      community.sops.sops_encrypt:
-        path: "{{ repository_path.stdout }}/{{ item.dest }}"
-        encrypted_regex: ^(data|stringData)$
-        age: ["{{ bootstrap_age_public_key }}"]
-        content_yaml: "{{ lookup('ansible.builtin.template', 'templates/' + item.src) | from_yaml }}"
-        mode: "0755"
-        force: true
-      loop: "{{ templates | selectattr('encrypt', 'equalto', true) | list }}"
-
-    - name: Ensure Ansible inventory directory exists
-      ansible.builtin.file:
-        path: "{{ repository_path.stdout }}/ansible/inventory/host_vars"
-        state: directory
-        mode: "0755"
-
-    - name: Template out node secrets
-      community.sops.sops_encrypt:
-        path: "{{ repository_path.stdout }}/ansible/inventory/host_vars/{{ item.name }}.sops.yaml"
-        age: ["{{ bootstrap_age_public_key }}"]
-        content_yaml: "{{ lookup('ansible.builtin.template', 'templates/ansible/node.sops.yaml.j2', template_vars=dict(password=item.password)) | from_yaml }}"
-        mode: "0755"
-        force: true
-      loop: "{{ bootstrap_hosts.master + bootstrap_hosts.worker | default([]) }}"
-      loop_control:
-        label: "{{ item.address }}"
-
-    # https://github.com/ansible-collections/community.sops/issues/153
-    - name: Restore sops configuration file
-      ansible.builtin.copy:
-        src: "{{ repository_path.stdout }}/.sops.yaml.orig"
-        dest: "{{ repository_path.stdout }}/.sops.yaml"
-        mode: "0755"
-    - name: Remove backup sops configuration file
-      ansible.builtin.file:
-        path: "{{ repository_path.stdout }}/.sops.yaml.orig"
-        state: absent

template/defaults/templates.yaml

@@ -1,44 +0,0 @@
----
-templates:
-  # Root
-  - src: .sops.yaml.j2
-    dest: ./.sops.yaml
-    encrypt: false
-  # Ansible dir
-  - src: ansible/supplemental.yaml.j2
-    dest: ./ansible/inventory/group_vars/kubernetes/supplemental.yaml
-    encrypt: false
-  - src: ansible/hosts.yaml.j2
-    dest: ./ansible/inventory/hosts.yaml
-    encrypt: false
-  # Kubernetes dir
-  - src: kubernetes/kube-prometheus-stack-helmrelease.yaml.j2
-    dest: ./kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml
-    encrypt: false
-  - src: kubernetes/cert-manager-secret.sops.yaml.j2
-    dest: ./kubernetes/apps/cert-manager/cert-manager/issuers/secret.sops.yaml
-    encrypt: true
-  - src: kubernetes/cloudflared-secret.sops.yaml.j2
-    dest: ./kubernetes/apps/networking/cloudflared/app/secret.sops.yaml
-    encrypt: true
-  - src: kubernetes/cluster-secrets.sops.yaml.j2
-    dest: ./kubernetes/flux/vars/cluster-secrets.sops.yaml
-    encrypt: true
-  - src: kubernetes/cluster-settings.yaml.j2
-    dest: ./kubernetes/flux/vars/cluster-settings.yaml
-    encrypt: false
-  - src: kubernetes/cluster.yaml.j2
-    dest: ./kubernetes/flux/config/cluster.yaml
-    encrypt: false
-  - src: kubernetes/external-dns-secret.sops.yaml.j2
-    dest: ./kubernetes/apps/networking/external-dns/app/secret.sops.yaml
-    encrypt: true
-  - src: kubernetes/github-webhook-token-secret.sops.yaml.j2
-    dest: ./kubernetes/apps/flux-system/addons/webhooks/github/secret.sops.yaml
-    encrypt: true
-  - src: kubernetes/grafana-admin-secret.sops.yaml.j2
-    dest: ./kubernetes/apps/monitoring/grafana/app/secret.sops.yaml
-    encrypt: true
-  - src: kubernetes/weave-gitops-secret.sops.yaml.j2
-    dest: ./kubernetes/apps/flux-system/weave-gitops/app/secret.sops.yaml
-    encrypt: true

template/tasks/ansible/main.yaml

@@ -0,0 +1,51 @@
+---
+- name: Get absolute path to this Git repository # noqa: command-instead-of-module
+  ansible.builtin.command: git rev-parse --show-toplevel
+  changed_when: false
+  check_mode: false
+  register: repository_path
+  failed_when: repository_path.rc != 0
+
+- name: Template Ansible configuration
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ repository_path.stdout }}/{{ item.dest }}"
+    mode: "0755"
+  loop:
+    - src: templates/ansible/supplemental.yaml.j2
+      dest: ./ansible/inventory/group_vars/kubernetes/supplemental.yaml
+    - src: templates/ansible/hosts.yaml.j2
+      dest: ./ansible/inventory/hosts.yaml
+
+# https://github.com/ansible-collections/community.sops/issues/153
+- name: Backup sops configuration file
+  ansible.builtin.copy:
+    src: "{{ repository_path.stdout }}/.sops.yaml"
+    dest: "{{ repository_path.stdout }}/.sops.yaml.orig"
+    mode: "0755"
+- name: Remove sops configuration file
+  ansible.builtin.file:
+    path: "{{ repository_path.stdout }}/.sops.yaml"
+    state: absent
+
+- name: Template out node secrets
+  community.sops.sops_encrypt:
+    path: "{{ repository_path.stdout }}/ansible/inventory/host_vars/{{ item.name }}.sops.yaml"
+    age: ["{{ bootstrap_age_public_key }}"]
+    content_yaml: "{{ lookup('ansible.builtin.template', 'templates/ansible/node.sops.yaml.j2', template_vars=dict(password=item.password)) | from_yaml }}"
+    mode: "0755"
+    force: true
+  loop: "{{ bootstrap_hosts.master + bootstrap_hosts.worker | default([]) }}"
+  loop_control:
+    label: "{{ item.address }}"
+
+# https://github.com/ansible-collections/community.sops/issues/153
+- name: Restore sops configuration file
+  ansible.builtin.copy:
+    src: "{{ repository_path.stdout }}/.sops.yaml.orig"
+    dest: "{{ repository_path.stdout }}/.sops.yaml"
+    mode: "0755"
+- name: Remove backup sops configuration file
+  ansible.builtin.file:
+    path: "{{ repository_path.stdout }}/.sops.yaml.orig"
+    state: absent

template/tasks/kubernetes/main.yaml

@@ -0,0 +1,58 @@
+---
+- name: Get absolute path to this Git repository # noqa: command-instead-of-module
+  ansible.builtin.command: git rev-parse --show-toplevel
+  changed_when: false
+  check_mode: false
+  register: repository_path
+  failed_when: repository_path.rc != 0
+
+- name: Ensure Kubernetes directories exist
+  ansible.builtin.file:
+    path: "{{ repository_path.stdout }}/kubernetes/{{ item.path }}"
+    state: directory
+    mode: "0755"
+  with_community.general.filetree:
+    - ../templates/kubernetes/
+  when: item.state == 'directory'
+
+- name: Template Kubernetes unencrypted files
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ repository_path.stdout }}/kubernetes/{{ item.path | replace('.j2', '') }}"
+  with_community.general.filetree:
+    - ../templates/kubernetes/
+  when: item.state == 'file' and 'sops' not in item.path
+
+# https://github.com/ansible-collections/community.sops/issues/153
+- name: Backup sops configuration file
+  ansible.builtin.copy:
+    src: "{{ repository_path.stdout }}/.sops.yaml"
+    dest: "{{ repository_path.stdout }}/.sops.yaml.orig"
+    mode: "0755"
+- name: Remove sops configuration file
+  ansible.builtin.file:
+    path: "{{ repository_path.stdout }}/.sops.yaml"
+    state: absent
+
+- name: Template Kubernetes unencrypted files
+  community.sops.sops_encrypt:
+    path: "{{ repository_path.stdout }}/kubernetes/{{ item.path | replace('.j2', '') }}"
+    encrypted_regex: ^(data|stringData)$
+    age: ["{{ bootstrap_age_public_key }}"]
+    content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}"
+    mode: "0755"
+    force: true
+  with_community.general.filetree:
+    - ../templates/kubernetes/
+  when: item.state == 'file' and 'sops' in item.path
+
+# https://github.com/ansible-collections/community.sops/issues/153
+- name: Restore sops configuration file
+  ansible.builtin.copy:
+    src: "{{ repository_path.stdout }}/.sops.yaml.orig"
+    dest: "{{ repository_path.stdout }}/.sops.yaml"
+    mode: "0755"
+- name: Remove backup sops configuration file
+  ansible.builtin.file:
+    path: "{{ repository_path.stdout }}/.sops.yaml.orig"
+    state: absent

template/tasks/sops/main.yaml

@@ -0,0 +1,13 @@
+---
+- name: Get absolute path to this Git repository # noqa: command-instead-of-module
+  ansible.builtin.command: git rev-parse --show-toplevel
+  changed_when: false
+  check_mode: false
+  register: repository_path
+  failed_when: repository_path.rc != 0
+
+- name: Template out sops configuration file
+  ansible.builtin.template:
+    src: "templates/.sops.yaml.j2"
+    dest: "{{ repository_path.stdout }}/.sops.yaml"
+    mode: "0755"

template/tasks/validation/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: Verify configuration
+  ansible.builtin.include_tasks: "{{ task }}.yaml"
+  loop: [age, cli, net, cloudflare, github]
+  loop_control:
+    loop_var: task

template/templates/kubernetes/apps/cert-manager/cert-manager/app/prometheusrule.yaml.j2

@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
@@ -15,54 +16,47 @@ spec:
           labels:
             severity: critical
           annotations:
-            description:
+            description: >
-              "New certificates will not be able to be minted, and existing
+              New certificates will not be able to be minted, and existing ones can't be renewed until cert-manager is back.
-              ones can't be renewed until cert-manager is back."
             runbook_url: https://gitlab.com/uneeq-oss/cert-manager-mixin/-/blob/master/RUNBOOK.md#certmanagerabsent
             summary: "Cert Manager has dissapeared from Prometheus service discovery."
     - name: certificates
       rules:
         - alert: CertManagerCertExpirySoon
           expr: |
-            avg by (exported_namespace, namespace, name) (
+            avg by (exported_namespace, namespace, name) (certmanager_certificate_expiration_timestamp_seconds - time()) < (21 * 24 * 3600)
-            certmanager_certificate_expiration_timestamp_seconds - time())
-              < (21 * 24 * 3600)
           for: 15m
           labels:
             severity: warning
           annotations:
-            description:
+            description: >
-              "The domain that this cert covers will be unavailable after
+              The domain that this cert covers will be unavailable after
-              {{ $value | humanizeDuration }}. Clients using endpoints that this cert
+              {% raw %}{{ $value | humanizeDuration }}{% endraw %}. Clients using endpoints that this cert
-              protects will start to fail in {{ $value | humanizeDuration }}."
+              protects will start to fail in {% raw %}{{ $value | humanizeDuration }}{% endraw %}.
             runbook_url: https://gitlab.com/uneeq-oss/cert-manager-mixin/-/blob/master/RUNBOOK.md#certmanagercertexpirysoon
-            summary:
+            summary: |
-              "The cert {{ $labels.name }} is {{ $value | humanizeDuration }}
+              The cert {% raw %}{{ $labels.name }}{% endraw %} is {% raw %}{{ $value | humanizeDuration }}{% endraw %} from expiry, it should have renewed over a week ago.
-              from expiry, it should have renewed over a week ago."
         - alert: CertManagerCertNotReady
           expr: |
-            max by (name, exported_namespace, namespace, condition) (
+            max by (name, exported_namespace, namespace, condition) (certmanager_certificate_ready_status{condition!="True"} == 1)
-            certmanager_certificate_ready_status{condition!="True"} == 1)
           for: 15m
           labels:
             severity: critical
           annotations:
-            description:
+            description: >
-              "This certificate has not been ready to serve traffic for at least
+              This certificate has not been ready to serve traffic for at least
               10m. If the cert is being renewed or there is another valid cert, the ingress
-              controller _may_ be able to serve that instead."
+              controller _may_ be able to serve that instead.
             runbook_url: https://gitlab.com/uneeq-oss/cert-manager-mixin/-/blob/master/RUNBOOK.md#certmanagercertnotready
-            summary: "The cert {{ $labels.name }} is not ready to serve traffic."
+            summary: "The cert {% raw %}{{ $labels.name }}{% endraw %} is not ready to serve traffic."
         - alert: CertManagerHittingRateLimits
           expr: |
-            sum by (host) (rate(certmanager_http_acme_client_request_count{status="429"}[5m]))
+            sum by (host) (rate(certmanager_http_acme_client_request_count{status="429"}[5m])) > 0
-              > 0
           for: 15m
           labels:
             severity: critical
           annotations:
-            description:
+            description: >
-              "Depending on the rate limit, cert-manager may be unable to generate
+              Depending on the rate limit, cert-manager may be unable to generate certificates for up to a week.
-              certificates for up to a week."
             runbook_url: https://gitlab.com/uneeq-oss/cert-manager-mixin/-/blob/master/RUNBOOK.md#certmanagerhittingratelimits
             summary: "Cert manager hitting LetsEncrypt rate limits."

template/templates/kubernetes/apps/default/echo-server/app/helmrelease.yaml.j2

@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
@@ -62,7 +63,7 @@ spec:
           external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
           hajimari.io/icon: video-input-antenna
         hosts:
-          - host: &host "{{ .Release.Name }}.${SECRET_DOMAIN}"
+          - host: &host "{% raw %}{{ .Release.Name }}{% endraw %}.${SECRET_DOMAIN}"
             paths:
               - path: /
                 pathType: Prefix

template/templates/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml.j2

@@ -19,4 +19,4 @@ metadata:
   name: pool
 spec:
   cidrs:
-    - cidr: "${NODE_CIDR}"
+    - cidr: "{{ bootstrap_node_cidr }}"

template/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2

@@ -73,9 +73,9 @@ spec:
                 - *host
     ipam:
       mode: kubernetes
-    k8sServiceHost: ${KUBE_VIP_ADDR}
+    k8sServiceHost: "{{ bootstrap_kube_vip_addr }}"
     k8sServicePort: 6443
-    ipv4NativeRoutingCIDR: ${CLUSTER_CIDR}
+    ipv4NativeRoutingCIDR: "{{ bootstrap_cluster_cidr }}"
     kubeProxyReplacement: strict
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
     operator:

template/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2

@@ -29,7 +29,7 @@ spec:
     fullnameOverride: coredns
     replicaCount: 1
     service:
-      clusterIP: "${COREDNS_ADDR}"
+      clusterIP: "{{ bootstrap_coredns_addr }}"
     serviceAccount:
       create: true
     deployment:

template/templates/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml.j2

@@ -36,7 +36,7 @@ spec:
       service:
         annotations:
           external-dns.alpha.kubernetes.io/hostname: "ingress.${SECRET_DOMAIN}"
-          io.cilium/lb-ipam-ips: "${INGRESS_NGINX_ADDR}"
+          io.cilium/lb-ipam-ips: "{{ bootstrap_ingress_nginx_addr }}"
         externalTrafficPolicy: Cluster
       publishService:
         enabled: true

template/templates/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml.j2

@@ -33,5 +33,5 @@ spec:
       type: LoadBalancer
       port: 53
       annotations:
-        io.cilium/lb-ipam-ips: "${K8S_GATEWAY_ADDR}"
+        io.cilium/lb-ipam-ips: "{{ bootstrap_k8s_gateway_addr }}"
       externalTrafficPolicy: Cluster