feat!: Rewrite configure script in ansible (#822)

* feat: smol changes

Signed-off-by: Devin Buhl <devin@buhl.casa>

* feat!: Rewrite configure script in ansible

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: add comment for real path

Signed-off-by: Devin Buhl <devin@buhl.casa>

* feat: verification is done

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: do not need net common

Signed-off-by: Devin Buhl <devin@buhl.casa>

* chore: clean up old comments

Signed-off-by: Devin Buhl <devin@buhl.casa>

* chore: clean up old comments

Signed-off-by: Devin Buhl <devin@buhl.casa>

* feat: finish up ansible host generation and more

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: backup/restore sops config between tasks

Signed-off-by: Devin Buhl <devin@buhl.casa>

* chore: clean up test comments

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: use ansible template on template

Signed-off-by: Devin Buhl <devin@buhl.casa>

* Update template/vars/config.sample.yaml

Co-authored-by: Manuel Rüger <manuel@rueg.eu>

* Update template/tasks/verify_cli.yaml

* chore: update readme

Signed-off-by: Devin Buhl <devin@buhl.casa>

* chore: update readme

Signed-off-by: Devin Buhl <devin@buhl.casa>

* Update README.md

Co-authored-by: Manuel Rüger <manuel@rueg.eu>

* fix: move bcrypt logic to template

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: derp

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>
Co-authored-by: Manuel Rüger <manuel@rueg.eu>

template/configure.yaml

@@ -0,0 +1,84 @@
+---
+- name: Cluster Installation
+  hosts: localhost
+  connection: local
+  vars_files:
+    - vars/config.yaml
+    - defaults/templates.yaml
+    - defaults/extras.yaml
+  tasks:
+    - name: Get absolute path to this Git repository # noqa: command-instead-of-module
+      ansible.builtin.command: git rev-parse --show-toplevel
+      changed_when: false
+      check_mode: false
+      register: repository_path
+      failed_when: repository_path.rc != 0
+
+    - name: Verify configuration
+      ansible.builtin.include_tasks: tasks/verify_{{ task }}.yaml
+      loop: [age, cli, net, cloudflare, github]
+      loop_control:
+        loop_var: task
+
+    - name: Ensure template directories exist
+      ansible.builtin.file:
+        path: "{{ repository_path.stdout }}/{{ item.dest | dirname }}"
+        state: directory
+        mode: "0755"
+      loop: "{{ templates }}"
+
+    - name: Template out non-encrypted files
+      ansible.builtin.template:
+        src: "templates/{{ item.src }}"
+        dest: "{{ repository_path.stdout }}/{{ item.dest }}"
+        mode: "0755"
+      loop: "{{ templates | selectattr('encrypt', 'equalto', false) | list }}"
+
+    # https://github.com/ansible-collections/community.sops/issues/153
+    - name: Backup sops configuration file
+      ansible.builtin.copy:
+        src: "{{ repository_path.stdout }}/.sops.yaml"
+        dest: "{{ repository_path.stdout }}/.sops.yaml.orig"
+        mode: "0755"
+    - name: Remove sops configuration file
+      ansible.builtin.file:
+        path: "{{ repository_path.stdout }}/.sops.yaml"
+        state: absent
+
+    - name: Template out encrypted files
+      community.sops.sops_encrypt:
+        path: "{{ repository_path.stdout }}/{{ item.dest }}"
+        encrypted_regex: ^(data|stringData)$
+        age: ["{{ bootstrap_age_public_key }}"]
+        content_yaml: "{{ lookup('ansible.builtin.template', 'templates/' + item.src) | from_yaml }}"
+        mode: "0755"
+        force: true
+      loop: "{{ templates | selectattr('encrypt', 'equalto', true) | list }}"
+
+    - name: Ensure Ansible inventory directory exists
+      ansible.builtin.file:
+        path: "{{ repository_path.stdout }}/ansible/inventory/host_vars"
+        state: directory
+        mode: "0755"
+
+    - name: Template out node secrets
+      community.sops.sops_encrypt:
+        path: "{{ repository_path.stdout }}/ansible/inventory/host_vars/{{ item.name }}.sops.yaml"
+        age: ["{{ bootstrap_age_public_key }}"]
+        content_yaml: "{{ lookup('ansible.builtin.template', 'templates/ansible/node.sops.yaml.j2', template_vars=dict(password=item.password)) | from_yaml }}"
+        mode: "0755"
+        force: true
+      loop: "{{ bootstrap_hosts.master + bootstrap_hosts.worker | default([]) }}"
+      loop_control:
+        label: "{{ item.address }}"
+
+    # https://github.com/ansible-collections/community.sops/issues/153
+    - name: Restore sops configuration file
+      ansible.builtin.copy:
+        src: "{{ repository_path.stdout }}/.sops.yaml.orig"
+        dest: "{{ repository_path.stdout }}/.sops.yaml"
+        mode: "0755"
+    - name: Remove backup sops configuration file
+      ansible.builtin.file:
+        path: "{{ repository_path.stdout }}/.sops.yaml.orig"
+        state: absent

template/defaults/extras.yaml

@@ -0,0 +1,2 @@
+---
+bootstrap_coredns_addr: "{{ bootstrap_service_cidr | ansible.utils.nthhost(10) }}"

template/defaults/templates.yaml

@@ -0,0 +1,44 @@
+---
+templates:
+  # Root
+  - src: .sops.yaml.j2
+    dest: ./.sops.yaml
+    encrypt: false
+  # Ansible dir
+  - src: ansible/supplemental.yaml.j2
+    dest: ./ansible/inventory/group_vars/kubernetes/supplemental.yaml
+    encrypt: false
+  - src: ansible/hosts.yaml.j2
+    dest: ./ansible/hosts.yaml
+    encrypt: false
+  # Kubernetes dir
+  - src: kubernetes/kube-prometheus-stack-helmrelease.yaml.j2
+    dest: ./kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml
+    encrypt: false
+  - src: kubernetes/cert-manager-secret.sops.yaml.j2
+    dest: ./kubernetes/apps/cert-manager/cert-manager/issuers/secret.sops.yaml
+    encrypt: true
+  - src: kubernetes/cloudflared-secret.sops.yaml.j2
+    dest: ./kubernetes/apps/networking/cloudflared/app/secret.sops.yaml
+    encrypt: true
+  - src: kubernetes/cluster-secrets.sops.yaml.j2
+    dest: ./kubernetes/flux/vars/cluster-secrets.sops.yaml
+    encrypt: true
+  - src: kubernetes/cluster-settings.yaml.j2
+    dest: ./kubernetes/flux/vars/cluster-settings.yaml
+    encrypt: false
+  - src: kubernetes/cluster.yaml.j2
+    dest: ./kubernetes/flux/config/cluster.yaml
+    encrypt: false
+  - src: kubernetes/external-dns-secret.sops.yaml.j2
+    dest: ./kubernetes/apps/networking/external-dns/app/secret.sops.yaml
+    encrypt: true
+  - src: kubernetes/github-webhook-token-secret.sops.yaml.j2
+    dest: ./kubernetes/apps/flux-system/addons/webhooks/github/secret.sops.yaml
+    encrypt: true
+  - src: kubernetes/grafana-admin-secret.sops.yaml.j2
+    dest: ./kubernetes/apps/monitoring/grafana/app/secret.sops.yaml
+    encrypt: true
+  - src: kubernetes/weave-gitops-secret.sops.yaml.j2
+    dest: ./kubernetes/apps/flux-system/weave-gitops/app/secret.sops.yaml
+    encrypt: true

template/tasks/verify_age.yaml

@@ -0,0 +1,22 @@
+---
+- name: Query age key file
+  ansible.builtin.stat:
+    path: ~/.config/sops/age/keys.txt
+  register: result
+
+- name: Check if age key file exists
+  ansible.builtin.assert:
+    that: [result.stat.exists]
+    success_msg: Age file ~/.config/sops/age/keys.txt exists
+    fail_msg: Age file ~/.config/sops/age/keys.txt does not exist
+
+- name: Query age key file contents
+  ansible.builtin.set_fact:
+    age_contents: "{{ lookup('file', '~/.config/sops/age/keys.txt') }}"
+
+- name: Check if age public keys match
+  ansible.builtin.assert:
+    that:
+      - "'{{ bootstrap_age_public_key }}' in age_contents"
+    success_msg: Age public key {{ bootstrap_age_public_key }} exists
+    fail_msg: Age public key {{ bootstrap_age_public_key }} does not exist

template/tasks/verify_cli.yaml

@@ -0,0 +1,8 @@
+---
+- name: Check for required CLI tools
+  ansible.builtin.command: "command -v {{ item }}"
+  loop: [age, flux, git, pip3, sops, task]
+  changed_when: false
+  check_mode: false
+  register: result
+  failed_when: result.rc != 0 and result.rc != 127

template/tasks/verify_cloudflare.yaml

@@ -0,0 +1,34 @@
+---
+- name: Query Cloudflare zone
+  ansible.builtin.uri:
+    url: "https://api.cloudflare.com/client/v4/zones?name={{ bootstrap_cloudflare_domain }}&status=active"
+    headers:
+      Authorization: "Bearer {{ bootstrap_cloudflare_token }}"
+      Content-Type: application/json
+    timeout: 5
+    return_content: true
+    body_format: json
+  register: result
+
+- name: Check if Cloudflare zone exists
+  ansible.builtin.assert:
+    that: ["result.json.success == true"]
+    success_msg: "Cloudflare zone {{ bootstrap_cloudflare_domain }} exists"
+    fail_msg: "Cloudflare zone {{ bootstrap_cloudflare_domain }} does not exist"
+
+- name: Query Cloudflared tunnel
+  ansible.builtin.uri:
+    url: "https://api.cloudflare.com/client/v4/accounts/{{ bootstrap_cloudflare_account_tag }}/cfd_tunnel/{{ bootstrap_cloudflare_tunnel_id }}"
+    headers:
+      Authorization: "Bearer {{ bootstrap_cloudflare_token }}"
+      Content-Type: application/json
+    timeout: 5
+    return_content: true
+    body_format: json
+  register: result
+
+- name: Check if Cloudflared tunnel exists
+  ansible.builtin.assert:
+    that: ["result.json.success == true"]
+    success_msg: "Cloudflared tunnel {{ bootstrap_cloudflare_tunnel_id }} exists"
+    fail_msg: "Cloudflared tunnel {{ bootstrap_cloudflare_tunnel_id }} does not exist"

template/tasks/verify_github.yaml

@@ -0,0 +1,30 @@
+---
+- name: Query Github username
+  ansible.builtin.uri:
+    url: "https://api.github.com/users/{{ bootstrap_github_username }}"
+    timeout: 5
+    return_content: true
+    body_format: json
+  register: result
+
+- name: Check if username exists
+  ansible.builtin.assert:
+    that:
+      - "result.json.login == '{{ bootstrap_github_username }}'"
+    success_msg: "Github user {{ bootstrap_github_username }} exists"
+    fail_msg: "Github user {{ bootstrap_github_username }} does not exist"
+
+- name: Query Github repo
+  ansible.builtin.uri:
+    url: "https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}"
+    timeout: 5
+    return_content: true
+    body_format: json
+  register: result
+
+- name: Check if repo exists
+  ansible.builtin.assert:
+    that:
+      - "result.json.full_name == '{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}'"
+    success_msg: "Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} exists"
+    fail_msg: "Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} does not exist"

template/tasks/verify_net.yaml

@@ -0,0 +1,94 @@
+---
+- name: Verify master node count
+  ansible.builtin.assert:
+    that:
+      - "{{ bootstrap_hosts.master | length > 0 }}"
+      - "{{ bootstrap_hosts.master | length is odd }}"
+    success_msg: "Master node count {{ bootstrap_hosts.master | length }} is correct."
+    fail_msg: "Master node count {{ bootstrap_hosts.master | length }} is not greater than 0 or is not odd."
+
+- name: Verify node CIDR
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_node_cidr is ansible.utils.ipv4 }}"]
+    success_msg: "Node CIDR {{ bootstrap_node_cidr }} is valid."
+    fail_msg: "Node CIDR {{ bootstrap_node_cidr }} is invalid."
+
+- name: Verify cluster CIDR
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_cluster_cidr is ansible.utils.ipv4 }}"]
+    success_msg: "Cluster CIDR {{ bootstrap_cluster_cidr }} is valid."
+    fail_msg: "Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid."
+
+- name: Verify service CIDR
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_service_cidr is ansible.utils.ipv4 }}"]
+    success_msg: "Service CIDR {{ bootstrap_service_cidr }} is valid."
+    fail_msg: "Service CIDR {{ bootstrap_service_cidr }} is invalid."
+
+- name: Verify k8s_gateway
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_k8s_gateway_addr is ansible.utils.ipv4 }}"]
+    success_msg: "k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is valid."
+    fail_msg: "k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is invalid."
+
+- name: Verify k8s_gateway in node CIDR
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_k8s_gateway_addr) }}"]
+    success_msg: "k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is within {{ bootstrap_node_cidr }}."
+    fail_msg: "k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is not within {{ bootstrap_node_cidr }}."
+
+- name: Verify ingress-nginx
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_ingress_nginx_addr is ansible.utils.ipv4 }}"]
+    success_msg: "ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is valid."
+    fail_msg: "ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is invalid."
+
+- name: Verify ingress-nginx in node CIDR
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_ingress_nginx_addr) }}"]
+    success_msg: "ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is within {{ bootstrap_node_cidr }}."
+    fail_msg: "ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is not within {{ bootstrap_node_cidr }}."
+
+- name: Verify kube-vip
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_kube_vip_addr is ansible.utils.ipv4 }}"]
+    success_msg: "kube-vip address {{ bootstrap_kube_vip_addr }} is valid."
+    fail_msg: "kube-vip address {{ bootstrap_kube_vip_addr }} is invalid."
+
+- name: Verify kube-vip in node CIDR
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kube_vip_addr) }}"]
+    success_msg: "kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}."
+    fail_msg: "kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}."
+
+- name: Verify nodes are not the same IPs as k8s_gateway, ingress-nginx or kube-vip
+  ansible.builtin.assert:
+    that:
+      - "{{ item.address not in (bootstrap_k8s_gateway_addr, bootstrap_ingress_nginx_addr, bootstrap_kube_vip_addr) }}"
+    success_msg: "Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip."
+    fail_msg: "Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip."
+    quiet: true
+  loop: "{{ bootstrap_hosts.master + bootstrap_hosts.worker | default([]) }}"
+  loop_control:
+     label: "{{ item.address }}"
+
+- name: Verify nodes are in node CIDR
+  ansible.builtin.assert:
+    that: ["{{ bootstrap_node_cidr | ansible.utils.network_in_usable(item.address) }}"]
+    success_msg: "Node address {{ item.address }} is within {{ bootstrap_node_cidr }}."
+    fail_msg: "Node address {{ item.address }} is not within {{ bootstrap_node_cidr }}."
+    quiet: true
+  loop: "{{ bootstrap_hosts.master + bootstrap_hosts.worker | default([]) }}"
+  loop_control:
+     label: "{{ item.address }}"
+
+- name: Verify SSH port is reachable
+  ansible.builtin.wait_for:
+    host: "{{ item.address }}"
+    port: 22
+    search_regex: OpenSSH
+    timeout: 10
+  connection: local
+  loop: "{{ bootstrap_hosts.master + bootstrap_hosts.worker | default([]) }}"
+  loop_control:
+     label: "{{ item.address }}"

template/templates/.sops.yaml.j2

@@ -0,0 +1,11 @@
+---
+creation_rules:
+  - path_regex: kubernetes/.*\.sops\.ya?ml
+    encrypted_regex: "^(data|stringData)$"
+    key_groups:
+      - age:
+          - "{{ bootstrap_age_public_key }}"
+  - path_regex: ansible/.*\.sops\.ya?ml
+    key_groups:
+      - age:
+          - "{{ bootstrap_age_public_key }}"

template/templates/ansible/hosts.yaml.j2

@@ -0,0 +1,20 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+kubernetes:
+  children:
+    master:
+      hosts:
+      {% for item in bootstrap_hosts.master %}
+        {{ item.name }}:
+          ansible_user: {{ item.username }}
+          ansible_host: {{ item.address }}
+      {% endfor %}
+    {% if bootstrap_hosts.worker | default([]) | length > 0 %}
+    worker:
+      hosts:
+      {% for item in bootstrap_hosts.worker %}
+        {{ item.name }}:
+          ansible_user: {{ item.username }}
+          ansible_host: {{ item.address }}
+      {% endfor %}
+    {% endif %}

template/templates/ansible/node.sops.yaml.j2

@@ -0,0 +1,2 @@
+---
+ansible_become_pass: "{{ password }}"

template/templates/ansible/supplemental.yaml.j2

@@ -0,0 +1,6 @@
+---
+timezone: "{{ bootstrap_timezone }}"
+coredns_addr: "{{ bootstrap_coredns_addr }}"
+kube_vip_addr: "{{ bootstrap_kube_vip_addr }}"
+cluster_cidr: "{{ bootstrap_cluster_cidr }}"
+service_cidr: "{{ bootstrap_service_cidr }}"

template/templates/kubernetes/cert-manager-secret.sops.yaml.j2

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cert-manager-secret
+  namespace: cert-manager
+stringData:
+  api-token: "{{ bootstrap_cloudflare_token }}"

template/templates/kubernetes/cloudflared-secret.sops.yaml.j2

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloudflared-secret
+  namespace: networking
+stringData:
+  TUNNEL_ID: "{{ bootstrap_cloudflare_tunnel_id }}"
+  credentials.json: |
+    {
+      "AccountTag": "{{ bootstrap_cloudflare_account_tag }}",
+      "TunnelSecret": "{{ bootstrap_cloudflare_tunnel_secret }}",
+      "TunnelID": "{{ bootstrap_cloudflare_tunnel_id }}"
+    }

template/templates/kubernetes/cluster-secrets.sops.yaml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cluster-secrets
+  namespace: flux-system
+stringData:
+  SECRET_DOMAIN: "{{ bootstrap_cloudflare_domain }}"
+  SECRET_ACME_EMAIL: "{{ bootstrap_acme_email }}"
+  SECRET_CLOUDFLARE_TUNNEL_ID: "{{ bootstrap_cloudflare_tunnel_id }}"

template/templates/kubernetes/cluster-settings.yaml.j2

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cluster-settings
+  namespace: flux-system
+data:
+  TIMEZONE: "{{ bootstrap_timezone }}"
+  COREDNS_ADDR: "{{ bootstrap_coredns_addr }}"
+  K8S_GATEWAY_ADDR: "{{ bootstrap_k8s_gateway_addr }}"
+  INGRESS_NGINX_ADDR: "{{ bootstrap_ingress_nginx_addr }}"
+  KUBE_VIP_ADDR: "{{ bootstrap_kube_vip_addr }}"
+  CLUSTER_CIDR: "{{ bootstrap_cluster_cidr }}"
+  SERVICE_CIDR: "{{ bootstrap_service_cidr }}"

template/templates/kubernetes/cluster.yaml.j2

@@ -0,0 +1,40 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: home-kubernetes
+  namespace: flux-system
+spec:
+  interval: 30m
+  ref:
+    branch: main
+  url: "https://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}.git"
+  ignore: |
+    # exclude all
+    /*
+    # include kubernetes directory
+    !/kubernetes
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./kubernetes/flux
+  prune: true
+  wait: false
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-age
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: cluster-settings
+      - kind: Secret
+        name: cluster-secrets

template/templates/kubernetes/external-dns-secret.sops.yaml.j2

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: external-dns-secret
+  namespace: networking
+stringData:
+  api-token: "{{ bootstrap_cloudflare_token }}"

template/templates/kubernetes/github-webhook-token-secret.sops.yaml.j2

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: github-webhook-token-secret
+  namespace: flux-system
+stringData:
+  token: "{{ bootstrap_flux_github_webhook_token }}"

template/templates/kubernetes/grafana-admin-secret.sops.yaml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: grafana-admin-secret
+  namespace: monitoring
+stringData:
+  admin-user: admin
+  admin-password: "{{ bootstrap_grafana_admin_password }}"

template/templates/kubernetes/kube-prometheus-stack-helmrelease.yaml.j2

@@ -0,0 +1,142 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: kube-prometheus-stack
+  namespace: monitoring
+spec:
+  interval: 30m
+  timeout: 15m
+  chart:
+    spec:
+      chart: kube-prometheus-stack
+      version: 47.1.0
+      sourceRef:
+        kind: HelmRepository
+        name: prometheus-community
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    createNamespace: true
+    crds: CreateReplace
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    crds: CreateReplace
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  dependsOn:
+    - name: local-path-provisioner
+      namespace: kube-system
+    - name: ingress-nginx
+      namespace: networking
+  values:
+    cleanPrometheusOperatorObjectNames: true
+    alertmanager:
+      enabled: false
+    grafana:
+      enabled: false
+      forceDeployDashboards: true
+      sidecar:
+        dashboards:
+          multicluster:
+            etcd:
+              enabled: true
+    kube-state-metrics:
+      metricLabelsAllowlist:
+        - "pods=[*]"
+        - "deployments=[*]"
+        - "persistentvolumeclaims=[*]"
+      prometheus:
+        monitor:
+          enabled: true
+          relabelings:
+            - action: replace
+              regex: (.*)
+              replacement: $1
+              sourceLabels: ["__meta_kubernetes_pod_node_name"]
+              targetLabel: kubernetes_node
+    kubelet:
+      enabled: true
+      serviceMonitor:
+        metricRelabelings:
+          # Remove duplicate metrics
+          - sourceLabels: ["__name__"]
+            regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authentication_token|cadvisor_version|container_blkio|container_cpu|container_fs|container_last|container_memory|container_network|container_oom|container_processes|container|csi_operations|disabled_metric|get_token|go|hidden_metric|kubelet_certificate|kubelet_cgroup|kubelet_container|kubelet_containers|kubelet_cpu|kubelet_device|kubelet_graceful|kubelet_http|kubelet_lifecycle|kubelet_managed|kubelet_node|kubelet_pleg|kubelet_pod|kubelet_run|kubelet_running|kubelet_runtime|kubelet_server|kubelet_started|kubelet_volume|kubernetes_build|kubernetes_feature|machine_cpu|machine_memory|machine_nvm|machine_scrape|node_namespace|plugin_manager|prober_probe|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|storage_operation|volume_manager|volume_operation|workqueue)_(.+)"
+            action: keep
+          - sourceLabels: ["node"]
+            targetLabel: instance
+            action: replace
+    kubeApiServer:
+      enabled: true
+      serviceMonitor:
+        metricRelabelings:
+          # Remove duplicate metrics
+          - sourceLabels: ["__name__"]
+            regex: "(aggregator_openapi|aggregator_unavailable|apiextensions_openapi|apiserver_admission|apiserver_audit|apiserver_cache|apiserver_cel|apiserver_client|apiserver_crd|apiserver_current|apiserver_envelope|apiserver_flowcontrol|apiserver_init|apiserver_kube|apiserver_longrunning|apiserver_request|apiserver_requested|apiserver_response|apiserver_selfrequest|apiserver_storage|apiserver_terminated|apiserver_tls|apiserver_watch|apiserver_webhooks|authenticated_user|authentication|disabled_metric|etcd_bookmark|etcd_lease|etcd_request|field_validation|get_token|go|grpc_client|hidden_metric|kube_apiserver|kubernetes_build|kubernetes_feature|node_authorizer|pod_security|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|serviceaccount_legacy|serviceaccount_stale|serviceaccount_valid|watch_cache|workqueue)_(.+)"
+            action: keep
+          # Remove high cardinality metrics
+          - sourceLabels: ["__name__"]
+            regex: (apiserver|etcd|rest_client)_request(|_sli|_slo)_duration_seconds_bucket
+            action: drop
+          - sourceLabels: ["__name__"]
+            regex: (apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket)
+            action: drop
+    kubeControllerManager:
+      enabled: true
+      endpoints: &cp
+      {% for item in bootstrap_hosts.master %}
+        - {{ item.address }}
+      {% endfor %}
+      serviceMonitor:
+        metricRelabelings:
+          # Remove duplicate metrics
+          - sourceLabels: ["__name__"]
+            regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|attachdetach_controller|authenticated_user|authentication|cronjob_controller|disabled_metric|endpoint_slice|ephemeral_volume|garbagecollector_controller|get_token|go|hidden_metric|job_controller|kubernetes_build|kubernetes_feature|leader_election|node_collector|node_ipam|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|pv_collector|registered_metric|replicaset_controller|rest_client|retroactive_storageclass|root_ca|running_managed|scrape_duration|scrape_samples|scrape_series|service_controller|storage_count|storage_operation|ttl_after|volume_operation|workqueue)_(.+)"
+            action: keep
+    kubeEtcd:
+      enabled: true
+      endpoints: *cp
+    kubeProxy:
+      enabled: false # Disabled because eBPF
+    kubeScheduler:
+      enabled: true
+      endpoints: *cp
+      serviceMonitor:
+        metricRelabelings:
+          # Remove duplicate metrics
+          - sourceLabels: ["__name__"]
+            regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authenticated_user|authentication|disabled_metric|go|hidden_metric|kubernetes_build|kubernetes_feature|leader_election|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scheduler|scrape_duration|scrape_samples|scrape_series|workqueue)_(.+)"
+            action: keep
+    prometheus:
+      ingress:
+        enabled: true
+        ingressClassName: nginx
+        annotations:
+          hajimari.io/appName: Prometheus
+          hajimari.io/icon: simple-icons:prometheus
+        pathType: Prefix
+        hosts:
+          - &host "prometheus.${SECRET_DOMAIN}"
+        tls:
+          - hosts:
+              - *host
+      prometheusSpec:
+        ruleSelectorNilUsesHelmValues: false
+        serviceMonitorSelectorNilUsesHelmValues: false
+        podMonitorSelectorNilUsesHelmValues: false
+        probeSelectorNilUsesHelmValues: false
+        enableAdminAPI: true
+        walCompression: true
+        retentionSize: 8GB
+        storageSpec:
+          volumeClaimTemplate:
+            spec:
+              storageClassName: local-path
+              resources:
+                requests:
+                  storage: 10Gi

template/templates/kubernetes/weave-gitops-secret.sops.yaml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cluster-user-auth
+  namespace: flux-system
+type: Opaque
+stringData:
+  username: admin
+  password: "{{ bootstrap_weave_gitops_admin_password | password_hash('bcrypt') }}"

template/vars/.gitignore

@@ -0,0 +1 @@
+config.yaml

template/vars/config.sample.yaml

@@ -0,0 +1,59 @@
+---
+#
+# Bootstrap configuration - This file is gitignored
+#
+
+# Github username (e.g. onedr0p)
+bootstrap_github_username:
+# Github repository (e.g. flux-cluster-template)
+bootstrap_github_repository_name:
+# Age Public Key (e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta)
+bootstrap_age_public_key:
+# Choose your timezone (e.g. America/New_York)
+bootstrap_timezone: Etc/UTC
+# Email you want to be associated with the ACME account (used for TLS certificates via letsencrypt.org)
+bootstrap_acme_email:
+
+# Flux github webhook token (openssl rand -hex 12)
+bootstrap_flux_github_webhook_token:
+# Grafana admin password (openssl rand -hex 12)
+bootstrap_grafana_admin_password:
+# Weave GitOps dashboard password (openssl rand -hex 12)
+bootstrap_weave_gitops_admin_password:
+
+# Cloudflare domain
+bootstrap_cloudflare_domain:
+# Cloudflare API Token (not API Key)
+bootstrap_cloudflare_token:
+# Cloudflared Account Tag (cat ~/.cloudflared/*.json | jq -r .AccountTag)
+bootstrap_cloudflare_account_tag:
+# Cloudflared Tunnel Secret (cat ~/.cloudflared/*.json | jq -r .TunnelSecret)
+bootstrap_cloudflare_tunnel_secret:
+# Cloudflared Tunnel ID (cat ~/.cloudflared/*.json | jq -r .TunnelID)
+bootstrap_cloudflare_tunnel_id:
+
+# CIDR your nodes are on (e.g. 192.168.1.0/24)
+bootstrap_node_cidr:
+# Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used
+bootstrap_k8s_gateway_addr:
+# Load balancer IP for ingress-nginx, choose an available IP in your nodes network that is not being used
+bootstrap_ingress_nginx_addr:
+# IP address to use with kube-vip, choose an available IP in your nodes network that is not being used
+bootstrap_kube_vip_addr:
+# Choose your cluster and service cidrs, leave these unchanged unless you know what you are doing
+bootstrap_cluster_cidr: 10.42.0.0/16
+bootstrap_service_cidr: 10.43.0.0/16
+
+# Node information
+bootstrap_hosts:
+  # Use only 1, 3 or more odd master nodes, recommended is 3
+  master:
+    # - name: kube0
+    #   address:
+    #   username:
+    #   password:
+  worker:
+    # - name: kube1
+    #   address:
+    #   username:
+    #   password: