fix: keep included k3s coredns (#1197)

* Delete bootstrap/templates/kubernetes/apps/kube-system/coredns directory

* Update kustomization.yaml.j2

* Delete bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2

* Update cluster-installation.yaml.j2

* Delete bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2

* Update main.yaml.j2

* Update main.yaml.j2

* Update main.yaml.j2

* Update main.yaml.j2

* Update cluster-installation.yaml.j2

bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2

@@ -1,10 +1,6 @@
 #% if bootstrap_distribution == 'k3s' %#
 ---
-#
+# https://github.com/PyratLabs/ansible-role-k3s
-# Below vars are for the xanmanning.k3s role
-# ...see https://github.com/PyratLabs/ansible-role-k3s
-#
-
 # renovate: datasource=github-releases depName=k3s-io/k3s
 k3s_release_version: "v1.29.0+k3s1"
 k3s_install_hard_links: true
@@ -15,7 +11,6 @@ k3s_registration_address: "{{ kube_api_addr }}"
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
   - custom-cilium-helmchart.yaml.j2
-  - custom-coredns-helmchart.yaml.j2
 #% if bootstrap_nodes.master | length > 1 or bootstrap_kube_api_addr | default('') %#
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_urls:

bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2

@@ -2,7 +2,6 @@
 ---
 # https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/
 # https://github.com/PyratLabs/ansible-role-k3s
-
 k3s_control_node: true
 k3s_server:
   #% if bootstrap_ipv6_enabled | default(false) %#
@@ -16,17 +15,11 @@ k3s_server:
     - "{{ kube_api_hostname }}"
     #% endif %#
   docker: false
-  flannel-backend: "none"             # This needs to be in quotes
+  flannel-backend: "none" # quote
-  disable:
+  disable: ["flannel", "local-storage", "metrics-server", "servicelb", "traefik"]
-    - coredns                         # Disable coredns                 - replaced with Coredns Helm Chart
-    - flannel                         # Disable flannel                 - replaced with Cilium Helm Chart
-    - local-storage                   # Disable local-path-provisioner  - replaced with democratic-csi
-    - metrics-server                  # Disable metrics-server          - installed with Flux
-    - servicelb                       # Disable servicelb               - replaced with Cilium Helm Chart
-    - traefik                         # Disable traefik                 - replaced with ingress-nginx and installed with Flux
   disable-network-policy: true
   disable-cloud-controller: true
-  disable-kube-proxy: true            # Cilium uses eBPF
+  disable-kube-proxy: true
   write-kubeconfig-mode: "644"
   pause-image: registry.k8s.io/pause:3.9
   secrets-encryption: true
@@ -37,14 +30,13 @@ k3s_server:
   cluster-cidr: "{{ cluster_cidr }}"
   service-cidr: "{{ service_cidr }}"
   #% endif %#
-  etcd-expose-metrics: true           # Required to monitor etcd with kube-prometheus-stack
+  etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack
   kube-controller-manager-arg:
-    - "bind-address=0.0.0.0"          # Required to monitor kube-controller-manager with kube-prometheus-stack
+    - "bind-address=0.0.0.0" # Required to monitor kube-controller-manager with kube-prometheus-stack
   kube-scheduler-arg:
-    - "bind-address=0.0.0.0"          # Required to monitor kube-scheduler with kube-prometheus-stack
+    - "bind-address=0.0.0.0" # Required to monitor kube-scheduler with kube-prometheus-stack
   kube-apiserver-arg:
-    - "anonymous-auth=true"           # Required for HAProxy health-checks
+    - "anonymous-auth=true" # Required for external LB health-checks
-  # TODO: Move these options to a kubelet config file
   kubelet-arg:
     - "image-gc-high-threshold=55"
     - "image-gc-low-threshold=50"

bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2

@@ -2,7 +2,6 @@
 ---
 # https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/
 # https://github.com/PyratLabs/ansible-role-k3s
-
 k3s_control_node: false
 k3s_agent:
   #% if bootstrap_ipv6_enabled | default(false) %#
@@ -11,7 +10,6 @@ k3s_agent:
   node-ip: "{{ ansible_host }}"
   #% endif %#
   pause-image: registry.k8s.io/pause:3.9
-  # TODO: Move these options to a kubelet config file
   kubelet-arg:
     - "image-gc-high-threshold=55"
     - "image-gc-low-threshold=50"

bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2

@@ -46,12 +46,7 @@
         wait_sleep: 10
         wait_timeout: 360
       loop:
-        - { name: cilium,  kind: HelmChart, namespace: kube-system } # noqa: yaml[commas]
+        - { name: cilium, kind: HelmChart, namespace: kube-system }
-        - { name: coredns, kind: HelmChart, namespace: kube-system }
-
-    - name: Coredns
-      when: k3s_primary_control_node
-      ansible.builtin.include_tasks: tasks/coredns.yaml
 
     - name: Cilium
       when: k3s_primary_control_node

bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2

@@ -1,58 +0,0 @@
-#% if bootstrap_distribution == 'k3s' %#
----
-- name: Coredns
-  block:
-    - name: Coredns | Check if Coredns HelmChart exists
-      kubernetes.core.k8s_info:
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-        name: coredns
-        kind: HelmChart
-        namespace: kube-system
-      register: coredns_helmchart
-
-    - name: Coredns | Wait for Coredns to rollout
-      when: coredns_helmchart.resources | count > 0
-      kubernetes.core.k8s_info:
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-        name: helm-install-coredns
-        kind: Job
-        namespace: kube-system
-        wait: true
-        wait_condition:
-          type: Complete
-          status: true
-        wait_timeout: 360
-
-    - name: Coredns | Patch the Coredns HelmChart to unmanage it
-      when: coredns_helmchart.resources | count > 0
-      kubernetes.core.k8s_json_patch:
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-        name: coredns
-        kind: HelmChart
-        namespace: kube-system
-        patch:
-          - op: add
-            path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged
-            value: "true"
-
-    - name: Coredns | Delete the Coredns HelmChart CR
-      when: coredns_helmchart.resources | count > 0
-      kubernetes.core.k8s:
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-        name: coredns
-        kind: HelmChart
-        namespace: kube-system
-        state: absent
-
-    - name: Coredns | Force delete the Coredns HelmChart
-      when: coredns_helmchart.resources | count > 0
-      kubernetes.core.k8s:
-        kubeconfig: /etc/rancher/k3s/k3s.yaml
-        name: coredns
-        kind: HelmChart
-        namespace: kube-system
-        state: patched
-        definition:
-          metadata:
-            finalizers: []
-#% endif %#

bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2

@@ -1,79 +0,0 @@
-#% if bootstrap_distribution == 'k3s' %#
----
-# https://docs.k3s.io/helm
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: coredns
-  namespace: kube-system
-spec:
-  # renovate: datasource=helm
-  repo: https://coredns.github.io/helm
-  chart: coredns
-  version: 1.29.0
-  targetNamespace: kube-system
-  bootstrap: true
-  valuesContent: |-
-    fullnameOverride: coredns
-    replicaCount: 1
-    k8sAppLabelOverride: kube-dns
-    service:
-      name: kube-dns
-      clusterIP: "{{ coredns_addr }}"
-    serviceAccount:
-      create: true
-    deployment:
-      annotations:
-        reloader.stakater.com/auto: "true"
-    servers:
-      - zones:
-          - zone: .
-            scheme: dns://
-            use_tcp: true
-        port: 53
-        plugins:
-          - name: log
-          - name: errors
-          - name: health
-            configBlock: |-
-              lameduck 5s
-          - name: ready
-          - name: kubernetes
-            parameters: cluster.local in-addr.arpa ip6.arpa
-            configBlock: |-
-              pods insecure
-              fallthrough in-addr.arpa ip6.arpa
-              ttl 30
-          - name: prometheus
-            parameters: 0.0.0.0:9153
-          - name: forward
-            parameters: . /etc/resolv.conf
-          - name: cache
-            parameters: 30
-          - name: loop
-          - name: reload
-          - name: loadbalance
-    affinity:
-      nodeAffinity:
-        requiredDuringSchedulingIgnoredDuringExecution:
-          nodeSelectorTerms:
-            - matchExpressions:
-                - key: node-role.kubernetes.io/control-plane
-                  operator: Exists
-    tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-        effect: NoSchedule
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-    topologySpreadConstraints:
-      - maxSkew: 1
-        topologyKey: kubernetes.io/hostname
-        whenUnsatisfiable: DoNotSchedule
-        labelSelector:
-          matchLabels:
-            app.kubernetes.io/instance: coredns
-#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2

@@ -1,89 +0,0 @@
-#% if bootstrap_distribution == "k3s" %#
----
-apiVersion: helm.toolkit.fluxcd.io/v2beta2
-kind: HelmRelease
-metadata:
-  name: coredns
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: coredns
-      version: 1.29.0
-      sourceRef:
-        kind: HelmRepository
-        name: coredns
-        namespace: flux-system
-  install:
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  uninstall:
-    keepHistory: false
-  values:
-    fullnameOverride: coredns
-    replicaCount: 1
-    k8sAppLabelOverride: kube-dns
-    service:
-      name: kube-dns
-      clusterIP: "${COREDNS_ADDR}"
-    serviceAccount:
-      create: true
-    deployment:
-      annotations:
-        reloader.stakater.com/auto: "true"
-    servers:
-      - zones:
-          - zone: .
-            scheme: dns://
-            use_tcp: true
-        port: 53
-        plugins:
-          - name: log
-          - name: errors
-          - name: health
-            configBlock: |-
-              lameduck 5s
-          - name: ready
-          - name: kubernetes
-            parameters: cluster.local in-addr.arpa ip6.arpa
-            configBlock: |-
-              pods insecure
-              fallthrough in-addr.arpa ip6.arpa
-              ttl 30
-          - name: prometheus
-            parameters: 0.0.0.0:9153
-          - name: forward
-            parameters: . /etc/resolv.conf
-          - name: cache
-            parameters: 30
-          - name: loop
-          - name: reload
-          - name: loadbalance
-    affinity:
-      nodeAffinity:
-        requiredDuringSchedulingIgnoredDuringExecution:
-          nodeSelectorTerms:
-            - matchExpressions:
-                - key: node-role.kubernetes.io/control-plane
-                  operator: Exists
-    tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-        effect: NoSchedule
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-    topologySpreadConstraints:
-      - maxSkew: 1
-        topologyKey: kubernetes.io/hostname
-        whenUnsatisfiable: DoNotSchedule
-        labelSelector:
-          matchLabels:
-            app.kubernetes.io/instance: coredns
-#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2

@@ -1,7 +0,0 @@
-#% if bootstrap_distribution == "k3s" %#
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ./helmrelease.yaml
-#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2

@@ -1,22 +0,0 @@
-#% if bootstrap_distribution == "k3s" %#
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: &app coredns
-  namespace: flux-system
-spec:
-  targetNamespace: kube-system
-  commonMetadata:
-    labels:
-      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/kube-system/coredns/app
-  prune: false # never should be deleted
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: false
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
-#% endif %#

bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2

@@ -4,9 +4,6 @@ kind: Kustomization
 resources:
   - ./namespace.yaml
   - ./cilium/ks.yaml
-  #% if bootstrap_distribution == "k3s" %#
-  - ./coredns/ks.yaml
-  #% endif %#
   #% if bootstrap_distribution == "talos" %#
   - ./kubelet-csr-approver/ks.yaml
   #% endif %#