From eed4faa9e6af5c47f1a6975ff16178cab64c23cf Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Fri, 19 Jan 2024 06:13:39 -0500 Subject: [PATCH] fix: keep included k3s coredns (#1197) * Delete bootstrap/templates/kubernetes/apps/kube-system/coredns directory * Update kustomization.yaml.j2 * Delete bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 * Update cluster-installation.yaml.j2 * Delete bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2 * Update main.yaml.j2 * Update main.yaml.j2 * Update main.yaml.j2 * Update main.yaml.j2 * Update cluster-installation.yaml.j2 --- .../group_vars/kubernetes/main.yaml.j2 | 7 +- .../inventory/group_vars/master/main.yaml.j2 | 22 ++--- .../inventory/group_vars/worker/main.yaml.j2 | 2 - .../playbooks/cluster-installation.yaml.j2 | 7 +- .../ansible/playbooks/tasks/coredns.yaml.j2 | 58 ------------ .../custom-coredns-helmchart.yaml.j2.j2 | 79 ---------------- .../coredns/app/helmrelease.yaml.j2 | 89 ------------------- .../coredns/app/kustomization.yaml.j2 | 7 -- .../apps/kube-system/coredns/ks.yaml.j2 | 22 ----- .../apps/kube-system/kustomization.yaml.j2 | 3 - 10 files changed, 9 insertions(+), 287 deletions(-) delete mode 100644 bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2 delete mode 100644 bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 delete mode 100644 bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 delete mode 100644 bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2 delete mode 100644 bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 index 7f4b30ea..2796de57 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 @@ -1,10 +1,6 @@ #% if bootstrap_distribution == 'k3s' %# --- -# -# Below vars are for the xanmanning.k3s role -# ...see https://github.com/PyratLabs/ansible-role-k3s -# - +# https://github.com/PyratLabs/ansible-role-k3s # renovate: datasource=github-releases depName=k3s-io/k3s k3s_release_version: "v1.29.0+k3s1" k3s_install_hard_links: true @@ -15,7 +11,6 @@ k3s_registration_address: "{{ kube_api_addr }}" # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_templates: - custom-cilium-helmchart.yaml.j2 - - custom-coredns-helmchart.yaml.j2 #% if bootstrap_nodes.master | length > 1 or bootstrap_kube_api_addr | default('') %# # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_urls: diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 index e4d09fc3..c87e81de 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 @@ -2,7 +2,6 @@ --- # https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/ # https://github.com/PyratLabs/ansible-role-k3s - k3s_control_node: true k3s_server: #% if bootstrap_ipv6_enabled | default(false) %# @@ -16,17 +15,11 @@ k3s_server: - "{{ kube_api_hostname }}" #% endif %# docker: false - flannel-backend: "none" # This needs to be in quotes - disable: - - coredns # Disable coredns - replaced with Coredns Helm Chart - - flannel # Disable flannel - replaced with Cilium Helm Chart - - local-storage # Disable local-path-provisioner - replaced with democratic-csi - - metrics-server # Disable metrics-server - installed with Flux - - servicelb # Disable servicelb - replaced with Cilium Helm Chart - - traefik # Disable traefik - replaced with ingress-nginx and installed with Flux + flannel-backend: "none" # quote + disable: ["flannel", "local-storage", "metrics-server", "servicelb", "traefik"] disable-network-policy: true disable-cloud-controller: true - disable-kube-proxy: true # Cilium uses eBPF + disable-kube-proxy: true write-kubeconfig-mode: "644" pause-image: registry.k8s.io/pause:3.9 secrets-encryption: true @@ -37,14 +30,13 @@ k3s_server: cluster-cidr: "{{ cluster_cidr }}" service-cidr: "{{ service_cidr }}" #% endif %# - etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack + etcd-expose-metrics: true # Required to monitor etcd with kube-prometheus-stack kube-controller-manager-arg: - - "bind-address=0.0.0.0" # Required to monitor kube-controller-manager with kube-prometheus-stack + - "bind-address=0.0.0.0" # Required to monitor kube-controller-manager with kube-prometheus-stack kube-scheduler-arg: - - "bind-address=0.0.0.0" # Required to monitor kube-scheduler with kube-prometheus-stack + - "bind-address=0.0.0.0" # Required to monitor kube-scheduler with kube-prometheus-stack kube-apiserver-arg: - - "anonymous-auth=true" # Required for HAProxy health-checks - # TODO: Move these options to a kubelet config file + - "anonymous-auth=true" # Required for external LB health-checks kubelet-arg: - "image-gc-high-threshold=55" - "image-gc-low-threshold=50" diff --git a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 index fd448dae..923365ee 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 @@ -2,7 +2,6 @@ --- # https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/ # https://github.com/PyratLabs/ansible-role-k3s - k3s_control_node: false k3s_agent: #% if bootstrap_ipv6_enabled | default(false) %# @@ -11,7 +10,6 @@ k3s_agent: node-ip: "{{ ansible_host }}" #% endif %# pause-image: registry.k8s.io/pause:3.9 - # TODO: Move these options to a kubelet config file kubelet-arg: - "image-gc-high-threshold=55" - "image-gc-low-threshold=50" diff --git a/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 index 0f0bd3ed..43ee78b8 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 @@ -46,12 +46,7 @@ wait_sleep: 10 wait_timeout: 360 loop: - - { name: cilium, kind: HelmChart, namespace: kube-system } # noqa: yaml[commas] - - { name: coredns, kind: HelmChart, namespace: kube-system } - - - name: Coredns - when: k3s_primary_control_node - ansible.builtin.include_tasks: tasks/coredns.yaml + - { name: cilium, kind: HelmChart, namespace: kube-system } - name: Cilium when: k3s_primary_control_node diff --git a/bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2 b/bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2 deleted file mode 100644 index e007eb4b..00000000 --- a/bootstrap/templates/ansible/playbooks/tasks/coredns.yaml.j2 +++ /dev/null @@ -1,58 +0,0 @@ -#% if bootstrap_distribution == 'k3s' %# ---- -- name: Coredns - block: - - name: Coredns | Check if Coredns HelmChart exists - kubernetes.core.k8s_info: - kubeconfig: /etc/rancher/k3s/k3s.yaml - name: coredns - kind: HelmChart - namespace: kube-system - register: coredns_helmchart - - - name: Coredns | Wait for Coredns to rollout - when: coredns_helmchart.resources | count > 0 - kubernetes.core.k8s_info: - kubeconfig: /etc/rancher/k3s/k3s.yaml - name: helm-install-coredns - kind: Job - namespace: kube-system - wait: true - wait_condition: - type: Complete - status: true - wait_timeout: 360 - - - name: Coredns | Patch the Coredns HelmChart to unmanage it - when: coredns_helmchart.resources | count > 0 - kubernetes.core.k8s_json_patch: - kubeconfig: /etc/rancher/k3s/k3s.yaml - name: coredns - kind: HelmChart - namespace: kube-system - patch: - - op: add - path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged - value: "true" - - - name: Coredns | Delete the Coredns HelmChart CR - when: coredns_helmchart.resources | count > 0 - kubernetes.core.k8s: - kubeconfig: /etc/rancher/k3s/k3s.yaml - name: coredns - kind: HelmChart - namespace: kube-system - state: absent - - - name: Coredns | Force delete the Coredns HelmChart - when: coredns_helmchart.resources | count > 0 - kubernetes.core.k8s: - kubeconfig: /etc/rancher/k3s/k3s.yaml - name: coredns - kind: HelmChart - namespace: kube-system - state: patched - definition: - metadata: - finalizers: [] -#% endif %# diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 deleted file mode 100644 index fb2da9ae..00000000 --- a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 +++ /dev/null @@ -1,79 +0,0 @@ -#% if bootstrap_distribution == 'k3s' %# ---- -# https://docs.k3s.io/helm -apiVersion: helm.cattle.io/v1 -kind: HelmChart -metadata: - name: coredns - namespace: kube-system -spec: - # renovate: datasource=helm - repo: https://coredns.github.io/helm - chart: coredns - version: 1.29.0 - targetNamespace: kube-system - bootstrap: true - valuesContent: |- - fullnameOverride: coredns - replicaCount: 1 - k8sAppLabelOverride: kube-dns - service: - name: kube-dns - clusterIP: "{{ coredns_addr }}" - serviceAccount: - create: true - deployment: - annotations: - reloader.stakater.com/auto: "true" - servers: - - zones: - - zone: . - scheme: dns:// - use_tcp: true - port: 53 - plugins: - - name: log - - name: errors - - name: health - configBlock: |- - lameduck 5s - - name: ready - - name: kubernetes - parameters: cluster.local in-addr.arpa ip6.arpa - configBlock: |- - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - - name: prometheus - parameters: 0.0.0.0:9153 - - name: forward - parameters: . /etc/resolv.conf - - name: cache - parameters: 30 - - name: loop - - name: reload - - name: loadbalance - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app.kubernetes.io/instance: coredns -#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 deleted file mode 100644 index 79dd3f18..00000000 --- a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 +++ /dev/null @@ -1,89 +0,0 @@ -#% if bootstrap_distribution == "k3s" %# ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta2 -kind: HelmRelease -metadata: - name: coredns -spec: - interval: 30m - chart: - spec: - chart: coredns - version: 1.29.0 - sourceRef: - kind: HelmRepository - name: coredns - namespace: flux-system - install: - remediation: - retries: 3 - upgrade: - cleanupOnFail: true - remediation: - retries: 3 - uninstall: - keepHistory: false - values: - fullnameOverride: coredns - replicaCount: 1 - k8sAppLabelOverride: kube-dns - service: - name: kube-dns - clusterIP: "${COREDNS_ADDR}" - serviceAccount: - create: true - deployment: - annotations: - reloader.stakater.com/auto: "true" - servers: - - zones: - - zone: . - scheme: dns:// - use_tcp: true - port: 53 - plugins: - - name: log - - name: errors - - name: health - configBlock: |- - lameduck 5s - - name: ready - - name: kubernetes - parameters: cluster.local in-addr.arpa ip6.arpa - configBlock: |- - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - - name: prometheus - parameters: 0.0.0.0:9153 - - name: forward - parameters: . /etc/resolv.conf - - name: cache - parameters: 30 - - name: loop - - name: reload - - name: loadbalance - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: DoNotSchedule - labelSelector: - matchLabels: - app.kubernetes.io/instance: coredns -#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2 deleted file mode 100644 index 07657024..00000000 --- a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2 +++ /dev/null @@ -1,7 +0,0 @@ -#% if bootstrap_distribution == "k3s" %# ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ./helmrelease.yaml -#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 deleted file mode 100644 index c9b0306d..00000000 --- a/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 +++ /dev/null @@ -1,22 +0,0 @@ -#% if bootstrap_distribution == "k3s" %# ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: &app coredns - namespace: flux-system -spec: - targetNamespace: kube-system - commonMetadata: - labels: - app.kubernetes.io/name: *app - path: ./kubernetes/apps/kube-system/coredns/app - prune: false # never should be deleted - sourceRef: - kind: GitRepository - name: home-kubernetes - wait: false - interval: 30m - retryInterval: 1m - timeout: 5m -#% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 index 0f1ee8ae..b79fdb0a 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 @@ -4,9 +4,6 @@ kind: Kustomization resources: - ./namespace.yaml - ./cilium/ks.yaml - #% if bootstrap_distribution == "k3s" %# - - ./coredns/ks.yaml - #% endif %# #% if bootstrap_distribution == "talos" %# - ./kubelet-csr-approver/ks.yaml #% endif %#