From d46e7f9663037fa781ef88986bf165b4a14e6505 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 18 Jan 2024 08:38:03 -0500 Subject: [PATCH] feat: add partial support, change k0s config filename, always use NLLB for k0s (#1180) * feat: add partial support, change k0s config filename, always use NLLB for k0s Signed-off-by: Devin Buhl * chore: update wording Signed-off-by: Devin Buhl --------- Signed-off-by: Devin Buhl --- .taskfiles/K0s/Taskfile.yaml | 4 +- Taskfile.yaml | 2 +- .../custom-cilium-helmchart.yaml.j2.j2 | 51 +------- .../cilium/app/helmrelease.yaml.j2 | 116 +----------------- .../{k0s-config.yaml.j2 => k0sctl.yaml.j2} | 59 +-------- .../partials/cilium-values-full.yaml.partial | 113 +++++++++++++++++ .../partials/cilium-values-init.yaml.partial | 57 +++++++++ bootstrap/vars/config.sample.yaml | 2 +- makejinja.toml | 5 + 9 files changed, 189 insertions(+), 220 deletions(-) rename bootstrap/templates/kubernetes/k0s/{k0s-config.yaml.j2 => k0sctl.yaml.j2} (60%) create mode 100644 bootstrap/templates/partials/cilium-values-full.yaml.partial create mode 100644 bootstrap/templates/partials/cilium-values-init.yaml.partial diff --git a/.taskfiles/K0s/Taskfile.yaml b/.taskfiles/K0s/Taskfile.yaml index 4d40ec7d..6b2db974 100644 --- a/.taskfiles/K0s/Taskfile.yaml +++ b/.taskfiles/K0s/Taskfile.yaml @@ -6,13 +6,13 @@ env: DISABLE_UPGRADE_CHECK: "true" vars: - K0S_CONFIG_FILE: "{{.KUBERNETES_DIR}}/k0s/k0s-config.yaml" + K0S_CONFIG_FILE: "{{.KUBERNETES_DIR}}/k0s/k0sctl.yaml" tasks: apply: desc: Apply k0s cluster config - dir: "{{.KUBERNETES_DIR}}/k0s" # Needed for uploading files in k0s-config.yaml + dir: "{{.KUBERNETES_DIR}}/k0s" # Needed for uploading files in k0sctl.yaml cmds: - k0sctl apply --config {{.K0S_CONFIG_FILE}} - task: kubeconfig diff --git a/Taskfile.yaml b/Taskfile.yaml index 8491e2b1..b76be486 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -77,7 +77,7 @@ tasks: .template: internal: true cmds: - - ./.venv/bin/makejinja --keep-trailing-newline --force + - ./.venv/bin/makejinja - task: sops:encrypt preconditions: - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" } diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 index 8346ffed..b81c0cca 100644 --- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 +++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 @@ -14,52 +14,7 @@ spec: targetNamespace: kube-system bootstrap: true valuesContent: |- - autoDirectNodeRoutes: true - bpf: - masquerade: true - bgp: - enabled: false - cluster: - name: home-cluster - id: 1 - containerRuntime: - integration: containerd - socketPath: /var/run/k3s/containerd/containerd.sock - endpointRoutes: - enabled: true - hubble: - enabled: false - ipam: - mode: kubernetes - ipv4NativeRoutingCIDR: "{{ cluster_cidr }}" - #% if bootstrap_ipv6_enabled | default(false) %# - ipv6NativeRoutingCIDR: "{{ cluster_cidr_v6 }}" - ipv6: - enabled: true - #% endif %# - k8sServiceHost: 127.0.0.1 - k8sServicePort: 6444 - kubeProxyReplacement: true - kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 - l2announcements: - #% if bootstrap_ipv6_enabled | default(false) %# - enabled: false - #% else %# - enabled: true - # https://github.com/cilium/cilium/issues/26586 - leaseDuration: 120s - leaseRenewDeadline: 60s - leaseRetryPeriod: 1s - #% endif %# - loadBalancer: - algorithm: maglev - mode: "#{ bootstrap_cilium_loadbalancer_mode | default('dsr', true) }#" - localRedirectPolicy: true - operator: - replicas: 1 - rollOutPods: true - rollOutCiliumPods: true - routingMode: native - securityContext: - privileged: true + #% filter indent(width=4, first=True) %# + #% include 'partials/cilium-values-init.yaml.partial' %# + #% endfilter %# #% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 index fa5e8a82..e4c2d68a 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 @@ -24,116 +24,6 @@ spec: uninstall: keepHistory: false values: - autoDirectNodeRoutes: true - bpf: - masquerade: true - bgp: - enabled: false - cluster: - name: home-cluster - id: 1 - containerRuntime: - integration: containerd - #% if bootstrap_distribution == "k3s" %# - socketPath: /var/run/k3s/containerd/containerd.sock - #% elif bootstrap_distribution == "k0s" %# - socketPath: /var/run/k0s/containerd.sock - #% endif %# - endpointRoutes: - enabled: true - hubble: - enabled: true - metrics: - enabled: - - dns:query - - drop - - tcp - - flow - - port-distribution - - icmp - - http - serviceMonitor: - enabled: true - dashboards: - enabled: true - annotations: - grafana_folder: Cilium - relay: - enabled: true - rollOutPods: true - prometheus: - serviceMonitor: - enabled: true - ui: - enabled: true - rollOutPods: true - ingress: - enabled: true - className: internal - #% if homepage.enabled | default(false) %# - annotations: - gethomepage.dev/enabled: "true" - gethomepage.dev/group: Network - gethomepage.dev/name: Cilium - gethomepage.dev/icon: cilium.png - #% endif %# - hosts: - - "hubble.${SECRET_DOMAIN}" - tls: - - hosts: - - "hubble.${SECRET_DOMAIN}" - ipam: - mode: kubernetes - ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}" - #% if bootstrap_ipv6_enabled | default(false) %# - ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}" - ipv6: - enabled: true - #% endif %# - #% if bootstrap_distribution == "k3s" %# - k8sServiceHost: 127.0.0.1 - k8sServicePort: 6444 - #% else %# - k8sServiceHost: "${KUBE_API_ADDR}" - k8sServicePort: 6443 - #% endif %# - kubeProxyReplacement: true - kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 - l2announcements: - #% if bootstrap_ipv6_enabled | default(false) %# - enabled: false - #% else %# - enabled: true - # https://github.com/cilium/cilium/issues/26586 - leaseDuration: 120s - leaseRenewDeadline: 60s - leaseRetryPeriod: 1s - #% endif %# - loadBalancer: - algorithm: maglev - mode: "#{ bootstrap_cilium_loadbalancer_mode | default('dsr', true) }#" - localRedirectPolicy: true - operator: - replicas: 1 - rollOutPods: true - prometheus: - enabled: true - serviceMonitor: - enabled: true - dashboards: - enabled: true - annotations: - grafana_folder: Cilium - prometheus: - enabled: true - serviceMonitor: - enabled: true - trustCRDsExist: true - dashboards: - enabled: true - annotations: - grafana_folder: Cilium - rollOutCiliumPods: true - routingMode: native - securityContext: - privileged: true + #% filter indent(width=4, first=True) %# + #% include 'partials/cilium-values-full.yaml.partial' %# + #% endfilter %# diff --git a/bootstrap/templates/kubernetes/k0s/k0s-config.yaml.j2 b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 similarity index 60% rename from bootstrap/templates/kubernetes/k0s/k0s-config.yaml.j2 rename to bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 index 1f9cf910..5a3ac8b7 100644 --- a/bootstrap/templates/kubernetes/k0s/k0s-config.yaml.j2 +++ b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 @@ -86,14 +86,12 @@ spec: #% endif %# #% endfor %# network: + provider: custom kubeProxy: disabled: true - #% if bootstrap_nodes.master | length > 1 %# nodeLocalLoadBalancing: enabled: true type: EnvoyProxy - #% endif %# - provider: custom extensions: helm: repositories: @@ -106,56 +104,7 @@ spec: version: "1.14.5" namespace: kube-system values: |2 - autoDirectNodeRoutes: true - bpf: - masquerade: true - bgp: - enabled: false - cluster: - name: home-cluster - id: 1 - containerRuntime: - integration: containerd - socketPath: /var/run/k0s/containerd.sock - endpointRoutes: - enabled: true - hubble: - enabled: false - ipam: - mode: kubernetes - ipv4NativeRoutingCIDR: "#{ bootstrap_cluster_cidr }#" - #% if bootstrap_ipv6_enabled | default(false) %# - ipv6NativeRoutingCIDR: "#{ bootstrap_cluster_cidr_v6 }#" - ipv6: - enabled: true - #% endif %# - #% if bootstrap_nodes.master | length > 1 or bootstrap_kube_api_addr | default('') %# - k8sServiceHost: "#{ bootstrap_kube_api_addr }#" - #% else %# - k8sServiceHost: "#{ bootstrap_nodes.master[0].address }#" - #% endif %# - k8sServicePort: 6443 - kubeProxyReplacement: true - kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 - l2announcements: - #% if bootstrap_ipv6_enabled | default(false) %# - enabled: false - #% else %# - enabled: true - # https://github.com/cilium/cilium/issues/26586 - leaseDuration: 120s - leaseRenewDeadline: 60s - leaseRetryPeriod: 1s - #% endif %# - loadBalancer: - algorithm: maglev - mode: "#{ bootstrap_cilium_loadbalancer_mode | default('dsr', true) }#" - localRedirectPolicy: true - operator: - replicas: 1 - rollOutPods: true - rollOutCiliumPods: true - routingMode: native - securityContext: - privileged: true + #% filter indent(width=18, first=True) %# + #% include 'partials/cilium-values-init.yaml.partial' %# + #% endfilter %# #% endif %# diff --git a/bootstrap/templates/partials/cilium-values-full.yaml.partial b/bootstrap/templates/partials/cilium-values-full.yaml.partial new file mode 100644 index 00000000..78ceb0e7 --- /dev/null +++ b/bootstrap/templates/partials/cilium-values-full.yaml.partial @@ -0,0 +1,113 @@ +autoDirectNodeRoutes: true +bpf: + masquerade: true +bgp: + enabled: false +cluster: + name: home-cluster + id: 1 +containerRuntime: + integration: containerd + #% if bootstrap_distribution == "k3s" %# + socketPath: /var/run/k3s/containerd/containerd.sock + #% elif bootstrap_distribution == "k0s" %# + socketPath: /var/run/k0s/containerd.sock + #% endif %# +endpointRoutes: + enabled: true +hubble: + enabled: true + metrics: + enabled: + - dns:query + - drop + - tcp + - flow + - port-distribution + - icmp + - http + serviceMonitor: + enabled: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + relay: + enabled: true + rollOutPods: true + prometheus: + serviceMonitor: + enabled: true + ui: + enabled: true + rollOutPods: true + ingress: + enabled: true + className: internal + #% if homepage.enabled | default(false) %# + annotations: + gethomepage.dev/enabled: "true" + gethomepage.dev/group: Network + gethomepage.dev/name: Cilium + gethomepage.dev/icon: cilium.png + #% endif %# + hosts: + - "hubble.${SECRET_DOMAIN}" + tls: + - hosts: + - "hubble.${SECRET_DOMAIN}" +ipam: + mode: kubernetes +ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}" +#% if bootstrap_ipv6_enabled | default(false) %# +ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}" +ipv6: + enabled: true +#% endif %# +#% if bootstrap_distribution == "k3s" %# +k8sServiceHost: 127.0.0.1 +k8sServicePort: 6444 +#% elif bootstrap_distribution == "k0s" %# +k8sServiceHost: localhost +k8sServicePort: 7443 +#% endif %# +kubeProxyReplacement: true +kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 +l2announcements: + #% if bootstrap_ipv6_enabled | default(false) %# + enabled: false + #% else %# + enabled: true + # https://github.com/cilium/cilium/issues/26586 + leaseDuration: 120s + leaseRenewDeadline: 60s + leaseRetryPeriod: 1s + #% endif %# +loadBalancer: + algorithm: maglev + mode: "#{ bootstrap_cilium_loadbalancer_mode | default('dsr', true) }#" +localRedirectPolicy: true +operator: + replicas: 1 + rollOutPods: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium +prometheus: + enabled: true + serviceMonitor: + enabled: true + trustCRDsExist: true +dashboards: + enabled: true + annotations: + grafana_folder: Cilium +rollOutCiliumPods: true +routingMode: native +securityContext: + privileged: true diff --git a/bootstrap/templates/partials/cilium-values-init.yaml.partial b/bootstrap/templates/partials/cilium-values-init.yaml.partial new file mode 100644 index 00000000..52e60eec --- /dev/null +++ b/bootstrap/templates/partials/cilium-values-init.yaml.partial @@ -0,0 +1,57 @@ +autoDirectNodeRoutes: true +bpf: + masquerade: true +bgp: + enabled: false +cluster: + name: home-cluster + id: 1 +containerRuntime: + integration: containerd + #% if bootstrap_distribution == "k3s" %# + socketPath: /var/run/k3s/containerd/containerd.sock + #% elif bootstrap_distribution == "k0s" %# + socketPath: /var/run/k0s/containerd.sock + #% endif %# +endpointRoutes: + enabled: true +hubble: + enabled: false +ipam: + mode: kubernetes +ipv4NativeRoutingCIDR: "#{ bootstrap_cluster_cidr }#" +#% if bootstrap_ipv6_enabled | default(false) %# +ipv6NativeRoutingCIDR: "#{ bootstrap_cluster_cidr_v6 }#" +ipv6: + enabled: true +#% endif %# +#% if bootstrap_distribution == "k3s" %# +k8sServiceHost: 127.0.0.1 +k8sServicePort: 6444 +#% elif bootstrap_distribution == "k0s" %# +k8sServiceHost: localhost +k8sServicePort: 7443 +#% endif %# +kubeProxyReplacement: true +kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 +l2announcements: + #% if bootstrap_ipv6_enabled | default(false) %# + enabled: false + #% else %# + enabled: true + # https://github.com/cilium/cilium/issues/26586 + leaseDuration: 120s + leaseRenewDeadline: 60s + leaseRetryPeriod: 1s + #% endif %# +loadBalancer: + algorithm: maglev + mode: "#{ bootstrap_cilium_loadbalancer_mode | default('dsr', true) }#" +localRedirectPolicy: true +operator: + replicas: 1 + rollOutPods: true +rollOutCiliumPods: true +routingMode: native +securityContext: + privileged: true diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index c144bec5..159eb0f4 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -46,7 +46,7 @@ bootstrap_node_cidr: # and keepalived in k0s and use the master node IP instead bootstrap_kube_api_addr: # (Optional) The hostname of the Kube API, set this if you intend to call the Kube API by -# hostname rather than IP. Hostname is added to the Kube API cert sans +# hostname rather than IP. If the hostname is set it is added to the Kube API cert sans bootstrap_kube_api_hostname: # The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used bootstrap_k8s_gateway_addr: diff --git a/makejinja.toml b/makejinja.toml index 0eeeba78..603c6c1b 100644 --- a/makejinja.toml +++ b/makejinja.toml @@ -1,10 +1,12 @@ [makejinja] inputs = ["./bootstrap/templates"] output = "./" +exclude_patterns = ["*.partial"] data = ["./bootstrap/vars/addons.yaml", "./bootstrap/vars/config.yaml"] import_paths = ["./bootstrap/scripts"] loaders = ["loader:Loader"] jinja_suffix = ".j2" +force = true # Block and comment delimiters are changed to avoid conflicts with Renovate # Variable delimiters are changed to avoid conflicts with Renovate and Go templates @@ -16,3 +18,6 @@ comment_start = "#|" comment_end = "|#" variable_start = "#{" variable_end = "}#" + +[makejinja.whitespace] +keep_trailing_newline = true