From 467af18b0f43189776933633deb98f241377171b Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 3 Jul 2023 14:47:51 -0400 Subject: [PATCH] feat!: Drop Metallb in favor of Cilium L2Announcements (#821) --- .config.sample.env | 23 ++--- README.md | 7 +- .../group_vars/kubernetes/kubernetes.yml | 2 +- .../group_vars/master/kubernetes.yml | 8 +- ansible/playbooks/cluster-installation.yml | 8 +- ansible/playbooks/templates/cilium-l2.yaml.j2 | 22 +++++ .../templates/custom-cilium-helmchart.yaml.j2 | 23 ++++- .../templates/kube-vip-static-pod.yaml.j2 | 2 +- configure | 87 +++---------------- .../cert-manager/app/helmrelease.yaml | 2 +- .../default/echo-server/app/helmrelease.yaml | 2 +- .../default/hajimari/app/helmrelease.yaml | 2 +- .../weave-gitops/app/helmrelease.yaml | 2 +- .../kube-system/cilium/app/helmrelease.yaml | 23 +++-- .../app/helmrelease.yaml | 2 +- .../metrics-server/app/helmrelease.yaml | 2 +- .../kube-system/reloader/app/helmrelease.yaml | 2 +- .../kubernetes-dashboard/app/helmrelease.yaml | 2 +- .../cloudflared/app/helmrelease.yaml | 2 +- .../external-dns/app/helmrelease.yaml | 2 +- .../ingress-nginx/app/helmrelease.yaml | 6 +- .../apps/networking/ingress-nginx/ks.yaml | 1 - .../k8s-gateway/app/helmrelease.yaml | 6 +- .../apps/networking/k8s-gateway/ks.yaml | 2 - kubernetes/apps/networking/kustomization.yaml | 1 - .../networking/metallb/app/helmrelease.yaml | 33 ------- .../networking/metallb/app/kustomization.yaml | 6 -- .../metallb/config/kustomization.yaml | 5 -- .../networking/metallb/config/resources.yaml | 18 ---- kubernetes/apps/networking/metallb/ks.yaml | 34 -------- .../flux/repositories/helm/kustomization.yaml | 1 - .../flux/repositories/helm/metallb.yaml | 9 -- tmpl/ansible/supplemental.yml | 2 +- tmpl/kubernetes/flux/cluster-settings.yaml | 5 +- 34 files changed, 111 insertions(+), 243 deletions(-) create mode 100644 ansible/playbooks/templates/cilium-l2.yaml.j2 delete mode 100644 kubernetes/apps/networking/metallb/app/helmrelease.yaml delete mode 100644 kubernetes/apps/networking/metallb/app/kustomization.yaml delete mode 100644 kubernetes/apps/networking/metallb/config/kustomization.yaml delete mode 100644 kubernetes/apps/networking/metallb/config/resources.yaml delete mode 100644 kubernetes/apps/networking/metallb/ks.yaml delete mode 100644 kubernetes/flux/repositories/helm/metallb.yaml diff --git a/.config.sample.env b/.config.sample.env index 9c19c41d..ccbaeb64 100644 --- a/.config.sample.env +++ b/.config.sample.env @@ -45,22 +45,11 @@ export BOOTSTRAP_CLOUDFLARE_ACCOUNT_TAG="" export BOOTSTRAP_CLOUDFLARE_TUNNEL_SECRET="" export BOOTSTRAP_CLOUDFLARE_TUNNEL_ID="" -# Pick a range of unused IPs that are on the same network as your nodes -# You don't need many IPs, just choose 10 IPs to start with -# e.g. 192.168.1.220-192.168.1.230 -export BOOTSTRAP_METALLB_LB_RANGE="" -# The load balancer IP for k8s_gateway, choose from one of the available IPs above -# e.g. 192.168.1.220 -export BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR="" -# The load balancer IP for the ingress controller, choose from one of the available IPs above -# that doesn't conflict with any other IP addresses here -# e.g. 192.168.1.221 -export BOOTSTRAP_METALLB_INGRESS_ADDR="" -# The IP Address to use with kube-vip -# Pick a unused IP that is on the same network as your nodes -# and outside the ${BOOTSTRAP_METALLB_LB_RANGE} range -# and doesn't conflict with any other IP addresses here -# e.g. 192.168.1.254 +# The load balancer IP for k8s_gateway, choose a available IP in your nodes network that is not being used +export BOOTSTRAP_K8S_GATEWAY_ADDR="" +# The load balancer IP for ingress-nginx, choose a available IP in your nodes network that is not being used +export BOOTSTRAP_INGRESS_NGINX_ADDR="" +# The IP address to use with kube-vip, choose a available IP in your nodes network that is not being used export BOOTSTRAP_KUBE_VIP_ADDR="" # Choose your cluster or service cidrs # Leave this unchanged unless you know what you are doing @@ -84,7 +73,7 @@ export BOOTSTRAP_ANSIBLE_DEFAULT_NODE_HOSTNAME_PREFIX="k8s-" # NOTE: Must only c # incrementing the last digit on the variable name for each node # -# Host IP Address to the control plane node +# IP Address to the node # That doesn't conflict with any other IP addresses here # e.g. 192.168.1.200 export BOOTSTRAP_ANSIBLE_HOST_ADDR_0="" diff --git a/README.md b/README.md index 8a830c74..29e91de4 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ The following components will be installed in your [k3s](https://k3s.io/) cluste - [flux](https://toolkit.fluxcd.io/) - GitOps operator for managing Kubernetes clusters from a Git repository - [kube-vip](https://kube-vip.io/) - Load balancer for the Kubernetes control plane nodes -- [metallb](https://metallb.universe.tf/) - Load balancer for Kubernetes services - [cert-manager](https://cert-manager.io/) - Operator to request SSL certificates and store them as Kubernetes resources - [cilium](https://cilium.io/) - Container networking interface for inter pod and service networking - [external-dns](https://github.com/kubernetes-sigs/external-dns) - Operator to publish DNS records to Cloudflare (and other providers) based on Kubernetes ingresses @@ -430,16 +429,16 @@ You should start to see your applications using the new certificate. The `external-dns` application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only public sub-domains exposed. In order to make additional applications public you must set an ingress annotation (`external-dns.alpha.kubernetes.io/target`) like done in the `HelmRelease` for `echo-server`. -For split DNS to work it is required to have `${SECRET_DOMAIN}` point to the `${METALLB_K8S_GATEWAY_ADDR}` load balancer IP address on your home DNS server. This will ensure DNS requests for `${SECRET_DOMAIN}` will only get routed to your `k8s_gateway` service thus providing **internal** DNS resolution to your cluster applications/ingresses from any device that uses your home DNS server. +For split DNS to work it is required to have `${SECRET_DOMAIN}` point to the `${K8S_GATEWAY_ADDR}` load balancer IP address on your home DNS server. This will ensure DNS requests for `${SECRET_DOMAIN}` will only get routed to your `k8s_gateway` service thus providing **internal** DNS resolution to your cluster applications/ingresses from any device that uses your home DNS server. For and example with Pi-Hole apply the following file and restart dnsmasq: ```sh # /etc/dnsmasq.d/99-k8s-gateway-forward.conf -server=/${SECRET_DOMAIN}/${METALLB_K8S_GATEWAY_ADDR} +server=/${SECRET_DOMAIN}/${K8S_GATEWAY_ADDR} ``` -Now try to resolve an internal-only domain with `dig @${pi-hole-ip} hajimari.${SECRET_DOMAIN}` it should resolve to your `${METALLB_INGRESS_ADDR}` IP. +Now try to resolve an internal-only domain with `dig @${pi-hole-ip} hajimari.${SECRET_DOMAIN}` it should resolve to your `${INGRESS_NGINX_ADDR}` IP. If having trouble you can ask for help in [this](https://github.com/onedr0p/flux-cluster-template/discussions/719) Github discussion. diff --git a/ansible/inventory/group_vars/kubernetes/kubernetes.yml b/ansible/inventory/group_vars/kubernetes/kubernetes.yml index 4ca217ea..9a3008ce 100644 --- a/ansible/inventory/group_vars/kubernetes/kubernetes.yml +++ b/ansible/inventory/group_vars/kubernetes/kubernetes.yml @@ -10,7 +10,7 @@ k3s_install_hard_links: true k3s_become: true k3s_etcd_datastore: true k3s_use_unsupported_config: true -k3s_registration_address: "{{ kubevip_address }}" +k3s_registration_address: "{{ kube_vip_addr }}" k3s_server_manifests_urls: # Kube-vip RBAC - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml diff --git a/ansible/inventory/group_vars/master/kubernetes.yml b/ansible/inventory/group_vars/master/kubernetes.yml index 6bbe206f..053969f7 100644 --- a/ansible/inventory/group_vars/master/kubernetes.yml +++ b/ansible/inventory/group_vars/master/kubernetes.yml @@ -6,15 +6,15 @@ k3s_control_node: true k3s_server: node-ip: "{{ ansible_host }}" tls-san: - - "{{ kubevip_address }}" + - "{{ kube_vip_addr }}" docker: false flannel-backend: "none" # This needs to be in quotes disable: - - coredns # Disable coredns - replaced with Helm Chart - - flannel # Disable flannel - replaced with Cilium + - coredns # Disable coredns - replaced with Coredns Helm Chart + - flannel # Disable flannel - replaced with Cilium Helm Chart - local-storage # Disable local-path-provisioner - installed with Flux - metrics-server # Disable metrics-server - installed with Flux - - servicelb # Disable servicelb - replaced with metallb and installed with Flux + - servicelb # Disable servicelb - replaced with Cilium Helm Chart - traefik # Disable traefik - replaced with ingress-nginx and installed with Flux disable-network-policy: true disable-cloud-controller: true diff --git a/ansible/playbooks/cluster-installation.yml b/ansible/playbooks/cluster-installation.yml index 2d1edb23..2d9a5f26 100644 --- a/ansible/playbooks/cluster-installation.yml +++ b/ansible/playbooks/cluster-installation.yml @@ -47,9 +47,11 @@ loop: - { name: cilium, kind: HelmChart, namespace: kube-system } - { name: coredns, kind: HelmChart, namespace: kube-system } - - { name: podmonitors.monitoring.coreos.com, kind: CustomResourceDefinition } - - { name: prometheusrules.monitoring.coreos.com, kind: CustomResourceDefinition } - - { name: servicemonitors.monitoring.coreos.com, kind: CustomResourceDefinition } + - { name: policy, kind: CiliumL2AnnouncementPolicy } + - { name: pool, kind: CiliumLoadBalancerIPPool } + - { name: podmonitors, kind: CustomResourceDefinition } + - { name: prometheusrules, kind: CustomResourceDefinition } + - { name: servicemonitors, kind: CustomResourceDefinition } - name: Coredns when: diff --git a/ansible/playbooks/templates/cilium-l2.yaml.j2 b/ansible/playbooks/templates/cilium-l2.yaml.j2 new file mode 100644 index 00000000..015398fd --- /dev/null +++ b/ansible/playbooks/templates/cilium-l2.yaml.j2 @@ -0,0 +1,22 @@ +--- +# https://docs.cilium.io/en/latest/network/l2-announcements +apiVersion: cilium.io/v2alpha1 +kind: CiliumL2AnnouncementPolicy +metadata: + name: policy +spec: + loadBalancerIPs: true + # NOTE: This might need to be set if you have more than one active NIC on your nodes + # interfaces: + # - ^eno[0-9]+ + nodeSelector: + matchLabels: + kubernetes.io/os: linux +--- +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: pool +spec: + cidrs: + - cidr: "{{ (ansible_default_ipv4.network + '/' + ansible_default_ipv4.netmask) | ansible.utils.ipaddr('network/prefix') }}" diff --git a/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 b/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 index ff82a487..77f844ef 100644 --- a/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 +++ b/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2 @@ -1,4 +1,5 @@ --- +# https://docs.k3s.io/helm apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: @@ -8,25 +9,41 @@ spec: repo: https://helm.cilium.io/ chart: cilium # renovate: datasource=helm depName=cilium registryUrl=https://helm.cilium.io - version: "1.13.4" + version: "1.14.0-rc.0" targetNamespace: kube-system bootstrap: true valuesContent: |- + autoDirectNodeRoutes: true bpf: masquerade: true + bgp: + enabled: false cluster: name: home-cluster id: 1 containerRuntime: integration: containerd socketPath: /var/run/k3s/containerd/containerd.sock + endpointRoutes: + enabled: true hubble: enabled: false ipam: mode: kubernetes - k8sServiceHost: "{{ kubevip_address }}" + ipv4NativeRoutingCIDR: "{{ k3s_server['cluster-cidr'] }}" + k8sServiceHost: "{{ kube_vip_addr }}" k8sServicePort: 6443 kubeProxyReplacement: strict kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + l2announcements: + enabled: true + loadBalancer: + algorithm: maglev + mode: dsr + localRedirectPolicy: true operator: - replicas: 1 + rollOutPods: true + rollOutCiliumPods: true + securityContext: + privileged: true + tunnel: disabled diff --git a/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 b/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 index 60ea8065..e1a56b3e 100644 --- a/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 +++ b/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 @@ -15,7 +15,7 @@ spec: args: ["manager"] env: - name: address - value: "{{ kubevip_address }}" + value: "{{ kube_vip_addr }}" - name: vip_arp value: "true" - name: port diff --git a/configure b/configure index 8876155a..3479ba1a 100755 --- a/configure +++ b/configure @@ -28,10 +28,9 @@ main() { verify_binaries verify_master_count verify_ansible_hosts - verify_metallb + verify_lbs verify_kubevip verify_cluster_service_cidrs - verify_addressing verify_age verify_git_repository verify_cloudflare @@ -139,66 +138,6 @@ _has_valid_ip() { fi } -verify_addressing() { - local found_kube_vip="false" - local found_k8s_gateway="false" - local found_ingress="false" - # Verify the metallb min and metallb ceiling are in the same network - metallb_subnet_min=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f1 | cut -d. -f1,2,3) - metallb_subnet_ceil=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f2 | cut -d. -f1,2,3) - if [[ "${metallb_subnet_min}" != "${metallb_subnet_ceil}" ]]; then - _log "ERROR(${FUNCNAME[0]})" "The provided MetalLB IP range '${BOOTSTRAP_METALLB_LB_RANGE}' is not in the same subnet" - exit 1 - fi - # Verify the node IP addresses are on the same network as the metallb range - for var in "${!BOOTSTRAP_ANSIBLE_HOST_ADDR_@}"; do - node_subnet=$(echo "${!var}" | cut -d. -f1,2,3) - if [[ "${node_subnet}" != "${metallb_subnet_min}" ]]; then - _log "ERROR(${FUNCNAME[0]})" "The subnet for node '${!var}' is not in the same subnet as the provided metallb range '${BOOTSTRAP_METALLB_LB_RANGE}'" - exit 1 - fi - done - # Verify the kube-vip IP is in the same network as the metallb range - kubevip_subnet=$(echo "${BOOTSTRAP_KUBE_VIP_ADDR}" | cut -d. -f1,2,3) - if [[ "${kubevip_subnet}" != "${metallb_subnet_min}" ]]; then - _log "ERROR(${FUNCNAME[0]})" "The subnet for kupe-vip '${BOOTSTRAP_KUBE_VIP_ADDR}' is not the same subnet as the provided metallb range '${BOOTSTRAP_METALLB_LB_RANGE}'" - exit 1 - fi - # Depending on the IP address, verify if it should be in the metallb range or not - metallb_octet_min=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f1 | cut -d. -f4) - metallb_octet_ceil=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f2 | cut -d. -f4) - for (( octet=metallb_octet_min; octet<=metallb_octet_ceil; octet++ )); do - addr="${metallb_subnet_min}.${octet}" - if [[ "${addr}" == "${BOOTSTRAP_KUBE_VIP_ADDR}" ]]; then - found_kube_vip="true" - fi - if [[ "${addr}" == "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}" ]]; then - found_k8s_gateway="true" - fi - if [[ "${addr}" == "${BOOTSTRAP_METALLB_INGRESS_ADDR}" ]]; then - found_ingress="true" - fi - for var in "${!BOOTSTRAP_ANSIBLE_HOST_ADDR_@}"; do - if [[ "${!var}" == "${addr}" ]]; then - _log "ERROR(${FUNCNAME[0]})" "The IP for node '${!var}' should NOT be in the provided metallb range '${BOOTSTRAP_METALLB_LB_RANGE}'" - exit 1 - fi - done - done - if [[ "${found_kube_vip}" == "true" ]]; then - _log "ERROR(${FUNCNAME[0]})" "The IP for kube-vip '${BOOTSTRAP_KUBE_VIP_ADDR}' should NOT be in the provided metallb range '${BOOTSTRAP_METALLB_LB_RANGE}'" - exit 1 - fi - if [[ "${found_k8s_gateway}" == "false" ]]; then - _log "ERROR(${FUNCNAME[0]})" "The IP for k8s_gateway '${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}' should be in the provided metallb range '${BOOTSTRAP_METALLB_LB_RANGE}'" - exit 1 - fi - if [[ "${found_ingress}" == "false" ]]; then - _log "ERROR(${FUNCNAME[0]})" "The IP for ingress '${BOOTSTRAP_METALLB_INGRESS_ADDR}' should be in the provided metallb range '${BOOTSTRAP_METALLB_LB_RANGE}'" - exit 1 - fi -} - verify_age() { _has_envar "BOOTSTRAP_AGE_PUBLIC_KEY" _has_envar "SOPS_AGE_KEY_FILE" @@ -245,14 +184,12 @@ verify_kubevip() { _has_valid_ip "${BOOTSTRAP_KUBE_VIP_ADDR}" "BOOTSTRAP_KUBE_VIP_ADDR" } -verify_metallb() { - _has_envar "BOOTSTRAP_METALLB_LB_RANGE" - _has_envar "BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR" - _has_envar "BOOTSTRAP_METALLB_INGRESS_ADDR" - _has_valid_ip "$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f1)" "BOOTSTRAP_METALLB_LB_RANGE" - _has_valid_ip "$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f2)" "BOOTSTRAP_METALLB_LB_RANGE" - _has_valid_ip "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}" "BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR" - _has_valid_ip "${BOOTSTRAP_METALLB_INGRESS_ADDR}" "BOOTSTRAP_METALLB_INGRESS_ADDR" +verify_lbs() { + #TODO: Validate IPs aren't already in use with arp + _has_envar "BOOTSTRAP_K8S_GATEWAY_ADDR" + _has_envar "BOOTSTRAP_INGRESS_NGINX_ADDR" + _has_valid_ip "${BOOTSTRAP_K8S_GATEWAY_ADDR}" "BOOTSTRAP_K8S_GATEWAY_ADDR" + _has_valid_ip "${BOOTSTRAP_INGRESS_NGINX_ADDR}" "BOOTSTRAP_INGRESS_NGINX_ADDR" } verify_cluster_service_cidrs() { @@ -352,16 +289,16 @@ verify_ansible_hosts() { _has_envar "${node_password}" "true" _has_envar "${node_control}" _has_optional_envar "${node_hostname}" - if [[ "${!node_addr}" == "${BOOTSTRAP_KUBE_VIP_ADDR}" && "${BOOTSTRAP_KUBE_VIP_ENABLED}" == "true" ]]; then + if [[ "${!node_addr}" == "${BOOTSTRAP_KUBE_VIP_ADDR}" ]]; then _log "ERROR(${FUNCNAME[0]})" "The kube-vip IP '${BOOTSTRAP_KUBE_VIP_ADDR}' should not be the same as the IP for node '${!node_addr}'" exit 1 fi - if [[ "${!node_addr}" == "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}" ]]; then - _log "ERROR(${FUNCNAME[0]})" "The k8s-gateway load balancer IP '${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}' should not be the same as the IP for node '${!node_addr}'" + if [[ "${!node_addr}" == "${BOOTSTRAP_K8S_GATEWAY_ADDR}" ]]; then + _log "ERROR(${FUNCNAME[0]})" "The k8s-gateway load balancer IP '${BOOTSTRAP_K8S_GATEWAY_ADDR}' should not be the same as the IP for node '${!node_addr}'" exit 1 fi - if [[ "${!node_addr}" == "${BOOTSTRAP_METALLB_INGRESS_ADDR}" ]]; then - _log "ERROR(${FUNCNAME[0]})" "The ingress load balancer IP '${BOOTSTRAP_METALLB_INGRESS_ADDR}' should not be the same as the IP for node '${!node_addr}'" + if [[ "${!node_addr}" == "${BOOTSTRAP_INGRESS_NGINX_ADDR}" ]]; then + _log "ERROR(${FUNCNAME[0]})" "The ingress load balancer IP '${BOOTSTRAP_INGRESS_NGINX_ADDR}' should not be the same as the IP for node '${!node_addr}'" exit 1 fi if ssh -q -o BatchMode=yes -o ConnectTimeout=5 "${!node_username}"@"${!var}" "true"; then diff --git a/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml b/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml index 7b992b50..0ca22f17 100644 --- a/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml +++ b/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: cert-manager namespace: cert-manager spec: - interval: 15m + interval: 30m chart: spec: chart: cert-manager diff --git a/kubernetes/apps/default/echo-server/app/helmrelease.yaml b/kubernetes/apps/default/echo-server/app/helmrelease.yaml index f79ea0a3..3860689b 100644 --- a/kubernetes/apps/default/echo-server/app/helmrelease.yaml +++ b/kubernetes/apps/default/echo-server/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: echo-server namespace: default spec: - interval: 15m + interval: 30m chart: spec: chart: app-template diff --git a/kubernetes/apps/default/hajimari/app/helmrelease.yaml b/kubernetes/apps/default/hajimari/app/helmrelease.yaml index 7c9868cb..a06ca044 100644 --- a/kubernetes/apps/default/hajimari/app/helmrelease.yaml +++ b/kubernetes/apps/default/hajimari/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: hajimari namespace: default spec: - interval: 15m + interval: 30m chart: spec: chart: hajimari diff --git a/kubernetes/apps/flux-system/weave-gitops/app/helmrelease.yaml b/kubernetes/apps/flux-system/weave-gitops/app/helmrelease.yaml index a43a7c50..2b1de751 100644 --- a/kubernetes/apps/flux-system/weave-gitops/app/helmrelease.yaml +++ b/kubernetes/apps/flux-system/weave-gitops/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: weave-gitops namespace: flux-system spec: - interval: 15m + interval: 30m chart: spec: chart: weave-gitops diff --git a/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml b/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml index 1a270252..7c7d32c0 100644 --- a/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml +++ b/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml @@ -5,11 +5,11 @@ metadata: name: cilium namespace: kube-system spec: - interval: 15m + interval: 30m chart: spec: chart: cilium - version: 1.13.4 + version: 1.14.0-rc.0 sourceRef: kind: HelmRepository name: cilium @@ -26,19 +26,24 @@ spec: uninstall: keepHistory: false values: + autoDirectNodeRoutes: true bpf: masquerade: true + bgp: + enabled: false cluster: name: home-cluster id: 1 containerRuntime: integration: containerd socketPath: /var/run/k3s/containerd/containerd.sock + endpointRoutes: + enabled: true hubble: enabled: true metrics: enabled: - - dns:query;ignoreAAAA + - dns:query - drop - tcp - flow @@ -66,12 +71,12 @@ spec: - *host ipam: mode: kubernetes - k8sServiceHost: "${KUBE_VIP_ADDR}" + k8sServiceHost: ${KUBE_VIP_ADDR} k8sServicePort: 6443 + ipv4NativeRoutingCIDR: ${CLUSTER_CIDR} kubeProxyReplacement: strict kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 operator: - replicas: 1 rollOutPods: true prometheus: enabled: true @@ -82,3 +87,11 @@ spec: serviceMonitor: enabled: true rollOutCiliumPods: true + l2announcements: + enabled: true + loadBalancer: + algorithm: maglev + mode: dsr + securityContext: + privileged: true + tunnel: disabled diff --git a/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml b/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml index 27f06fae..fe363fff 100644 --- a/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml +++ b/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: local-path-provisioner namespace: kube-system spec: - interval: 15m + interval: 30m chart: spec: chart: ./deploy/chart/local-path-provisioner diff --git a/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml b/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml index 9a68692b..ce92031b 100644 --- a/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml +++ b/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: metrics-server namespace: kube-system spec: - interval: 15m + interval: 30m chart: spec: chart: metrics-server diff --git a/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml b/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml index 1de73c43..2948edaa 100644 --- a/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml +++ b/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: reloader namespace: &namespace kube-system spec: - interval: 15m + interval: 30m chart: spec: chart: reloader diff --git a/kubernetes/apps/monitoring/kubernetes-dashboard/app/helmrelease.yaml b/kubernetes/apps/monitoring/kubernetes-dashboard/app/helmrelease.yaml index 432ff396..02303312 100644 --- a/kubernetes/apps/monitoring/kubernetes-dashboard/app/helmrelease.yaml +++ b/kubernetes/apps/monitoring/kubernetes-dashboard/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: kubernetes-dashboard namespace: monitoring spec: - interval: 15m + interval: 30m chart: spec: chart: kubernetes-dashboard diff --git a/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml b/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml index fc08802a..557f0c6f 100644 --- a/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml +++ b/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: cloudflared namespace: networking spec: - interval: 15m + interval: 30m chart: spec: chart: app-template diff --git a/kubernetes/apps/networking/external-dns/app/helmrelease.yaml b/kubernetes/apps/networking/external-dns/app/helmrelease.yaml index 40c40876..8474475a 100644 --- a/kubernetes/apps/networking/external-dns/app/helmrelease.yaml +++ b/kubernetes/apps/networking/external-dns/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: &app external-dns namespace: networking spec: - interval: 15m + interval: 30m chart: spec: chart: external-dns diff --git a/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml b/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml index 872797f8..dfde4edf 100644 --- a/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml +++ b/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: ingress-nginx namespace: networking spec: - interval: 15m + interval: 30m chart: spec: chart: ingress-nginx @@ -36,8 +36,8 @@ spec: service: annotations: external-dns.alpha.kubernetes.io/hostname: "ingress.${SECRET_DOMAIN}" - loadBalancerIP: "${METALLB_INGRESS_ADDR}" - externalTrafficPolicy: Local + io.cilium/lb-ipam-ips: "${INGRESS_NGINX_ADDR}" + externalTrafficPolicy: Cluster publishService: enabled: true ingressClassResource: diff --git a/kubernetes/apps/networking/ingress-nginx/ks.yaml b/kubernetes/apps/networking/ingress-nginx/ks.yaml index 89ffa390..00ab322f 100644 --- a/kubernetes/apps/networking/ingress-nginx/ks.yaml +++ b/kubernetes/apps/networking/ingress-nginx/ks.yaml @@ -25,7 +25,6 @@ metadata: spec: dependsOn: - name: cluster-apps-ingress-nginx-certificates - - name: cluster-apps-metallb-config path: ./kubernetes/apps/networking/ingress-nginx/app prune: true sourceRef: diff --git a/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml b/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml index b77f6885..df6009c0 100644 --- a/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml +++ b/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml @@ -5,7 +5,7 @@ metadata: name: k8s-gateway namespace: networking spec: - interval: 15m + interval: 30m chart: spec: chart: k8s-gateway @@ -33,5 +33,5 @@ spec: type: LoadBalancer port: 53 annotations: - metallb.universe.tf/loadBalancerIPs: "${METALLB_K8S_GATEWAY_ADDR}" - externalTrafficPolicy: Local + io.cilium/lb-ipam-ips: "${K8S_GATEWAY_ADDR}" + externalTrafficPolicy: Cluster diff --git a/kubernetes/apps/networking/k8s-gateway/ks.yaml b/kubernetes/apps/networking/k8s-gateway/ks.yaml index 9af8a36a..6f6ee444 100644 --- a/kubernetes/apps/networking/k8s-gateway/ks.yaml +++ b/kubernetes/apps/networking/k8s-gateway/ks.yaml @@ -5,8 +5,6 @@ metadata: name: cluster-apps-k8s-gateway namespace: flux-system spec: - dependsOn: - - name: cluster-apps-metallb-config path: ./kubernetes/apps/networking/k8s-gateway/app prune: true sourceRef: diff --git a/kubernetes/apps/networking/kustomization.yaml b/kubernetes/apps/networking/kustomization.yaml index 01e2814e..dd8ed3eb 100644 --- a/kubernetes/apps/networking/kustomization.yaml +++ b/kubernetes/apps/networking/kustomization.yaml @@ -7,4 +7,3 @@ resources: - ./external-dns/ks.yaml - ./ingress-nginx/ks.yaml - ./k8s-gateway/ks.yaml - - ./metallb/ks.yaml diff --git a/kubernetes/apps/networking/metallb/app/helmrelease.yaml b/kubernetes/apps/networking/metallb/app/helmrelease.yaml deleted file mode 100644 index b0565f08..00000000 --- a/kubernetes/apps/networking/metallb/app/helmrelease.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: metallb - namespace: networking -spec: - interval: 15m - chart: - spec: - chart: metallb - version: 0.13.10 - sourceRef: - kind: HelmRepository - name: metallb - namespace: flux-system - maxHistory: 2 - install: - createNamespace: true - remediation: - retries: 3 - upgrade: - cleanupOnFail: true - remediation: - retries: 3 - uninstall: - keepHistory: false - values: - crds: - enabled: true - speaker: - frr: - enabled: false diff --git a/kubernetes/apps/networking/metallb/app/kustomization.yaml b/kubernetes/apps/networking/metallb/app/kustomization.yaml deleted file mode 100644 index c83d92a8..00000000 --- a/kubernetes/apps/networking/metallb/app/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: networking -resources: - - ./helmrelease.yaml diff --git a/kubernetes/apps/networking/metallb/config/kustomization.yaml b/kubernetes/apps/networking/metallb/config/kustomization.yaml deleted file mode 100644 index b739ea95..00000000 --- a/kubernetes/apps/networking/metallb/config/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: networking -resources: - - resources.yaml diff --git a/kubernetes/apps/networking/metallb/config/resources.yaml b/kubernetes/apps/networking/metallb/config/resources.yaml deleted file mode 100644 index 7c646df4..00000000 --- a/kubernetes/apps/networking/metallb/config/resources.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: l2-ip - namespace: networking -spec: - ipAddressPools: - - default-pool ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: default-pool - namespace: networking -spec: - addresses: - - "${METALLB_LB_RANGE}" diff --git a/kubernetes/apps/networking/metallb/ks.yaml b/kubernetes/apps/networking/metallb/ks.yaml deleted file mode 100644 index 50bf4597..00000000 --- a/kubernetes/apps/networking/metallb/ks.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: cluster-apps-metallb - namespace: flux-system -spec: - path: ./kubernetes/apps/networking/metallb/app - prune: true - sourceRef: - kind: GitRepository - name: home-kubernetes - wait: true - interval: 30m - retryInterval: 1m - timeout: 5m ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: cluster-apps-metallb-config - namespace: flux-system -spec: - dependsOn: - - name: cluster-apps-metallb - path: ./kubernetes/apps/networking/metallb/config - prune: true - sourceRef: - kind: GitRepository - name: home-kubernetes - wait: true - interval: 30m - retryInterval: 1m - timeout: 5m diff --git a/kubernetes/flux/repositories/helm/kustomization.yaml b/kubernetes/flux/repositories/helm/kustomization.yaml index 4c488aa2..11e9ef5c 100644 --- a/kubernetes/flux/repositories/helm/kustomization.yaml +++ b/kubernetes/flux/repositories/helm/kustomization.yaml @@ -12,7 +12,6 @@ resources: - ./jetstack.yaml - ./k8s-gateway.yaml - ./kubernetes-dashboard.yaml - - ./metallb.yaml - ./metrics-server.yaml - ./prometheus-community.yaml - ./stakater.yaml diff --git a/kubernetes/flux/repositories/helm/metallb.yaml b/kubernetes/flux/repositories/helm/metallb.yaml deleted file mode 100644 index 61bf8079..00000000 --- a/kubernetes/flux/repositories/helm/metallb.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: source.toolkit.fluxcd.io/v1beta2 -kind: HelmRepository -metadata: - name: metallb - namespace: flux-system -spec: - interval: 1h - url: https://metallb.github.io/metallb diff --git a/tmpl/ansible/supplemental.yml b/tmpl/ansible/supplemental.yml index 7b803b72..042310cd 100644 --- a/tmpl/ansible/supplemental.yml +++ b/tmpl/ansible/supplemental.yml @@ -1,5 +1,5 @@ --- timezone: "${BOOTSTRAP_TIMEZONE}" -kubevip_address: "${BOOTSTRAP_KUBE_VIP_ADDR}" +kube_vip_addr: "${BOOTSTRAP_KUBE_VIP_ADDR}" cluster_cidr: "${BOOTSTRAP_CLUSTER_CIDR}" service_cidr: "${BOOTSTRAP_SERVICE_CIDR}" diff --git a/tmpl/kubernetes/flux/cluster-settings.yaml b/tmpl/kubernetes/flux/cluster-settings.yaml index ee04dd13..6e156bc8 100644 --- a/tmpl/kubernetes/flux/cluster-settings.yaml +++ b/tmpl/kubernetes/flux/cluster-settings.yaml @@ -6,9 +6,8 @@ metadata: namespace: flux-system data: TIMEZONE: "${BOOTSTRAP_TIMEZONE}" - METALLB_LB_RANGE: "${BOOTSTRAP_METALLB_LB_RANGE}" - METALLB_K8S_GATEWAY_ADDR: "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}" - METALLB_INGRESS_ADDR: "${BOOTSTRAP_METALLB_INGRESS_ADDR}" + K8S_GATEWAY_ADDR: "${BOOTSTRAP_K8S_GATEWAY_ADDR}" + INGRESS_NGINX_ADDR: "${BOOTSTRAP_INGRESS_NGINX_ADDR}" KUBE_VIP_ADDR: "${BOOTSTRAP_KUBE_VIP_ADDR}" CLUSTER_CIDR: "${BOOTSTRAP_CLUSTER_CIDR}" SERVICE_CIDR: "${BOOTSTRAP_SERVICE_CIDR}"