diff --git a/README.md b/README.md index 5aa2bc73..232ffc93 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The following components will be installed in your [k3s](https://k3s.io/) cluste - [kube-vip](https://kube-vip.io/) - Load balancer for the Kubernetes control plane nodes - [metallb](https://metallb.universe.tf/) - Load balancer for Kubernetes services - [cert-manager](https://cert-manager.io/) - Operator to request SSL certificates and store them as Kubernetes resources -- [calico](https://www.tigera.io/project-calico/) - Container networking interface for inter pod and service networking +- [cilium](https://cilium.io/) - Container networking interface for inter pod and service networking - [external-dns](https://github.com/kubernetes-sigs/external-dns) - Operator to publish DNS records to Cloudflare (and other providers) based on Kubernetes ingresses - [k8s_gateway](https://github.com/ori-edge/k8s_gateway) - DNS resolver that provides local DNS to your Kubernetes ingresses - [ingress-nginx](https://kubernetes.github.io/ingress-nginx/) - Kubernetes ingress controller used for a HTTP reverse proxy of Kubernetes ingresses diff --git a/ansible/inventory/group_vars/kubernetes/k3s.yml b/ansible/inventory/group_vars/kubernetes/k3s.yml index 04e12181..802ce95f 100644 --- a/ansible/inventory/group_vars/kubernetes/k3s.yml +++ b/ansible/inventory/group_vars/kubernetes/k3s.yml @@ -31,9 +31,6 @@ k3s_server_manifests_urls: # Kube-vip - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml filename: custom-kube-vip-rbac.yaml - # Tigera Operator - - url: https://raw.githubusercontent.com/projectcalico/calico/v3.25.1/manifests/tigera-operator.yaml - filename: custom-calico-tigera-operator.yaml # Essential Prometheus Operator CRDs - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml filename: custom-prometheus-podmonitors.yaml @@ -42,8 +39,12 @@ k3s_server_manifests_urls: - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml filename: custom-prometheus-servicemonitors.yaml -# (list) A flat list of templates to deploy on the primary control plane +# (list) A flat list of templates to deploy on the primary control plane node # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_templates: - - custom-calico-installation.yaml.j2 - - custom-kube-vip-daemonset.yaml.j2 + - cilium-helmchart.yaml.j2 + +# (list) A flat list of templates to deploy as static pods on all the control plane nodes +# /var/lib/rancher/k3s/agent/pod-manifests +k3s_server_pod_manifests_templates: + - kube-vip-static-pod.yaml.j2 diff --git a/ansible/inventory/group_vars/master/k3s.yml b/ansible/inventory/group_vars/master/k3s.yml index 14bda64f..48ed222e 100644 --- a/ansible/inventory/group_vars/master/k3s.yml +++ b/ansible/inventory/group_vars/master/k3s.yml @@ -26,6 +26,7 @@ k3s_server: - traefik disable-network-policy: true disable-cloud-controller: true + disable-kube-proxy: true write-kubeconfig-mode: "644" # Network CIDR to use for pod IPs cluster-cidr: "10.42.0.0/16" @@ -34,9 +35,6 @@ k3s_server: kube-controller-manager-arg: # Required to monitor kube-controller-manager with kube-prometheus-stack - "bind-address=0.0.0.0" - kube-proxy-arg: - # Required to monitor kube-proxy with kube-prometheus-stack - - "metrics-bind-address=0.0.0.0" kube-scheduler-arg: # Required to monitor kube-scheduler with kube-prometheus-stack - "bind-address=0.0.0.0" diff --git a/ansible/playbooks/cluster-installation.yml b/ansible/playbooks/cluster-installation.yml index b23b89e6..8e813dcb 100644 --- a/ansible/playbooks/cluster-installation.yml +++ b/ansible/playbooks/cluster-installation.yml @@ -68,14 +68,9 @@ wait_sleep: 10 wait_timeout: 360 loop: - - name: tigera-operator - kind: Deployment - namespace: tigera-operator - - name: kube-vip - kind: DaemonSet + - name: cilium + kind: HelmChart namespace: kube-system - - kind: Installation - name: default - name: podmonitors.monitoring.coreos.com kind: CustomResourceDefinition - name: prometheusrules.monitoring.coreos.com @@ -89,7 +84,7 @@ - k3s_control_node # NOTE - # Cleaning up the manifests from the /var/lib/rancher/k3s/server/manifests directory + # Cleaning up certain manifests from the /var/lib/rancher/k3s/server/manifests directory # is needed because k3s has an awesome "feature" to always re-deploy them when the k3s # service is restarted. Removing them does not uninstall the manifests from your cluster. diff --git a/ansible/playbooks/templates/cilium-helmchart.yaml.j2 b/ansible/playbooks/templates/cilium-helmchart.yaml.j2 new file mode 100644 index 00000000..6d171f53 --- /dev/null +++ b/ansible/playbooks/templates/cilium-helmchart.yaml.j2 @@ -0,0 +1,54 @@ +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cilium + namespace: kube-system +spec: + repo: https://helm.cilium.io/ + chart: cilium + version: 1.13.2 + targetNamespace: kube-system + bootstrap: true + valuesContent: |- + bpf: + masquerade: true + cluster: + name: home-cluster + id: 1 + containerRuntime: + integration: containerd + socketPath: /var/run/k3s/containerd/containerd.sock + hubble: + enabled: true + metrics: + enabled: + - dns:query;ignoreAAAA + - drop + - tcp + - flow + - port-distribution + - icmp + - http + relay: + enabled: true + rollOutPods: true + ui: + enabled: true + rollOutPods: true + ingress: + enabled: false + ipam: + mode: cluster-pool + operator: + clusterPoolIPv4PodCIDRList: ["{{ k3s_server['cluster-cidr'] }}"] + clusterPoolIPv4MaskSize: 24 + k8sServiceHost: "{{ kubevip_address }}" + k8sServicePort: 6443 + kubeProxyReplacement: strict + kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + operator: + enabled: true + replicas: 1 + rollOutPods: true + rollOutCiliumPods: true diff --git a/ansible/playbooks/templates/custom-calico-installation.yaml.j2 b/ansible/playbooks/templates/custom-calico-installation.yaml.j2 deleted file mode 100644 index ca127b7c..00000000 --- a/ansible/playbooks/templates/custom-calico-installation.yaml.j2 +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: operator.tigera.io/v1 -kind: Installation -metadata: - name: default -spec: - registry: quay.io - imagePath: calico - calicoNetwork: - # https://docs.k3s.io/installation/network-options#custom-cni - containerIPForwarding: "Enabled" - # https://projectcalico.docs.tigera.io/networking/ip-autodetection - nodeAddressAutodetectionV4: - cidrs: - - "{{ (ansible_default_ipv4.network + '/' + ansible_default_ipv4.netmask) | ansible.utils.ipaddr('network/prefix') }}" - # Note: The ipPools section cannot be modified post-install. - ipPools: - - blockSize: 26 - cidr: "{{ k3s_server['cluster-cidr'] }}" - encapsulation: "VXLANCrossSubnet" - natOutgoing: Enabled - nodeSelector: all() - nodeMetricsPort: 9091 - typhaMetricsPort: 9093 diff --git a/ansible/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 b/ansible/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 deleted file mode 100644 index 4a506766..00000000 --- a/ansible/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 +++ /dev/null @@ -1,72 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-vip - namespace: kube-system - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip -spec: - selector: - matchLabels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip - template: - metadata: - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip - spec: - containers: - - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.6.0 - imagePullPolicy: IfNotPresent - args: ["manager"] - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "false" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: address - value: "{{ k3s_registration_address }}" - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - serviceAccountName: kube-vip - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists diff --git a/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 b/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 new file mode 100644 index 00000000..8c81e398 --- /dev/null +++ b/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 @@ -0,0 +1,58 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: kube-vip + namespace: kube-system + labels: + app.kubernetes.io/instance: kube-vip + app.kubernetes.io/name: kube-vip +spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.6.0 + imagePullPolicy: IfNotPresent + args: ["manager"] + env: + - name: address + value: "{{ kubevip_address }}" + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: prometheus_server + value: :2112 + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW"] + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + serviceAccountName: kube-vip + volumes: + - name: kubeconfig + hostPath: + path: /etc/rancher/k3s/k3s.yaml diff --git a/kubernetes/apps/kube-system/kube-vip/app/daemonset.yaml b/kubernetes/apps/kube-system/kube-vip/app/daemonset.yaml deleted file mode 100644 index b22dab7d..00000000 --- a/kubernetes/apps/kube-system/kube-vip/app/daemonset.yaml +++ /dev/null @@ -1,72 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-vip - namespace: kube-system - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip -spec: - selector: - matchLabels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip - template: - metadata: - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip - spec: - containers: - - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.6.0 - imagePullPolicy: IfNotPresent - args: ["manager"] - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "false" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: address - value: "${KUBE_VIP_ADDR}" - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - serviceAccountName: kube-vip - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists diff --git a/kubernetes/apps/kube-system/kube-vip/app/kustomization.yaml b/kubernetes/apps/kube-system/kube-vip/app/kustomization.yaml deleted file mode 100644 index 1d0fbe56..00000000 --- a/kubernetes/apps/kube-system/kube-vip/app/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kube-system -resources: - - ./rbac.yaml - - ./daemonset.yaml -labels: - - pairs: - kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/kubernetes/apps/kube-system/kube-vip/app/rbac.yaml b/kubernetes/apps/kube-system/kube-vip/app/rbac.yaml deleted file mode 100644 index 3cd70968..00000000 --- a/kubernetes/apps/kube-system/kube-vip/app/rbac.yaml +++ /dev/null @@ -1,44 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system -secrets: - - name: kube-vip ---- -apiVersion: v1 -kind: Secret -type: kubernetes.io/service-account-token -metadata: - name: kube-vip - namespace: kube-system - annotations: - kubernetes.io/service-account.name: kube-vip ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services", "services/status", "nodes"] - verbs: ["list", "get", "watch", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role -subjects: - - kind: ServiceAccount - name: kube-vip - namespace: kube-system diff --git a/kubernetes/apps/kube-system/kube-vip/ks.yaml b/kubernetes/apps/kube-system/kube-vip/ks.yaml deleted file mode 100644 index 5007cb7c..00000000 --- a/kubernetes/apps/kube-system/kube-vip/ks.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: cluster-apps-kube-vip - namespace: flux-system -spec: - path: ./kubernetes/apps/kube-system/kube-vip/app - prune: true - sourceRef: - kind: GitRepository - name: home-kubernetes - wait: true - interval: 30m - retryInterval: 1m - timeout: 5m diff --git a/kubernetes/apps/kube-system/kustomization.yaml b/kubernetes/apps/kube-system/kustomization.yaml index 9f5916e3..9981ad8c 100644 --- a/kubernetes/apps/kube-system/kustomization.yaml +++ b/kubernetes/apps/kube-system/kustomization.yaml @@ -3,7 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - - ./kube-vip/ks.yaml - ./local-path-provisioner/ks.yaml - ./metrics-server/ks.yaml - ./reloader/ks.yaml diff --git a/tmpl/kubernetes/flux/cluster-settings.yaml b/tmpl/kubernetes/flux/cluster-settings.yaml index 5e126097..279ac4ba 100644 --- a/tmpl/kubernetes/flux/cluster-settings.yaml +++ b/tmpl/kubernetes/flux/cluster-settings.yaml @@ -6,7 +6,6 @@ metadata: namespace: flux-system data: TIMEZONE: "${BOOTSTRAP_TIMEZONE}" - KUBE_VIP_ADDR: "${BOOTSTRAP_KUBE_VIP_ADDR}" METALLB_LB_RANGE: "${BOOTSTRAP_METALLB_LB_RANGE}" METALLB_K8S_GATEWAY_ADDR: "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}" METALLB_INGRESS_ADDR: "${BOOTSTRAP_METALLB_INGRESS_ADDR}"