feat!: replace calico with cilium (#740)

README.md

@@ -31,7 +31,7 @@ The following components will be installed in your [k3s](https://k3s.io/) cluste
 - [kube-vip](https://kube-vip.io/) - Load balancer for the Kubernetes control plane nodes
 - [metallb](https://metallb.universe.tf/) - Load balancer for Kubernetes services
 - [cert-manager](https://cert-manager.io/) - Operator to request SSL certificates and store them as Kubernetes resources
-- [calico](https://www.tigera.io/project-calico/) - Container networking interface for inter pod and service networking
+- [cilium](https://cilium.io/) - Container networking interface for inter pod and service networking
 - [external-dns](https://github.com/kubernetes-sigs/external-dns) - Operator to publish DNS records to Cloudflare (and other providers) based on Kubernetes ingresses
 - [k8s_gateway](https://github.com/ori-edge/k8s_gateway) - DNS resolver that provides local DNS to your Kubernetes ingresses
 - [ingress-nginx](https://kubernetes.github.io/ingress-nginx/) - Kubernetes ingress controller used for a HTTP reverse proxy of Kubernetes ingresses

ansible/inventory/group_vars/kubernetes/k3s.yml

@@ -31,9 +31,6 @@ k3s_server_manifests_urls:
   # Kube-vip
   - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml
     filename: custom-kube-vip-rbac.yaml
-  # Tigera Operator
-  - url: https://raw.githubusercontent.com/projectcalico/calico/v3.25.1/manifests/tigera-operator.yaml
-    filename: custom-calico-tigera-operator.yaml
   # Essential Prometheus Operator CRDs
   - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
     filename: custom-prometheus-podmonitors.yaml
@@ -42,8 +39,12 @@ k3s_server_manifests_urls:
   - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
     filename: custom-prometheus-servicemonitors.yaml
 
-# (list) A flat list of templates to deploy on the primary control plane
+# (list) A flat list of templates to deploy on the primary control plane node
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
-  - custom-calico-installation.yaml.j2
-  - custom-kube-vip-daemonset.yaml.j2
+  - cilium-helmchart.yaml.j2
+
+# (list) A flat list of templates to deploy as static pods on all the control plane nodes
+# /var/lib/rancher/k3s/agent/pod-manifests
+k3s_server_pod_manifests_templates:
+  - kube-vip-static-pod.yaml.j2

ansible/inventory/group_vars/master/k3s.yml

@@ -26,6 +26,7 @@ k3s_server:
     - traefik
   disable-network-policy: true
   disable-cloud-controller: true
+  disable-kube-proxy: true
   write-kubeconfig-mode: "644"
   # Network CIDR to use for pod IPs
   cluster-cidr: "10.42.0.0/16"
@@ -34,9 +35,6 @@ k3s_server:
   kube-controller-manager-arg:
     # Required to monitor kube-controller-manager with kube-prometheus-stack
     - "bind-address=0.0.0.0"
-  kube-proxy-arg:
-    # Required to monitor kube-proxy with kube-prometheus-stack
-    - "metrics-bind-address=0.0.0.0"
   kube-scheduler-arg:
     # Required to monitor kube-scheduler with kube-prometheus-stack
     - "bind-address=0.0.0.0"

ansible/playbooks/cluster-installation.yml

@@ -68,14 +68,9 @@
         wait_sleep: 10
         wait_timeout: 360
       loop:
-        - name: tigera-operator
-          kind: Deployment
-          namespace: tigera-operator
-        - name: kube-vip
-          kind: DaemonSet
+        - name: cilium
+          kind: HelmChart
           namespace: kube-system
-        - kind: Installation
-          name: default
         - name: podmonitors.monitoring.coreos.com
           kind: CustomResourceDefinition
         - name: prometheusrules.monitoring.coreos.com
@@ -89,7 +84,7 @@
         - k3s_control_node
 
     # NOTE
-    # Cleaning up the manifests from the /var/lib/rancher/k3s/server/manifests directory
+    # Cleaning up certain manifests from the /var/lib/rancher/k3s/server/manifests directory
     # is needed because k3s has an awesome "feature" to always re-deploy them when the k3s
     # service is restarted. Removing them does not uninstall the manifests from your cluster.
 

ansible/playbooks/templates/cilium-helmchart.yaml.j2

@@ -0,0 +1,54 @@
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: cilium
+  namespace: kube-system
+spec:
+  repo: https://helm.cilium.io/
+  chart: cilium
+  version: 1.13.2
+  targetNamespace: kube-system
+  bootstrap: true
+  valuesContent: |-
+    bpf:
+      masquerade: true
+    cluster:
+      name: home-cluster
+      id: 1
+    containerRuntime:
+      integration: containerd
+      socketPath: /var/run/k3s/containerd/containerd.sock
+    hubble:
+      enabled: true
+      metrics:
+        enabled:
+          - dns:query;ignoreAAAA
+          - drop
+          - tcp
+          - flow
+          - port-distribution
+          - icmp
+          - http
+      relay:
+        enabled: true
+        rollOutPods: true
+      ui:
+        enabled: true
+        rollOutPods: true
+        ingress:
+          enabled: false
+    ipam:
+      mode: cluster-pool
+      operator:
+        clusterPoolIPv4PodCIDRList: ["{{ k3s_server['cluster-cidr'] }}"]
+        clusterPoolIPv4MaskSize: 24
+    k8sServiceHost: "{{ kubevip_address }}"
+    k8sServicePort: 6443
+    kubeProxyReplacement: strict
+    kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+    operator:
+      enabled: true
+      replicas: 1
+      rollOutPods: true
+    rollOutCiliumPods: true

ansible/playbooks/templates/custom-calico-installation.yaml.j2

@@ -1,24 +0,0 @@
----
-apiVersion: operator.tigera.io/v1
-kind: Installation
-metadata:
-  name: default
-spec:
-  registry: quay.io
-  imagePath: calico
-  calicoNetwork:
-    # https://docs.k3s.io/installation/network-options#custom-cni
-    containerIPForwarding: "Enabled"
-    # https://projectcalico.docs.tigera.io/networking/ip-autodetection
-    nodeAddressAutodetectionV4:
-      cidrs:
-        - "{{ (ansible_default_ipv4.network + '/' + ansible_default_ipv4.netmask) | ansible.utils.ipaddr('network/prefix') }}"
-    # Note: The ipPools section cannot be modified post-install.
-    ipPools:
-      - blockSize: 26
-        cidr: "{{ k3s_server['cluster-cidr'] }}"
-        encapsulation: "VXLANCrossSubnet"
-        natOutgoing: Enabled
-        nodeSelector: all()
-  nodeMetricsPort: 9091
-  typhaMetricsPort: 9093

ansible/playbooks/templates/custom-kube-vip-daemonset.yaml.j2

@@ -1,72 +0,0 @@
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-vip
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/instance: kube-vip
-    app.kubernetes.io/name: kube-vip
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: kube-vip
-      app.kubernetes.io/name: kube-vip
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: kube-vip
-        app.kubernetes.io/name: kube-vip
-    spec:
-      containers:
-        - name: kube-vip
-          image: ghcr.io/kube-vip/kube-vip:v0.6.0
-          imagePullPolicy: IfNotPresent
-          args: ["manager"]
-          env:
-            - name: vip_arp
-              value: "true"
-            - name: port
-              value: "6443"
-            - name: vip_cidr
-              value: "32"
-            - name: cp_enable
-              value: "true"
-            - name: cp_namespace
-              value: kube-system
-            - name: svc_enable
-              value: "false"
-            - name: vip_leaderelection
-              value: "true"
-            - name: vip_leaseduration
-              value: "15"
-            - name: vip_renewdeadline
-              value: "10"
-            - name: vip_retryperiod
-              value: "2"
-            - name: address
-              value: "{{ k3s_registration_address }}"
-          securityContext:
-            capabilities:
-              add: ["NET_ADMIN", "NET_RAW"]
-      hostAliases:
-        - hostnames:
-            - kubernetes
-          ip: 127.0.0.1
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/master
-                    operator: Exists
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists

ansible/playbooks/templates/kube-vip-static-pod.yaml.j2

@@ -0,0 +1,58 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-vip
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/instance: kube-vip
+    app.kubernetes.io/name: kube-vip
+spec:
+  containers:
+    - name: kube-vip
+      image: ghcr.io/kube-vip/kube-vip:v0.6.0
+      imagePullPolicy: IfNotPresent
+      args: ["manager"]
+      env:
+        - name: address
+          value: "{{ kubevip_address }}"
+        - name: vip_arp
+          value: "true"
+        - name: port
+          value: "6443"
+        - name: vip_cidr
+          value: "32"
+        - name: cp_enable
+          value: "true"
+        - name: cp_namespace
+          value: kube-system
+        - name: vip_ddns
+          value: "false"
+        - name: svc_enable
+          value: "false"
+        - name: vip_leaderelection
+          value: "true"
+        - name: vip_leaseduration
+          value: "15"
+        - name: vip_renewdeadline
+          value: "10"
+        - name: vip_retryperiod
+          value: "2"
+        - name: prometheus_server
+          value: :2112
+      securityContext:
+        capabilities:
+          add: ["NET_ADMIN", "NET_RAW"]
+      volumeMounts:
+        - mountPath: /etc/kubernetes/admin.conf
+          name: kubeconfig
+  hostAliases:
+    - hostnames:
+        - kubernetes
+      ip: 127.0.0.1
+  hostNetwork: true
+  serviceAccountName: kube-vip
+  volumes:
+    - name: kubeconfig
+      hostPath:
+        path: /etc/rancher/k3s/k3s.yaml

kubernetes/apps/kube-system/kube-vip/app/daemonset.yaml

@@ -1,72 +0,0 @@
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-vip
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/instance: kube-vip
-    app.kubernetes.io/name: kube-vip
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: kube-vip
-      app.kubernetes.io/name: kube-vip
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: kube-vip
-        app.kubernetes.io/name: kube-vip
-    spec:
-      containers:
-        - name: kube-vip
-          image: ghcr.io/kube-vip/kube-vip:v0.6.0
-          imagePullPolicy: IfNotPresent
-          args: ["manager"]
-          env:
-            - name: vip_arp
-              value: "true"
-            - name: port
-              value: "6443"
-            - name: vip_cidr
-              value: "32"
-            - name: cp_enable
-              value: "true"
-            - name: cp_namespace
-              value: kube-system
-            - name: svc_enable
-              value: "false"
-            - name: vip_leaderelection
-              value: "true"
-            - name: vip_leaseduration
-              value: "15"
-            - name: vip_renewdeadline
-              value: "10"
-            - name: vip_retryperiod
-              value: "2"
-            - name: address
-              value: "${KUBE_VIP_ADDR}"
-          securityContext:
-            capabilities:
-              add: ["NET_ADMIN", "NET_RAW"]
-      hostAliases:
-        - hostnames:
-            - kubernetes
-          ip: 127.0.0.1
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/master
-                    operator: Exists
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists

kubernetes/apps/kube-system/kube-vip/app/kustomization.yaml

@@ -1,10 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: kube-system
-resources:
-  - ./rbac.yaml
-  - ./daemonset.yaml
-labels:
-  - pairs:
-      kustomize.toolkit.fluxcd.io/prune: disabled

kubernetes/apps/kube-system/kube-vip/app/rbac.yaml

@@ -1,44 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
-secrets:
-  - name: kube-vip
----
-apiVersion: v1
-kind: Secret
-type: kubernetes.io/service-account-token
-metadata:
-  name: kube-vip
-  namespace: kube-system
-  annotations:
-    kubernetes.io/service-account.name: kube-vip
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status", "nodes"]
-    verbs: ["list", "get", "watch", "update"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["list", "get", "watch", "update", "create"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-  - kind: ServiceAccount
-    name: kube-vip
-    namespace: kube-system

kubernetes/apps/kube-system/kube-vip/ks.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: cluster-apps-kube-vip
-  namespace: flux-system
-spec:
-  path: ./kubernetes/apps/kube-system/kube-vip/app
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: true
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m

kubernetes/apps/kube-system/kustomization.yaml

@@ -3,7 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
-  - ./kube-vip/ks.yaml
   - ./local-path-provisioner/ks.yaml
   - ./metrics-server/ks.yaml
   - ./reloader/ks.yaml

tmpl/kubernetes/flux/cluster-settings.yaml

@@ -6,7 +6,6 @@ metadata:
   namespace: flux-system
 data:
   TIMEZONE: "${BOOTSTRAP_TIMEZONE}"
-  KUBE_VIP_ADDR: "${BOOTSTRAP_KUBE_VIP_ADDR}"
   METALLB_LB_RANGE: "${BOOTSTRAP_METALLB_LB_RANGE}"
   METALLB_K8S_GATEWAY_ADDR: "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}"
   METALLB_INGRESS_ADDR: "${BOOTSTRAP_METALLB_INGRESS_ADDR}"