infra/home-cluster
2
0 Fork 0
Code Issues Pull Requests 40 Actions Packages Projects Releases Wiki Activity

feat!: replace calico with cilium (#740)

Browse Source
This commit is contained in:
Devin Buhl authored and GitHub committed 2023-05-28 11:23:09 -04:00
1 parent edb3b21310
commit 2899466303
14 files changed
+124 -258
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

ansible/inventory/group_vars/kubernetes/k3s.yml
+7 -6
View File
@@ -31,9 +31,6 @@ k3s_server_manifests_urls:
# Kube-vip
- url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml
filename: custom-kube-vip-rbac.yaml
# Tigera Operator
- url: https://raw.githubusercontent.com/projectcalico/calico/v3.25.1/manifests/tigera-operator.yaml
filename: custom-calico-tigera-operator.yaml
# Essential Prometheus Operator CRDs
- url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
filename: custom-prometheus-podmonitors.yaml
@@ -42,8 +39,12 @@ k3s_server_manifests_urls:
- url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
filename: custom-prometheus-servicemonitors.yaml
# (list) A flat list of templates to deploy on the primary control plane
# (list) A flat list of templates to deploy on the primary control plane node
# /var/lib/rancher/k3s/server/manifests
k3s_server_manifests_templates:
- custom-calico-installation.yaml.j2
- custom-kube-vip-daemonset.yaml.j2
- cilium-helmchart.yaml.j2
# (list) A flat list of templates to deploy as static pods on all the control plane nodes
# /var/lib/rancher/k3s/agent/pod-manifests
k3s_server_pod_manifests_templates:
- kube-vip-static-pod.yaml.j2
ansible/inventory/group_vars/master/k3s.yml
+1 -3
View File
@@ -26,6 +26,7 @@ k3s_server:
- traefik
disable-network-policy: true
disable-cloud-controller: true
disable-kube-proxy: true
write-kubeconfig-mode: "644"
# Network CIDR to use for pod IPs
cluster-cidr: "10.42.0.0/16"
@@ -34,9 +35,6 @@ k3s_server:
kube-controller-manager-arg:
# Required to monitor kube-controller-manager with kube-prometheus-stack
- "bind-address=0.0.0.0"
kube-proxy-arg:
# Required to monitor kube-proxy with kube-prometheus-stack
- "metrics-bind-address=0.0.0.0"
kube-scheduler-arg:
# Required to monitor kube-scheduler with kube-prometheus-stack
- "bind-address=0.0.0.0"
ansible/playbooks/cluster-installation.yml
+3 -8
View File
@@ -68,14 +68,9 @@
wait_sleep: 10
wait_timeout: 360
loop:
- name: tigera-operator
kind: Deployment
namespace: tigera-operator
- name: kube-vip
kind: DaemonSet
- name: cilium
kind: HelmChart
namespace: kube-system
- kind: Installation
name: default
- name: podmonitors.monitoring.coreos.com
kind: CustomResourceDefinition
- name: prometheusrules.monitoring.coreos.com
@@ -89,7 +84,7 @@
- k3s_control_node
# NOTE
# Cleaning up the manifests from the /var/lib/rancher/k3s/server/manifests directory
# Cleaning up certain manifests from the /var/lib/rancher/k3s/server/manifests directory
# is needed because k3s has an awesome "feature" to always re-deploy them when the k3s
# service is restarted. Removing them does not uninstall the manifests from your cluster.
ansible/playbooks/templates/cilium-helmchart.yaml.j2
+54
View File
@@ -0,0 +1,54 @@
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: cilium
namespace: kube-system
spec:
repo: https://helm.cilium.io/
chart: cilium
version: 1.13.2
targetNamespace: kube-system
bootstrap: true
valuesContent: |-
bpf:
masquerade: true
cluster:
name: home-cluster
id: 1
containerRuntime:
integration: containerd
socketPath: /var/run/k3s/containerd/containerd.sock
hubble:
enabled: true
metrics:
enabled:
- dns:query;ignoreAAAA
- drop
- tcp
- flow
- port-distribution
- icmp
- http
relay:
enabled: true
rollOutPods: true
ui:
enabled: true
rollOutPods: true
ingress:
enabled: false
ipam:
mode: cluster-pool
operator:
clusterPoolIPv4PodCIDRList: ["{{ k3s_server['cluster-cidr'] }}"]
clusterPoolIPv4MaskSize: 24
k8sServiceHost: "{{ kubevip_address }}"
k8sServicePort: 6443
kubeProxyReplacement: strict
kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
operator:
enabled: true
replicas: 1
rollOutPods: true
rollOutCiliumPods: true
ansible/playbooks/templates/custom-calico-installation.yaml.j2
-24
View File
@@ -1,24 +0,0 @@
---
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
name: default
spec:
registry: quay.io
imagePath: calico
calicoNetwork:
# https://docs.k3s.io/installation/network-options#custom-cni
containerIPForwarding: "Enabled"
# https://projectcalico.docs.tigera.io/networking/ip-autodetection
nodeAddressAutodetectionV4:
cidrs:
- "{{ (ansible_default_ipv4.network + '/' + ansible_default_ipv4.netmask) | ansible.utils.ipaddr('network/prefix') }}"
# Note: The ipPools section cannot be modified post-install.
ipPools:
- blockSize: 26
cidr: "{{ k3s_server['cluster-cidr'] }}"
encapsulation: "VXLANCrossSubnet"
natOutgoing: Enabled
nodeSelector: all()
nodeMetricsPort: 9091
typhaMetricsPort: 9093
ansible/playbooks/templates/custom-kube-vip-daemonset.yaml.j2
-72
View File
@@ -1,72 +0,0 @@
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-vip
namespace: kube-system
labels:
app.kubernetes.io/instance: kube-vip
app.kubernetes.io/name: kube-vip
spec:
selector:
matchLabels:
app.kubernetes.io/instance: kube-vip
app.kubernetes.io/name: kube-vip
template:
metadata:
labels:
app.kubernetes.io/instance: kube-vip
app.kubernetes.io/name: kube-vip
spec:
containers:
- name: kube-vip
image: ghcr.io/kube-vip/kube-vip:v0.6.0
imagePullPolicy: IfNotPresent
args: ["manager"]
env:
- name: vip_arp
value: "true"
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: svc_enable
value: "false"
- name: vip_leaderelection
value: "true"
- name: vip_leaseduration
value: "15"
- name: vip_renewdeadline
value: "10"
- name: vip_retryperiod
value: "2"
- name: address
value: "{{ k3s_registration_address }}"
securityContext:
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
serviceAccountName: kube-vip
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/master
operator: Exists
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
ansible/playbooks/templates/kube-vip-static-pod.yaml.j2
+58
View File
@@ -0,0 +1,58 @@
---
apiVersion: v1
kind: Pod
metadata:
name: kube-vip
namespace: kube-system
labels:
app.kubernetes.io/instance: kube-vip
app.kubernetes.io/name: kube-vip
spec:
containers:
- name: kube-vip
image: ghcr.io/kube-vip/kube-vip:v0.6.0
imagePullPolicy: IfNotPresent
args: ["manager"]
env:
- name: address
value: "{{ kubevip_address }}"
- name: vip_arp
value: "true"
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_ddns
value: "false"
- name: svc_enable
value: "false"
- name: vip_leaderelection
value: "true"
- name: vip_leaseduration
value: "15"
- name: vip_renewdeadline
value: "10"
- name: vip_retryperiod
value: "2"
- name: prometheus_server
value: :2112
securityContext:
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
serviceAccountName: kube-vip
volumes:
- name: kubeconfig
hostPath:
path: /etc/rancher/k3s/k3s.yaml