diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 index 12b32ca3..f8fcceff 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 @@ -1,4 +1,3 @@ -#jinja2: trim_blocks: True, lstrip_blocks: True --- apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease @@ -29,120 +28,7 @@ spec: dependsOn: - name: local-path-provisioner namespace: kube-system - values: - crds: - enabled: true - cleanPrometheusOperatorObjectNames: true - alertmanager: - enabled: false - kube-state-metrics: - metricLabelsAllowlist: - - "deployments=[*]" - - "persistentvolumeclaims=[*]" - - "pods=[*]" - prometheus: - monitor: - enabled: true - relabelings: - - action: replace - sourceLabels: ["__meta_kubernetes_pod_node_name"] - regex: ^(.*)$ - replacement: $1 - targetLabel: kubernetes_node - kubelet: - enabled: true - serviceMonitor: - metricRelabelings: - # Remove duplicate labels provided by k3s - - action: keep - sourceLabels: ["__name__"] - regex: (apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authentication_token|cadvisor_version|container_blkio|container_cpu|container_fs|container_last|container_memory|container_network|container_oom|container_processes|container|csi_operations|disabled_metric|get_token|go|hidden_metric|kubelet_certificate|kubelet_cgroup|kubelet_container|kubelet_containers|kubelet_cpu|kubelet_device|kubelet_graceful|kubelet_http|kubelet_lifecycle|kubelet_managed|kubelet_node|kubelet_pleg|kubelet_pod|kubelet_run|kubelet_running|kubelet_runtime|kubelet_server|kubelet_started|kubelet_volume|kubernetes_build|kubernetes_feature|machine_cpu|machine_memory|machine_nvm|machine_scrape|node_namespace|plugin_manager|prober_probe|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|storage_operation|volume_manager|volume_operation|workqueue)_(.+) - - action: replace - sourceLabels: ["node"] - targetLabel: instance - # Drop high cardinality labels - - action: labeldrop - regex: (uid) - - action: labeldrop - regex: (id|name) - - action: drop - sourceLabels: ["__name__"] - regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count) - kubeApiServer: - enabled: true - serviceMonitor: - metricRelabelings: - # Remove duplicate labels provided by k3s - - action: keep - sourceLabels: ["__name__"] - regex: (aggregator_openapi|aggregator_unavailable|apiextensions_openapi|apiserver_admission|apiserver_audit|apiserver_cache|apiserver_cel|apiserver_client|apiserver_crd|apiserver_current|apiserver_envelope|apiserver_flowcontrol|apiserver_init|apiserver_kube|apiserver_longrunning|apiserver_request|apiserver_requested|apiserver_response|apiserver_selfrequest|apiserver_storage|apiserver_terminated|apiserver_tls|apiserver_watch|apiserver_webhooks|authenticated_user|authentication|disabled_metric|etcd_bookmark|etcd_lease|etcd_request|field_validation|get_token|go|grpc_client|hidden_metric|kube_apiserver|kubernetes_build|kubernetes_feature|node_authorizer|pod_security|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|serviceaccount_legacy|serviceaccount_stale|serviceaccount_valid|watch_cache|workqueue)_(.+) - # Drop high cardinality labels - - action: drop - sourceLabels: ["__name__"] - regex: (apiserver|etcd|rest_client)_request(|_sli|_slo)_duration_seconds_bucket - - action: drop - sourceLabels: ["__name__"] - regex: (apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket) - kubeControllerManager: - enabled: true - endpoints: &cp - {% for item in bootstrap_nodes.master %} - - {{ item.address }} - {% endfor %} - serviceMonitor: - metricRelabelings: - # Remove duplicate labels provided by k3s - - action: keep - sourceLabels: ["__name__"] - regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|attachdetach_controller|authenticated_user|authentication|cronjob_controller|disabled_metric|endpoint_slice|ephemeral_volume|garbagecollector_controller|get_token|go|hidden_metric|job_controller|kubernetes_build|kubernetes_feature|leader_election|node_collector|node_ipam|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|pv_collector|registered_metric|replicaset_controller|rest_client|retroactive_storageclass|root_ca|running_managed|scrape_duration|scrape_samples|scrape_series|service_controller|storage_count|storage_operation|ttl_after|volume_operation|workqueue)_(.+)" - kubeEtcd: - enabled: true - endpoints: *cp - kubeScheduler: - enabled: true - endpoints: *cp - serviceMonitor: - metricRelabelings: - # Remove duplicate labels provided by k3s - - action: keep - sourceLabels: ["__name__"] - regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authenticated_user|authentication|disabled_metric|go|hidden_metric|kubernetes_build|kubernetes_feature|leader_election|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scheduler|scrape_duration|scrape_samples|scrape_series|workqueue)_(.+)" - kubeProxy: - enabled: false # Disabled due to eBPF - prometheus: - ingress: - enabled: true - ingressClassName: internal - annotations: - hajimari.io/appName: Prometheus - hajimari.io/icon: simple-icons:prometheus - pathType: Prefix - hosts: - - &host "prometheus.${SECRET_DOMAIN}" - tls: - - hosts: - - *host - prometheusSpec: - ruleSelectorNilUsesHelmValues: false - serviceMonitorSelectorNilUsesHelmValues: false - podMonitorSelectorNilUsesHelmValues: false - probeSelectorNilUsesHelmValues: false - scrapeConfigSelectorNilUsesHelmValues: false - enableAdminAPI: true - walCompression: true - retentionSize: 8GB - storageSpec: - volumeClaimTemplate: - spec: - storageClassName: local-path - resources: - requests: - storage: 10Gi - grafana: - enabled: false - forceDeployDashboards: true - sidecar: - dashboards: - multicluster: - etcd: - enabled: true + valuesFrom: + - kind: ConfigMap + name: kube-prometheus-stack-values + valuesKey: values.yaml diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 new file mode 100644 index 00000000..43ce08ec --- /dev/null +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 @@ -0,0 +1,125 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-prometheus-stack-values + namespace: monitoring +data: + values.yaml: | + crds: + enabled: true + cleanPrometheusOperatorObjectNames: true + alertmanager: + enabled: false + kube-state-metrics: + metricLabelsAllowlist: + - "deployments=[*]" + - "persistentvolumeclaims=[*]" + - "pods=[*]" + prometheus: + monitor: + enabled: true + relabelings: + - action: replace + sourceLabels: ["__meta_kubernetes_pod_node_name"] + regex: ^(.*)$ + replacement: $1 + targetLabel: kubernetes_node + kubelet: + enabled: true + serviceMonitor: + metricRelabelings: + # Remove duplicate labels provided by k3s + - action: keep + sourceLabels: ["__name__"] + regex: (apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authentication_token|cadvisor_version|container_blkio|container_cpu|container_fs|container_last|container_memory|container_network|container_oom|container_processes|container|csi_operations|disabled_metric|get_token|go|hidden_metric|kubelet_certificate|kubelet_cgroup|kubelet_container|kubelet_containers|kubelet_cpu|kubelet_device|kubelet_graceful|kubelet_http|kubelet_lifecycle|kubelet_managed|kubelet_node|kubelet_pleg|kubelet_pod|kubelet_run|kubelet_running|kubelet_runtime|kubelet_server|kubelet_started|kubelet_volume|kubernetes_build|kubernetes_feature|machine_cpu|machine_memory|machine_nvm|machine_scrape|node_namespace|plugin_manager|prober_probe|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|storage_operation|volume_manager|volume_operation|workqueue)_(.+) + - action: replace + sourceLabels: ["node"] + targetLabel: instance + # Drop high cardinality labels + - action: labeldrop + regex: (uid) + - action: labeldrop + regex: (id|name) + - action: drop + sourceLabels: ["__name__"] + regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count) + kubeApiServer: + enabled: true + serviceMonitor: + metricRelabelings: + # Remove duplicate labels provided by k3s + - action: keep + sourceLabels: ["__name__"] + regex: (aggregator_openapi|aggregator_unavailable|apiextensions_openapi|apiserver_admission|apiserver_audit|apiserver_cache|apiserver_cel|apiserver_client|apiserver_crd|apiserver_current|apiserver_envelope|apiserver_flowcontrol|apiserver_init|apiserver_kube|apiserver_longrunning|apiserver_request|apiserver_requested|apiserver_response|apiserver_selfrequest|apiserver_storage|apiserver_terminated|apiserver_tls|apiserver_watch|apiserver_webhooks|authenticated_user|authentication|disabled_metric|etcd_bookmark|etcd_lease|etcd_request|field_validation|get_token|go|grpc_client|hidden_metric|kube_apiserver|kubernetes_build|kubernetes_feature|node_authorizer|pod_security|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|serviceaccount_legacy|serviceaccount_stale|serviceaccount_valid|watch_cache|workqueue)_(.+) + # Drop high cardinality labels + - action: drop + sourceLabels: ["__name__"] + regex: (apiserver|etcd|rest_client)_request(|_sli|_slo)_duration_seconds_bucket + - action: drop + sourceLabels: ["__name__"] + regex: (apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket) + kubeControllerManager: + enabled: true + endpoints: &cp + {% for item in bootstrap_nodes.master %} + - {{ item.address }} + {% endfor %} + serviceMonitor: + metricRelabelings: + # Remove duplicate labels provided by k3s + - action: keep + sourceLabels: ["__name__"] + regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|attachdetach_controller|authenticated_user|authentication|cronjob_controller|disabled_metric|endpoint_slice|ephemeral_volume|garbagecollector_controller|get_token|go|hidden_metric|job_controller|kubernetes_build|kubernetes_feature|leader_election|node_collector|node_ipam|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|pv_collector|registered_metric|replicaset_controller|rest_client|retroactive_storageclass|root_ca|running_managed|scrape_duration|scrape_samples|scrape_series|service_controller|storage_count|storage_operation|ttl_after|volume_operation|workqueue)_(.+)" + kubeEtcd: + enabled: true + endpoints: *cp + kubeScheduler: + enabled: true + endpoints: *cp + serviceMonitor: + metricRelabelings: + # Remove duplicate labels provided by k3s + - action: keep + sourceLabels: ["__name__"] + regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authenticated_user|authentication|disabled_metric|go|hidden_metric|kubernetes_build|kubernetes_feature|leader_election|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scheduler|scrape_duration|scrape_samples|scrape_series|workqueue)_(.+)" + kubeProxy: + enabled: false # Disabled due to eBPF + prometheus: + ingress: + enabled: true + ingressClassName: internal + annotations: + hajimari.io/appName: Prometheus + hajimari.io/icon: simple-icons:prometheus + pathType: Prefix + hosts: + - &host "prometheus.${SECRET_DOMAIN}" + tls: + - hosts: + - *host + prometheusSpec: + ruleSelectorNilUsesHelmValues: false + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + probeSelectorNilUsesHelmValues: false + scrapeConfigSelectorNilUsesHelmValues: false + enableAdminAPI: true + walCompression: true + retentionSize: 8GB + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: local-path + resources: + requests: + storage: 10Gi + grafana: + enabled: false + forceDeployDashboards: true + sidecar: + dashboards: + multicluster: + etcd: + enabled: true diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/kustomization.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/kustomization.yaml.j2 index 2469b52a..56c7f044 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/app/kustomization.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/kustomization.yaml.j2 @@ -3,4 +3,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: monitoring resources: + - ./helmvalues.yaml - ./helmrelease.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 index 0e2b2125..9ad9954b 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2 @@ -1,4 +1,3 @@ -#jinja2: trim_blocks: True, lstrip_blocks: True --- apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease @@ -25,103 +24,7 @@ spec: retries: 3 uninstall: keepHistory: false - values: - autoDirectNodeRoutes: true - bpf: - masquerade: true - bgp: - enabled: false - cluster: - name: home-cluster - id: 1 - containerRuntime: - integration: containerd - socketPath: /var/run/k3s/containerd/containerd.sock - endpointRoutes: - enabled: true - hubble: - enabled: true - metrics: - enabled: - - dns:query - - drop - - tcp - - flow - - port-distribution - - icmp - - http - serviceMonitor: - enabled: true - dashboards: - enabled: true - annotations: - grafana_folder: Cilium - relay: - enabled: true - rollOutPods: true - prometheus: - serviceMonitor: - enabled: true - ui: - enabled: true - rollOutPods: true - ingress: - enabled: true - className: internal - annotations: - hajimari.io/icon: simple-icons:cilium - hosts: - - &host "hubble.${SECRET_DOMAIN}" - tls: - - hosts: - - *host - ipam: - mode: kubernetes - ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}" - {% if bootstrap_ipv6_enabled | default(false) %} - ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}" - ipv6: - enabled: true - {% endif %} - k8sServiceHost: "${KUBE_VIP_ADDR}" - k8sServicePort: 6443 - kubeProxyReplacement: true - kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 - l2announcements: - {% if bootstrap_ipv6_enabled | default(false) %} - enabled: false - {% else %} - enabled: true - # https://github.com/cilium/cilium/issues/26586 - leaseDuration: 120s - leaseRenewDeadline: 60s - leaseRetryPeriod: 1s - {% endif %} - loadBalancer: - algorithm: maglev - mode: dsr - localRedirectPolicy: true - operator: - replicas: 1 - rollOutPods: true - prometheus: - enabled: true - serviceMonitor: - enabled: true - dashboards: - enabled: true - annotations: - grafana_folder: Cilium - prometheus: - enabled: true - serviceMonitor: - enabled: true - trustCRDsExist: true - dashboards: - enabled: true - annotations: - grafana_folder: Cilium - rollOutCiliumPods: true - securityContext: - privileged: true - tunnel: disabled + valuesFrom: + - kind: ConfigMap + name: cilium-values + valuesKey: values.yaml diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 new file mode 100644 index 00000000..798e02d5 --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 @@ -0,0 +1,108 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-values + namespace: kube-system +data: + values.yaml: | + autoDirectNodeRoutes: true + bpf: + masquerade: true + bgp: + enabled: false + cluster: + name: home-cluster + id: 1 + containerRuntime: + integration: containerd + socketPath: /var/run/k3s/containerd/containerd.sock + endpointRoutes: + enabled: true + hubble: + enabled: true + metrics: + enabled: + - dns:query + - drop + - tcp + - flow + - port-distribution + - icmp + - http + serviceMonitor: + enabled: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + relay: + enabled: true + rollOutPods: true + prometheus: + serviceMonitor: + enabled: true + ui: + enabled: true + rollOutPods: true + ingress: + enabled: true + className: internal + annotations: + hajimari.io/icon: simple-icons:cilium + hosts: + - &host "hubble.${SECRET_DOMAIN}" + tls: + - hosts: + - *host + ipam: + mode: kubernetes + ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}" + {% if bootstrap_ipv6_enabled | default(false) %} + ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}" + ipv6: + enabled: true + {% endif %} + k8sServiceHost: "${KUBE_VIP_ADDR}" + k8sServicePort: 6443 + kubeProxyReplacement: true + kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + l2announcements: + {% if bootstrap_ipv6_enabled | default(false) %} + enabled: false + {% else %} + enabled: true + # https://github.com/cilium/cilium/issues/26586 + leaseDuration: 120s + leaseRenewDeadline: 60s + leaseRetryPeriod: 1s + {% endif %} + loadBalancer: + algorithm: maglev + mode: dsr + localRedirectPolicy: true + operator: + replicas: 1 + rollOutPods: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + prometheus: + enabled: true + serviceMonitor: + enabled: true + trustCRDsExist: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + rollOutCiliumPods: true + securityContext: + privileged: true + tunnel: disabled diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 index aa7aace1..5d10cf6f 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/kustomization.yaml.j2 @@ -7,4 +7,5 @@ resources: {% if not bootstrap_ipv6_enabled | default(false) %} - ./cilium-l2.yaml {% endif %} + - ./helmvalues.yaml - ./helmrelease.yaml