From 3a74be1eeb9e2bb3726d1b09e2ab5c0c8e939a47 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Sat, 22 Jul 2023 08:54:03 -0400 Subject: [PATCH] feat: Update KPS and include scrapeconfigs in bootstrap Signed-off-by: Devin Buhl --- .../app/helmrelease.yaml.j2 | 49 ++++++++++++------- .../group_vars/kubernetes/main.yaml.j2 | 2 + .../playbooks/cluster-installation.yaml.j2 | 1 + 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 index ea41f6ad..cac91282 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 @@ -11,7 +11,7 @@ spec: chart: spec: chart: kube-prometheus-stack - version: 47.1.0 + version: 48.1.2 sourceRef: kind: HelmRepository name: prometheus-community @@ -35,17 +35,11 @@ spec: - name: ingress-nginx namespace: networking values: + crds: + enabled: true cleanPrometheusOperatorObjectNames: true alertmanager: enabled: false - grafana: - enabled: false - forceDeployDashboards: true - sidecar: - dashboards: - multicluster: - etcd: - enabled: true kube-state-metrics: metricLabelsAllowlist: - "pods=[*]" @@ -56,7 +50,7 @@ spec: enabled: true relabelings: - action: replace - regex: (.*) + regex: ^(.*)$ replacement: $1 sourceLabels: ["__meta_kubernetes_pod_node_name"] targetLabel: kubernetes_node @@ -64,22 +58,30 @@ spec: enabled: true serviceMonitor: metricRelabelings: - # Remove duplicate metrics + # Remove duplicate labels provided by k3s - sourceLabels: ["__name__"] - regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authentication_token|cadvisor_version|container_blkio|container_cpu|container_fs|container_last|container_memory|container_network|container_oom|container_processes|container|csi_operations|disabled_metric|get_token|go|hidden_metric|kubelet_certificate|kubelet_cgroup|kubelet_container|kubelet_containers|kubelet_cpu|kubelet_device|kubelet_graceful|kubelet_http|kubelet_lifecycle|kubelet_managed|kubelet_node|kubelet_pleg|kubelet_pod|kubelet_run|kubelet_running|kubelet_runtime|kubelet_server|kubelet_started|kubelet_volume|kubernetes_build|kubernetes_feature|machine_cpu|machine_memory|machine_nvm|machine_scrape|node_namespace|plugin_manager|prober_probe|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|storage_operation|volume_manager|volume_operation|workqueue)_(.+)" + regex: (apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authentication_token|cadvisor_version|container_blkio|container_cpu|container_fs|container_last|container_memory|container_network|container_oom|container_processes|container|csi_operations|disabled_metric|get_token|go|hidden_metric|kubelet_certificate|kubelet_cgroup|kubelet_container|kubelet_containers|kubelet_cpu|kubelet_device|kubelet_graceful|kubelet_http|kubelet_lifecycle|kubelet_managed|kubelet_node|kubelet_pleg|kubelet_pod|kubelet_run|kubelet_running|kubelet_runtime|kubelet_server|kubelet_started|kubelet_volume|kubernetes_build|kubernetes_feature|machine_cpu|machine_memory|machine_nvm|machine_scrape|node_namespace|plugin_manager|prober_probe|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|storage_operation|volume_manager|volume_operation|workqueue)_(.+) action: keep - sourceLabels: ["node"] targetLabel: instance action: replace + # Drop high cardinality labels + - action: labeldrop + regex: (uid) + - action: labeldrop + regex: (id|name) + - action: drop + source_labels: ["__name__"] + regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count) kubeApiServer: enabled: true serviceMonitor: metricRelabelings: - # Remove duplicate metrics + # Remove duplicate labels provided by k3s - sourceLabels: ["__name__"] - regex: "(aggregator_openapi|aggregator_unavailable|apiextensions_openapi|apiserver_admission|apiserver_audit|apiserver_cache|apiserver_cel|apiserver_client|apiserver_crd|apiserver_current|apiserver_envelope|apiserver_flowcontrol|apiserver_init|apiserver_kube|apiserver_longrunning|apiserver_request|apiserver_requested|apiserver_response|apiserver_selfrequest|apiserver_storage|apiserver_terminated|apiserver_tls|apiserver_watch|apiserver_webhooks|authenticated_user|authentication|disabled_metric|etcd_bookmark|etcd_lease|etcd_request|field_validation|get_token|go|grpc_client|hidden_metric|kube_apiserver|kubernetes_build|kubernetes_feature|node_authorizer|pod_security|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|serviceaccount_legacy|serviceaccount_stale|serviceaccount_valid|watch_cache|workqueue)_(.+)" + regex: (aggregator_openapi|aggregator_unavailable|apiextensions_openapi|apiserver_admission|apiserver_audit|apiserver_cache|apiserver_cel|apiserver_client|apiserver_crd|apiserver_current|apiserver_envelope|apiserver_flowcontrol|apiserver_init|apiserver_kube|apiserver_longrunning|apiserver_request|apiserver_requested|apiserver_response|apiserver_selfrequest|apiserver_storage|apiserver_terminated|apiserver_tls|apiserver_watch|apiserver_webhooks|authenticated_user|authentication|disabled_metric|etcd_bookmark|etcd_lease|etcd_request|field_validation|get_token|go|grpc_client|hidden_metric|kube_apiserver|kubernetes_build|kubernetes_feature|node_authorizer|pod_security|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scrape_duration|scrape_samples|scrape_series|serviceaccount_legacy|serviceaccount_stale|serviceaccount_valid|watch_cache|workqueue)_(.+) action: keep - # Remove high cardinality metrics + # Drop high cardinality labels - sourceLabels: ["__name__"] regex: (apiserver|etcd|rest_client)_request(|_sli|_slo)_duration_seconds_bucket action: drop @@ -94,24 +96,24 @@ spec: {% endfor %} serviceMonitor: metricRelabelings: - # Remove duplicate metrics + # Remove duplicate labels provided by k3s - sourceLabels: ["__name__"] regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|attachdetach_controller|authenticated_user|authentication|cronjob_controller|disabled_metric|endpoint_slice|ephemeral_volume|garbagecollector_controller|get_token|go|hidden_metric|job_controller|kubernetes_build|kubernetes_feature|leader_election|node_collector|node_ipam|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|pv_collector|registered_metric|replicaset_controller|rest_client|retroactive_storageclass|root_ca|running_managed|scrape_duration|scrape_samples|scrape_series|service_controller|storage_count|storage_operation|ttl_after|volume_operation|workqueue)_(.+)" action: keep kubeEtcd: enabled: true endpoints: *cp - kubeProxy: - enabled: false # Disabled because eBPF kubeScheduler: enabled: true endpoints: *cp serviceMonitor: metricRelabelings: - # Remove duplicate metrics + # Remove duplicate labels provided by k3s - sourceLabels: ["__name__"] regex: "(apiserver_audit|apiserver_client|apiserver_delegated|apiserver_envelope|apiserver_storage|apiserver_webhooks|authenticated_user|authentication|disabled_metric|go|hidden_metric|kubernetes_build|kubernetes_feature|leader_election|process_cpu|process_max|process_open|process_resident|process_start|process_virtual|registered_metric|rest_client|scheduler|scrape_duration|scrape_samples|scrape_series|workqueue)_(.+)" action: keep + kubeProxy: + enabled: false # Disabled due to eBPF prometheus: ingress: enabled: true @@ -130,6 +132,7 @@ spec: serviceMonitorSelectorNilUsesHelmValues: false podMonitorSelectorNilUsesHelmValues: false probeSelectorNilUsesHelmValues: false + scrapeConfigSelectorNilUsesHelmValues: false enableAdminAPI: true walCompression: true retentionSize: 8GB @@ -140,3 +143,11 @@ spec: resources: requests: storage: 10Gi + grafana: + enabled: false + forceDeployDashboards: true + sidecar: + dashboards: + multicluster: + etcd: + enabled: true diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 index eb6b0bc1..8aa8b9c1 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 @@ -21,6 +21,8 @@ k3s_server_manifests_urls: filename: custom-prometheus-podmonitors.yaml - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml filename: custom-prometheus-prometheusrules.yaml + - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml + filename: custom-prometheus-scrapeconfigs.yaml - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml filename: custom-prometheus-servicemonitors.yaml # /var/lib/rancher/k3s/server/manifests diff --git a/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 index 91ee083a..24e502bb 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-installation.yaml.j2 @@ -54,6 +54,7 @@ {% endif %} - { name: podmonitors.monitoring.coreos.com, kind: CustomResourceDefinition } - { name: prometheusrules.monitoring.coreos.com, kind: CustomResourceDefinition } + - { name: scrapeconfigs.monitoring.coreos.com, kind: CustomResourceDefinition } - { name: servicemonitors.monitoring.coreos.com, kind: CustomResourceDefinition } - name: Coredns