feat!: refactor namespaces and add descheduler (#1110)

Signed-off-by: Devin Buhl <devin@buhl.casa>
2024-01-02 16:48:33 -05:00
1 parent a8bc633ff1
commit e167882e64
@@ -2,7 +2,7 @@
 - name: Set addon facts
  ansible.builtin.set_fact:
    addon_name: csi-driver-nfs
-    addon_namespace: kube-system
+    addon_namespace: storage

 - name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }}
  when: item.state == 'directory'
@@ -2,7 +2,7 @@
 - name: Set addon facts
  ansible.builtin.set_fact:
    addon_name: grafana
-    addon_namespace: monitoring
+    addon_namespace: observability

 - name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }}
  when: item.state == 'directory'
@@ -2,7 +2,7 @@
 - name: Set addon facts
  ansible.builtin.set_fact:
    addon_name: kube-prometheus-stack
-    addon_namespace: monitoring
+    addon_namespace: observability

 - name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }}
  when: item.state == 'directory'
@@ -2,7 +2,7 @@
 - name: Set addon facts
  ansible.builtin.set_fact:
    addon_name: kubernetes-dashboard
-    addon_namespace: monitoring
+    addon_namespace: observability

 - name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }}
  when: item.state == 'directory'
@@ -2,7 +2,7 @@
 - name: Set addon facts
  ansible.builtin.set_fact:
    addon_name: system-upgrade-controller
-    addon_namespace: kube-system
+    addon_namespace: tools

 - name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }}
  when: item.state == 'directory'
@@ -5,11 +5,11 @@ metadata:
  name: &app csi-driver-nfs
  namespace: flux-system
 spec:
-  targetNamespace: kube-system
+  targetNamespace: storage
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/kube-system/csi-driver-nfs/app
+  path: ./kubernetes/apps/storage/csi-driver-nfs/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -25,7 +25,7 @@ spec:
    keepHistory: false
  dependsOn:
    - name: local-path-provisioner
-      namespace: kube-system
+      namespace: storage
  values:
    deploymentStrategy:
      type: Recreate
@@ -85,7 +85,7 @@ spec:
            type: prometheus
            uid: prometheus
            access: proxy
-            url: http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090
+            url: http://kube-prometheus-stack-prometheus.observability.svc.cluster.local:9090
            jsonData:
              prometheusType: Prometheus
            isDefault: true
@@ -5,11 +5,11 @@ metadata:
  name: &app grafana
  namespace: flux-system
 spec:
-  targetNamespace: monitoring
+  targetNamespace: observability
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/monitoring/grafana/app
+  path: ./kubernetes/apps/observability/grafana/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -38,7 +38,7 @@ spec:
      namespaceSelector:
        matchNames:
          - default
-          - monitoring
+          - observability
    ingress:
      main:
        enabled: true
@@ -28,7 +28,7 @@ spec:
    keepHistory: false
  dependsOn:
    - name: local-path-provisioner
-      namespace: kube-system
+      namespace: storage
  valuesFrom:
    - name: kube-prometheus-stack-values
      kind: ConfigMap
@@ -5,11 +5,11 @@ metadata:
  name: &app kube-prometheus-stack
  namespace: flux-system
 spec:
-  targetNamespace: monitoring
+  targetNamespace: observability
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/monitoring/kube-prometheus-stack/app
+  path: ./kubernetes/apps/observability/kube-prometheus-stack/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -1,5 +1,5 @@
 # For dashboard sign in token:
-# kubectl -n monitoring get secret kubernetes-dashboard -o jsonpath='{.data.token}' | base64 -d
+# kubectl -n observability get secret kubernetes-dashboard -o jsonpath='{.data.token}' | base64 -d
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -9,7 +9,7 @@ metadata:
    app.kubernetes.io/managed-by: Helm
  annotations:
    meta.helm.sh/release-name: kubernetes-dashboard
-    meta.helm.sh/release-namespace: monitoring
+    meta.helm.sh/release-namespace: observability
 secrets:
  - name: kubernetes-dashboard
 ---
@@ -22,7 +22,7 @@ metadata:
    app.kubernetes.io/managed-by: Helm
  annotations:
    meta.helm.sh/release-name: kubernetes-dashboard
-    meta.helm.sh/release-namespace: monitoring
+    meta.helm.sh/release-namespace: observability
    kubernetes.io/service-account.name: kubernetes-dashboard
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -36,4 +36,4 @@ roleRef:
 subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
-    namespace: monitoring
+    namespace: observability
@@ -5,14 +5,14 @@ metadata:
  name: &app kubernetes-dashboard
  namespace: flux-system
 spec:
-  targetNamespace: monitoring
+  targetNamespace: observability
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
  dependsOn:
    - name: cert-manager
    - name: metrics-server
-  path: ./kubernetes/apps/monitoring/kubernetes-dashboard/app
+  path: ./kubernetes/apps/observability/kubernetes-dashboard/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -25,4 +25,4 @@ roleRef:
 subjects:
  - kind: ServiceAccount
    name: system-upgrade
-    namespace: kube-system
+    namespace: tools
@@ -5,11 +5,11 @@ metadata:
  name: &app system-upgrade-controller
  namespace: flux-system
 spec:
-  targetNamespace: kube-system
+  targetNamespace: tools
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/kube-system/system-upgrade-controller/app
+  path: ./kubernetes/apps/tools/system-upgrade-controller/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -25,13 +25,13 @@ metadata:
  name: &app system-upgrade-controller-plans
  namespace: flux-system
 spec:
-  targetNamespace: kube-system
+  targetNamespace: tools
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
  dependsOn:
    - name: system-upgrade-controller
-  path: ./kubernetes/apps/kube-system/system-upgrade-controller/plans
+  path: ./kubernetes/apps/tools/system-upgrade-controller/plans
  prune: true
  sourceRef:
    kind: GitRepository
@@ -37,4 +37,4 @@ spec:
      enabled: true
      servicemonitor:
        enabled: true
-        prometheusInstance: monitoring
+        prometheusInstance: observability
@@ -6,13 +6,4 @@ resources:
  - ./namespace.yaml
  - ./cilium/ks.yaml
  - ./coredns/ks.yaml
-  - ./local-path-provisioner/ks.yaml
  - ./metrics-server/ks.yaml
-  - ./reloader/ks.yaml
-  - ./snapshot-controller/ks.yaml
-  {% if csi_driver_nfs.enabled | default(false) %}
-  - ./csi-driver-nfs/ks.yaml
-  {% endif %}
-  {% if system_upgrade_controller.enabled | default(false) %}
-  - ./system-upgrade-controller/ks.yaml
-  {% endif %}
@@ -1,22 +0,0 @@
---
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: &app local-path-provisioner
-  namespace: flux-system
-  labels:
-    substitution.flux.home.arpa/disabled: "true"
-spec:
-  targetNamespace: kube-system
-  commonMetadata:
-    labels:
-      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/kube-system/local-path-provisioner/app
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: false
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
@@ -4,11 +4,11 @@ originRequest:

 ingress:
  - hostname: "${SECRET_DOMAIN}"
-    service: https://nginx-external-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.network.svc.cluster.local:443
    originRequest:
      originServerName: "external.${SECRET_DOMAIN}"
  - hostname: "*.${SECRET_DOMAIN}"
-    service: https://nginx-external-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.network.svc.cluster.local:443
    originRequest:
      originServerName: "external.${SECRET_DOMAIN}"
  - service: http_status:404
@@ -5,13 +5,13 @@ metadata:
  name: &app cloudflared
  namespace: flux-system
 spec:
-  targetNamespace: networking
+  targetNamespace: network
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
  dependsOn:
    - name: external-dns
-  path: ./kubernetes/apps/networking/cloudflared/app
+  path: ./kubernetes/apps/network/cloudflared/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -5,11 +5,11 @@ metadata:
  name: &app echo-server
  namespace: flux-system
 spec:
-  targetNamespace: networking
+  targetNamespace: network
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/networking/echo-server/app
+  path: ./kubernetes/apps/network/echo-server/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -5,11 +5,11 @@ metadata:
  name: &app external-dns
  namespace: flux-system
 spec:
-  targetNamespace: networking
+  targetNamespace: network
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/networking/external-dns/app
+  path: ./kubernetes/apps/network/external-dns/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -5,11 +5,11 @@ metadata:
  name: &app k8s-gateway
  namespace: flux-system
 spec:
-  targetNamespace: networking
+  targetNamespace: network
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/networking/k8s-gateway/app
+  path: ./kubernetes/apps/network/k8s-gateway/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -2,6 +2,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: volsync
+  name: network
  labels:
    kustomize.toolkit.fluxcd.io/prune: disabled
@@ -4,7 +4,6 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta2
 kind: HelmRelease
 metadata:
  name: nginx-external
-  namespace: &namespace networking
 spec:
  interval: 30m
  chart:
@@ -27,7 +26,7 @@ spec:
    keepHistory: false
  dependsOn:
    - name: cloudflared
-      namespace: networking
+      namespace: network
  values:
    fullnameOverride: nginx-external
    controller:
@@ -70,14 +69,13 @@ spec:
        enabled: true
        serviceMonitor:
          enabled: true
-          namespace: *namespace
          namespaceSelector:
            any: true
      extraArgs:
        {% if bootstrap_acme_production_enabled | default(false) %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
+        default-ssl-certificate: "network/${SECRET_DOMAIN/./-}-production-tls"
        {% else %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
+        default-ssl-certificate: "network/${SECRET_DOMAIN/./-}-staging-tls"
        {% endif %}
      topologySpreadConstraints:
        - maxSkew: 1
@@ -4,7 +4,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta2
 kind: HelmRelease
 metadata:
  name: nginx-internal
-  namespace: &namespace networking
+  namespace: &namespace network
 spec:
  interval: 30m
  chart:
@@ -67,14 +67,13 @@ spec:
        enabled: true
        serviceMonitor:
          enabled: true
-          namespace: *namespace
          namespaceSelector:
            any: true
      extraArgs:
        {% if bootstrap_acme_production_enabled | default(false) %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
+        default-ssl-certificate: "network/${SECRET_DOMAIN/./-}-production-tls"
        {% else %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
+        default-ssl-certificate: "network/${SECRET_DOMAIN/./-}-staging-tls"
        {% endif %}
      topologySpreadConstraints:
        - maxSkew: 1
@@ -5,13 +5,13 @@ metadata:
  name: &app nginx-certificates
  namespace: flux-system
 spec:
-  targetNamespace: networking
+  targetNamespace: network
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
  dependsOn:
    - name: cert-manager-issuers
-  path: ./kubernetes/apps/networking/nginx/certificates
+  path: ./kubernetes/apps/network/nginx/certificates
  prune: true
  sourceRef:
    kind: GitRepository
@@ -27,13 +27,13 @@ metadata:
  name: &app nginx-external
  namespace: flux-system
 spec:
-  targetNamespace: networking
+  targetNamespace: network
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
  dependsOn:
    - name: nginx-certificates
-  path: ./kubernetes/apps/networking/nginx/external
+  path: ./kubernetes/apps/network/nginx/external
  prune: true
  sourceRef:
    kind: GitRepository
@@ -49,13 +49,13 @@ metadata:
  name: &app nginx-internal
  namespace: flux-system
 spec:
-  targetNamespace: networking
+  targetNamespace: network
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
  dependsOn:
    - name: nginx-certificates
-  path: ./kubernetes/apps/networking/nginx/internal
+  path: ./kubernetes/apps/network/nginx/internal
  prune: true
  sourceRef:
    kind: GitRepository
@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: observability
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
@@ -0,0 +1,12 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./namespace.yaml
+  - ./local-path-provisioner/ks.yaml
+  - ./snapshot-controller/ks.yaml
+  - ./volsync/ks.yaml
+  {% if csi_driver_nfs.enabled | default(false) %}
+  - ./csi-driver-nfs/ks.yaml
+  {% endif %}
@@ -0,0 +1,20 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app local-path-provisioner
+  namespace: flux-system
+spec:
+  targetNamespace: storage
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/storage/local-path-provisioner/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
@@ -2,6 +2,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: monitoring
+  name: storage
  labels:
    kustomize.toolkit.fluxcd.io/prune: disabled
@@ -5,11 +5,11 @@ metadata:
  name: &app snapshot-controller
  namespace: flux-system
 spec:
-  targetNamespace: kube-system
+  targetNamespace: storage
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/kube-system/snapshot-controller/app
+  path: ./kubernetes/apps/storage/snapshot-controller/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -25,7 +25,7 @@ spec:
    keepHistory: false
  dependsOn:
    - name: snapshot-controller
-      namespace: kube-system
+      namespace: storage
  values:
    manageCRDs: true
    image:
@@ -5,11 +5,11 @@ metadata:
  name: &app volsync
  namespace: flux-system
 spec:
-  targetNamespace: volsync
+  targetNamespace: storage
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/volsync/volsync/app
+  path: ./kubernetes/apps/storage/volsync/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -0,0 +1,62 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: descheduler
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: descheduler
+      version: 0.29.0
+      sourceRef:
+        kind: HelmRepository
+        name: descheduler
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    kind: Deployment
+    deschedulerPolicyAPIVersion: descheduler/v1alpha2
+    deschedulerPolicy:
+      profiles:
+        - name: Default
+          pluginConfig:
+            - name: DefaultEvictor
+              args:
+                evictFailedBarePods: true
+                evictLocalStoragePods: true
+                evictSystemCriticalPods: true
+                nodeFit: true
+            - name: RemovePodsViolatingInterPodAntiAffinity
+            - name: RemovePodsViolatingNodeAffinity
+              args:
+                nodeAffinityType:
+                  - requiredDuringSchedulingIgnoredDuringExecution
+            - name: RemovePodsViolatingNodeTaints
+            - name: RemovePodsViolatingTopologySpreadConstraint
+              args:
+                constraints:
+                  - DoNotSchedule
+                  - ScheduleAnyway
+          plugins:
+            balance:
+              enabled:
+                - RemovePodsViolatingTopologySpreadConstraint
+            deschedule:
+              enabled:
+                - RemovePodsViolatingInterPodAntiAffinity
+                - RemovePodsViolatingNodeAffinity
+                - RemovePodsViolatingNodeTaints
+    service:
+      enabled: true
+    serviceMonitor:
+      enabled: true
@@ -0,0 +1,20 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app descheduler
+  namespace: flux-system
+spec:
+  targetNamespace: tools
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/tools/descheduler/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
@@ -0,0 +1,11 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./namespace.yaml
+  - ./descheduler/ks.yaml
+  - ./reloader/ks.yaml
+  {% if system_upgrade_controller.enabled | default(false) %}
+  - ./system-upgrade-controller/ks.yaml
+  {% endif %}
@@ -2,6 +2,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: networking
+  name: tools
  labels:
    kustomize.toolkit.fluxcd.io/prune: disabled
@@ -3,7 +3,6 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta2
 kind: HelmRelease
 metadata:
  name: reloader
-  namespace: &namespace kube-system
 spec:
  interval: 30m
  chart:
@@ -27,7 +26,7 @@ spec:
  values:
    fullnameOverride: reloader
    reloader:
-      reloadStrategy: annotations
+      readOnlyRootFileSystem: true
      podMonitor:
        enabled: true
-        namespace: *namespace
+        namespace: "{{ .Release.Namespace }}"
@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
@@ -5,11 +5,11 @@ metadata:
  name: &app reloader
  namespace: flux-system
 spec:
-  targetNamespace: kube-system
+  targetNamespace: tools
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/kube-system/reloader/app
+  path: ./kubernetes/apps/tools/reloader/app
  prune: true
  sourceRef:
    kind: GitRepository
@@ -1,8 +0,0 @@
---
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  # Pre Flux-Kustomizations
-  - ./namespace.yaml
-  # Flux-Kustomizations
-  - ./volsync/ks.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: descheduler
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://kubernetes-sigs.github.io/descheduler
@@ -9,6 +9,7 @@ resources:
  - ./coredns.yaml
  - ./csi-driver-nfs.yaml
  - ./democratic-csi.yaml
+  - ./descheduler.yaml
  - ./external-dns.yaml
  - ./grafana.yaml
  - ./hajimari.yaml