feat!: Separate out internal and external nginx controllers (#880)

2023-07-31 13:40:35 -04:00
1 parent 639fc37328
commit 377bdb3d72
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -5,10 +5,10 @@ metadata:
  name: flux-webhook
  namespace: flux-system
  annotations:
-    external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
+    external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}"
    hajimari.io/enable: "false"
 spec:
-  ingressClassName: nginx
+  ingressClassName: external
  rules:
    - host: &host "flux-webhook.${SECRET_DOMAIN}"
      http:
@@ -17,7 +17,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -17,7 +17,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -15,7 +15,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -12,7 +12,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -2,14 +2,13 @@
 originRequest:
  http2Origin: true

-# https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ingress
 ingress:
  - hostname: "${SECRET_DOMAIN}"
-    service: https://ingress-nginx-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.networking.svc.cluster.local:443
    originRequest:
      originServerName: "ingress.${SECRET_DOMAIN}"
  - hostname: "*.${SECRET_DOMAIN}"
-    service: https://ingress-nginx-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.networking.svc.cluster.local:443
    originRequest:
      originServerName: "ingress.${SECRET_DOMAIN}"
  - service: http_status:404
@@ -4,10 +4,8 @@ kind: DNSEndpoint
 metadata:
  name: cloudflared
  namespace: networking
-  annotations:
-    external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
 spec:
  endpoints:
-    - dnsName: "ingress.${SECRET_DOMAIN}"
+    - dnsName: "external.${SECRET_DOMAIN}"
      recordType: CNAME
      targets: ["${SECRET_CLOUDFLARE_TUNNEL_ID}.cfargotunnel.com"]
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -12,7 +12,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -17,7 +17,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -26,9 +25,6 @@ spec:
      retries: 3
  uninstall:
    keepHistory: false
-  dependsOn:
-    - name: ingress-nginx
-      namespace: networking
  values:
    controller:
      strategy: RollingUpdate
@@ -58,9 +54,9 @@ spec:
    ingress:
      main:
        enabled: true
-        ingressClassName: nginx
+        ingressClassName: external
        annotations:
-          external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
+          external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}"
          hajimari.io/icon: video-input-antenna
        hosts:
          - host: &host "{% raw %}{{ .Release.Name }}{% endraw %}.${SECRET_DOMAIN}"
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -35,7 +34,7 @@ spec:
            name: external-dns-secret
            key: api-token
    extraArgs:
-      - --annotation-filter=external-dns.alpha.kubernetes.io/target
+      - --ingress-class=external
      - --cloudflare-proxied
      - --crd-source-apiversion=externaldns.k8s.io/v1alpha1
      - --crd-source-kind=DNSEndpoint
@@ -1,76 +0,0 @@
-#jinja2: trim_blocks: True, lstrip_blocks: True
---
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
-  name: ingress-nginx
-  namespace: networking
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: ingress-nginx
-      version: 4.7.1
-      sourceRef:
-        kind: HelmRepository
-        name: ingress-nginx
-        namespace: flux-system
-  maxHistory: 2
-  install:
-    createNamespace: true
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  uninstall:
-    keepHistory: false
-  dependsOn:
-    - name: cloudflared
-      namespace: networking
-  values:
-    controller:
-      extraEnvs:
-        - name: TZ
-          value: "${TIMEZONE}"
-      service:
-        annotations:
-          external-dns.alpha.kubernetes.io/hostname: "ingress.${SECRET_DOMAIN}"
-          io.cilium/lb-ipam-ips: "{{ bootstrap_ingress_nginx_addr }}"
-        externalTrafficPolicy: Cluster
-      publishService:
-        enabled: true
-      ingressClassResource:
-        default: true
-      config:
-        client-header-timeout: 120
-        client-body-buffer-size: "100M"
-        client-body-timeout: 120
-        enable-brotli: "true"
-        hsts-max-age: "31449600"
-        keep-alive: 120
-        keep-alive-requests: 10000
-        proxy-body-size: "100M"
-        ssl-protocols: "TLSv1.3 TLSv1.2"
-      metrics:
-        enabled: true
-        serviceMonitor:
-          enabled: true
-          namespace: networking
-          namespaceSelector:
-            any: true
-      extraArgs:
-        {% if bootstrap_acme_production_enabled | default(false) %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
-        {% else %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
-        {% endif %}
-      resources:
-        requests:
-          cpu: 10m
-          memory: 250Mi
-        limits:
-          memory: 500Mi
-    defaultBackend:
-      enabled: false
@@ -1,36 +0,0 @@
---
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: cluster-apps-ingress-nginx-certificates
-  namespace: flux-system
-spec:
-  dependsOn:
-    - name: cluster-apps-cert-manager-issuers
-  path: ./kubernetes/apps/networking/ingress-nginx/certificates
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: true
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
---
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: cluster-apps-ingress-nginx
-  namespace: flux-system
-spec:
-  dependsOn:
-    - name: cluster-apps-ingress-nginx-certificates
-  path: ./kubernetes/apps/networking/ingress-nginx/app
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: false # no flux ks dependents
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -6,5 +6,5 @@ resources:
  - ./cloudflared/ks.yaml
  - ./echo-server/ks.yaml
  - ./external-dns/ks.yaml
-  - ./ingress-nginx/ks.yaml
  - ./k8s-gateway/ks.yaml
+  - ./nginx/ks.yaml
@@ -0,0 +1,97 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: nginx-external
+  namespace: networking
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: ingress-nginx
+      version: 4.7.1
+      sourceRef:
+        kind: HelmRepository
+        name: ingress-nginx
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  dependsOn:
+    - name: cloudflared
+      namespace: networking
+  values:
+    fullnameOverride: nginx-external
+    controller:
+      replicaCount: 1
+      service:
+        annotations:
+          external-dns.alpha.kubernetes.io/hostname: "external.${SECRET_DOMAIN}"
+          io.cilium/lb-ipam-ips: "{{ bootstrap_external_nginx_addr }}"
+        externalTrafficPolicy: Cluster
+      ingressClassResource:
+        name: external
+        default: true
+        controllerValue: k8s.io/external
+      admissionWebhooks:
+        objectSelector:
+          matchExpressions:
+            - key: ingress-class
+              operator: In
+              values: ["external"]
+      config:
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-real-ip: "true"
+        hsts-max-age: 31449600
+        keep-alive-requests: 10000
+        keep-alive: 120
+        log-format-escape-json: "true"
+        log-format-upstream: >
+          {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for",
+          "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time,
+          "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args",
+          "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer",
+          "http_user_agent": "$http_user_agent"}
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+      metrics:
+        enabled: true
+        serviceMonitor:
+          enabled: true
+          namespace: networking
+          namespaceSelector:
+            any: true
+      extraArgs:
+        {% if bootstrap_acme_production_enabled | default(false) %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
+        {% else %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
+        {% endif %}
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: nginx-external
+              app.kubernetes.io/component: controller
+      resources:
+        requests:
+          cpu: 10m
+          memory: 250Mi
+        limits:
+          memory: 500Mi
+    defaultBackend:
+      enabled: false
@@ -0,0 +1,94 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: nginx-internal
+  namespace: networking
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: ingress-nginx
+      version: 4.7.1
+      sourceRef:
+        kind: HelmRepository
+        name: ingress-nginx
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    fullnameOverride: nginx-internal
+    controller:
+      replicaCount: 1
+      service:
+        annotations:
+          external-dns.alpha.kubernetes.io/hostname: "internal.${SECRET_DOMAIN}"
+          io.cilium/lb-ipam-ips: "{{ bootstrap_internal_nginx_addr }}"
+        externalTrafficPolicy: Cluster
+      ingressClassResource:
+        name: internal
+        default: true
+        controllerValue: k8s.io/internal
+      admissionWebhooks:
+        objectSelector:
+          matchExpressions:
+            - key: ingress-class
+              operator: In
+              values: ["internal"]
+      config:
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-real-ip: "true"
+        hsts-max-age: 31449600
+        keep-alive-requests: 10000
+        keep-alive: 120
+        log-format-escape-json: "true"
+        log-format-upstream: >
+          {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for",
+          "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time,
+          "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args",
+          "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer",
+          "http_user_agent": "$http_user_agent"}
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+      metrics:
+        enabled: true
+        serviceMonitor:
+          enabled: true
+          namespace: networking
+          namespaceSelector:
+            any: true
+      extraArgs:
+        {% if bootstrap_acme_production_enabled | default(false) %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
+        {% else %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
+        {% endif %}
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: nginx-internal
+              app.kubernetes.io/component: controller
+      resources:
+        requests:
+          cpu: 10m
+          memory: 250Mi
+        limits:
+          memory: 500Mi
+    defaultBackend:
+      enabled: false
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: networking
+resources:
+  - ./helmrelease.yaml
@@ -0,0 +1,54 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-nginx-certificates
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cluster-apps-cert-manager-issuers
+  path: ./kubernetes/apps/networking/nginx/certificates
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-nginx-external
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cluster-apps-nginx-certificates
+  path: ./kubernetes/apps/networking/nginx/external
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-nginx-internal
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cluster-apps-nginx-certificates
+  path: ./kubernetes/apps/networking/nginx/internal
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m