feat!: Separate out internal and external nginx controllers (#880)

2023-07-31 13:40:35 -04:00
1 parent 639fc37328
commit 377bdb3d72
@@ -67,17 +67,29 @@
    success_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is within {{ bootstrap_node_cidr }}.
    fail_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is not within {{ bootstrap_node_cidr }}.

- name: Verify ingress-nginx
+- name: Verify internal nginx
  ansible.builtin.assert:
-    that: bootstrap_ingress_nginx_addr is ansible.utils.ipv4
-    success_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is valid.
-    fail_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is invalid.
+    that: bootstrap_internal_nginx_addr is ansible.utils.ipv4
+    success_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is valid.
+    fail_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is invalid.

- name: Verify ingress-nginx in node CIDR
+- name: Verify internal nginx in node CIDR
  ansible.builtin.assert:
-    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_ingress_nginx_addr)
-    success_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is within {{ bootstrap_node_cidr }}.
-    fail_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is not within {{ bootstrap_node_cidr }}.
+    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_internal_nginx_addr)
+    success_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is within {{ bootstrap_node_cidr }}.
+    fail_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is not within {{ bootstrap_node_cidr }}.
+
+- name: Verify external nginx
+  ansible.builtin.assert:
+    that: bootstrap_external_nginx_addr is ansible.utils.ipv4
+    success_msg: external nginx address {{ bootstrap_external_nginx_addr }} is valid.
+    fail_msg: external nginx address {{ bootstrap_external_nginx_addr }} is invalid.
+
+- name: Verify external nginx in node CIDR
+  ansible.builtin.assert:
+    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_external_nginx_addr)
+    success_msg: external nginx address {{ bootstrap_external_nginx_addr }} is within {{ bootstrap_node_cidr }}.
+    fail_msg: external nginx address {{ bootstrap_external_nginx_addr }} is not within {{ bootstrap_node_cidr }}.

 - name: Verify kube-vip
  ansible.builtin.assert:
@@ -91,9 +103,21 @@
    success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}.
    fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}.

- name: Verify nodes are not the same IPs as k8s_gateway, ingress-nginx or kube-vip
+- name: Verify all IP addresses are unique
  ansible.builtin.assert:
-    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_ingress_nginx_addr, bootstrap_kube_vip_addr)
+    that: >
+      [
+        bootstrap_k8s_gateway_addr,
+        bootstrap_external_nginx_addr,
+        bootstrap_internal_nginx_addr,
+        bootstrap_kube_vip_addr
+      ] | unique | length == 4
+    success_msg: All IP addresses are unique.
+    fail_msg: All IP addresses are not unique.
+
+- name: Verify nodes are not the same IPs as k8s_gateway, nginx external/internal or kube-vip
+  ansible.builtin.assert:
+    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_nginx_addr, bootstrap_internal_nginx_addr, bootstrap_kube_vip_addr)
    success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip.
    fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip.
    quiet: true
@@ -17,7 +17,8 @@
    - bootstrap_flux_github_webhook_token
    - bootstrap_github_repository_name
    - bootstrap_github_username
-    - bootstrap_ingress_nginx_addr
+    - bootstrap_external_nginx_addr
+    - bootstrap_internal_nginx_addr
    - bootstrap_ipv6_enabled
    - bootstrap_k8s_gateway_addr
    - bootstrap_kube_vip_addr
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -28,8 +27,6 @@ spec:
  dependsOn:
    - name: local-path-provisioner
      namespace: kube-system
-    - name: ingress-nginx
-      namespace: networking
  values:
    deploymentStrategy:
      type: Recreate
@@ -162,7 +159,7 @@ spec:
      enabled: true
    ingress:
      enabled: true
-      ingressClassName: nginx
+      ingressClassName: internal
      annotations:
        hajimari.io/icon: simple-icons:grafana
      hosts:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -25,9 +24,6 @@ spec:
      retries: 3
  uninstall:
    keepHistory: false
-  dependsOn:
-    - name: ingress-nginx
-      namespace: networking
  values:
    hajimari:
      title: Apps
@@ -47,7 +43,7 @@ spec:
    ingress:
      main:
        enabled: true
-        ingressClassName: nginx
+        ingressClassName: internal
        annotations:
          hajimari.io/enable: "false"
        hosts:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -18,7 +18,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    crds: CreateReplace
    remediation:
      retries: 3
@@ -32,8 +31,6 @@ spec:
  dependsOn:
    - name: local-path-provisioner
      namespace: kube-system
-    - name: ingress-nginx
-      namespace: networking
  values:
    crds:
      enabled: true
@@ -117,7 +114,7 @@ spec:
    prometheus:
      ingress:
        enabled: true
-        ingressClassName: nginx
+        ingressClassName: internal
        annotations:
          hajimari.io/appName: Prometheus
          hajimari.io/icon: simple-icons:prometheus
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -25,12 +24,7 @@ spec:
      retries: 3
  uninstall:
    keepHistory: false
-  dependsOn:
-    - name: ingress-nginx
-      namespace: networking
  values:
-    env:
-      TZ: "${TIMEZONE}"
    api:
      containers:
        args:
@@ -7,14 +7,13 @@ metadata:
 spec:
  dependsOn:
    - name: cluster-apps-cert-manager
-    - name: cluster-apps-ingress-nginx
    - name: cluster-apps-metrics-server
  path: ./kubernetes/apps/monitoring/kubernetes-dashboard/app
  prune: true
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -28,7 +28,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -5,10 +5,10 @@ metadata:
  name: flux-webhook
  namespace: flux-system
  annotations:
-    external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
+    external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}"
    hajimari.io/enable: "false"
 spec:
-  ingressClassName: nginx
+  ingressClassName: external
  rules:
    - host: &host "flux-webhook.${SECRET_DOMAIN}"
      http:
@@ -17,7 +17,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -17,7 +17,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -15,7 +15,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -12,7 +12,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -2,14 +2,13 @@
 originRequest:
  http2Origin: true

-# https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ingress
 ingress:
  - hostname: "${SECRET_DOMAIN}"
-    service: https://ingress-nginx-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.networking.svc.cluster.local:443
    originRequest:
      originServerName: "ingress.${SECRET_DOMAIN}"
  - hostname: "*.${SECRET_DOMAIN}"
-    service: https://ingress-nginx-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.networking.svc.cluster.local:443
    originRequest:
      originServerName: "ingress.${SECRET_DOMAIN}"
  - service: http_status:404
@@ -4,10 +4,8 @@ kind: DNSEndpoint
 metadata:
  name: cloudflared
  namespace: networking
-  annotations:
-    external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
 spec:
  endpoints:
-    - dnsName: "ingress.${SECRET_DOMAIN}"
+    - dnsName: "external.${SECRET_DOMAIN}"
      recordType: CNAME
      targets: ["${SECRET_CLOUDFLARE_TUNNEL_ID}.cfargotunnel.com"]
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -12,7 +12,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -17,7 +17,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -26,9 +25,6 @@ spec:
      retries: 3
  uninstall:
    keepHistory: false
-  dependsOn:
-    - name: ingress-nginx
-      namespace: networking
  values:
    controller:
      strategy: RollingUpdate
@@ -58,9 +54,9 @@ spec:
    ingress:
      main:
        enabled: true
-        ingressClassName: nginx
+        ingressClassName: external
        annotations:
-          external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
+          external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}"
          hajimari.io/icon: video-input-antenna
        hosts:
          - host: &host "{% raw %}{{ .Release.Name }}{% endraw %}.${SECRET_DOMAIN}"
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -35,7 +34,7 @@ spec:
            name: external-dns-secret
            key: api-token
    extraArgs:
-      - --annotation-filter=external-dns.alpha.kubernetes.io/target
+      - --ingress-class=external
      - --cloudflare-proxied
      - --crd-source-apiversion=externaldns.k8s.io/v1alpha1
      - --crd-source-kind=DNSEndpoint
@@ -1,76 +0,0 @@
-#jinja2: trim_blocks: True, lstrip_blocks: True
---
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
-  name: ingress-nginx
-  namespace: networking
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: ingress-nginx
-      version: 4.7.1
-      sourceRef:
-        kind: HelmRepository
-        name: ingress-nginx
-        namespace: flux-system
-  maxHistory: 2
-  install:
-    createNamespace: true
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  uninstall:
-    keepHistory: false
-  dependsOn:
-    - name: cloudflared
-      namespace: networking
-  values:
-    controller:
-      extraEnvs:
-        - name: TZ
-          value: "${TIMEZONE}"
-      service:
-        annotations:
-          external-dns.alpha.kubernetes.io/hostname: "ingress.${SECRET_DOMAIN}"
-          io.cilium/lb-ipam-ips: "{{ bootstrap_ingress_nginx_addr }}"
-        externalTrafficPolicy: Cluster
-      publishService:
-        enabled: true
-      ingressClassResource:
-        default: true
-      config:
-        client-header-timeout: 120
-        client-body-buffer-size: "100M"
-        client-body-timeout: 120
-        enable-brotli: "true"
-        hsts-max-age: "31449600"
-        keep-alive: 120
-        keep-alive-requests: 10000
-        proxy-body-size: "100M"
-        ssl-protocols: "TLSv1.3 TLSv1.2"
-      metrics:
-        enabled: true
-        serviceMonitor:
-          enabled: true
-          namespace: networking
-          namespaceSelector:
-            any: true
-      extraArgs:
-        {% if bootstrap_acme_production_enabled | default(false) %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
-        {% else %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
-        {% endif %}
-      resources:
-        requests:
-          cpu: 10m
-          memory: 250Mi
-        limits:
-          memory: 500Mi
-    defaultBackend:
-      enabled: false
@@ -1,36 +0,0 @@
---
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: cluster-apps-ingress-nginx-certificates
-  namespace: flux-system
-spec:
-  dependsOn:
-    - name: cluster-apps-cert-manager-issuers
-  path: ./kubernetes/apps/networking/ingress-nginx/certificates
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: true
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
---
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: cluster-apps-ingress-nginx
-  namespace: flux-system
-spec:
-  dependsOn:
-    - name: cluster-apps-ingress-nginx-certificates
-  path: ./kubernetes/apps/networking/ingress-nginx/app
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: false # no flux ks dependents
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
@@ -16,7 +16,6 @@ spec:
        namespace: flux-system
  maxHistory: 2
  install:
-    createNamespace: true
    remediation:
      retries: 3
  upgrade:
@@ -10,7 +10,7 @@ spec:
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -6,5 +6,5 @@ resources:
  - ./cloudflared/ks.yaml
  - ./echo-server/ks.yaml
  - ./external-dns/ks.yaml
-  - ./ingress-nginx/ks.yaml
  - ./k8s-gateway/ks.yaml
+  - ./nginx/ks.yaml
@@ -0,0 +1,97 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: nginx-external
+  namespace: networking
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: ingress-nginx
+      version: 4.7.1
+      sourceRef:
+        kind: HelmRepository
+        name: ingress-nginx
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  dependsOn:
+    - name: cloudflared
+      namespace: networking
+  values:
+    fullnameOverride: nginx-external
+    controller:
+      replicaCount: 1
+      service:
+        annotations:
+          external-dns.alpha.kubernetes.io/hostname: "external.${SECRET_DOMAIN}"
+          io.cilium/lb-ipam-ips: "{{ bootstrap_external_nginx_addr }}"
+        externalTrafficPolicy: Cluster
+      ingressClassResource:
+        name: external
+        default: true
+        controllerValue: k8s.io/external
+      admissionWebhooks:
+        objectSelector:
+          matchExpressions:
+            - key: ingress-class
+              operator: In
+              values: ["external"]
+      config:
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-real-ip: "true"
+        hsts-max-age: 31449600
+        keep-alive-requests: 10000
+        keep-alive: 120
+        log-format-escape-json: "true"
+        log-format-upstream: >
+          {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for",
+          "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time,
+          "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args",
+          "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer",
+          "http_user_agent": "$http_user_agent"}
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+      metrics:
+        enabled: true
+        serviceMonitor:
+          enabled: true
+          namespace: networking
+          namespaceSelector:
+            any: true
+      extraArgs:
+        {% if bootstrap_acme_production_enabled | default(false) %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
+        {% else %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
+        {% endif %}
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: nginx-external
+              app.kubernetes.io/component: controller
+      resources:
+        requests:
+          cpu: 10m
+          memory: 250Mi
+        limits:
+          memory: 500Mi
+    defaultBackend:
+      enabled: false
@@ -0,0 +1,94 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: nginx-internal
+  namespace: networking
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: ingress-nginx
+      version: 4.7.1
+      sourceRef:
+        kind: HelmRepository
+        name: ingress-nginx
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    fullnameOverride: nginx-internal
+    controller:
+      replicaCount: 1
+      service:
+        annotations:
+          external-dns.alpha.kubernetes.io/hostname: "internal.${SECRET_DOMAIN}"
+          io.cilium/lb-ipam-ips: "{{ bootstrap_internal_nginx_addr }}"
+        externalTrafficPolicy: Cluster
+      ingressClassResource:
+        name: internal
+        default: true
+        controllerValue: k8s.io/internal
+      admissionWebhooks:
+        objectSelector:
+          matchExpressions:
+            - key: ingress-class
+              operator: In
+              values: ["internal"]
+      config:
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-real-ip: "true"
+        hsts-max-age: 31449600
+        keep-alive-requests: 10000
+        keep-alive: 120
+        log-format-escape-json: "true"
+        log-format-upstream: >
+          {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for",
+          "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time,
+          "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args",
+          "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer",
+          "http_user_agent": "$http_user_agent"}
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+      metrics:
+        enabled: true
+        serviceMonitor:
+          enabled: true
+          namespace: networking
+          namespaceSelector:
+            any: true
+      extraArgs:
+        {% if bootstrap_acme_production_enabled | default(false) %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
+        {% else %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
+        {% endif %}
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: nginx-internal
+              app.kubernetes.io/component: controller
+      resources:
+        requests:
+          cpu: 10m
+          memory: 250Mi
+        limits:
+          memory: 500Mi
+    defaultBackend:
+      enabled: false
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: networking
+resources:
+  - ./helmrelease.yaml
@@ -0,0 +1,54 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-nginx-certificates
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cluster-apps-cert-manager-issuers
+  path: ./kubernetes/apps/networking/nginx/certificates
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-nginx-external
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cluster-apps-nginx-certificates
+  path: ./kubernetes/apps/networking/nginx/external
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-nginx-internal
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cluster-apps-nginx-certificates
+  path: ./kubernetes/apps/networking/nginx/internal
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
@@ -40,8 +40,10 @@ bootstrap_node_cidr:
 bootstrap_kube_vip_addr:
 # The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used
 bootstrap_k8s_gateway_addr:
-# The Load balancer IP for ingress-nginx, choose an available IP in your nodes network that is not being used
-bootstrap_ingress_nginx_addr:
+# The Load balancer IP for external nginx, choose an available IP in your nodes network that is not being used
+bootstrap_external_nginx_addr:
+# The Load balancer IP for internal nginx, choose an available IP in your nodes network that is not being used
+bootstrap_internal_nginx_addr:

 # Keep the next three options default unless you know what you are doing
 # (Advanced) Enable ipv6