feat!: Separate out internal and external nginx controllers (#880)

README.md

@@ -359,7 +359,7 @@ _Mic check, 1, 2_ - In a few moments applications should be lighting up like Chr
 
 #### 🌐 DNS
 
-The `external-dns` application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only public sub-domains exposed. In order to make additional applications public you must set an ingress annotation (`external-dns.alpha.kubernetes.io/target`) like done in the `HelmRelease` for `echo-server`.
+The `external-dns` application created in the `networking` namespace will handle creating public DNS records. By default, `echo-server` and the `flux-webhook` are the only public sub-domains exposed. In order to make additional applications public you must set the correct ingress class name and ingress annotations like done in the `HelmRelease` for `echo-server`.
 
 For split DNS to work it is required to have `${bootstrap_cloudflare_domain}` point to the `${bootstrap_k8s_gateway_addr}` load balancer IP address on your home DNS server. This will ensure DNS requests for `${bootstrap_cloudflare_domain}` will only get routed to your `k8s_gateway` service thus providing **internal** DNS resolution to your cluster applications/ingresses from any device that uses your home DNS server.
 
@@ -370,7 +370,7 @@ For and example with Pi-Hole apply the following file and restart dnsmasq:
 server=/${bootstrap_cloudflare_domain}/${bootstrap_k8s_gateway_addr}
 ```
 
-Now try to resolve an internal-only domain with `dig @${pi-hole-ip} hajimari.${bootstrap_cloudflare_domain}` it should resolve to your `${bootstrap_ingress_nginx_addr}` IP.
+Now try to resolve an internal-only domain with `dig @${pi-hole-ip} hajimari.${bootstrap_cloudflare_domain}` it should resolve to your `${bootstrap_internal_nginx_addr}` IP.
 
 If you're having trouble with DNS be sure to check out these two Github discussions, [Internal DNS](https://github.com/onedr0p/flux-cluster-template/discussions/719) and [Pod DNS resolution broken](https://github.com/onedr0p/flux-cluster-template/discussions/635).
 

bootstrap/tasks/validation/net.yaml

@@ -67,17 +67,29 @@
     success_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is within {{ bootstrap_node_cidr }}.
     fail_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is not within {{ bootstrap_node_cidr }}.
 
-- name: Verify ingress-nginx
+- name: Verify internal nginx
   ansible.builtin.assert:
-    that: bootstrap_ingress_nginx_addr is ansible.utils.ipv4
+    that: bootstrap_internal_nginx_addr is ansible.utils.ipv4
-    success_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is valid.
+    success_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is valid.
-    fail_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is invalid.
+    fail_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is invalid.
 
-- name: Verify ingress-nginx in node CIDR
+- name: Verify internal nginx in node CIDR
   ansible.builtin.assert:
-    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_ingress_nginx_addr)
+    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_internal_nginx_addr)
-    success_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is within {{ bootstrap_node_cidr }}.
+    success_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is within {{ bootstrap_node_cidr }}.
-    fail_msg: ingress-nginx address {{ bootstrap_ingress_nginx_addr }} is not within {{ bootstrap_node_cidr }}.
+    fail_msg: internal nginx address {{ bootstrap_internal_nginx_addr }} is not within {{ bootstrap_node_cidr }}.
+
+- name: Verify external nginx
+  ansible.builtin.assert:
+    that: bootstrap_external_nginx_addr is ansible.utils.ipv4
+    success_msg: external nginx address {{ bootstrap_external_nginx_addr }} is valid.
+    fail_msg: external nginx address {{ bootstrap_external_nginx_addr }} is invalid.
+
+- name: Verify external nginx in node CIDR
+  ansible.builtin.assert:
+    that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_external_nginx_addr)
+    success_msg: external nginx address {{ bootstrap_external_nginx_addr }} is within {{ bootstrap_node_cidr }}.
+    fail_msg: external nginx address {{ bootstrap_external_nginx_addr }} is not within {{ bootstrap_node_cidr }}.
 
 - name: Verify kube-vip
   ansible.builtin.assert:
@@ -91,9 +103,21 @@
     success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}.
     fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}.
 
-- name: Verify nodes are not the same IPs as k8s_gateway, ingress-nginx or kube-vip
+- name: Verify all IP addresses are unique
   ansible.builtin.assert:
-    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_ingress_nginx_addr, bootstrap_kube_vip_addr)
+    that: >
+      [
+        bootstrap_k8s_gateway_addr,
+        bootstrap_external_nginx_addr,
+        bootstrap_internal_nginx_addr,
+        bootstrap_kube_vip_addr
+      ] | unique | length == 4
+    success_msg: All IP addresses are unique.
+    fail_msg: All IP addresses are not unique.
+
+- name: Verify nodes are not the same IPs as k8s_gateway, nginx external/internal or kube-vip
+  ansible.builtin.assert:
+    that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_nginx_addr, bootstrap_internal_nginx_addr, bootstrap_kube_vip_addr)
     success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip.
     fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip.
     quiet: true

bootstrap/tasks/validation/vars.yaml

@@ -17,7 +17,8 @@
     - bootstrap_flux_github_webhook_token
     - bootstrap_github_repository_name
     - bootstrap_github_username
-    - bootstrap_ingress_nginx_addr
+    - bootstrap_external_nginx_addr
+    - bootstrap_internal_nginx_addr
     - bootstrap_ipv6_enabled
     - bootstrap_k8s_gateway_addr
     - bootstrap_kube_vip_addr

bootstrap/templates/addons/csi-driver-nfs/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:
@@ -28,8 +27,6 @@ spec:
   dependsOn:
     - name: local-path-provisioner
       namespace: kube-system
-    - name: ingress-nginx
-      namespace: networking
   values:
     deploymentStrategy:
       type: Recreate
@@ -162,7 +159,7 @@ spec:
       enabled: true
     ingress:
       enabled: true
-      ingressClassName: nginx
+      ingressClassName: internal
       annotations:
         hajimari.io/icon: simple-icons:grafana
       hosts:

bootstrap/templates/addons/grafana/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/hajimari/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:
@@ -25,9 +24,6 @@ spec:
       retries: 3
   uninstall:
     keepHistory: false
-  dependsOn:
-    - name: ingress-nginx
-      namespace: networking
   values:
     hajimari:
       title: Apps
@@ -47,7 +43,7 @@ spec:
     ingress:
       main:
         enabled: true
-        ingressClassName: nginx
+        ingressClassName: internal
         annotations:
           hajimari.io/enable: "false"
         hosts:

bootstrap/templates/addons/hajimari/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2

@@ -18,7 +18,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     crds: CreateReplace
     remediation:
       retries: 3
@@ -32,8 +31,6 @@ spec:
   dependsOn:
     - name: local-path-provisioner
       namespace: kube-system
-    - name: ingress-nginx
-      namespace: networking
   values:
     crds:
       enabled: true
@@ -117,7 +114,7 @@ spec:
     prometheus:
       ingress:
         enabled: true
-        ingressClassName: nginx
+        ingressClassName: internal
         annotations:
           hajimari.io/appName: Prometheus
           hajimari.io/icon: simple-icons:prometheus

bootstrap/templates/addons/kube-prometheus-stack/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/kubernetes-dashboard/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:
@@ -25,12 +24,7 @@ spec:
       retries: 3
   uninstall:
     keepHistory: false
-  dependsOn:
-    - name: ingress-nginx
-      namespace: networking
   values:
-    env:
-      TZ: "${TIMEZONE}"
     api:
       containers:
         args:

bootstrap/templates/addons/kubernetes-dashboard/ks.yaml.j2

@@ -7,14 +7,13 @@ metadata:
 spec:
   dependsOn:
     - name: cluster-apps-cert-manager
-    - name: cluster-apps-ingress-nginx
     - name: cluster-apps-metrics-server
   path: ./kubernetes/apps/monitoring/kubernetes-dashboard/app
   prune: true
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/system-upgrade-controller/ks.yaml.j2

@@ -28,7 +28,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/addons/weave-gitops/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/addons/weave-gitops/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/flux-system/addons/webhooks/github/ingress.yaml.j2

@@ -5,10 +5,10 @@ metadata:
   name: flux-webhook
   namespace: flux-system
   annotations:
-    external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
+    external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}"
     hajimari.io/enable: "false"
 spec:
-  ingressClassName: nginx
+  ingressClassName: external
   rules:
     - host: &host "flux-webhook.${SECRET_DOMAIN}"
       http:

bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml.j2

@@ -17,7 +17,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/cilium/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2

@@ -17,7 +17,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/app/helmrelease.yaml.j2

@@ -15,7 +15,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml.j2

@@ -12,7 +12,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/metrics-server/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/kube-system/reloader/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/kube-system/reloader/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/networking/cloudflared/app/configs/config.yaml.j2

@@ -2,14 +2,13 @@
 originRequest:
   http2Origin: true
 
-# https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ingress
 ingress:
   - hostname: "${SECRET_DOMAIN}"
-    service: https://ingress-nginx-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.networking.svc.cluster.local:443
     originRequest:
       originServerName: "ingress.${SECRET_DOMAIN}"
   - hostname: "*.${SECRET_DOMAIN}"
-    service: https://ingress-nginx-controller.networking.svc.cluster.local:443
+    service: https://nginx-external-controller.networking.svc.cluster.local:443
     originRequest:
       originServerName: "ingress.${SECRET_DOMAIN}"
   - service: http_status:404

bootstrap/templates/kubernetes/apps/networking/cloudflared/app/dnsendpoint.yaml.j2

@@ -4,10 +4,8 @@ kind: DNSEndpoint
 metadata:
   name: cloudflared
   namespace: networking
-  annotations:
-    external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
 spec:
   endpoints:
-    - dnsName: "ingress.${SECRET_DOMAIN}"
+    - dnsName: "external.${SECRET_DOMAIN}"
       recordType: CNAME
       targets: ["${SECRET_CLOUDFLARE_TUNNEL_ID}.cfargotunnel.com"]

bootstrap/templates/kubernetes/apps/networking/cloudflared/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/networking/cloudflared/ks.yaml.j2

@@ -12,7 +12,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/networking/echo-server/app/helmrelease.yaml.j2

@@ -17,7 +17,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:
@@ -26,9 +25,6 @@ spec:
       retries: 3
   uninstall:
     keepHistory: false
-  dependsOn:
-    - name: ingress-nginx
-      namespace: networking
   values:
     controller:
       strategy: RollingUpdate
@@ -58,9 +54,9 @@ spec:
     ingress:
       main:
         enabled: true
-        ingressClassName: nginx
+        ingressClassName: external
         annotations:
-          external-dns.alpha.kubernetes.io/target: "ingress.${SECRET_DOMAIN}"
+          external-dns.alpha.kubernetes.io/target: "external.${SECRET_DOMAIN}"
           hajimari.io/icon: video-input-antenna
         hosts:
           - host: &host "{% raw %}{{ .Release.Name }}{% endraw %}.${SECRET_DOMAIN}"

bootstrap/templates/kubernetes/apps/networking/echo-server/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/networking/external-dns/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:
@@ -35,7 +34,7 @@ spec:
             name: external-dns-secret
             key: api-token
     extraArgs:
-      - --annotation-filter=external-dns.alpha.kubernetes.io/target
+      - --ingress-class=external
       - --cloudflare-proxied
       - --crd-source-apiversion=externaldns.k8s.io/v1alpha1
       - --crd-source-kind=DNSEndpoint

bootstrap/templates/kubernetes/apps/networking/ingress-nginx/app/helmrelease.yaml.j2

@@ -1,76 +0,0 @@
-#jinja2: trim_blocks: True, lstrip_blocks: True
----
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
-  name: ingress-nginx
-  namespace: networking
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: ingress-nginx
-      version: 4.7.1
-      sourceRef:
-        kind: HelmRepository
-        name: ingress-nginx
-        namespace: flux-system
-  maxHistory: 2
-  install:
-    createNamespace: true
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  uninstall:
-    keepHistory: false
-  dependsOn:
-    - name: cloudflared
-      namespace: networking
-  values:
-    controller:
-      extraEnvs:
-        - name: TZ
-          value: "${TIMEZONE}"
-      service:
-        annotations:
-          external-dns.alpha.kubernetes.io/hostname: "ingress.${SECRET_DOMAIN}"
-          io.cilium/lb-ipam-ips: "{{ bootstrap_ingress_nginx_addr }}"
-        externalTrafficPolicy: Cluster
-      publishService:
-        enabled: true
-      ingressClassResource:
-        default: true
-      config:
-        client-header-timeout: 120
-        client-body-buffer-size: "100M"
-        client-body-timeout: 120
-        enable-brotli: "true"
-        hsts-max-age: "31449600"
-        keep-alive: 120
-        keep-alive-requests: 10000
-        proxy-body-size: "100M"
-        ssl-protocols: "TLSv1.3 TLSv1.2"
-      metrics:
-        enabled: true
-        serviceMonitor:
-          enabled: true
-          namespace: networking
-          namespaceSelector:
-            any: true
-      extraArgs:
-        {% if bootstrap_acme_production_enabled | default(false) %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
-        {% else %}
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
-        {% endif %}
-      resources:
-        requests:
-          cpu: 10m
-          memory: 250Mi
-        limits:
-          memory: 500Mi
-    defaultBackend:
-      enabled: false

bootstrap/templates/kubernetes/apps/networking/ingress-nginx/ks.yaml.j2

@@ -1,36 +0,0 @@
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: cluster-apps-ingress-nginx-certificates
-  namespace: flux-system
-spec:
-  dependsOn:
-    - name: cluster-apps-cert-manager-issuers
-  path: ./kubernetes/apps/networking/ingress-nginx/certificates
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: true
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: cluster-apps-ingress-nginx
-  namespace: flux-system
-spec:
-  dependsOn:
-    - name: cluster-apps-ingress-nginx-certificates
-  path: ./kubernetes/apps/networking/ingress-nginx/app
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: false # no flux ks dependents
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m

bootstrap/templates/kubernetes/apps/networking/k8s-gateway/app/helmrelease.yaml.j2

@@ -16,7 +16,6 @@ spec:
         namespace: flux-system
   maxHistory: 2
   install:
-    createNamespace: true
     remediation:
       retries: 3
   upgrade:

bootstrap/templates/kubernetes/apps/networking/k8s-gateway/ks.yaml.j2

@@ -10,7 +10,7 @@ spec:
   sourceRef:
     kind: GitRepository
     name: home-kubernetes
-  wait: false # no flux ks dependents
+  wait: false
   interval: 30m
   retryInterval: 1m
   timeout: 5m

bootstrap/templates/kubernetes/apps/networking/kustomization.yaml.j2

@@ -6,5 +6,5 @@ resources:
   - ./cloudflared/ks.yaml
   - ./echo-server/ks.yaml
   - ./external-dns/ks.yaml
-  - ./ingress-nginx/ks.yaml
   - ./k8s-gateway/ks.yaml
+  - ./nginx/ks.yaml

bootstrap/templates/kubernetes/apps/networking/nginx/external/helmrelease.yaml.j2

@@ -0,0 +1,97 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: nginx-external
+  namespace: networking
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: ingress-nginx
+      version: 4.7.1
+      sourceRef:
+        kind: HelmRepository
+        name: ingress-nginx
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  dependsOn:
+    - name: cloudflared
+      namespace: networking
+  values:
+    fullnameOverride: nginx-external
+    controller:
+      replicaCount: 1
+      service:
+        annotations:
+          external-dns.alpha.kubernetes.io/hostname: "external.${SECRET_DOMAIN}"
+          io.cilium/lb-ipam-ips: "{{ bootstrap_external_nginx_addr }}"
+        externalTrafficPolicy: Cluster
+      ingressClassResource:
+        name: external
+        default: true
+        controllerValue: k8s.io/external
+      admissionWebhooks:
+        objectSelector:
+          matchExpressions:
+            - key: ingress-class
+              operator: In
+              values: ["external"]
+      config:
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-real-ip: "true"
+        hsts-max-age: 31449600
+        keep-alive-requests: 10000
+        keep-alive: 120
+        log-format-escape-json: "true"
+        log-format-upstream: >
+          {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for",
+          "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time,
+          "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args",
+          "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer",
+          "http_user_agent": "$http_user_agent"}
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+      metrics:
+        enabled: true
+        serviceMonitor:
+          enabled: true
+          namespace: networking
+          namespaceSelector:
+            any: true
+      extraArgs:
+        {% if bootstrap_acme_production_enabled | default(false) %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
+        {% else %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
+        {% endif %}
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: nginx-external
+              app.kubernetes.io/component: controller
+      resources:
+        requests:
+          cpu: 10m
+          memory: 250Mi
+        limits:
+          memory: 500Mi
+    defaultBackend:
+      enabled: false

bootstrap/templates/kubernetes/apps/networking/nginx/internal/helmrelease.yaml.j2

@@ -0,0 +1,94 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: nginx-internal
+  namespace: networking
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: ingress-nginx
+      version: 4.7.1
+      sourceRef:
+        kind: HelmRepository
+        name: ingress-nginx
+        namespace: flux-system
+  maxHistory: 2
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    fullnameOverride: nginx-internal
+    controller:
+      replicaCount: 1
+      service:
+        annotations:
+          external-dns.alpha.kubernetes.io/hostname: "internal.${SECRET_DOMAIN}"
+          io.cilium/lb-ipam-ips: "{{ bootstrap_internal_nginx_addr }}"
+        externalTrafficPolicy: Cluster
+      ingressClassResource:
+        name: internal
+        default: true
+        controllerValue: k8s.io/internal
+      admissionWebhooks:
+        objectSelector:
+          matchExpressions:
+            - key: ingress-class
+              operator: In
+              values: ["internal"]
+      config:
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-real-ip: "true"
+        hsts-max-age: 31449600
+        keep-alive-requests: 10000
+        keep-alive: 120
+        log-format-escape-json: "true"
+        log-format-upstream: >
+          {"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for",
+          "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time,
+          "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args",
+          "request_length": $request_length, "duration": $request_time, "method": "$request_method", "http_referrer": "$http_referer",
+          "http_user_agent": "$http_user_agent"}
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+      metrics:
+        enabled: true
+        serviceMonitor:
+          enabled: true
+          namespace: networking
+          namespaceSelector:
+            any: true
+      extraArgs:
+        {% if bootstrap_acme_production_enabled | default(false) %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-production-tls"
+        {% else %}
+        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-staging-tls"
+        {% endif %}
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: nginx-internal
+              app.kubernetes.io/component: controller
+      resources:
+        requests:
+          cpu: 10m
+          memory: 250Mi
+        limits:
+          memory: 500Mi
+    defaultBackend:
+      enabled: false

bootstrap/templates/kubernetes/apps/networking/nginx/internal/kustomization.yaml.j2

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: networking
+resources:
+  - ./helmrelease.yaml

bootstrap/templates/kubernetes/apps/networking/nginx/ks.yaml.j2

@@ -0,0 +1,54 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-nginx-certificates
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cluster-apps-cert-manager-issuers
+  path: ./kubernetes/apps/networking/nginx/certificates
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-nginx-external
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cluster-apps-nginx-certificates
+  path: ./kubernetes/apps/networking/nginx/external
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-nginx-internal
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cluster-apps-nginx-certificates
+  path: ./kubernetes/apps/networking/nginx/internal
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m

bootstrap/vars/config.sample.yaml

@@ -40,8 +40,10 @@ bootstrap_node_cidr:
 bootstrap_kube_vip_addr:
 # The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used
 bootstrap_k8s_gateway_addr:
-# The Load balancer IP for ingress-nginx, choose an available IP in your nodes network that is not being used
+# The Load balancer IP for external nginx, choose an available IP in your nodes network that is not being used
-bootstrap_ingress_nginx_addr:
+bootstrap_external_nginx_addr:
+# The Load balancer IP for internal nginx, choose an available IP in your nodes network that is not being used
+bootstrap_internal_nginx_addr:
 
 # Keep the next three options default unless you know what you are doing
 # (Advanced) Enable ipv6