feat!: switch from ingress-nginx to traefik

.github/renovate.json5

@@ -19,19 +19,37 @@
     ],
   },
   "regexManagers": [
-    // regexManager to read and process HelmReleases and CRDs
+    // regexManager to read and process HelmRelease files
     {
       "fileMatch": [
         "cluster/.+\\.yaml$"
       ],
       "matchStrings": [
         // helm releases
-        "registryUrl=(?<registryUrl>.*?)\n *chart: (?<depName>.*?)\n *version: (?<currentValue>.*)\n",
+        "registryUrl=(?<registryUrl>.*?)\n *chart: (?<depName>.*?)\n *version: (?<currentValue>.*)\n"
-        // cert-manager crd
+      ],
+      "datasourceTemplate": "helm"
+    },
+    // regexManager to read and process cert-manager CRD's
+    {
+      "fileMatch": [
+        "cluster/crds/cert-manager/.+\\.yaml$"
+      ],
+      "matchStrings": [
         "registryUrl=(?<registryUrl>.*?) chart=(?<depName>.*?)\n.*\\/(?<currentValue>.*?)\\/"
       ],
       "datasourceTemplate": "helm"
     },
+    // regexManager to read and process Traefik CRD's
+    {
+      "fileMatch": [
+        "cluster/crds/traefik/.+\\.yaml$"
+      ],
+      "matchStrings": [
+        "registryUrl=(?<registryUrl>.*?) chart=(?<depName>.*?)\n *tag: v(?<currentValue>.*)\n"
+      ],
+      "datasourceTemplate": "helm"
+    },
   ],
   "packageRules": [
     // setup datasources

README.md

@@ -24,7 +24,7 @@ Feel free to read up on any of these technologies before you get started to be m
 - [flux](https://toolkit.fluxcd.io/) - GitOps tool for deploying manifests from the `cluster` directory
 - [metallb](https://metallb.universe.tf/) - bare metal load balancer
 - [cert-manager](https://cert-manager.io/) - SSL certificates - with Cloudflare DNS challenge
-- [ingress-nginx](https://kubernetes.github.io/ingress-nginx/) - ingress controller
+- [traefik](https://traefik.io) - ingress controller
 - [hajimari](https://github.com/toboshii/hajimari) - start page with ingress discovery
 - [system-upgrade-controller](https://github.com/rancher/system-upgrade-controller) - upgrade k3s
 
@@ -267,8 +267,8 @@ export BOOTSTRAP_GITHUB_REPOSITORY="https://github.com/k8s-at-home/home-cluster"
 export BOOTSTRAP_DOMAIN="k8s-at-home.com"
 # Pick a range of unused IPs that are on the same network as your nodes
 export BOOTSTRAP_METALLB_LB_RANGE="169.254.1.10-169.254.1.20"
-# The load balancer IP for ingress-nginx, choose from one of the available IPs above
+# The load balancer IP for traefik, choose from one of the available IPs above
-export BOOTSTRAP_INGRESS_NGINX_LB="169.254.1.10"
+export BOOTSTRAP_SVC_TRAEFIK_ADDR="169.254.1.10"
 ```
 
 5. Create required files based on ALL exported environment variables.
@@ -340,10 +340,10 @@ kubectl --kubeconfig=./kubeconfig get pods -n flux-system
 If your cluster is not accessible to outside world you can update your hosts file to verify the ingress controller is working.
 
 ```sh
-echo "${BOOTSTRAP_INGRESS_NGINX_LB} ${BOOTSTRAP_DOMAIN} homer.${BOOTSTRAP_DOMAIN}" | sudo tee -a /etc/hosts
+echo "${BOOTSTRAP_SVC_TRAEFIK_ADDR} ${BOOTSTRAP_DOMAIN} hajimari.${BOOTSTRAP_DOMAIN}" | sudo tee -a /etc/hosts
 ```
 
-Head over to your browser and you _should_ be able to access `https://homer.${BOOTSTRAP_DOMAIN}`
+Head over to your browser and you _should_ be able to access `https://hajimari.${BOOTSTRAP_DOMAIN}`
 
 ### direnv
 

cluster/apps/default/hajimari/helm-release.yaml

@@ -25,6 +25,7 @@ spec:
       namespaceSelector:
         matchNames:
           - default
+          - networking
       name: "you"
       customApps:
         - name: Some External Cluster Service
@@ -98,8 +99,9 @@ spec:
     ingress:
       main:
         enabled: true
-        ingressClassName: nginx
+        ingressClassName: "traefik"
         annotations:
+          traefik.ingress.kubernetes.io/router.entrypoints: "websecure"
           hajimari.io/enable: "true"
           hajimari.io/icon: "weather-sunset"
         hosts:

cluster/apps/networking/ingress-nginx/helm-release.yaml

@@ -1,41 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
-  name: ingress-nginx
-  namespace: networking
-spec:
-  interval: 5m
-  chart:
-    spec:
-      # renovate: registryUrl=https://kubernetes.github.io/ingress-nginx
-      chart: ingress-nginx
-      version: 4.0.1
-      sourceRef:
-        kind: HelmRepository
-        name: ingress-nginx-charts
-        namespace: flux-system
-  values:
-    controller:
-      ingressClass: nginx
-      replicaCount: 1
-      service:
-        type: LoadBalancer
-        loadBalancerIP: "${INGRESS_NGINX_LB}"
-        externalTrafficPolicy: Local
-      config:
-        ssl-protocols: "TLSv1.3 TLSv1.2"
-        proxy-body-size: "100m"
-        use-forwarded-headers: "true"
-      metrics:
-        enabled: false
-        serviceMonitor:
-          enabled: false
-      extraArgs:
-        default-ssl-certificate: "networking/${SECRET_DOMAIN/./-}-tls"
-      resources:
-        requests:
-          memory: 100Mi
-          cpu: 100m
-        limits:
-          memory: 500Mi

cluster/apps/networking/kustomization.yaml

@@ -1,5 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ingress-nginx
+  - traefik
   - wildcard-certificate

cluster/apps/networking/traefik/dashboard/ingress.yaml

@@ -0,0 +1,28 @@
+  
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: traefik-dashboard
+  namespace: networking
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: "websecure"
+    hajimari.io/enable: "true"
+    hajimari.io/icon: "web"
+    hajimari.io/appName: "traefik"
+spec:
+  tls:
+    - hosts:
+        - "traefik.${SECRET_DOMAIN}"
+      secretName: "${SECRET_DOMAIN/./-}-tls"
+  rules:
+    - host: "traefik.${SECRET_DOMAIN}"
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: traefik
+                port:
+                  number: 9000

cluster/apps/networking/traefik/dashboard/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ingress.yaml

cluster/apps/networking/traefik/helm-release.yaml

@@ -0,0 +1,85 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: traefik
+  namespace: networking
+spec:
+  interval: 5m
+  chart:
+    spec:
+      # renovate: registryUrl=https://helm.traefik.io/traefik
+      chart: traefik
+      version: 10.3.2
+      sourceRef:
+        kind: HelmRepository
+        name: traefik-charts
+        namespace: flux-system
+      interval: 5m
+  dependsOn:
+    - name: cert-manager
+      namespace: cert-manager
+  values:
+    image:
+      name: ghcr.io/k8s-at-home/traefik
+    deployment:
+      kind: Deployment
+      replicas: 1
+    service:
+      enabled: true
+      type: LoadBalancer
+      spec:
+        loadBalancerIP: "${SVC_TRAEFIK_ADDR}"
+        externalTrafficPolicy: Local
+    logs:
+      general:
+        format: json
+        level: DEBUG
+      access:
+        enabled: true
+        format: json
+    ingressClass:
+      enabled: true
+      isDefaultClass: true
+      fallbackApiVersion: v1
+    ingressRoute:
+      dashboard:
+        enabled: false
+    globalArguments:
+      - "--api.insecure=true"
+      - "--serverstransport.insecureskipverify=true"
+      - "--providers.kubernetesingress.ingressclass=traefik"
+      - "--metrics.prometheus=true"
+      - "--metrics.prometheus.entryPoint=metrics"
+    additionalArguments:
+      - "--providers.kubernetesingress.ingressendpoint.ip=${SVC_TRAEFIK_ADDR}"
+      - "--providers.kubernetescrd.allowexternalnameservices=true"
+    ports:
+      traefik:
+        expose: true
+      web:
+        redirectTo: websecure
+      websecure:
+        tls:
+          enabled: true
+          options: "default"
+      metrics:
+        port: 8082
+        expose: true
+        exposedPort: 8082
+    tlsOptions:
+      default:
+        minVersion: VersionTLS12
+        maxVersion: VersionTLS13
+        sniStrict: true
+    pilot:
+      enabled: false
+    experimental:
+      plugins:
+        enabled: false
+    resources:
+      requests:
+        memory: 100Mi
+        cpu: 500m
+      limits:
+        memory: 500Mi

cluster/apps/networking/traefik/kustomization.yaml

@@ -2,3 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - helm-release.yaml
+  - tls-store
+  - dashboard

cluster/apps/networking/traefik/tls-store/default.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: TLSStore
+metadata:
+  name: default
+  namespace: networking
+spec:
+  defaultCertificate:
+    secretName: "${SECRET_DOMAIN/./-}-tls"

cluster/apps/networking/traefik/tls-store/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - default.yaml

cluster/base/flux-system/charts/helm/authentik-charts.yaml

@@ -2,9 +2,9 @@
 apiVersion: source.toolkit.fluxcd.io/v1beta1
 kind: HelmRepository
 metadata:
-  name: ingress-nginx-charts
+  name: authentik-charts
   namespace: flux-system
 spec:
   interval: 1h
-  url: https://kubernetes.github.io/ingress-nginx
+  url: https://docker.beryju.org/chartrepo/authentik
   timeout: 3m

cluster/base/flux-system/charts/helm/deliveryhero-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: deliveryhero-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://charts.deliveryhero.io/
+  timeout: 3m

cluster/base/flux-system/charts/helm/descheduler-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: descheduler-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://kubernetes-sigs.github.io/descheduler
+  timeout: 3m

cluster/base/flux-system/charts/helm/external-dns-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: external-dns-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://kubernetes-sigs.github.io/external-dns
+  timeout: 3m

cluster/base/flux-system/charts/helm/grafana-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: grafana-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://grafana.github.io/helm-charts
+  timeout: 3m

cluster/base/flux-system/charts/helm/infracloudio-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: infracloudio-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://infracloudio.github.io/charts
+  timeout: 3m

cluster/base/flux-system/charts/helm/kustomization.yaml

@@ -1,9 +1,23 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - authentik-charts.yaml
   - bitnami-charts.yaml
+  - deliveryhero-charts.yaml
+  - descheduler-charts.yaml
+  - external-dns-charts.yaml
+  - grafana-charts.yaml
   - hajimari-charts.yaml
-  - ingress-nginx-charts.yaml
+  - infracloudio-charts.yaml
   - jetstack-charts.yaml
   - k8s-at-home-charts.yaml
   - metallb-charts.yaml
+  - minecraft-server-charts.yaml
+  - minio-charts.yaml
+  - nfs-subdir-external-provisioner-charts.yaml
+  - node-feature-discovery-charts.yaml
+  - prometheus-community-charts.yaml
+  - rook-ceph-charts.yaml
+  - stakater-charts.yaml
+  - traefik-charts.yaml
+  - weaveworks-kured-charts.yaml

cluster/base/flux-system/charts/helm/minecraft-server-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: minecraft-server-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://itzg.github.io/minecraft-server-charts/
+  timeout: 3m

cluster/base/flux-system/charts/helm/minio-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: minio-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://operator.min.io
+  timeout: 3m

cluster/base/flux-system/charts/helm/nfs-subdir-external-provisioner-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: nfs-subdir-external-provisioner-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
+  timeout: 3m

cluster/base/flux-system/charts/helm/node-feature-discovery-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: node-feature-discovery-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://kubernetes-sigs.github.io/node-feature-discovery/charts
+  timeout: 3m

cluster/base/flux-system/charts/helm/prometheus-community-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: prometheus-community-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://prometheus-community.github.io/helm-charts
+  timeout: 3m

cluster/base/flux-system/charts/helm/rook-ceph-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: rook-ceph-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://charts.rook.io/release
+  timeout: 3m

cluster/base/flux-system/charts/helm/stakater-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: stakater-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://stakater.github.io/stakater-charts
+  timeout: 3m

cluster/base/flux-system/charts/helm/traefik-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: traefik-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://helm.traefik.io/traefik
+  timeout: 3m

cluster/base/flux-system/charts/helm/weaveworks-kured-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: weaveworks-kured-charts
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://weaveworks.github.io/kured
+  timeout: 3m

cluster/crds/kustomization.yaml

@@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - cert-manager
+  - traefik

cluster/crds/traefik/crds.yaml

@@ -0,0 +1,57 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: GitRepository
+metadata:
+  name: traefik-crd-source
+  namespace: flux-system
+spec:
+  interval: 30m
+  url: https://github.com/traefik/traefik-helm-chart.git
+  ref:
+    # renovate: registryUrl=https://helm.traefik.io/traefik chart=traefik
+    tag: v10.3.2
+  ignore: |
+    # exclude all
+    /*
+    # path to crds
+    !/traefik/crds/
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+kind: Kustomization
+metadata:
+  name: traefik-crds
+  namespace: flux-system
+spec:
+  interval: 15m
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: traefik-crd-source
+  healthChecks:
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: ingressroutes.traefik.containo.us
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: ingressroutetcps.traefik.containo.us
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: ingressrouteudps.traefik.containo.us
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: middlewares.traefik.containo.us
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: middlewaretcps.traefik.containo.us
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: serverstransports.traefik.containo.us
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: tlsoptions.traefik.containo.us
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: tlsstores.traefik.containo.us
+    - apiVersion: apiextensions.k8s.io/v1
+      kind: CustomResourceDefinition
+      name: traefikservices.traefik.containo.us

cluster/crds/traefik/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - crds.yaml

tmpl/cluster-settings.yaml

@@ -6,4 +6,4 @@ metadata:
   namespace: flux-system
 data:
   METALLB_LB_RANGE: ${BOOTSTRAP_METALLB_LB_RANGE}
-  INGRESS_NGINX_LB: ${BOOTSTRAP_INGRESS_NGINX_LB}
+  SVC_TRAEFIK_ADDR: ${BOOTSTRAP_SVC_TRAEFIK_ADDR}