From c62dcbe8da29b1681c0dbd2092c5de27709be185 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Fri, 15 Apr 2022 23:21:26 -0400 Subject: [PATCH] feat: add k8s_gateway for accessing ingresses locally over DNS (#287) --- .config.sample.env | 5 ++- .github/ISSUE_TEMPLATE/config.yml | 1 + .github/linters/.yamllint.yaml | 1 + README.md | 21 ++++++++-- .../default/echo-server/kustomization.yaml | 1 + .../apps/default/hajimari/kustomization.yaml | 1 + cluster/apps/default/kustomization.yaml | 1 + cluster/apps/kustomization.yaml | 1 + .../networking/k8s-gateway/helm-release.yaml | 41 +++++++++++++++++++ .../networking/k8s-gateway/kustomization.yaml | 5 +++ cluster/apps/networking/kustomization.yaml | 2 + .../traefik/dashboard/kustomization.yaml | 1 + .../networking/traefik/kustomization.yaml | 1 + .../charts/helm/k8s-gateway-charts.yaml | 10 +++++ .../charts/helm/kustomization.yaml | 1 + .../flux-system/charts/kustomization.yaml | 1 + cluster/base/flux-system/kustomization.yaml | 1 + .../kube-system/kube-vip/kustomization.yaml | 1 + cluster/core/kube-system/kustomization.yaml | 1 + cluster/core/kustomization.yaml | 1 + .../core/metallb-system/kustomization.yaml | 1 + cluster/core/namespaces/kustomization.yaml | 1 + cluster/crds/kustomization.yaml | 1 + cluster/crds/traefik/kustomization.yaml | 1 + configure.sh | 3 ++ tmpl/cluster/cluster-settings.yaml | 1 + tmpl/cluster/kube-vip-daemonset.yaml | 8 ++-- 27 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 cluster/apps/networking/k8s-gateway/helm-release.yaml create mode 100644 cluster/apps/networking/k8s-gateway/kustomization.yaml create mode 100644 cluster/base/flux-system/charts/helm/k8s-gateway-charts.yaml diff --git a/.config.sample.env b/.config.sample.env index f183ce87..4af3d064 100644 --- a/.config.sample.env +++ b/.config.sample.env @@ -16,8 +16,11 @@ export BOOTSTRAP_CLOUDFLARE_APIKEY="" # You don't need many IPs, just choose 10 IPs to start with # e.g. 192.168.1.220-192.168.1.230 export BOOTSTRAP_METALLB_LB_RANGE="" -# The load balancer IP for traefik, choose from one of the available IPs above +# The load balancer IP for k8s_gateway, choose from one of the available IPs above # e.g. 192.168.1.220 +export BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR="" +# The load balancer IP for traefik, choose from one of the available IPs above +# e.g. 192.168.1.221 export BOOTSTRAP_METALLB_TRAEFIK_ADDR="" # Age Public Key - string should start with age # e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 20705b23..782cffae 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,3 +1,4 @@ +--- blank_issues_enabled: false contact_links: - name: Discuss on Discord diff --git a/.github/linters/.yamllint.yaml b/.github/linters/.yamllint.yaml index 3ed5366b..e13c88db 100644 --- a/.github/linters/.yamllint.yaml +++ b/.github/linters/.yamllint.yaml @@ -1,3 +1,4 @@ +--- ignore: | *.sops.* gotk-components.yaml diff --git a/README.md b/README.md index 8cd09aa3..3194774e 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Feel free to read up on any of these technologies before you get started to be m - [echo-server](https://github.com/Ealenn/Echo-Server) - REST Server Tests (Echo-Server) API (useful for debugging HTTP issues) - [flux](https://toolkit.fluxcd.io/) - GitOps tool for deploying manifests from the `cluster` directory - [hajimari](https://github.com/toboshii/hajimari) - start page with ingress discovery +- [k8s_gateway](https://github.com/ori-edge/k8s_gateway) - DNS resolver for all types of external Kubernetes resources - [kube-vip](https://kube-vip.io/) - layer 2 load balancer for the Kubernetes control plane - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) - default storage class provided by k3s - [metallb](https://metallb.universe.tf/) - bare metal load balancer @@ -306,19 +307,31 @@ kubectl --kubeconfig=./provision/kubeconfig get pods -n flux-system # source-controller-7d6875bcb4-zqw9f 1/1 Running 0 1h ``` -If all goes well and you have port forwarded `80` and `443` in your router to the `${BOOTSTRAP_METALLB_TRAEFIK_ADDR}` IP, in a few moments head over to your browser and you _should_ be able to access `https://hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` +🎉 **Congratulations** if all goes smooth you'll have a Kubernetes cluster managed by Flux, your Git repository is driving the state of your cluster. -🎉 **Congratulations** you have a Kubernetes cluster managed by Flux, your Git repository is driving the state of your cluster. +Now it's time to pause and go get some coffee ☕ ## 📣 Post installation +### 🌐 DNS + +`k8s_gateway` is deployed on the IP choosen for `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}`. Inorder to test DNS you can point your clients DNS to the `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}` IP address and load `https://hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` in your browser. + +You can also try debugging with the command `dig`, e.g. `dig @${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR} hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` and you should get a valid answer containing your `${BOOTSTRAP_METALLB_TRAEFIK_ADDR}` IP address. + +If your router (or Pi-Hole, Adguard Home or whatever) supports conditional DNS forwarding (also know as split-horizon DNS) you may have DNS requests for `${SECRET_DOMAIN}` only point to the `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}` IP address. This will ensure only DNS requests for `${SECRET_DOMAIN}` will get routed to your `k8s_gateway` service. + +To access services from the outside world port forwarded `80` and `443` in your router to the `${BOOTSTRAP_METALLB_TRAEFIK_ADDR}` IP, in a few moments head over to your browser and you _should_ be able to access `https://hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` from a device outside your LAN. + ### 👉 Troubleshooting -Our [wiki](https://github.com/k8s-at-home/template-cluster-k3s/wiki) is a good place to start troubleshooting issues. If that doesn't cover your issue, start a new thread in the #support channel on our [Discord](https://discord.gg/k8s-at-home). +Our [wiki](https://github.com/k8s-at-home/template-cluster-k3s/wiki) (WIP, contributions welcome) is a good place to start troubleshooting issues. If that doesn't cover your issue, come join and say Hi in our [Discord](https://discord.gg/k8s-at-home) server by starting a new thread in the #kubernetes support channel. + +You may also open a issue on this GitHub repo or open a [discussion on GitHub](https://github.com/k8s-at-home/organization/discussions). ### 🤖 Integrations -Our Check out our [wiki](https://github.com/k8s-at-home/template-cluster-k3s/wiki) (WIP) for more integrations! +Our Check out our [wiki](https://github.com/k8s-at-home/template-cluster-k3s/wiki) (WIP, contributions welcome) for more integrations! ## ❔ What's next diff --git a/cluster/apps/default/echo-server/kustomization.yaml b/cluster/apps/default/echo-server/kustomization.yaml index 34a8531c..2fa2de20 100644 --- a/cluster/apps/default/echo-server/kustomization.yaml +++ b/cluster/apps/default/echo-server/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/apps/default/hajimari/kustomization.yaml b/cluster/apps/default/hajimari/kustomization.yaml index d443f6cb..e06d74e5 100644 --- a/cluster/apps/default/hajimari/kustomization.yaml +++ b/cluster/apps/default/hajimari/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/apps/default/kustomization.yaml b/cluster/apps/default/kustomization.yaml index 6915e671..4c098db6 100644 --- a/cluster/apps/default/kustomization.yaml +++ b/cluster/apps/default/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/apps/kustomization.yaml b/cluster/apps/kustomization.yaml index aa6355c2..40789815 100644 --- a/cluster/apps/kustomization.yaml +++ b/cluster/apps/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/apps/networking/k8s-gateway/helm-release.yaml b/cluster/apps/networking/k8s-gateway/helm-release.yaml new file mode 100644 index 00000000..009f3edf --- /dev/null +++ b/cluster/apps/networking/k8s-gateway/helm-release.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: k8s-gateway + namespace: networking +spec: + interval: 5m + chart: + spec: + chart: k8s-gateway + version: 1.1.6 + sourceRef: + kind: HelmRepository + name: k8s-gateway + namespace: flux-system + interval: 5m + values: + fullnameOverride: k8s-gateway + domain: ${SECRET_DOMAIN} + ttl: 1 + service: + type: LoadBalancer + port: 53 + annotations: + metallb.universe.tf/loadBalancerIPs: "${METALLB_K8S_GATEWAY_ADDR}" + externalTrafficPolicy: Local + # Increase replicaCount and set an anti-affinity if you wish to scale this + # replicaCount: 1 + # affinity: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/name + # operator: In + # values: + # - k8s-gateway + # topologyKey: kubernetes.io/hostname diff --git a/cluster/apps/networking/k8s-gateway/kustomization.yaml b/cluster/apps/networking/k8s-gateway/kustomization.yaml new file mode 100644 index 00000000..2fa2de20 --- /dev/null +++ b/cluster/apps/networking/k8s-gateway/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - helm-release.yaml diff --git a/cluster/apps/networking/kustomization.yaml b/cluster/apps/networking/kustomization.yaml index ff2f69c5..2696558b 100644 --- a/cluster/apps/networking/kustomization.yaml +++ b/cluster/apps/networking/kustomization.yaml @@ -1,5 +1,7 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cloudflare-ddns + - k8s-gateway - traefik diff --git a/cluster/apps/networking/traefik/dashboard/kustomization.yaml b/cluster/apps/networking/traefik/dashboard/kustomization.yaml index 972f3ed0..afec11c2 100644 --- a/cluster/apps/networking/traefik/dashboard/kustomization.yaml +++ b/cluster/apps/networking/traefik/dashboard/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/apps/networking/traefik/kustomization.yaml b/cluster/apps/networking/traefik/kustomization.yaml index 8b500613..93d91fd2 100644 --- a/cluster/apps/networking/traefik/kustomization.yaml +++ b/cluster/apps/networking/traefik/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/base/flux-system/charts/helm/k8s-gateway-charts.yaml b/cluster/base/flux-system/charts/helm/k8s-gateway-charts.yaml new file mode 100644 index 00000000..740de15a --- /dev/null +++ b/cluster/base/flux-system/charts/helm/k8s-gateway-charts.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: k8s-gateway + namespace: flux-system +spec: + interval: 30m + url: https://ori-edge.github.io/k8s_gateway/ + timeout: 3m diff --git a/cluster/base/flux-system/charts/helm/kustomization.yaml b/cluster/base/flux-system/charts/helm/kustomization.yaml index 371b50bb..32af60f7 100644 --- a/cluster/base/flux-system/charts/helm/kustomization.yaml +++ b/cluster/base/flux-system/charts/helm/kustomization.yaml @@ -13,6 +13,7 @@ resources: - infracloudio-charts.yaml - jetstack-charts.yaml - k8s-at-home-charts.yaml + - k8s-gateway-charts.yaml - metallb-charts.yaml - metrics-server-charts.yaml - minecraft-server-charts.yaml diff --git a/cluster/base/flux-system/charts/kustomization.yaml b/cluster/base/flux-system/charts/kustomization.yaml index 0a03f298..96b157b7 100644 --- a/cluster/base/flux-system/charts/kustomization.yaml +++ b/cluster/base/flux-system/charts/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/base/flux-system/kustomization.yaml b/cluster/base/flux-system/kustomization.yaml index 6cf83ade..9f1cbea5 100644 --- a/cluster/base/flux-system/kustomization.yaml +++ b/cluster/base/flux-system/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/core/kube-system/kube-vip/kustomization.yaml b/cluster/core/kube-system/kube-vip/kustomization.yaml index e6255de2..a6c942f2 100644 --- a/cluster/core/kube-system/kube-vip/kustomization.yaml +++ b/cluster/core/kube-system/kube-vip/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/core/kube-system/kustomization.yaml b/cluster/core/kube-system/kustomization.yaml index 65ea8bdc..639fc97c 100644 --- a/cluster/core/kube-system/kustomization.yaml +++ b/cluster/core/kube-system/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/core/kustomization.yaml b/cluster/core/kustomization.yaml index 9882a235..654b1332 100644 --- a/cluster/core/kustomization.yaml +++ b/cluster/core/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/core/metallb-system/kustomization.yaml b/cluster/core/metallb-system/kustomization.yaml index 34a8531c..2fa2de20 100644 --- a/cluster/core/metallb-system/kustomization.yaml +++ b/cluster/core/metallb-system/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/core/namespaces/kustomization.yaml b/cluster/core/namespaces/kustomization.yaml index feecf3d0..71aa049b 100644 --- a/cluster/core/namespaces/kustomization.yaml +++ b/cluster/core/namespaces/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/crds/kustomization.yaml b/cluster/crds/kustomization.yaml index 5b7db55d..8b0a42ba 100644 --- a/cluster/crds/kustomization.yaml +++ b/cluster/crds/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/cluster/crds/traefik/kustomization.yaml b/cluster/crds/traefik/kustomization.yaml index 2ed3b351..7d6dc05e 100644 --- a/cluster/crds/traefik/kustomization.yaml +++ b/cluster/crds/traefik/kustomization.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: diff --git a/configure.sh b/configure.sh index 0580ca67..23a6f7df 100755 --- a/configure.sh +++ b/configure.sh @@ -173,13 +173,16 @@ verify_metallb() { local ip_floor= local ip_ceil= _has_envar "BOOTSTRAP_METALLB_LB_RANGE" + _has_envar "BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR" _has_envar "BOOTSTRAP_METALLB_TRAEFIK_ADDR" ip_floor=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f1) ip_ceil=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f2) + # TODO(configure.sh): More checks on valid IP addressing _has_valid_ip "${ip_floor}" "BOOTSTRAP_METALLB_LB_RANGE" _has_valid_ip "${ip_ceil}" "BOOTSTRAP_METALLB_LB_RANGE" + _has_valid_ip "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}" "BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR" _has_valid_ip "${BOOTSTRAP_METALLB_TRAEFIK_ADDR}" "BOOTSTRAP_METALLB_TRAEFIK_ADDR" } diff --git a/tmpl/cluster/cluster-settings.yaml b/tmpl/cluster/cluster-settings.yaml index bf9595de..18cd40eb 100644 --- a/tmpl/cluster/cluster-settings.yaml +++ b/tmpl/cluster/cluster-settings.yaml @@ -6,4 +6,5 @@ metadata: namespace: flux-system data: METALLB_LB_RANGE: ${BOOTSTRAP_METALLB_LB_RANGE} + METALLB_K8S_GATEWAY_ADDR: ${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR} METALLB_TRAEFIK_ADDR: ${BOOTSTRAP_METALLB_TRAEFIK_ADDR} diff --git a/tmpl/cluster/kube-vip-daemonset.yaml b/tmpl/cluster/kube-vip-daemonset.yaml index 0f69c28d..726032ea 100644 --- a/tmpl/cluster/kube-vip-daemonset.yaml +++ b/tmpl/cluster/kube-vip-daemonset.yaml @@ -57,11 +57,11 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists tolerations: - effect: NoSchedule operator: Exists