feat!: Remove support for k3s and go full-on Talos (#1482)

2024-05-21 08:29:59 -04:00
1 parent f4eb701ac6
commit 21afc0baa2
@@ -0,0 +1,68 @@
+---
+autoDirectNodeRoutes: true
+bgpControlPlane:
+  enabled: true
+bpf:
+  masquerade: false
+cgroup:
+  automount:
+    enabled: false
+  hostRoot: /sys/fs/cgroup
+cluster:
+  id: 1
+  name: #{ bootstrap_cluster_name|default('home-kubernetes', true) }#
+cni:
+  exclusive: false
+containerRuntime:
+  integration: containerd
+# NOTE: devices might need to be set if you have more than one active NIC on your hosts
+# devices: eno+ eth+
+endpointRoutes:
+  enabled: true
+hubble:
+  enabled: false
+ipam:
+  mode: kubernetes
+ipv4NativeRoutingCIDR: "${CLUSTER_CIDR}"
+#% if bootstrap_feature_gates.dual_stack_ipv4_first %#
+ipv6NativeRoutingCIDR: "${CLUSTER_CIDR_V6}"
+ipv6:
+  enabled: true
+#% endif %#
+k8sServiceHost: 127.0.0.1
+k8sServicePort: 7445
+kubeProxyReplacement: true
+kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+l2announcements:
+  #% if ((bootstrap_bgp.enabled) or (bootstrap_feature_gates.dual_stack_ipv4_first)) %#
+  enabled: false # https://github.com/cilium/cilium/issues/28985
+  #% else %#
+  enabled: true
+  #% endif %#
+loadBalancer:
+  algorithm: maglev
+  mode: snat
+localRedirectPolicy: true
+operator:
+  replicas: 1
+  rollOutPods: true
+rollOutCiliumPods: true
+routingMode: native
+securityContext:
+  capabilities:
+    ciliumAgent:
+      - CHOWN
+      - KILL
+      - NET_ADMIN
+      - NET_RAW
+      - IPC_LOCK
+      - SYS_ADMIN
+      - SYS_RESOURCE
+      - DAC_OVERRIDE
+      - FOWNER
+      - SETGID
+      - SETUID
+    cleanCiliumState:
+      - NET_ADMIN
+      - SYS_ADMIN
+      - SYS_RESOURCE
@@ -20,7 +20,57 @@ spec:
    cleanupOnFail: true
    remediation:
      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: cilium-helm-values
  values:
-    #% filter indent(width=4, first=True) %#
-    #% include 'partials/cilium-values-full.partial.yaml.j2' %#
-    #% endfilter %#
+    #% if bootstrap_cloudflare.enabled %#
+    hubble:
+      enabled: true
+      metrics:
+        enabled:
+          - dns:query
+          - drop
+          - tcp
+          - flow
+          - port-distribution
+          - icmp
+          - http
+        serviceMonitor:
+          enabled: true
+        dashboards:
+          enabled: true
+          annotations:
+            grafana_folder: Cilium
+      relay:
+        enabled: true
+        rollOutPods: true
+        prometheus:
+          serviceMonitor:
+            enabled: true
+      ui:
+        enabled: true
+        rollOutPods: true
+        ingress:
+          enabled: true
+          className: internal
+          hosts: ["hubble.${SECRET_DOMAIN}"]
+    #% endif %#
+    operator:
+      prometheus:
+        enabled: true
+        serviceMonitor:
+          enabled: true
+      dashboards:
+        enabled: true
+        annotations:
+          grafana_folder: Cilium
+    prometheus:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        trustCRDsExist: true
+    dashboards:
+      enabled: true
+      annotations:
+        grafana_folder: Cilium
@@ -3,3 +3,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - ./helmrelease.yaml
+configMapGenerator:
+  - name: cilium-helm-values
+    files:
+      - values.yaml=./helm-values.yaml
+configurations:
+  - kustomizeconfig.yaml
@@ -0,0 +1,7 @@
+---
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease
@@ -0,0 +1,58 @@
+---
+fullnameOverride: coredns
+replicaCount: 1
+k8sAppLabelOverride: kube-dns
+serviceAccount:
+  create: true
+service:
+  name: kube-dns
+  clusterIP: "#{ bootstrap_service_network | nthhost(10) }#"
+servers:
+  - zones:
+      - zone: .
+        scheme: dns://
+        use_tcp: true
+    port: 53
+    plugins:
+      - name: errors
+      - name: health
+        configBlock: |-
+          lameduck 5s
+      - name: ready
+      - name: log
+        configBlock: |-
+          class error
+      - name: prometheus
+        parameters: 0.0.0.0:9153
+      - name: kubernetes
+        parameters: cluster.local in-addr.arpa ip6.arpa
+        configBlock: |-
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+      - name: forward
+        parameters: . /etc/resolv.conf
+      - name: cache
+        parameters: 30
+      - name: loop
+      - name: reload
+      - name: loadbalance
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+topologySpreadConstraints:
+  - maxSkew: 1
+    topologyKey: kubernetes.io/hostname
+    whenUnsatisfiable: DoNotSchedule
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/instance: coredns
@@ -0,0 +1,27 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: coredns
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: coredns
+      version: 1.29.0
+      sourceRef:
+        kind: HelmRepository
+        name: coredns
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: coredns-helm-values
@@ -0,0 +1,12 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+configMapGenerator:
+  - name: coredns-helm-values
+    files:
+      - values.yaml=./helm-values.yaml
+configurations:
+  - kustomizeconfig.yaml
@@ -0,0 +1,7 @@
+---
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease
@@ -1,20 +1,21 @@
 ---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: &app system-upgrade-controller
+  name: &app coredns
  namespace: flux-system
 spec:
-  targetNamespace: system-upgrade
+  targetNamespace: kube-system
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/system-upgrade/system-upgrade-controller/app
-  prune: true
+  path: ./kubernetes/apps/kube-system/coredns/app
+  prune: false # never should be deleted
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
-  wait: true
+  wait: false
  interval: 30m
  retryInterval: 1m
  timeout: 5m
@@ -1 +0,0 @@
-main = lambda data: data.get("bootstrap_distribution", "k3s") in ["k3s"]
@@ -1,2 +0,0 @@
---
-#% include 'partials/kube-vip-ds.partial.yaml.j2' %#
@@ -1,6 +0,0 @@
---
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ./rbac.yaml
-  - ./daemonset.yaml
@@ -1,2 +0,0 @@
---
-#% include 'partials/kube-vip-rbac.partial.yaml.j2' %#
@@ -1 +0,0 @@
-main = lambda data: data.get("bootstrap_distribution", "k3s") in ["talos"]
@@ -0,0 +1,3 @@
+---
+providerRegex: ^(#{ (bootstrap_node_inventory | map(attribute='name') | join('|')) }#)$
+bypassDnsResolution: true
@@ -20,10 +20,10 @@ spec:
    cleanupOnFail: true
    remediation:
      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: kubelet-csr-approver-helm-values
  values:
-    #% filter indent(width=4, first=True) %#
-    #% include 'partials/kubelet-csr-approver-values.partial.yaml.j2' %#
-    #% endfilter %#
    metrics:
      enable: true
      serviceMonitor:
@@ -3,3 +3,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - ./helmrelease.yaml
+configMapGenerator:
+  - name: kubelet-csr-approver-helm-values
+    files:
+      - values.yaml=./helm-values.yaml
+configurations:
+  - kustomizeconfig.yaml
@@ -0,0 +1,7 @@
+---
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease
@@ -4,12 +4,8 @@ kind: Kustomization
 resources:
  - ./namespace.yaml
  - ./cilium/ks.yaml
+  - ./coredns/ks.yaml
  - ./metrics-server/ks.yaml
  - ./reloader/ks.yaml
-  #% if bootstrap_distribution in ["talos"] %#
  - ./kubelet-csr-approver/ks.yaml
  - ./spegel/ks.yaml
-  #% endif %#
-  #% if bootstrap_distribution in ["k3s"] %#
-  - ./kube-vip/ks.yaml
-  #% endif %#
@@ -25,9 +25,6 @@ spec:
      - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
      - --kubelet-use-node-status-port
      - --metric-resolution=15s
-      #% if bootstrap_distribution in ["k3s"] %#
-      - --kubelet-insecure-tls
-      #% endif %#
    metrics:
      enabled: true
    serviceMonitor:
@@ -1 +0,0 @@
-main = lambda data: data.get("bootstrap_distribution", "k3s") in ["talos"]
@@ -0,0 +1,7 @@
+---
+spegel:
+  containerdSock: /run/containerd/containerd.sock
+  containerdRegistryConfigPath: /etc/cri/conf.d/hosts
+service:
+  registry:
+    hostPort: 29999
@@ -20,12 +20,9 @@ spec:
    cleanupOnFail: true
    remediation:
      retries: 3
+  valuesFrom:
+    - kind: ConfigMap
+      name: spegel-helm-values
  values:
-    spegel:
-      containerdSock: /run/containerd/containerd.sock
-      containerdRegistryConfigPath: /etc/cri/conf.d/hosts
-    service:
-      registry:
-        hostPort: 29999
    serviceMonitor:
      enabled: true
@@ -3,3 +3,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - ./helmrelease.yaml
+configMapGenerator:
+  - name: spegel-helm-values
+    files:
+      - values.yaml=./helm-values.yaml
+configurations:
+  - kustomizeconfig.yaml
@@ -0,0 +1,7 @@
+---
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease
@@ -3,5 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - ./namespace.yaml
-  - ./system-upgrade-controller/ks.yaml
-  - ./k3s/ks.yaml
+  - ./prometheus-operator-crds/ks.yaml
@@ -2,6 +2,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: system-upgrade
+  name: observability
  labels:
    kustomize.toolkit.fluxcd.io/prune: disabled
@@ -0,0 +1,22 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: prometheus-operator-crds
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: prometheus-operator-crds
+      version: 11.0.0
+      sourceRef:
+        kind: HelmRepository
+        name: prometheus-community
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
@@ -2,4 +2,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ./plan.yaml
+  - ./helmrelease.yaml
@@ -2,15 +2,15 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: &app kube-vip
+  name: &app prometheus-operator-crds
  namespace: flux-system
 spec:
-  targetNamespace: kube-system
+  targetNamespace: observability
  commonMetadata:
    labels:
      app.kubernetes.io/name: *app
-  path: ./kubernetes/apps/kube-system/kube-vip/app
-  prune: true
+  path: ./kubernetes/apps/observability/prometheus-operator-crds/app
+  prune: false # never should be deleted
  sourceRef:
    kind: GitRepository
    name: home-kubernetes
@@ -1 +0,0 @@
-main = lambda data: data.get("bootstrap_distribution", "k3s") in ["k3s"]
@@ -1,50 +0,0 @@
---
-apiVersion: upgrade.cattle.io/v1
-kind: Plan
-metadata:
-  name: controllers
-spec:
-  version: "${KUBE_VERSION}"
-  upgrade:
-    image: rancher/k3s-upgrade
-  serviceAccountName: system-upgrade
-  concurrency: 1
-  cordon: true
-  nodeSelector:
-    matchExpressions:
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-  tolerations:
-    - effect: NoSchedule
-      operator: Exists
-    - effect: NoExecute
-      operator: Exists
-    - key: node-role.kubernetes.io/control-plane
-      effect: NoSchedule
-      operator: Exists
-    - key: node-role.kubernetes.io/master
-      effect: NoSchedule
-      operator: Exists
-    - key: node-role.kubernetes.io/etcd
-      effect: NoExecute
-      operator: Exists
-    - key: CriticalAddonsOnly
-      operator: Exists
---
-apiVersion: upgrade.cattle.io/v1
-kind: Plan
-metadata:
-  name: workers
-spec:
-  version: "${KUBE_VERSION}"
-  serviceAccountName: system-upgrade
-  concurrency: 1
-  nodeSelector:
-    matchExpressions:
-      - key: node-role.kubernetes.io/control-plane
-        operator: DoesNotExist
-  prepare:
-    image: rancher/k3s-upgrade
-    args: ["prepare", "controllers"]
-  upgrade:
-    image: rancher/k3s-upgrade
@@ -1,26 +0,0 @@
---
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: &app system-upgrade-k3s
-  namespace: flux-system
-spec:
-  targetNamespace: system-upgrade
-  commonMetadata:
-    labels:
-      app.kubernetes.io/name: *app
-  dependsOn:
-    - name: system-upgrade-controller
-  path: ./kubernetes/apps/system-upgrade/k3s/app
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: home-kubernetes
-  wait: false
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
-  postBuild:
-    substitute:
-      # renovate: datasource=github-releases depName=k3s-io/k3s
-      KUBE_VERSION: v1.30.0+k3s1
@@ -1,101 +0,0 @@
---
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: &app system-upgrade-controller
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: app-template
-      version: 3.1.0
-      sourceRef:
-        kind: HelmRepository
-        name: bjw-s
-        namespace: flux-system
-  install:
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  values:
-    controllers:
-      system-upgrade-controller:
-        strategy: RollingUpdate
-        containers:
-          app:
-            image:
-              repository: docker.io/rancher/system-upgrade-controller
-              tag: v0.13.4
-            env:
-              SYSTEM_UPGRADE_CONTROLLER_DEBUG: false
-              SYSTEM_UPGRADE_CONTROLLER_THREADS: 2
-              SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: 900
-              SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: 99
-              SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: IfNotPresent
-              SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: registry.k8s.io/kubectl:v1.30.1
-              SYSTEM_UPGRADE_JOB_PRIVILEGED: true
-              SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: 900
-              SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m
-              SYSTEM_UPGRADE_CONTROLLER_NAME: *app
-              SYSTEM_UPGRADE_CONTROLLER_NAMESPACE:
-                valueFrom:
-                  fieldRef:
-                    fieldPath: metadata.namespace
-            securityContext:
-              allowPrivilegeEscalation: false
-              readOnlyRootFilesystem: true
-              capabilities: { drop: ["ALL"] }
-    defaultPodOptions:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-        runAsGroup: 65534
-        seccompProfile: { type: RuntimeDefault }
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
-      tolerations:
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/master
-          operator: Exists
-          effect: NoSchedule
-    serviceAccount:
-      create: true
-      name: system-upgrade
-    persistence:
-      tmp:
-        type: emptyDir
-        globalMounts:
-          - path: /tmp
-      etc-ssl:
-        type: hostPath
-        hostPath: /etc/ssl
-        hostPathType: DirectoryOrCreate
-        globalMounts:
-          - path: /etc/ssl
-            readOnly: true
-      etc-pki:
-        type: hostPath
-        hostPath: /etc/pki
-        hostPathType: DirectoryOrCreate
-        globalMounts:
-          - path: /etc/pki
-            readOnly: true
-      etc-ca-certificates:
-        type: hostPath
-        hostPath: /etc/ca-certificates
-        hostPathType: DirectoryOrCreate
-        globalMounts:
-          - path: /etc/ca-certificates
-            readOnly: true
@@ -1,8 +0,0 @@
---
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  # renovate: datasource=github-releases depName=rancher/system-upgrade-controller
-  - https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml
-  - helmrelease.yaml
-  - rbac.yaml
@@ -1,13 +0,0 @@
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: system-upgrade
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
-  - kind: ServiceAccount
-    name: system-upgrade
-    namespace: system-upgrade
				`@@ -1 +0,0 @@`
				`main = lambda data: data.get("bootstrap_distribution", "k3s") in ["k3s"]`