fix: cilium not exclusive CNI, remove ingress tls block and use defaultPodOptions in app-template workloads

Signed-off-by: Devin Buhl <devin@buhl.casa>

bootstrap/templates/partials/cilium-values-full.partial.yaml.j2

@@ -10,11 +10,12 @@ cgroup:
 cluster:
   id: 1
   name: #{ bootstrap_cluster_name|default('home-kubernetes', true) }#
-#% if bootstrap_distribution in ["k3s"] %#
 cni:
+  exclusive: false
+  #% if bootstrap_distribution in ["k3s"] %#
   binPath: /var/lib/rancher/k3s/data/current/bin
   confPath: /var/lib/rancher/k3s/agent/etc/cni/net.d
-#% endif %#
+  #% endif %#
 containerRuntime:
   integration: containerd
   #% if bootstrap_distribution in ["k3s"] %#
@@ -54,11 +55,7 @@ hubble:
     ingress:
       enabled: true
       className: internal
-      hosts:
-        - "hubble.${SECRET_DOMAIN}"
-      tls:
-        - hosts:
-            - "hubble.${SECRET_DOMAIN}"
+      hosts: ["hubble.${SECRET_DOMAIN}"]
 #% else %#
 hubble:
   enabled: false

bootstrap/templates/partials/cilium-values-init.partial.yaml.j2

@@ -10,11 +10,12 @@ cgroup:
 cluster:
   id: 1
   name: #{ bootstrap_cluster_name|default('home-kubernetes', true) }#
-#% if bootstrap_distribution in ["k3s"] %#
 cni:
+  exclusive: false
+  #% if bootstrap_distribution in ["k3s"] %#
   binPath: /var/lib/rancher/k3s/data/current/bin
   confPath: /var/lib/rancher/k3s/agent/etc/cni/net.d
-#% endif %#
+  #% endif %#
 containerRuntime:
   integration: containerd
   #% if bootstrap_distribution in ["k3s"] %#