feat!: Use Ansible and Terraform to provision Nodes and DNS and create configure script (#84)

provision/ansible/inventory/group_vars/kubernetes/k3s.yml

@@ -0,0 +1,32 @@
+---
+#
+# Below vars are for the xanmanning.k3s role
+# ...see https://github.com/PyratLabs/ansible-role-k3s
+#
+
+# (string) Use a specific version of k3s
+k3s_release_version: "v1.21.5+k3s1"
+
+# (bool) Install using hard links rather than symbolic links.
+k3s_install_hard_links: true
+
+# (bool) Escalate user privileges for all tasks
+k3s_become_for_all: true
+
+# (bool) Enable debug logging on the k3s service
+k3s_debug: false
+
+# (bool) Enable etcd embedded datastore
+k3s_etcd_datastore: true
+
+# (bool) Allow the use of unsupported configurations in k3s
+k3s_use_unsupported_config: false
+
+# (string) Control Plane registration address
+k3s_registration_address: "{{ kubevip_address }}"
+
+# (list) A flat list of templates to deploy on the primary control plane
+# /var/lib/rancher/k3s/server/manifests
+k3s_server_manifests_templates:
+  - "kube-vip/kube-vip-rbac.yaml.j2"
+  - "kube-vip/kube-vip-daemonset.yaml.j2"

provision/ansible/inventory/group_vars/kubernetes/ubuntu.yml

@@ -0,0 +1,6 @@
+---
+# (string) Timezone for the servers
+# timezone: "America/New_York"
+
+# (list) Additional ssh public keys to add to the nodes
+# ssh_authorized_keys:

provision/ansible/inventory/group_vars/master/k3s.yml

@@ -0,0 +1,42 @@
+---
+# https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/
+# https://github.com/PyratLabs/ansible-role-k3s
+
+# (bool) Specify if a host (or host group) are part of the control plane
+k3s_control_node: true
+
+# (dict) k3s settings for all control-plane nodes
+k3s_server:
+  node-ip: "{{ ansible_host }}"
+  tls-san:
+    # kube-vip
+    - "{{ kubevip_address }}"
+  docker: false
+  flannel-backend: "host-gw" # This needs to be in quotes
+  disable:
+    - traefik
+    - servicelb
+    - metrics-server
+  disable-network-policy: true
+  disable-cloud-controller: true
+  write-kubeconfig-mode: "644"
+  # Network CIDR to use for pod IPs
+  cluster-cidr: "10.42.0.0/16"
+  # Network CIDR to use for service IPs
+  service-cidr: "10.43.0.0/16"
+  kubelet-arg:
+    - "feature-gates=GracefulNodeShutdown=true"
+    - "feature-gates=MixedProtocolLBService=true"
+  # Required to use monitor these components with kube-prometheus-stack
+  kube-controller-manager-arg:
+    - "address=0.0.0.0"
+    - "bind-address=0.0.0.0"
+  kube-proxy-arg:
+    - "metrics-bind-address=0.0.0.0"
+  kube-scheduler-arg:
+    - "address=0.0.0.0"
+    - "bind-address=0.0.0.0"
+  etcd-expose-metrics: true
+  # Required for HAProxy health-checks
+  kube-apiserver-arg:
+    - "anonymous-auth=true"

provision/ansible/inventory/group_vars/worker/k3s.yml

@@ -0,0 +1,13 @@
+---
+# https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/
+# https://github.com/PyratLabs/ansible-role-k3s
+
+# (bool) Specify if a host (or host group) are part of the control plane
+k3s_control_node: false
+
+# (dict) k3s settings for all worker nodes
+k3s_agent:
+  node-ip: "{{ ansible_host }}"
+  kubelet-arg:
+    - "feature-gates=GracefulNodeShutdown=true"
+    - "feature-gates=MixedProtocolLBService=true"

provision/ansible/inventory/hosts.yml

@@ -0,0 +1,11 @@
+---
+kubernetes:
+  children:
+    master:
+      hosts:
+        k8s-0:
+          ansible_host: 192.168.42.10
+    worker:
+      hosts:
+        k8s-1:
+          ansible_host: 192.168.42.13

provision/ansible/playbooks/k3s-install.yml

@@ -0,0 +1,65 @@
+---
+- hosts:
+    - master
+    - worker
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      pause:
+        seconds: 5
+  tasks:
+    - name: Check if cluster is installed
+      ansible.builtin.stat:
+        path: "/etc/rancher/k3s/config.yaml"
+      register: k3s_check_installed
+      check_mode: false
+
+    - name: Set manifest facts
+      ansible.builtin.set_fact:
+        k3s_server_manifests_templates: []
+        k3s_server_manifests_urls: []
+      when: k3s_check_installed.stat.exists
+
+    - name: Install Kubernetes
+      include_role:
+        name: xanmanning.k3s
+        public: true
+
+    - name: Copy kubeconfig locally to /tmp
+      run_once: true
+      ansible.builtin.fetch:
+        src: "/etc/rancher/k3s/k3s.yaml"
+        dest: "/tmp/kubeconfig"
+        flat: true
+      when:
+        - k3s_control_node is defined
+        - k3s_control_node
+
+    - name: Update kubeconfig with the right IPv4 address
+      delegate_to: localhost
+      become: false
+      run_once: true
+      ansible.builtin.replace:
+        path: "/tmp/kubeconfig"
+        regexp: "https://127.0.0.1:6443"
+        replace: "https://{{ k3s_registration_address }}:6443"
+
+    - name: Remove deployed manifest templates
+      ansible.builtin.file:
+        path: "{{ k3s_server_manifests_dir }}/{{ item | basename | regex_replace('\\.j2$', '') }}"
+        state: absent
+      loop: "{{ k3s_server_manifests_templates }}"
+      when:
+        - k3s_server_manifests_templates
+        - k3s_server_manifests_templates | length > 0
+
+    - name: Remove deployed manifest urls
+      ansible.builtin.file:
+        path: "{{ k3s_server_manifests_dir }}/{{ item.filename }}"
+        state: absent
+      loop: "{{ k3s_server_manifests_urls }}"
+      when:
+        - k3s_server_manifests_urls
+        - k3s_server_manifests_urls | length > 0

provision/ansible/playbooks/k3s-nuke.yml

@@ -0,0 +1,32 @@
+---
+- hosts:
+    - master
+    - worker
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      pause:
+        seconds: 5
+  tasks:
+    - name: Kill k3s
+      ansible.builtin.command: /usr/local/bin/k3s-killall.sh
+    - name: Uninstall k3s
+      ansible.builtin.command:
+        cmd: /usr/local/bin/k3s-uninstall.sh
+        removes: /usr/local/bin/k3s-uninstall.sh
+    - name: Uninstall k3s agent
+      ansible.builtin.command:
+        cmd: /usr/local/bin/k3s-agent-uninstall.sh
+        removes: /usr/local/bin/k3s-agent-uninstall.sh
+    - name: Gather list of CNI files to delete
+      find:
+        paths: /etc/cni/net.d
+        patterns: "*"
+      register: files_to_delete
+    - name: Delete CNI files
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        state: absent
+      loop: "{{ files_to_delete.files }}"

provision/ansible/playbooks/ubuntu-prepare.yml

@@ -0,0 +1,13 @@
+---
+- hosts:
+    - master
+    - worker
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      pause:
+        seconds: 5
+  roles:
+    - ubuntu

provision/ansible/playbooks/ubuntu-upgrade.yml

@@ -0,0 +1,18 @@
+---
+- hosts:
+    - master
+    - worker
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  tasks:
+    - name: upgrade
+      ansible.builtin.apt:
+        upgrade: full
+        update_cache: true
+        cache_valid_time: 3600
+        autoclean: true
+        autoremove: true
+      register: apt_upgrade
+      retries: 5
+      until: apt_upgrade is success

provision/ansible/requirements.yml

@@ -0,0 +1,9 @@
+---
+collections:
+  - name: community.general
+    version: 3.7.0
+  - name: community.sops
+    version: 1.1.0
+roles:
+  - src: xanmanning.k3s
+    version: v2.11.0

provision/ansible/roles/ubuntu/tasks/filesystem.yml

@@ -0,0 +1,20 @@
+---
+- name: Update max_user_watches
+  ansible.posix.sysctl:
+    name: fs.inotify.max_user_watches
+    value: "65536"
+    state: present
+    sysctl_file: /etc/sysctl.d/98-kubernetes-fs.conf
+
+- name: Disable swap at runtime
+  ansible.builtin.command: swapoff -a
+  when: ansible_swaptotal_mb > 0
+
+- name: Disable swap on boot
+  ansible.posix.mount:
+    name: "{{ item }}"
+    fstype: swap
+    state: absent
+  loop:
+    - swap
+    - none

provision/ansible/roles/ubuntu/tasks/locale.yml

@@ -0,0 +1,6 @@
+---
+- name: Set timezone
+  community.general.timezone:
+    name: "{{ timezone }}"
+  when:
+    - timezone is defined

provision/ansible/roles/ubuntu/tasks/main.yml

@@ -0,0 +1,24 @@
+---
+- include: locale.yml
+  tags:
+    - locale
+
+- include: packages.yml
+  tags:
+    - packages
+
+- include: network.yml
+  tags:
+    - network
+
+- include: filesystem.yml
+  tags:
+    - filesystem
+
+- include: unattended-upgrades.yml
+  tags:
+    - unattended-upgrades
+
+- include: user.yml
+  tags:
+    - user

provision/ansible/roles/ubuntu/tasks/network.yml

@@ -0,0 +1,47 @@
+---
+- name: Set hostname to inventory hostname
+  ansible.builtin.hostname:
+    name: "{{ inventory_hostname }}"
+  when:
+    - ansible_hostname != inventory_hostname
+
+- name: Update /etc/hosts to include hostname
+  ansible.builtin.blockinfile:
+    path: /etc/hosts
+    create: true
+    block: |
+      127.0.0.1 localhost
+      127.0.1.1 {{ inventory_hostname }}
+
+      # The following lines are desirable for IPv6 capable hosts
+      ::1 ip6-localhost ip6-loopback
+      fe00::0 ip6-localnet
+      ff00::0 ip6-mcastprefix
+      ff02::1 ip6-allnodes
+      ff02::2 ip6-allrouters
+      ff02::3 ip6-allhosts
+
+- name: Check for bridge-nf-call-iptables
+  ansible.builtin.stat:
+    path: /proc/sys/net/bridge/bridge-nf-call-iptables
+  register: bridge_nf_call_iptables_result
+
+- name: Set Kubernetes network configuration
+  ansible.builtin.blockinfile:
+    path: /etc/sysctl.d/99-kubernetes-cri.conf
+    mode: 0644
+    create: true
+    block: |
+      net.ipv4.ip_forward = 1
+      net.bridge.bridge-nf-call-iptables = 1
+      net.ipv6.conf.all.forwarding = 1
+      net.bridge.bridge-nf-call-ip6tables = 1
+  when:
+    - bridge_nf_call_iptables_result.stat.exists
+  register: sysctl_network
+
+- name: Reload Kubernetes network configuration
+  ansible.builtin.shell: sysctl -p /etc/sysctl.d/99-kubernetes-cri.conf
+  when:
+    - sysctl_network.changed
+    - bridge_nf_call_iptables_result.stat.exists

provision/ansible/roles/ubuntu/tasks/packages.yml

@@ -0,0 +1,76 @@
+---
+- name: Upgrade all system packages
+  ansible.builtin.apt:
+    upgrade: full
+    update_cache: true
+    cache_valid_time: 3600
+    autoclean: true
+    autoremove: true
+  register: apt_upgrade
+  retries: 5
+  until: apt_upgrade is success
+
+- name: Install common packages
+  ansible.builtin.apt:
+    name:
+      - apt-transport-https
+      - arptables
+      - ca-certificates
+      - curl
+      - ebtables
+      - iputils-ping
+      - ipvsadm
+      - lvm2
+      - net-tools
+      - nfs-common
+      - open-iscsi
+      - psmisc
+      - smartmontools
+      - software-properties-common
+      - unattended-upgrades
+    install_recommends: false
+    update_cache: true
+    cache_valid_time: 3600
+    autoclean: true
+    autoremove: true
+  register: apt_install_common
+  retries: 5
+  until: apt_install_common is success
+
+- name: Copy 20auto-upgrades unattended-upgrades config
+  ansible.builtin.blockinfile:
+    path: /etc/apt/apt.conf.d/20auto-upgrades
+    mode: 0644
+    create: true
+    block: |
+      APT::Periodic::Update-Package-Lists "1";
+      APT::Periodic::Download-Upgradeable-Packages "1";
+      APT::Periodic::AutocleanInterval "7";
+      APT::Periodic::Unattended-Upgrade "1";
+
+- name: Copy 50unattended-upgrades unattended-upgrades config
+  ansible.builtin.blockinfile:
+    path: /etc/apt/apt.conf.d/50unattended-upgrades
+    mode: 0644
+    create: true
+    block: |
+      Unattended-Upgrade::Automatic-Reboot "false";
+      Unattended-Upgrade::Remove-Unused-Dependencies "true";
+      Unattended-Upgrade::Allowed-Origins {
+          "${distro_id} stable";
+          "${distro_id} ${distro_codename}-security";
+          "${distro_id} ${distro_codename}-updates";
+      };
+
+- name: Start unattended-upgrades service
+  ansible.builtin.systemd:
+    name: unattended-upgrades
+    enabled: true
+    state: started
+
+- name: Restart unattended-upgrades service
+  ansible.builtin.service:
+    name: unattended-upgrades.service
+    daemon_reload: true
+    enabled: true
+    state: restarted

provision/ansible/roles/ubuntu/tasks/user.yml

@@ -0,0 +1,16 @@
+---
+- name: Add user to sudoers
+  ansible.builtin.copy:
+    content: "{{ ansible_user }} ALL=(ALL:ALL) NOPASSWD:ALL"
+    dest: "/etc/sudoers.d/{{ ansible_user }}_nopasswd"
+    mode: "0440"
+
+- name: Add additional user SSH public keys
+  ansible.posix.authorized_key:
+    user: "{{ ansible_user }}"
+    key: "{{ item }}"
+  loop: "{{ ssh_authorized_keys }}"
+  when:
+    - ssh_authorized_keys is defined
+    - ssh_authorized_keys is iterable
+    - ssh_authorized_keys | length > 0

provision/ansible/templates/kube-vip-daemonset.yaml.j2

@@ -0,0 +1,57 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-vip
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/instance: kube-vip
+    app.kubernetes.io/name: kube-vip
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: kube-vip
+      app.kubernetes.io/name: kube-vip
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: kube-vip
+        app.kubernetes.io/name: kube-vip
+    spec:
+      containers:
+        - name: kube-vip
+          image: ghcr.io/kube-vip/kube-vip:v0.3.7
+          imagePullPolicy: IfNotPresent
+          args:
+            - manager
+          env:
+            - name: vip_arp
+              value: "true"
+            - name: vip_interface
+              value: "{{ kubevip_interface }}"
+            - name: port
+              value: "6443"
+            - name: vip_cidr
+              value: "32"
+            - name: cp_enable
+              value: "true"
+            - name: cp_namespace
+              value: kube-system
+            - name: svc_enable
+              value: "false"
+            - name: vip_address
+              value: "{{ k3s_registration_address }}"
+          securityContext:
+            capabilities:
+              add:
+              - NET_ADMIN
+              - NET_RAW
+              - SYS_TIME
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: "true"
+      serviceAccountName: kube-vip
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule

provision/ansible/templates/kube-vip-rbac.yaml.j2

@@ -0,0 +1,33 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-vip
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  name: system:kube-vip-role
+rules:
+  - apiGroups: [""]
+    resources: ["services", "services/status", "nodes"]
+    verbs: ["list", "get", "watch", "update"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["list", "get", "watch", "update", "create"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:kube-vip-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-vip-role
+subjects:
+- kind: ServiceAccount
+  name: kube-vip
+  namespace: kube-system

provision/terraform/cloudflare/main.tf

@@ -0,0 +1,110 @@
+terraform {
+
+  required_providers {
+    cloudflare = {
+      source  = "cloudflare/cloudflare"
+      version = "3.1.0"
+    }
+    http = {
+      source  = "hashicorp/http"
+      version = "2.1.0"
+    }
+    sops = {
+      source  = "carlpett/sops"
+      version = "0.6.3"
+    }
+  }
+}
+
+data "sops_file" "cloudflare_secrets" {
+  source_file = "secret.sops.yaml"
+}
+
+provider "cloudflare" {
+  email   = data.sops_file.cloudflare_secrets.data["cloudflare_email"]
+  api_key = data.sops_file.cloudflare_secrets.data["cloudflare_apikey"]
+}
+
+data "cloudflare_zones" "domain" {
+  filter {
+    name = data.sops_file.cloudflare_secrets.data["cloudflare_domain"]
+  }
+}
+
+resource "cloudflare_zone_settings_override" "cloudflare_settings" {
+  zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id")
+  settings {
+    # /ssl-tls
+    ssl = "strict"
+    # /ssl-tls/edge-certificates
+    always_use_https         = "on"
+    min_tls_version          = "1.0"
+    opportunistic_encryption = "on"
+    tls_1_3                  = "zrt"
+    automatic_https_rewrites = "on"
+    universal_ssl            = "on"
+    # /firewall/settings
+    browser_check  = "on"
+    challenge_ttl  = 1800
+    privacy_pass   = "on"
+    security_level = "medium"
+    # /speed/optimization
+    brotli = "on"
+    minify {
+      css  = "on"
+      js   = "on"
+      html = "on"
+    }
+    rocket_loader = "on"
+    # /caching/configuration
+    always_online    = "off"
+    development_mode = "off"
+    # /network
+    http3               = "on"
+    zero_rtt            = "on"
+    ipv6                = "on"
+    websockets          = "on"
+    opportunistic_onion = "on"
+    pseudo_ipv4         = "off"
+    ip_geolocation      = "on"
+    # /content-protection
+    email_obfuscation   = "on"
+    server_side_exclude = "on"
+    hotlink_protection  = "off"
+    # /workers
+    security_header {
+      enabled = false
+    }
+  }
+}
+
+data "http" "ipv4" {
+  url = "http://ipv4.icanhazip.com"
+}
+
+resource "cloudflare_record" "ipv4" {
+  name    = "ipv4"
+  zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id")
+  value   = chomp(data.http.ipv4.body)
+  proxied = true
+  type    = "A"
+  ttl     = 1
+}
+
+resource "cloudflare_record" "root" {
+  name    = data.sops_file.cloudflare_secrets.data["cloudflare_domain"]
+  zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id")
+  value   = "ipv4.${data.sops_file.cloudflare_secrets.data["cloudflare_domain"]}"
+  proxied = true
+  type    = "CNAME"
+  ttl     = 1
+}
+
+resource "cloudflare_record" "hajimari" {
+  name    = "hajimari"
+  zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id")
+  value   = "ipv4.${data.sops_file.cloudflare_secrets.data["cloudflare_domain"]}"
+  proxied = true
+  type    = "CNAME"
+  ttl     = 1
+}