diff --git a/.github/workflows/invalid-template.yaml b/.github/workflows/invalid-template.yaml index 35174316..ba9d21ec 100644 --- a/.github/workflows/invalid-template.yaml +++ b/.github/workflows/invalid-template.yaml @@ -10,7 +10,7 @@ on: jobs: support: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: dessant/support-requests@v2 with: diff --git a/.github/workflows/support.yaml b/.github/workflows/support.yaml index 223a9367..44305dac 100644 --- a/.github/workflows/support.yaml +++ b/.github/workflows/support.yaml @@ -10,7 +10,7 @@ on: jobs: support: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: dessant/support-requests@v2 with: diff --git a/README.md b/README.md index c6eb7c8c..46437922 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ _Additional applications include [hajimari](https://github.com/toboshii/hajimari For provisioning the following tools will be used: -- [Ubuntu](https://ubuntu.com/download/server) - Universal operating system that supports running all kinds of home related workloads in Kubernetes -- [Ansible](https://www.ansible.com) - Provision the Ubuntu OS and install k3s +- [Fedora 36 Server](https://getfedora.org/en/server/download/) - Universal operating system that supports running all kinds of home related workloads in Kubernetes and has a faster release cycle +- [Ansible](https://www.ansible.com) - Provision Fedora Server and install k3s - [Terraform](https://www.terraform.io) - Provision an already existing Cloudflare domain and certain DNS records to be used with your k3s cluster ## 📝 Prerequisites @@ -43,7 +43,7 @@ For provisioning the following tools will be used: ### 💻 Systems -- One or more nodes with a fresh install of [Ubuntu Server 22.04](https://ubuntu.com/download/server). +- One or more nodes with a fresh install of [Fedora Server 36](https://getfedora.org/en/server/download/). - These nodes can be ARM64/AMD64 bare metal or VMs. - An odd number of control plane nodes, greater than or equal to 3 is required if deploying more than one control plane node. - A [Cloudflare](https://www.cloudflare.com/) account with a domain, this will be managed by Terraform and external-dns. You can [register new domains](https://www.cloudflare.com/products/registrar/) directly thru Cloudflare. @@ -174,11 +174,11 @@ In order to use Terraform and `cert-manager` with the Cloudflare DNS challenge y task configure ``` -### ⚡ Preparing Ubuntu with Ansible +### ⚡ Preparing Fedora Server with Ansible -📍 Here we will be running a Ansible Playbook to prepare Ubuntu for running a Kubernetes cluster. +📍 Here we will be running a Ansible Playbook to prepare Fedora Server for running a Kubernetes cluster. -📍 Nodes are not security hardened by default, you can do this with [dev-sec/ansible-collection-hardening](https://github.com/dev-sec/ansible-collection-hardening) or similar if it supports Ubuntu 22.04. +📍 Nodes are not security hardened by default, you can do this with [dev-sec/ansible-collection-hardening](https://github.com/dev-sec/ansible-collection-hardening) or similar if it supports Fedora Server. 1. Ensure you are able to SSH into your nodes from your workstation using a private SSH key **without a passphrase**. This is how Ansible is able to connect to your remote nodes. @@ -202,7 +202,7 @@ In order to use Terraform and `cert-manager` with the Cloudflare DNS challenge y task ansible:ping ``` -5. Run the Ubuntu Prepare Ansible playbook +5. Run the Fedora Server Ansible prepare playbook ```sh task ansible:prepare diff --git a/provision/ansible/inventory/group_vars/kubernetes/os.yml b/provision/ansible/inventory/group_vars/kubernetes/os.yml new file mode 100644 index 00000000..3bdcc674 --- /dev/null +++ b/provision/ansible/inventory/group_vars/kubernetes/os.yml @@ -0,0 +1,29 @@ +--- +# (string) Timezone for the servers +# timezone: "America/New_York" + +# (list) Additional ssh public keys to add to the nodes +# ssh_authorized_keys: + +packages: + - "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-{{ ansible_distribution_major_version }}.noarch.rpm" + - "https://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-{{ ansible_distribution_major_version }}.noarch.rpm" + - dnf-automatic + - dnf-utils + - hdparm + - htop + - intel-gpu-tools + - ipvsadm + - lm_sensors + - nano + - nvme-cli + - socat + # TODO(ansible): Might be required for newer Intel CPU generations + # https://ask.fedoraproject.org/t/intel-graphics-best-practices-and-settings-for-hardware-acceleration/21119 + # - intel-media-driver + # - mesa-dri-drivers + # - libva-intel-driver + # - libva-intel-hybrid-driver + # - libva-utils + # - libva-vdpau-driver + # - libvdpau-va-gl diff --git a/provision/ansible/inventory/group_vars/kubernetes/ubuntu.yml b/provision/ansible/inventory/group_vars/kubernetes/ubuntu.yml deleted file mode 100644 index 188d1300..00000000 --- a/provision/ansible/inventory/group_vars/kubernetes/ubuntu.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# (string) Timezone for the servers -# timezone: "America/New_York" - -# (list) Additional ssh public keys to add to the nodes -# ssh_authorized_keys: diff --git a/provision/ansible/playbooks/cluster-installation.yml b/provision/ansible/playbooks/cluster-installation.yml index 678a6207..2ce3ce64 100644 --- a/provision/ansible/playbooks/cluster-installation.yml +++ b/provision/ansible/playbooks/cluster-installation.yml @@ -7,7 +7,7 @@ any_errors_fatal: true pre_tasks: - name: Pausing for 5 seconds... - pause: + ansible.builtin.pause: seconds: 5 tasks: - name: Check if cluster is installed @@ -23,9 +23,11 @@ when: k3s_check_installed.stat.exists - name: Install Kubernetes - include_role: + ansible.builtin.include_role: name: xanmanning.k3s public: true + vars: + k3s_state: installed - name: Get absolute path to this Git repository delegate_to: localhost @@ -46,7 +48,7 @@ - k3s_control_node is defined - k3s_control_node - - name: Update kubeconfig with the right IPv4 address + - name: Update kubeconfig with the correct IPv4 address delegate_to: localhost become: false run_once: true @@ -55,6 +57,17 @@ regexp: "https://127.0.0.1:6443" replace: "https://{{ k3s_registration_address }}:6443" + # Cleaning up the manifests from the /var/lib/rancher/k3s/server/manifests + # directory is needed because k3s has an awesome + # "feature" to always deploy these on restarting + # the k3s systemd service. Removing them does + # not uninstall the manifests. + + # Removing them means we can manage the lifecycle + # of these components outside of the + # /var/lib/rancher/k3s/server/manifests directory + + # FIXME(ansible): Check for deployments to be happy rather than waiting - name: Wait for k3s to finish installing the deployed manifests ansible.builtin.wait_for: timeout: 15 diff --git a/provision/ansible/playbooks/cluster-nuke.yml b/provision/ansible/playbooks/cluster-nuke.yml index edafac4f..eada1970 100644 --- a/provision/ansible/playbooks/cluster-nuke.yml +++ b/provision/ansible/playbooks/cluster-nuke.yml @@ -22,23 +22,20 @@ ansible.builtin.pause: seconds: 5 tasks: - - name: Kill k3s - ansible.builtin.command: /usr/local/bin/k3s-killall.sh - name: Uninstall k3s - ansible.builtin.command: - cmd: /usr/local/bin/k3s-uninstall.sh - removes: /usr/local/bin/k3s-uninstall.sh - - name: Uninstall k3s agent - ansible.builtin.command: - cmd: /usr/local/bin/k3s-agent-uninstall.sh - removes: /usr/local/bin/k3s-agent-uninstall.sh - - name: Gather list of CNI files to delete + ansible.builtin.include_role: + name: xanmanning.k3s + public: true + vars: + k3s_state: uninstalled + - name: Gather list of CNI files ansible.builtin.find: paths: /etc/cni/net.d patterns: "*" - register: files_to_delete + hidden: true + register: directory_contents - name: Delete CNI files ansible.builtin.file: path: "{{ item.path }}" state: absent - loop: "{{ files_to_delete.files }}" + loop: "{{ directory_contents.files }}" diff --git a/provision/ansible/playbooks/cluster-prepare.yml b/provision/ansible/playbooks/cluster-prepare.yml index 0fc815b2..c4a30047 100644 --- a/provision/ansible/playbooks/cluster-prepare.yml +++ b/provision/ansible/playbooks/cluster-prepare.yml @@ -1,4 +1,5 @@ --- +--- - hosts: - master - worker @@ -7,7 +8,114 @@ any_errors_fatal: true pre_tasks: - name: Pausing for 5 seconds... - pause: + ansible.builtin.pause: seconds: 5 - roles: - - prepare + tasks: + - name: Locale + block: + - name: Locale | Set timezone + community.general.timezone: + name: "{{ timezone | default('Etc/UTC') }}" + - name: Networking + block: + - name: Networking | Set hostname to inventory hostname + ansible.builtin.hostname: + name: "{{ inventory_hostname }}" + - name: Networking | Update /etc/hosts to include inventory hostname + ansible.builtin.blockinfile: + path: /etc/hosts + block: | + 127.0.1.1 {{ inventory_hostname }} + - name: Packages + block: + - name: Packages | Improve dnf performance + ansible.builtin.blockinfile: + path: /etc/dnf/dnf.conf + block: | + defaultyes=True + deltarpm=True + install_weak_deps=False + max_parallel_downloads={{ ansible_processor_vcpus | default('8') }} + - name: Packages | Import rpmfusion keys + ansible.builtin.rpm_key: + state: present + key: "{{ item }}" + loop: + - https://rpmfusion.org/keys?action=AttachFile&do=get&target=RPM-GPG-KEY-rpmfusion-free-fedora-2020 + - https://rpmfusion.org/keys?action=AttachFile&do=get&target=RPM-GPG-KEY-rpmfusion-nonfree-fedora-2020 + - name: Packages | Install required packages + ansible.builtin.dnf: + name: "{{ packages | default([]) }}" + state: present + update_cache: true + - name: Packages | Remove leaf packages + ansible.builtin.dnf: + autoremove: true + - name: Packages | Enable automatic downloads of updates + ansible.builtin.systemd: + service: dnf-automatic-download.timer + enabled: true + state: started + - name: User Configuration + block: + - name: User Configuration | Add additional SSH public keys + ansible.posix.authorized_key: + user: "{{ ansible_user }}" + key: "{{ item }}" + loop: "{{ public_ssh_keys | default([]) }}" + - name: System Configuration (1) + block: + - name: System Configuration (1) | Disable firewalld + ansible.builtin.systemd: + service: firewalld.service + enabled: false + masked: true + state: stopped + - name: System Configuration (1) | Enable fstrim + ansible.builtin.systemd: + service: fstrim.timer + enabled: true + - name: System Configuration (2) + block: + - name: System Configuration (2) | Enable kernel modules now + community.general.modprobe: + name: "{{ item }}" + state: present + loop: [br_netfilter, overlay, rbd] + - name: System Configuration (2) | Enable kernel modules on boot + ansible.builtin.copy: + mode: 0644 + content: "{{ item }}" + dest: "/etc/modules-load.d/{{ item }}.conf" + loop: [br_netfilter, overlay, rbd] + - name: System Configuration (2) | Set sysctls + ansible.posix.sysctl: + name: "{{ item.key }}" + value: "{{ item.value }}" + sysctl_file: /etc/sysctl.d/99-kubernetes.conf + reload: true + with_dict: "{{ sysctl_config }}" + vars: + sysctl_config: + net.ipv4.ip_forward: 1 + net.ipv4.conf.all.forwarding: 1 + net.ipv4.conf.all.rp_filter: 0 + net.ipv4.conf.default.rp_filter: 0 + net.ipv6.conf.all.forwarding: 1 + net.bridge.bridge-nf-call-iptables: 1 + net.bridge.bridge-nf-call-ip6tables: 1 + fs.inotify.max_user_watches: 524288 + fs.inotify.max_user_instances: 512 + - name: System Configuration (2) | Disable swap + ansible.builtin.dnf: + name: zram-generator-defaults + state: absent + - name: System Configuration (2) | Permissive SELinux + ansible.posix.selinux: + state: permissive + notify: Reboot + + handlers: + - name: Reboot + ansible.builtin.reboot: + msg: Rebooting nodes diff --git a/provision/ansible/roles/prepare/handlers/main.yml b/provision/ansible/roles/prepare/handlers/main.yml deleted file mode 100644 index 98f5f24d..00000000 --- a/provision/ansible/roles/prepare/handlers/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Restart node - ansible.builtin.reboot: - msg: Restart node due to system changes - -- name: Restart unattended-upgrades - ansible.builtin.service: - name: unattended-upgrades.service - daemon_reload: true - enabled: true - state: restarted diff --git a/provision/ansible/roles/prepare/tasks/filesystem.yml b/provision/ansible/roles/prepare/tasks/filesystem.yml deleted file mode 100644 index d9e155ba..00000000 --- a/provision/ansible/roles/prepare/tasks/filesystem.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Update max_user_watches - ansible.posix.sysctl: - name: fs.inotify.max_user_watches - value: "65536" - state: present - sysctl_file: /etc/sysctl.d/98-kubernetes-fs.conf - -- name: Disable swap at runtime - ansible.builtin.command: swapoff -a - when: ansible_swaptotal_mb > 0 - -- name: Disable swap on boot - ansible.posix.mount: - name: "{{ item }}" - fstype: swap - state: absent - loop: - - swap - - none diff --git a/provision/ansible/roles/prepare/tasks/kernel.yml b/provision/ansible/roles/prepare/tasks/kernel.yml deleted file mode 100644 index acbd6337..00000000 --- a/provision/ansible/roles/prepare/tasks/kernel.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: Enable kernel modules runtime - community.general.modprobe: - name: "{{ item }}" - state: present - loop: - - br_netfilter - - overlay - - rbd - -- name: Enable kernel modules on boot - ansible.builtin.copy: - mode: 0644 - content: "{{ item }}" - dest: "/etc/modules-load.d/{{ item }}.conf" - loop: - - br_netfilter - - overlay - - rbd diff --git a/provision/ansible/roles/prepare/tasks/locale.yml b/provision/ansible/roles/prepare/tasks/locale.yml deleted file mode 100644 index bbef8b0f..00000000 --- a/provision/ansible/roles/prepare/tasks/locale.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Set timezone - community.general.timezone: - name: "{{ timezone }}" - when: - - timezone is defined diff --git a/provision/ansible/roles/prepare/tasks/main.yml b/provision/ansible/roles/prepare/tasks/main.yml deleted file mode 100644 index 6d59372a..00000000 --- a/provision/ansible/roles/prepare/tasks/main.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- include: locale.yml - tags: - - locale - -- include: packages.yml - tags: - - packages - -- import_tasks: kernel.yml - tags: - - kernel - -- include: network.yml - tags: - - network - -- include: filesystem.yml - tags: - - filesystem - -- include: unattended-upgrades.yml - tags: - - unattended-upgrades - -- include: user.yml - tags: - - user diff --git a/provision/ansible/roles/prepare/tasks/network.yml b/provision/ansible/roles/prepare/tasks/network.yml deleted file mode 100644 index 9061ba16..00000000 --- a/provision/ansible/roles/prepare/tasks/network.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: Set hostname to inventory hostname - ansible.builtin.hostname: - name: "{{ inventory_hostname }}" - when: - - ansible_hostname != inventory_hostname - -- name: Update /etc/hosts to include hostname - ansible.builtin.blockinfile: - path: /etc/hosts - create: true - block: | - 127.0.0.1 localhost - 127.0.1.1 {{ inventory_hostname }} - - # The following lines are desirable for IPv6 capable hosts - ::1 ip6-localhost ip6-loopback - fe00::0 ip6-localnet - ff00::0 ip6-mcastprefix - ff02::1 ip6-allnodes - ff02::2 ip6-allrouters - ff02::3 ip6-allhosts - -- name: Set system controls for Kubernetes - ansible.posix.sysctl: - name: "{{ item.key }}" - value: "{{ item.value }}" - sysctl_file: /etc/sysctl.d/99-kubernetes.conf - reload: true - with_dict: "{{ sysctl_config }}" - vars: - sysctl_config: - net.ipv4.ip_forward: 1 - net.ipv4.conf.all.forwarding: 1 - net.ipv6.conf.all.forwarding: 1 - net.bridge.bridge-nf-call-iptables: 1 - net.bridge.bridge-nf-call-ip6tables: 1 - net.ipv4.conf.default.rp_filter: 0 - net.ipv4.conf.all.rp_filter: 0 diff --git a/provision/ansible/roles/prepare/tasks/packages.yml b/provision/ansible/roles/prepare/tasks/packages.yml deleted file mode 100644 index 670aab97..00000000 --- a/provision/ansible/roles/prepare/tasks/packages.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -- name: Upgrade all system packages - ansible.builtin.apt: - upgrade: full - update_cache: true - -- name: Install common packages - ansible.builtin.apt: - name: - - apt-transport-https - - arptables - - ebtables - - ca-certificates - - curl - - ebtables - - gdisk - - hdparm - - htop - - iputils-ping - - ipvsadm - - lvm2 - - nano - - net-tools - - nfs-common - - nvme-cli - - open-iscsi - - psmisc - - smartmontools - - socat - - software-properties-common - - unattended-upgrades - - unzip - install_recommends: false - -- name: Install RasPi specific packages - ansible.builtin.apt: - name: - - linux-modules-extra-raspi - install_recommends: false - notify: Restart node - when: "'raspi' in ansible_kernel" - -- name: Restart node if necessary - ansible.builtin.meta: flush_handlers diff --git a/provision/ansible/roles/prepare/tasks/unattended-upgrades.yml b/provision/ansible/roles/prepare/tasks/unattended-upgrades.yml deleted file mode 100644 index 7d215bb3..00000000 --- a/provision/ansible/roles/prepare/tasks/unattended-upgrades.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- block: - - name: Copy 20auto-upgrades unattended-upgrades config - ansible.builtin.blockinfile: - path: /etc/apt/apt.conf.d/20auto-upgrades - mode: 0644 - create: true - block: | - APT::Periodic::Update-Package-Lists "1"; - APT::Periodic::Download-Upgradeable-Packages "1"; - APT::Periodic::AutocleanInterval "7"; - APT::Periodic::Unattended-Upgrade "1"; - - name: Copy 50unattended-upgrades unattended-upgrades config - ansible.builtin.blockinfile: - path: /etc/apt/apt.conf.d/50unattended-upgrades - mode: 0644 - create: true - block: | - Unattended-Upgrade::Automatic-Reboot "false"; - Unattended-Upgrade::Remove-Unused-Dependencies "true"; - Unattended-Upgrade::Allowed-Origins { - "${distro_id} stable"; - "${distro_id} ${distro_codename}-security"; - "${distro_id} ${distro_codename}-updates"; - }; - notify: Restart unattended-upgrades diff --git a/provision/ansible/roles/prepare/tasks/user.yml b/provision/ansible/roles/prepare/tasks/user.yml deleted file mode 100644 index 9b5766ff..00000000 --- a/provision/ansible/roles/prepare/tasks/user.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Add user to sudoers - ansible.builtin.copy: - content: "{{ ansible_user }} ALL=(ALL:ALL) NOPASSWD:ALL" - dest: "/etc/sudoers.d/{{ ansible_user }}_nopasswd" - mode: "0440" - -- name: Add additional user SSH public keys - ansible.posix.authorized_key: - user: "{{ ansible_user }}" - key: "{{ item }}" - loop: "{{ ssh_authorized_keys }}" - when: - - ssh_authorized_keys is defined - - ssh_authorized_keys is iterable - - ssh_authorized_keys | length > 0