From 8d5516a5462cd757c10d1e26e5edc07bd5630304 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 27 Sep 2021 11:24:55 -0400 Subject: [PATCH] feat!: Use Ansible and Terraform to provision Nodes and DNS and create configure script (#84) --- .config.sample.env | 54 ++++ .gitignore | 7 + .taskfiles/ansible.yml | 71 +++++ .taskfiles/terraform.yml | 22 ++ README.md | 221 +++++-------- Taskfile.yml | 4 + ansible.cfg | 57 ++++ .../cert-manager/letsencrypt-production.yaml | 6 +- .../cert-manager/letsencrypt-staging.yaml | 6 +- configure.sh | 292 ++++++++++++++++++ .../inventory/group_vars/kubernetes/k3s.yml | 32 ++ .../group_vars/kubernetes/ubuntu.yml | 6 + .../inventory/group_vars/master/k3s.yml | 42 +++ .../inventory/group_vars/worker/k3s.yml | 13 + .../ansible/inventory/host_vars/.gitkeep | 0 provision/ansible/inventory/hosts.yml | 11 + provision/ansible/playbooks/k3s-install.yml | 65 ++++ provision/ansible/playbooks/k3s-nuke.yml | 32 ++ .../ansible/playbooks/ubuntu-prepare.yml | 13 + .../ansible/playbooks/ubuntu-upgrade.yml | 18 ++ provision/ansible/requirements.yml | 9 + .../ansible/roles/ubuntu/tasks/filesystem.yml | 20 ++ .../ansible/roles/ubuntu/tasks/locale.yml | 6 + provision/ansible/roles/ubuntu/tasks/main.yml | 24 ++ .../ansible/roles/ubuntu/tasks/network.yml | 47 +++ .../ansible/roles/ubuntu/tasks/packages.yml | 76 +++++ provision/ansible/roles/ubuntu/tasks/user.yml | 16 + .../templates/kube-vip-daemonset.yaml.j2 | 57 ++++ .../ansible/templates/kube-vip-rbac.yaml.j2 | 33 ++ provision/terraform/cloudflare/main.tf | 110 +++++++ tmpl/.sops.yaml | 11 +- tmpl/ansible/kube-vip.yml | 5 + .../cert-manager-secret.sops.yaml} | 4 +- tmpl/{ => cluster}/cluster-secrets.sops.yaml | 2 +- tmpl/{ => cluster}/cluster-settings.yaml | 2 +- tmpl/{ => cluster}/gotk-sync.yaml | 2 +- tmpl/terraform/secret.sops.yaml | 4 + 37 files changed, 1247 insertions(+), 153 deletions(-) create mode 100644 .config.sample.env create mode 100644 .taskfiles/ansible.yml create mode 100644 .taskfiles/terraform.yml create mode 100644 ansible.cfg create mode 100755 configure.sh create mode 100644 provision/ansible/inventory/group_vars/kubernetes/k3s.yml create mode 100644 provision/ansible/inventory/group_vars/kubernetes/ubuntu.yml create mode 100644 provision/ansible/inventory/group_vars/master/k3s.yml create mode 100644 provision/ansible/inventory/group_vars/worker/k3s.yml create mode 100644 provision/ansible/inventory/host_vars/.gitkeep create mode 100644 provision/ansible/inventory/hosts.yml create mode 100644 provision/ansible/playbooks/k3s-install.yml create mode 100644 provision/ansible/playbooks/k3s-nuke.yml create mode 100644 provision/ansible/playbooks/ubuntu-prepare.yml create mode 100644 provision/ansible/playbooks/ubuntu-upgrade.yml create mode 100644 provision/ansible/requirements.yml create mode 100644 provision/ansible/roles/ubuntu/tasks/filesystem.yml create mode 100644 provision/ansible/roles/ubuntu/tasks/locale.yml create mode 100644 provision/ansible/roles/ubuntu/tasks/main.yml create mode 100644 provision/ansible/roles/ubuntu/tasks/network.yml create mode 100644 provision/ansible/roles/ubuntu/tasks/packages.yml create mode 100644 provision/ansible/roles/ubuntu/tasks/user.yml create mode 100644 provision/ansible/templates/kube-vip-daemonset.yaml.j2 create mode 100644 provision/ansible/templates/kube-vip-rbac.yaml.j2 create mode 100644 provision/terraform/cloudflare/main.tf create mode 100644 tmpl/ansible/kube-vip.yml rename tmpl/{secret.sops.yaml => cluster/cert-manager-secret.sops.yaml} (53%) rename tmpl/{ => cluster}/cluster-secrets.sops.yaml (78%) rename tmpl/{ => cluster}/cluster-settings.yaml (72%) rename tmpl/{ => cluster}/gotk-sync.yaml (93%) create mode 100644 tmpl/terraform/secret.sops.yaml diff --git a/.config.sample.env b/.config.sample.env new file mode 100644 index 00000000..96b962ee --- /dev/null +++ b/.config.sample.env @@ -0,0 +1,54 @@ +# +# Cluster related variables +# + +# The repo you created from this template +# e.g. https://github.com/k8s-at-home/home-cluster +export BOOTSTRAP_GIT_REPOSITORY="" +# Choose one of your cloudflare domains +# e.g. k8s-at-home.com +export BOOTSTRAP_CLOUDFLARE_DOMAIN="" +# The email you use to sign into Cloudflare with +export BOOTSTRAP_CLOUDFLARE_EMAIL="" +# Your global Cloudflare API Key +export BOOTSTRAP_CLOUDFLARE_APIKEY="" +# Pick a range of unused IPs that are on the same network as your nodes +# You don't need many IPs, just choose 10 IPs to start with +# e.g. 192.168.1.220-192.168.1.230 +export BOOTSTRAP_METALLB_LB_RANGE="" +# The load balancer IP for traefik, choose from one of the available IPs above +# e.g. 192.168.1.220 +export BOOTSTRAP_METALLB_TRAEFIK_ADDR="" +# Your personal pgp key +# e.g. 772154FFF783DE317KLCA0EC77149AC618D75581 +export BOOTSTRAP_PERSONAL_KEY_FP="" +# The flux pgp key +# e.g. AB675CE4CC64251G3S9AE1DAA88ARRTY2C009E2D +export BOOTSTRAP_FLUX_KEY_FP="" + +# +# Ansible related variables +# + +# The network interface on the host kube-vip should attach to +# e.g. eno1 +export BOOTSTRAP_ANSIBLE_KUBE_VIP_INTERFACE="" +# IP Address to use with KubeVIP +# Pick a unused IP that is on the same network as your nodes +# e.g. 192.168.1.254 +export BOOTSTRAP_ANSIBLE_KUBE_VIP_ADDRESS="" + +# +# Ansible hosts - repeat this block as many times as you need, +# incrementing the last digit on the variable for each node +# + +# Host IP Address to the control plane node +# e.g. 192.168.1.200 +export BOOTSTRAP_ANSIBLE_HOST_ADDR_0="" +# User Ansible will log into the nodes +export BOOTSTRAP_ANSIBLE_SSH_USERNAME_0="" +# Password Ansible will log into the nodes +export BOOTSTRAP_ANSIBLE_SSH_PASSWORD_0="" +# Set this node as a control node (true/false) +export BOOTSTRAP_ANSIBLE_CONTROL_NODE_0="" diff --git a/.gitignore b/.gitignore index 7fc3827c..a197cafc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,10 @@ Thumbs.db kubeconfig # vscode-sops .decrypted~*.yaml +.config.env +# Ansible +xanmanning.k3s* +# Terraform +.terraform +.terraform.tfstate* +terraform.tfstate* diff --git a/.taskfiles/ansible.yml b/.taskfiles/ansible.yml new file mode 100644 index 00000000..4578b671 --- /dev/null +++ b/.taskfiles/ansible.yml @@ -0,0 +1,71 @@ +--- +version: "3" + +env: + ANSIBLE_CONFIG: "{{.PROJECT_DIR}}/ansible.cfg" + +vars: + ANSIBLE_PLAYBOOK_DIR: "{{.ANSIBLE_DIR}}/playbooks" + ANSIBLE_INVENTORY_DIR: "{{.ANSIBLE_DIR}}/inventory" + +tasks: + + deps: + desc: Upgrade Ansible galaxy deps + dir: provision/ansible + cmds: + - "ansible-galaxy install -r requirements.yml --force" + + list: + desc: List all the hosts + dir: provision/ansible + cmds: + - "ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml --list-hosts" + + playbook:ubuntu-prepare: + desc: Prepare all the k8s nodes for running k3s + dir: provision/ansible + cmds: + - "ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/kubernetes/ubuntu-prepare.yml" + + playbook:ubuntu-upgrade: + desc: Upgrade all the k8s nodes operating system + dir: provision/ansible + cmds: + - "ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/kubernetes/ubuntu-upgrade.yml" + + playbook:k3s-install: + desc: Install Kubernetes on the nodes + dir: provision/ansible + cmds: + - "ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/kubernetes/k3s-install.yml" + + playbook:k3s-nuke: + desc: Install Kubernetes on the nodes + dir: provision/ansible + cmds: + - "ansible-playbook -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml {{.ANSIBLE_PLAYBOOK_DIR}}/kubernetes/k3s-nuke.yml" + + adhoc:ping: + desc: Ping all the hosts + dir: provision/ansible + cmds: + - "ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml --one-line -m 'ping'" + + adhoc:uptime: + desc: Uptime of all the hosts + dir: provision/ansible + cmds: + - ansible all -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml --one-line -a 'uptime' + + adhoc:reboot: + desc: Reboot all the k8s nodes + dir: provision/ansible + cmds: + - "ansible kubernetes -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml -a '/usr/bin/systemctl reboot' --become" + + adhoc:poweroff: + desc: Shutdown all the k8s nodes + dir: provision/ansible + cmds: + - "ansible kubernetes -i {{.ANSIBLE_INVENTORY_DIR}}/hosts.yml -a '/usr/bin/systemctl poweroff' --become" diff --git a/.taskfiles/terraform.yml b/.taskfiles/terraform.yml new file mode 100644 index 00000000..3bd16eaf --- /dev/null +++ b/.taskfiles/terraform.yml @@ -0,0 +1,22 @@ +--- +version: "3" + +tasks: + + init:cloudflare: + desc: Initialize terraform + dir: provision/terraform/cloudflare + cmds: + - "terraform init" + + plan:cloudflare: + desc: Prepare all the k8s nodes for running k3s + dir: provision/terraform/cloudflare + cmds: + - "terraform plan" + + apply:cloudflare: + desc: Prepare all the k8s nodes for running k3s + dir: provision/terraform/cloudflare + cmds: + - "terraform apply" diff --git a/README.md b/README.md index dc94aebe..ae03515d 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Template for deploying k3s backed by Flux -Highly opinionated template for deploying a single [k3s](https://k3s.io/) cluster with [k3sup](https://github.com/alexellis/k3sup) backed by [Flux](https://toolkit.fluxcd.io/) and [SOPS](https://toolkit.fluxcd.io/guides/mozilla-sops/). +Highly opinionated template for deploying a single [k3s](https://k3s.io) cluster with [Ansible](https://www.ansible.com) and [Terraform](https://www.terraform.io) backed by [Flux](https://toolkit.fluxcd.io/) and [SOPS](https://toolkit.fluxcd.io/guides/mozilla-sops/). -The purpose here is to showcase how you can deploy an entire Kubernetes cluster and show it off to the world using the [GitOps](https://www.weave.works/blog/what-is-gitops-really) tool [Flux](https://toolkit.fluxcd.io/). When completed, your Git repository will be driving the state of your Kubernetes cluster. In addition with the help of the [Flux SOPS integration](https://toolkit.fluxcd.io/guides/mozilla-sops/) you'll be able to commit GPG encrypted secrets to your public repo. +The purpose here is to showcase how you can deploy an entire Kubernetes cluster and show it off to the world using the [GitOps](https://www.weave.works/blog/what-is-gitops-really) tool [Flux](https://toolkit.fluxcd.io/). When completed, your Git repository will be driving the state of your Kubernetes cluster. In addition with the help of the [Ansible](https://github.com/ansible-collections/community.sops), [Terraform](https://github.com/carlpett/terraform-provider-sops) and [Flux](https://toolkit.fluxcd.io/guides/mozilla-sops/) SOPS integrations you'll be able to commit GPG encrypted secrets to your public repo. ## Overview @@ -31,30 +31,11 @@ Feel free to read up on any of these technologies before you get started to be m ## :memo:  Prerequisites -### :computer:  Nodes +### :computer:  Systems -Already provisioned Bare metal or VMs with any modern operating system like Ubuntu, Debian or CentOS. - -If coming from a fresh install of Linux make sure you do the following steps. - -- Copy over your SSH keys to all the hosts - -- Enable packet forwarding on the hosts and increase max_user_watches - -```sh -cat < # sub rsa4096 2021-03-11 [E] - -export PERSONAL_KEY_FP=772154FFF783DE317KLCA0EC77149AC618D75581 ``` 2. Create a Flux GPG Key and export the fingerprint @@ -169,67 +149,63 @@ gpg --list-secret-keys "${FLUX_KEY_NAME}" # AB675CE4CC64251G3S9AE1DAA88ARRTY2C009E2D # uid [ultimate] Home cluster (Flux) # sub rsa4096 2021-03-11 [E] - -export FLUX_KEY_FP=AB675CE4CC64251G3S9AE1DAA88ARRTY2C009E2D ``` -### :sailboat:  Installing k3s with k3sup +3. You will need the Fingerprints in the configuration section below. For example, in the above steps you will need `772154FFF783DE317KLCA0EC77149AC618D75581` and `AB675CE4CC64251G3S9AE1DAA88ARRTY2C009E2D` -:round_pushpin: Here we will be install [k3s](https://k3s.io/) with [k3sup](https://github.com/alexellis/k3sup). After completion, k3sup will drop a `kubeconfig` in your present working directory for use with interacting with your cluster with `kubectl`. +### :cloud:  Global Cloudflare API Key -1. Ensure you are able to SSH into you nodes with using your private ssh key. This is how k3sup is able to connect to your remote node. +In order to use Terraform and `cert-manager` with the Cloudflare DNS challenge you will need to create a API key. -2. Install the master node(s) +1. Head over to Cloudflare and create a API key by going [here](https://dash.cloudflare.com/profile/api-tokens). -_We will be installing metallb instead of servicelb, traefik and metrics-server will be installed with Flux._ +2. Under the `API Keys` section, create a global API Key. -```sh -k3sup install \ - --host=169.254.1.1 \ - --user=k8s-at-home \ - --k3s-version=v1.21.4+k3s1 \ - --k3s-extra-args="--disable servicelb --disable traefik --disable metrics-server" -``` +3. Use the API Key in the configuration section below. -3. Join worker nodes (optional) +### :page_facing_up:  Configuration -```sh -k3sup join \ - --host=169.254.1.2 \ - --server-host=169.254.1.1 \ - --k3s-version=v1.21.4+k3s1 \ - --user=k8s-at-home -``` +:round_pushpin: The `.config.env` file contains necessary configuration files that are needed by Ansible, Terraform and Flux. -4. Verify the nodes are online +1. Copy the `.config.sample.env` to `.config.env` and start filling out all the environment variables. **All are required** and read the comments they will explain further what is required. + +2. Once that is done, verify the configuration is correct by running `./configure.sh --verify` + +3. If you do not encounter any errors run `./configure.sh` to start having the script wire up the templated files and place them where they need to be. + +### :zap:  Preparing Ubuntu with Ansible + +:round_pushpin: Here we will be install [k3s](https://k3s.io/) with [this](https://galaxy.ansible.com/xanmanning/k3s) wonderful k3s Ansible galaxy role. After completion, Ansible will drop a `kubeconfig` in `/tmp/kubeconfig` for use with interacting with your cluster with `kubectl`. This file should be manually copied to the root of your repository. + +1. Ensure you are able to SSH into you nodes from your workstation with using your private ssh key. This is how Ansible is able to connect to your remote nodes. + +2. Verify Ansible can view your config by running `task ansible:list` + +3. Verify Ansible can ping your nodes by running `task ansible:ping` + +4. Finally, run the Ubuntu Prepare playbook by running `task ansible:playbook:ubuntu-prepare` + +5. If everything goes as planned you should see Ansible running the Ubuntu Prepare Playbook against your nodes. + +### :sailboat:  Installing k3s with Ansible + +1. Verify Ansible can view your config by running `task ansible:list` + +2. Verify Ansible can ping your nodes by running `task ansible:ping` + +3. Run the k3s install playbook by running `task ansible:playbook:k3s-install` + +4. If everything goes as planned you should see Ansible running the k3s install Playbook against your nodes. + +5. Copy the `kubeconfig` file from `/tmp` to your repository. + +6. Verify the nodes are online ```sh kubectl --kubeconfig=./kubeconfig get nodes # NAME STATUS ROLES AGE VERSION -# k8s-master-a Ready control-plane,master 4d20h v1.21.4+k3s1 -# k8s-worker-a Ready worker 4d20h v1.21.4+k3s1 -``` - -### :cloud:  Cloudflare API Token - -:round_pushpin: You may skip this step, **however** make sure to `export` dummy data **on item 8** in the below list. - -...Be aware you **will not** have a valid SSL cert until `cert-manager` is configured correctly - -In order to use `cert-manager` with the Cloudflare DNS challenge you will need to create a API token. - -1. Head over to Cloudflare and create a API token by going [here](https://dash.cloudflare.com/profile/api-tokens). -2. Click the blue `Create Token` button -3. Scroll down and create a Custom Token by choosing `Get started` -4. Give your token a name like `cert-manager` -5. Under `Permissions` give **read** access to `Zone` : `Zone` and **write** access to `Zone` : `DNS` -6. Under `Zone Resources` set it to `Include` : `All Zones` -7. Click `Continue to summary` and then `Create Token` -8. Export this token and your Cloudflare email address to an environment variable on your system to be used in the following steps - -```sh -export BOOTSTRAP_CLOUDFLARE_EMAIL="k8s-at-home@gmail.com" -export BOOTSTRAP_CLOUDFLARE_TOKEN="kpG6iyg3FS_du_8KRShdFuwfbwu3zMltbvmJV6cD" +# k8s-0 Ready control-plane,master 4d20h v1.21.5+k3s1 +# k8s-1 Ready worker 4d20h v1.21.5+k3s1 ``` ### :small_blue_diamond:  GitOps with Flux @@ -241,8 +217,8 @@ export BOOTSTRAP_CLOUDFLARE_TOKEN="kpG6iyg3FS_du_8KRShdFuwfbwu3zMltbvmJV6cD" ```sh flux --kubeconfig=./kubeconfig check --pre # ► checking prerequisites -# ✔ kubectl 1.21.4 >=1.18.0-0 -# ✔ Kubernetes 1.21.4+k3s1 >=1.16.0-0 +# ✔ kubectl 1.21.5 >=1.18.0-0 +# ✔ Kubernetes 1.21.5+k3s1 >=1.16.0-0 # ✔ prerequisites checks passed ``` @@ -255,53 +231,20 @@ kubectl --kubeconfig=./kubeconfig create namespace flux-system --dry-run=client 3. Add the Flux GPG key in-order for Flux to decrypt SOPS secrets ```sh -gpg --export-secret-keys --armor "${FLUX_KEY_FP}" | +source .config.env +gpg --export-secret-keys --armor "${BOOTSTRAP_FLUX_KEY_FP}" | kubectl --kubeconfig=./kubeconfig create secret generic sops-gpg \ --namespace=flux-system \ --from-file=sops.asc=/dev/stdin ``` -4. Export more environment variables for application configuration +:round_pushpin: Variables defined in `./cluster/base/cluster-secrets.sops.yaml` and `./cluster/base/cluster-settings.sops.yaml` will be usable anywhere in your YAML manifests under `./cluster` -```sh -# The repo you created from this template -export BOOTSTRAP_GITHUB_REPOSITORY="https://github.com/k8s-at-home/home-cluster" -# Choose one of your domains or use a made up one -export BOOTSTRAP_DOMAIN="k8s-at-home.com" -# Pick a range of unused IPs that are on the same network as your nodes -# You don't need many IPs, just choose 10 to start with -export BOOTSTRAP_METALLB_LB_RANGE="169.254.1.10-169.254.1.20" -# The load balancer IP for traefik, choose from one of the available IPs above -export BOOTSTRAP_SVC_TRAEFIK_ADDR="169.254.1.10" -``` +4. **Verify** all the above files are **encrypted** with SOPS -5. Create required files based on ALL exported environment variables. +5. If you verified all the secrets are encrypted, you can delete the `tmpl` directory now -```sh -envsubst < ./tmpl/.sops.yaml > ./.sops.yaml -envsubst < ./tmpl/cluster-secrets.sops.yaml > ./cluster/base/cluster-secrets.sops.yaml -envsubst < ./tmpl/cluster-settings.yaml > ./cluster/base/cluster-settings.yaml -envsubst < ./tmpl/gotk-sync.yaml > ./cluster/base/flux-system/gotk-sync.yaml -envsubst < ./tmpl/secret.sops.yaml > ./cluster/core/cert-manager/secret.sops.yaml -``` - -6. **Verify** all the above files have the correct information present - -7. Encrypt `cluster/cluster-secrets.sops.yaml` and `cert-manager/secret.sops.yaml` with SOPS - -```sh -export GPG_TTY=$(tty) -sops --encrypt --in-place ./cluster/base/cluster-secrets.sops.yaml -sops --encrypt --in-place ./cluster/core/cert-manager/secret.sops.yaml -``` - -:round_pushpin: Variables defined in `cluster-secrets.sops.yaml` and `cluster-settings.sops.yaml` will be usable anywhere in your YAML manifests under `./cluster` - -8. **Verify** all the above files are **encrypted** with SOPS - -9. If you verified all the secrets are encrypted, you can delete the `tmpl` directory now - -10. Push you changes to git +6. Push you changes to git ```sh git add -A @@ -309,7 +252,7 @@ git commit -m "initial commit" git push ``` -11. Install Flux +7. Install Flux :round_pushpin: Due to race conditions with the Flux CRDs you will have to run the below command twice. There should be no errors on this second run. @@ -326,11 +269,7 @@ kubectl --kubeconfig=./kubeconfig apply --kustomize=./cluster/base/flux-system # unable to recognize "./cluster/base/flux-system": no matches for kind "HelmRepository" in version "source.toolkit.fluxcd.io/v1beta1" ``` -:tada: **Congratulations** you have a Kubernetes cluster managed by Flux, your Git repository is driving the state of your cluster. - -## :mega:  Post installation - -### Verify Flux +8. Verify Flux components are running in the cluster ```sh kubectl --kubeconfig=./kubeconfig get pods -n flux-system @@ -341,17 +280,21 @@ kubectl --kubeconfig=./kubeconfig get pods -n flux-system # source-controller-7d6875bcb4-zqw9f 1/1 Running 0 1h ``` -### Verify ingress +:tada: **Congratulations** you have a Kubernetes cluster managed by Flux, your Git repository is driving the state of your cluster. -If your cluster is not accessible to outside world you can update your hosts file to verify the ingress controller is working. +### :cloud:  Configure Cloudflare DNS with Terraform -This will only be temporary and you should set up DNS to handle these records either manually or automated with [external-dns](https://github.com/kubernetes-sigs/external-dns). +:round_pushpin: Review the Terraform scripts under `./terraform/cloudflare/` and make sure you understand what it's doing (no really review it). If your domain already has existing DNS records be sure to export those DNS settings before you continue. Ideally you can update the terraform script to manage DNS for all records if you so choose to. -```sh -echo "${BOOTSTRAP_SVC_TRAEFIK_ADDR} ${BOOTSTRAP_DOMAIN} hajimari.${BOOTSTRAP_DOMAIN}" | sudo tee -a /etc/hosts -``` +1. Pull in the Terraform deps by running `task terraform:init` -Head over to your browser and you _should_ be able to access `https://hajimari.${BOOTSTRAP_DOMAIN}` +2. Review the changes Terraform will make to your Cloudflare domain by running `task terraform:plan` + +3. Finally have Terraform execute the task by running `task terraform:apply` + +If Terraform was ran successfully head over to your browser and you _should_ be able to access `https://hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` + +## :mega:  Post installation ### direnv diff --git a/Taskfile.yml b/Taskfile.yml index 700f9f85..54f866cc 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -5,12 +5,16 @@ vars: PROJECT_DIR: sh: "git rev-parse --show-toplevel" CLUSTER_DIR: "{{.PROJECT_DIR}}/cluster" + ANSIBLE_DIR: "{{.PROJECT_DIR}}/provision/ansible" + TERRAFORM_DIR: "{{.PROJECT_DIR}}/provision/terraform" env: KUBECONFIG: "{{.PROJECT_DIR}}/kubeconfig" includes: + ansible: .taskfiles/ansible.yml flux: .taskfiles/flux.yml format: .taskfiles/format.yml lint: .taskfiles/lint.yml pre-commit: .taskfiles/pre-commit.yml + terraform: .taskfiles/terraform.yml diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 00000000..30aac298 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,57 @@ +[defaults] + +#--- General settings +nocows = True +forks = 8 +module_name = command +deprecation_warnings = True +executable = /bin/bash +stdout_callback = yaml + +#--- Files/Directory settings +log_path = ~/ansible.log +inventory = ./provision/ansible/inventory +library = /usr/share/my_modules +remote_tmp = /tmp/.ansible/tmp +local_tmp = /tmp/.ansible/tmp +roles_path = ./provision/ansible/roles +retry_files_enabled = False + +#--- Fact Caching settings +fact_caching = jsonfile +fact_caching_connection = ~/.ansible/facts_cache +fact_caching_timeout = 7200 + +#--- SSH settings +remote_port = 22 +timeout = 60 +host_key_checking = False +ssh_executable = /usr/bin/ssh +private_key_file = ~/.ssh/id_rsa + +force_valid_group_names = ignore + +#--- Speed +callback_enabled = true +internal_poll_interval = 0.001 + +#--- Plugin settings +vars_plugins_enabled = host_group_vars,community.sops.sops + +[inventory] +unparsed_is_failed = true + +[privilege_escalation] +become = True +become_method = sudo +become_user = root +become_ask_pass = False + +[ssh_connection] +scp_if_ssh = smart +transfer_method = smart +retries = 3 +timeout = 10 +ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o Compression=yes -o ServerAliveInterval=15s +pipelining = True +control_path = %(directory)s/%%h-%%r diff --git a/cluster/core/cert-manager/letsencrypt-production.yaml b/cluster/core/cert-manager/letsencrypt-production.yaml index 348fc492..1f880a85 100644 --- a/cluster/core/cert-manager/letsencrypt-production.yaml +++ b/cluster/core/cert-manager/letsencrypt-production.yaml @@ -13,9 +13,9 @@ spec: - dns01: cloudflare: email: "${SECRET_CLOUDFLARE_EMAIL}" - apiTokenSecretRef: - name: cloudflare-token-secret - key: cloudflare-token + apiKeySecretRef: + name: cloudflare-api-key + key: api-key selector: dnsZones: - ${SECRET_DOMAIN} diff --git a/cluster/core/cert-manager/letsencrypt-staging.yaml b/cluster/core/cert-manager/letsencrypt-staging.yaml index e9146701..0d08a792 100644 --- a/cluster/core/cert-manager/letsencrypt-staging.yaml +++ b/cluster/core/cert-manager/letsencrypt-staging.yaml @@ -13,9 +13,9 @@ spec: - dns01: cloudflare: email: "${SECRET_CLOUDFLARE_EMAIL}" - apiTokenSecretRef: - name: cloudflare-token-secret - key: cloudflare-token + apiKeySecretRef: + name: cloudflare-api-key + key: api-key selector: dnsZones: - ${SECRET_DOMAIN} diff --git a/configure.sh b/configure.sh new file mode 100755 index 00000000..7725078e --- /dev/null +++ b/configure.sh @@ -0,0 +1,292 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +# shellcheck disable=SC2155 +export PROJECT_DIR=$(git rev-parse --show-toplevel) + +# shellcheck disable=SC2155 +export GPG_TTY=$(tty) + +# shellcheck disable=SC1091 +source "${PROJECT_DIR}/.config.env" + +show_help() { +cat << EOF +Usage: $(basename "$0") + -h, --help Display help + --verify Verify .config.env settings +EOF +} + +main() { + local verify= + + parse_command_line "$@" + + verify_binaries + + if [[ "${verify}" == 1 ]]; then + verify_ansible_hosts + verify_metallb + verify_kubevip + verify_gpg_fp + verify_git_repository + verify_cloudflare + else + # sops configuration file + envsubst < "${PROJECT_DIR}/tmpl/.sops.yaml" \ + > "${PROJECT_DIR}/.sops.yaml" + # cluster + envsubst < "${PROJECT_DIR}/tmpl/cluster/cluster-secrets.sops.yaml" \ + > "${PROJECT_DIR}/cluster/base/cluster-secrets.sops.yaml" + envsubst < "${PROJECT_DIR}/tmpl/cluster/cluster-settings.yaml" \ + > "${PROJECT_DIR}/cluster/base/cluster-settings.yaml" + envsubst < "${PROJECT_DIR}/tmpl/cluster/gotk-sync.yaml" \ + > "${PROJECT_DIR}/cluster/base/flux-system/gotk-sync.yaml" + envsubst < "${PROJECT_DIR}/tmpl/cluster/cert-manager-secret.sops.yaml" \ + > "${PROJECT_DIR}/cluster/core/cert-manager/secret.sops.yaml" + sops --encrypt --in-place "${PROJECT_DIR}/cluster/base/cluster-secrets.sops.yaml" + sops --encrypt --in-place "${PROJECT_DIR}/cluster/core/cert-manager/secret.sops.yaml" + # terraform + envsubst < "${PROJECT_DIR}/tmpl/terraform/secret.sops.yaml" \ + > "${PROJECT_DIR}/provision/terraform/cloudflare/secret.sops.yaml" + sops --encrypt --in-place "${PROJECT_DIR}/provision/terraform/cloudflare/secret.sops.yaml" + # ansible + envsubst < "${PROJECT_DIR}/tmpl/ansible/kube-vip.yml" \ + > "${PROJECT_DIR}/provision/ansible/inventory/group_vars/kubernetes/kube-vip.yml" + generate_ansible_hosts + generate_ansible_host_secrets + fi +} + +parse_command_line() { + while :; do + case "${1:-}" in + -h|--help) + show_help + exit + ;; + --verify) + verify=1 + ;; + *) + break + ;; + esac + + shift + done + + if [[ -z "$verify" ]]; then + verify=0 + fi +} + +_has_binary() { + command -v "${1}" >/dev/null 2>&1 || { + _log "ERROR" "${1} is not installed or not found in \$PATH" + exit 1 + } +} + +_has_envar() { + local option="${1}" + # shellcheck disable=SC2015 + [[ "${!option}" == "" ]] && { + _log "ERROR" "Unset variable ${option}" + exit 1 + } || { + _log "DEBUG" "Found variable '${option}' with value '${!option}'" + } +} + +_has_valid_ip() { + local ip="${1}" + local variable_name="${2}" + + if ! ipcalc "${ip}" | awk 'BEGIN{FS=":"; is_invalid=0} /^INVALID/ {is_invalid=1; print $1} END{exit is_invalid}' >/dev/null 2>&1; then + _log "DEBUG" "Variable '${variable_name}' has an invalid IP address '${ip}'" + exit 1 + else + _log "DEBUG" "Variable '${variable_name}' has a valid IP address '${ip}'" + fi +} + +verify_gpg_fp() { + _has_envar "BOOTSTRAP_PERSONAL_KEY_FP" + _has_envar "BOOTSTRAP_FLUX_KEY_FP" + + if ! gpg --list-keys "${BOOTSTRAP_PERSONAL_KEY_FP}" >/dev/null 2>&1; then + _log "ERROR" "Invalid Personal GPG FP ${BOOTSTRAP_PERSONAL_KEY_FP}" + exit 1 + else + _log "DEBUG" "Found Personal GPG Fingerprint '${BOOTSTRAP_PERSONAL_KEY_FP}'" + fi + + if ! gpg --list-keys "${BOOTSTRAP_FLUX_KEY_FP}" >/dev/null 2>&1; then + _log "ERROR" "Invalid Flux GPG FP '${BOOTSTRAP_FLUX_KEY_FP}'" + exit 1 + else + _log "DEBUG" "Found Flux GPG Fingerprint '${BOOTSTRAP_FLUX_KEY_FP}'" + fi +} + +verify_binaries() { + _has_binary "ansible" + _has_binary "envsubst" + _has_binary "flux" + _has_binary "git" + _has_binary "gpg" + _has_binary "helm" + _has_binary "ipcalc" + _has_binary "jq" + _has_binary "sops" + _has_binary "task" + _has_binary "terraform" +} + +verify_kubevip() { + _has_envar "BOOTSTRAP_ANSIBLE_KUBE_VIP_ADDRESS" + _has_valid_ip "${BOOTSTRAP_ANSIBLE_KUBE_VIP_ADDRESS}" "BOOTSTRAP_ANSIBLE_KUBE_VIP_ADDRESS" +} + +verify_metallb() { + local ip_floor= + local ip_ceil= + _has_envar "BOOTSTRAP_METALLB_LB_RANGE" + _has_envar "BOOTSTRAP_METALLB_TRAEFIK_ADDR" + + ip_floor=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f1) + ip_ceil=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f2) + + _has_valid_ip "${ip_floor}" "BOOTSTRAP_METALLB_LB_RANGE" + _has_valid_ip "${ip_ceil}" "BOOTSTRAP_METALLB_LB_RANGE" + _has_valid_ip "${BOOTSTRAP_METALLB_TRAEFIK_ADDR}" "BOOTSTRAP_METALLB_TRAEFIK_ADDR" +} + +verify_git_repository() { + _has_envar "BOOTSTRAP_GIT_REPOSITORY" + + export GIT_TERMINAL_PROMPT=0 + pushd "$(mktemp -d)" >/dev/null 2>&1 + [ "$(git ls-remote "${BOOTSTRAP_GIT_REPOSITORY}" 2> /dev/null)" ] || { + _log "ERROR" "Unable to find the remote Git repository '${BOOTSTRAP_GIT_REPOSITORY}'" + exit 1 + } + popd >/dev/null 2>&1 + export GIT_TERMINAL_PROMPT=1 +} + +verify_cloudflare() { + local account_zone= + local errors= + + _has_envar "BOOTSTRAP_CLOUDFLARE_APIKEY" + _has_envar "BOOTSTRAP_CLOUDFLARE_DOMAIN" + _has_envar "BOOTSTRAP_CLOUDFLARE_EMAIL" + + # Try to retrieve zone information from Cloudflare's API + account_zone=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=${BOOTSTRAP_CLOUDFLARE_DOMAIN}&status=active" \ + -H "X-Auth-Email: ${BOOTSTRAP_CLOUDFLARE_EMAIL}" \ + -H "X-Auth-Key: ${BOOTSTRAP_CLOUDFLARE_APIKEY}" \ + -H "Content-Type: application/json" + ) + + if [[ "$(echo "${account_zone}" | jq ".success")" == "true" ]]; then + _log "DEBUG" "Verified Cloudflare Account and Zone information" + exit 0 + else + errors=$(echo "${account_zone}" | jq -c ".errors") + _log "ERROR" "Unable to get Cloudflare Account and Zone information ${errors}" + exit 1 + fi +} + +verify_ansible_hosts() { + local node_id= + local node_addr= + local node_username= + local node_password= + local node_control= + + for var in "${!BOOTSTRAP_ANSIBLE_HOST_ADDR_@}"; do + node_id=$(echo "${var}" | awk -F"_" '{print $5}') + node_addr="BOOTSTRAP_ANSIBLE_HOST_ADDR_${node_id}" + node_username="BOOTSTRAP_ANSIBLE_SSH_USERNAME_${node_id}" + node_password="BOOTSTRAP_ANSIBLE_SSH_PASSWORD_${node_id}" + node_control="BOOTSTRAP_ANSIBLE_CONTROL_NODE_${node_id}" + + _has_envar "${node_addr}" + _has_envar "${node_username}" + _has_envar "${node_password}" + _has_envar "${node_control}" + + test="$(ssh -q -o BatchMode=yes -o ConnectTimeout=5 "${!node_username}"@"${!var}" echo ok 2>&1)" + if [[ "${test}" != "ok" ]]; then + _log "ERROR" "Unable to SSH into host '${!var}' with username '${!node_username}'" + exit 1 + else + _log "DEBUG" "Successfully SSH'ed into host '${!var}' with username '${!node_username}'" + fi + done +} + +generate_ansible_host_secrets() { + local node_id= + local node_username= + local node_password= + for var in "${!BOOTSTRAP_ANSIBLE_HOST_ADDR_@}"; do + node_id=$(echo "${var}" | awk -F"_" '{print $5}') + { + node_username="BOOTSTRAP_ANSIBLE_SSH_USERNAME_${node_id}" + node_password="BOOTSTRAP_ANSIBLE_SSH_PASSWORD_${node_id}" + printf "kind: Secret\n" + printf "ansible_user: %s\n" "${!node_username}" + printf "ansible_become_pass: %s\n" "${!node_password}" + } > "${PROJECT_DIR}/provision/ansible/inventory/host_vars/k8s-${node_id}.sops.yml" + sops --encrypt --in-place "${PROJECT_DIR}/provision/ansible/inventory/host_vars/k8s-${node_id}.sops.yml" + done +} + +generate_ansible_hosts() { + local worker_node_count= + { + printf "kubernetes:\n" + printf " children:\n" + printf " master:\n" + printf " hosts:\n" + worker_node_count=0 + for var in "${!BOOTSTRAP_ANSIBLE_HOST_ADDR_@}"; do + node_id=$(echo "${var}" | awk -F"_" '{print $5}') + node_control="BOOTSTRAP_ANSIBLE_CONTROL_NODE_${node_id}" + if [[ "${!node_control}" == "true" ]]; then + printf " k8s-%s:\n" "${node_id}" + printf " ansible_host: %s\n" "${!var}" + else + worker_node_count=$((worker_node_count+1)) + fi + done + if [[ ${worker_node_count} -gt 0 ]]; then + printf " worker:\n" + printf " hosts:\n" + for var in "${!BOOTSTRAP_ANSIBLE_HOST_ADDR_@}"; do + node_id=$(echo "${var}" | awk -F"_" '{print $5}') + node_control="BOOTSTRAP_ANSIBLE_CONTROL_NODE_${node_id}" + if [[ "${!node_control}" == "false" ]]; then + printf " k8s-%s:\n" "${node_id}" + printf " ansible_host: %s\n" "${!var}" + fi + done + fi + } > "${PROJECT_DIR}/provision/ansible/inventory/hosts.yml" +} + +function _log() { + local type="${1}" + local msg="${2}" + printf "[%s] [%s] %s\n" "$(date -u)" "${type}" "${msg}" +} + +main "$@" diff --git a/provision/ansible/inventory/group_vars/kubernetes/k3s.yml b/provision/ansible/inventory/group_vars/kubernetes/k3s.yml new file mode 100644 index 00000000..7c34472a --- /dev/null +++ b/provision/ansible/inventory/group_vars/kubernetes/k3s.yml @@ -0,0 +1,32 @@ +--- +# +# Below vars are for the xanmanning.k3s role +# ...see https://github.com/PyratLabs/ansible-role-k3s +# + +# (string) Use a specific version of k3s +k3s_release_version: "v1.21.5+k3s1" + +# (bool) Install using hard links rather than symbolic links. +k3s_install_hard_links: true + +# (bool) Escalate user privileges for all tasks +k3s_become_for_all: true + +# (bool) Enable debug logging on the k3s service +k3s_debug: false + +# (bool) Enable etcd embedded datastore +k3s_etcd_datastore: true + +# (bool) Allow the use of unsupported configurations in k3s +k3s_use_unsupported_config: false + +# (string) Control Plane registration address +k3s_registration_address: "{{ kubevip_address }}" + +# (list) A flat list of templates to deploy on the primary control plane +# /var/lib/rancher/k3s/server/manifests +k3s_server_manifests_templates: + - "kube-vip/kube-vip-rbac.yaml.j2" + - "kube-vip/kube-vip-daemonset.yaml.j2" diff --git a/provision/ansible/inventory/group_vars/kubernetes/ubuntu.yml b/provision/ansible/inventory/group_vars/kubernetes/ubuntu.yml new file mode 100644 index 00000000..188d1300 --- /dev/null +++ b/provision/ansible/inventory/group_vars/kubernetes/ubuntu.yml @@ -0,0 +1,6 @@ +--- +# (string) Timezone for the servers +# timezone: "America/New_York" + +# (list) Additional ssh public keys to add to the nodes +# ssh_authorized_keys: diff --git a/provision/ansible/inventory/group_vars/master/k3s.yml b/provision/ansible/inventory/group_vars/master/k3s.yml new file mode 100644 index 00000000..19bbaf11 --- /dev/null +++ b/provision/ansible/inventory/group_vars/master/k3s.yml @@ -0,0 +1,42 @@ +--- +# https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/ +# https://github.com/PyratLabs/ansible-role-k3s + +# (bool) Specify if a host (or host group) are part of the control plane +k3s_control_node: true + +# (dict) k3s settings for all control-plane nodes +k3s_server: + node-ip: "{{ ansible_host }}" + tls-san: + # kube-vip + - "{{ kubevip_address }}" + docker: false + flannel-backend: "host-gw" # This needs to be in quotes + disable: + - traefik + - servicelb + - metrics-server + disable-network-policy: true + disable-cloud-controller: true + write-kubeconfig-mode: "644" + # Network CIDR to use for pod IPs + cluster-cidr: "10.42.0.0/16" + # Network CIDR to use for service IPs + service-cidr: "10.43.0.0/16" + kubelet-arg: + - "feature-gates=GracefulNodeShutdown=true" + - "feature-gates=MixedProtocolLBService=true" + # Required to use monitor these components with kube-prometheus-stack + kube-controller-manager-arg: + - "address=0.0.0.0" + - "bind-address=0.0.0.0" + kube-proxy-arg: + - "metrics-bind-address=0.0.0.0" + kube-scheduler-arg: + - "address=0.0.0.0" + - "bind-address=0.0.0.0" + etcd-expose-metrics: true + # Required for HAProxy health-checks + kube-apiserver-arg: + - "anonymous-auth=true" diff --git a/provision/ansible/inventory/group_vars/worker/k3s.yml b/provision/ansible/inventory/group_vars/worker/k3s.yml new file mode 100644 index 00000000..8adba9a2 --- /dev/null +++ b/provision/ansible/inventory/group_vars/worker/k3s.yml @@ -0,0 +1,13 @@ +--- +# https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/ +# https://github.com/PyratLabs/ansible-role-k3s + +# (bool) Specify if a host (or host group) are part of the control plane +k3s_control_node: false + +# (dict) k3s settings for all worker nodes +k3s_agent: + node-ip: "{{ ansible_host }}" + kubelet-arg: + - "feature-gates=GracefulNodeShutdown=true" + - "feature-gates=MixedProtocolLBService=true" diff --git a/provision/ansible/inventory/host_vars/.gitkeep b/provision/ansible/inventory/host_vars/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/provision/ansible/inventory/hosts.yml b/provision/ansible/inventory/hosts.yml new file mode 100644 index 00000000..e4dbb6aa --- /dev/null +++ b/provision/ansible/inventory/hosts.yml @@ -0,0 +1,11 @@ +--- +kubernetes: + children: + master: + hosts: + k8s-0: + ansible_host: 192.168.42.10 + worker: + hosts: + k8s-1: + ansible_host: 192.168.42.13 diff --git a/provision/ansible/playbooks/k3s-install.yml b/provision/ansible/playbooks/k3s-install.yml new file mode 100644 index 00000000..e407c97c --- /dev/null +++ b/provision/ansible/playbooks/k3s-install.yml @@ -0,0 +1,65 @@ +--- +- hosts: + - master + - worker + become: true + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Pausing for 5 seconds... + pause: + seconds: 5 + tasks: + - name: Check if cluster is installed + ansible.builtin.stat: + path: "/etc/rancher/k3s/config.yaml" + register: k3s_check_installed + check_mode: false + + - name: Set manifest facts + ansible.builtin.set_fact: + k3s_server_manifests_templates: [] + k3s_server_manifests_urls: [] + when: k3s_check_installed.stat.exists + + - name: Install Kubernetes + include_role: + name: xanmanning.k3s + public: true + + - name: Copy kubeconfig locally to /tmp + run_once: true + ansible.builtin.fetch: + src: "/etc/rancher/k3s/k3s.yaml" + dest: "/tmp/kubeconfig" + flat: true + when: + - k3s_control_node is defined + - k3s_control_node + + - name: Update kubeconfig with the right IPv4 address + delegate_to: localhost + become: false + run_once: true + ansible.builtin.replace: + path: "/tmp/kubeconfig" + regexp: "https://127.0.0.1:6443" + replace: "https://{{ k3s_registration_address }}:6443" + + - name: Remove deployed manifest templates + ansible.builtin.file: + path: "{{ k3s_server_manifests_dir }}/{{ item | basename | regex_replace('\\.j2$', '') }}" + state: absent + loop: "{{ k3s_server_manifests_templates }}" + when: + - k3s_server_manifests_templates + - k3s_server_manifests_templates | length > 0 + + - name: Remove deployed manifest urls + ansible.builtin.file: + path: "{{ k3s_server_manifests_dir }}/{{ item.filename }}" + state: absent + loop: "{{ k3s_server_manifests_urls }}" + when: + - k3s_server_manifests_urls + - k3s_server_manifests_urls | length > 0 diff --git a/provision/ansible/playbooks/k3s-nuke.yml b/provision/ansible/playbooks/k3s-nuke.yml new file mode 100644 index 00000000..a320a2e7 --- /dev/null +++ b/provision/ansible/playbooks/k3s-nuke.yml @@ -0,0 +1,32 @@ +--- +- hosts: + - master + - worker + become: true + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Pausing for 5 seconds... + pause: + seconds: 5 + tasks: + - name: Kill k3s + ansible.builtin.command: /usr/local/bin/k3s-killall.sh + - name: Uninstall k3s + ansible.builtin.command: + cmd: /usr/local/bin/k3s-uninstall.sh + removes: /usr/local/bin/k3s-uninstall.sh + - name: Uninstall k3s agent + ansible.builtin.command: + cmd: /usr/local/bin/k3s-agent-uninstall.sh + removes: /usr/local/bin/k3s-agent-uninstall.sh + - name: Gather list of CNI files to delete + find: + paths: /etc/cni/net.d + patterns: "*" + register: files_to_delete + - name: Delete CNI files + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ files_to_delete.files }}" diff --git a/provision/ansible/playbooks/ubuntu-prepare.yml b/provision/ansible/playbooks/ubuntu-prepare.yml new file mode 100644 index 00000000..719ae06f --- /dev/null +++ b/provision/ansible/playbooks/ubuntu-prepare.yml @@ -0,0 +1,13 @@ +--- +- hosts: + - master + - worker + become: true + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Pausing for 5 seconds... + pause: + seconds: 5 + roles: + - ubuntu diff --git a/provision/ansible/playbooks/ubuntu-upgrade.yml b/provision/ansible/playbooks/ubuntu-upgrade.yml new file mode 100644 index 00000000..88701b5d --- /dev/null +++ b/provision/ansible/playbooks/ubuntu-upgrade.yml @@ -0,0 +1,18 @@ +--- +- hosts: + - master + - worker + become: true + gather_facts: true + any_errors_fatal: true + tasks: + - name: upgrade + ansible.builtin.apt: + upgrade: full + update_cache: true + cache_valid_time: 3600 + autoclean: true + autoremove: true + register: apt_upgrade + retries: 5 + until: apt_upgrade is success diff --git a/provision/ansible/requirements.yml b/provision/ansible/requirements.yml new file mode 100644 index 00000000..a4b5a2dc --- /dev/null +++ b/provision/ansible/requirements.yml @@ -0,0 +1,9 @@ +--- +collections: + - name: community.general + version: 3.7.0 + - name: community.sops + version: 1.1.0 +roles: + - src: xanmanning.k3s + version: v2.11.0 diff --git a/provision/ansible/roles/ubuntu/tasks/filesystem.yml b/provision/ansible/roles/ubuntu/tasks/filesystem.yml new file mode 100644 index 00000000..d9e155ba --- /dev/null +++ b/provision/ansible/roles/ubuntu/tasks/filesystem.yml @@ -0,0 +1,20 @@ +--- +- name: Update max_user_watches + ansible.posix.sysctl: + name: fs.inotify.max_user_watches + value: "65536" + state: present + sysctl_file: /etc/sysctl.d/98-kubernetes-fs.conf + +- name: Disable swap at runtime + ansible.builtin.command: swapoff -a + when: ansible_swaptotal_mb > 0 + +- name: Disable swap on boot + ansible.posix.mount: + name: "{{ item }}" + fstype: swap + state: absent + loop: + - swap + - none diff --git a/provision/ansible/roles/ubuntu/tasks/locale.yml b/provision/ansible/roles/ubuntu/tasks/locale.yml new file mode 100644 index 00000000..bbef8b0f --- /dev/null +++ b/provision/ansible/roles/ubuntu/tasks/locale.yml @@ -0,0 +1,6 @@ +--- +- name: Set timezone + community.general.timezone: + name: "{{ timezone }}" + when: + - timezone is defined diff --git a/provision/ansible/roles/ubuntu/tasks/main.yml b/provision/ansible/roles/ubuntu/tasks/main.yml new file mode 100644 index 00000000..8666f18d --- /dev/null +++ b/provision/ansible/roles/ubuntu/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- include: locale.yml + tags: + - locale + +- include: packages.yml + tags: + - packages + +- include: network.yml + tags: + - network + +- include: filesystem.yml + tags: + - filesystem + +- include: unattended-upgrades.yml + tags: + - unattended-upgrades + +- include: user.yml + tags: + - user diff --git a/provision/ansible/roles/ubuntu/tasks/network.yml b/provision/ansible/roles/ubuntu/tasks/network.yml new file mode 100644 index 00000000..a097fc9d --- /dev/null +++ b/provision/ansible/roles/ubuntu/tasks/network.yml @@ -0,0 +1,47 @@ +--- +- name: Set hostname to inventory hostname + ansible.builtin.hostname: + name: "{{ inventory_hostname }}" + when: + - ansible_hostname != inventory_hostname + +- name: Update /etc/hosts to include hostname + ansible.builtin.blockinfile: + path: /etc/hosts + create: true + block: | + 127.0.0.1 localhost + 127.0.1.1 {{ inventory_hostname }} + + # The following lines are desirable for IPv6 capable hosts + ::1 ip6-localhost ip6-loopback + fe00::0 ip6-localnet + ff00::0 ip6-mcastprefix + ff02::1 ip6-allnodes + ff02::2 ip6-allrouters + ff02::3 ip6-allhosts + +- name: Check for bridge-nf-call-iptables + ansible.builtin.stat: + path: /proc/sys/net/bridge/bridge-nf-call-iptables + register: bridge_nf_call_iptables_result + +- name: Set Kubernetes network configuration + ansible.builtin.blockinfile: + path: /etc/sysctl.d/99-kubernetes-cri.conf + mode: 0644 + create: true + block: | + net.ipv4.ip_forward = 1 + net.bridge.bridge-nf-call-iptables = 1 + net.ipv6.conf.all.forwarding = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + when: + - bridge_nf_call_iptables_result.stat.exists + register: sysctl_network + +- name: Reload Kubernetes network configuration + ansible.builtin.shell: sysctl -p /etc/sysctl.d/99-kubernetes-cri.conf + when: + - sysctl_network.changed + - bridge_nf_call_iptables_result.stat.exists diff --git a/provision/ansible/roles/ubuntu/tasks/packages.yml b/provision/ansible/roles/ubuntu/tasks/packages.yml new file mode 100644 index 00000000..c5092e1f --- /dev/null +++ b/provision/ansible/roles/ubuntu/tasks/packages.yml @@ -0,0 +1,76 @@ +--- +- name: Upgrade all system packages + ansible.builtin.apt: + upgrade: full + update_cache: true + cache_valid_time: 3600 + autoclean: true + autoremove: true + register: apt_upgrade + retries: 5 + until: apt_upgrade is success + +- name: Install common packages + ansible.builtin.apt: + name: + - apt-transport-https + - arptables + - ca-certificates + - curl + - ebtables + - iputils-ping + - ipvsadm + - lvm2 + - net-tools + - nfs-common + - open-iscsi + - psmisc + - smartmontools + - software-properties-common + - unattended-upgrades + install_recommends: false + update_cache: true + cache_valid_time: 3600 + autoclean: true + autoremove: true + register: apt_install_common + retries: 5 + until: apt_install_common is success + +- name: Copy 20auto-upgrades unattended-upgrades config + ansible.builtin.blockinfile: + path: /etc/apt/apt.conf.d/20auto-upgrades + mode: 0644 + create: true + block: | + APT::Periodic::Update-Package-Lists "1"; + APT::Periodic::Download-Upgradeable-Packages "1"; + APT::Periodic::AutocleanInterval "7"; + APT::Periodic::Unattended-Upgrade "1"; + +- name: Copy 50unattended-upgrades unattended-upgrades config + ansible.builtin.blockinfile: + path: /etc/apt/apt.conf.d/50unattended-upgrades + mode: 0644 + create: true + block: | + Unattended-Upgrade::Automatic-Reboot "false"; + Unattended-Upgrade::Remove-Unused-Dependencies "true"; + Unattended-Upgrade::Allowed-Origins { + "${distro_id} stable"; + "${distro_id} ${distro_codename}-security"; + "${distro_id} ${distro_codename}-updates"; + }; + +- name: Start unattended-upgrades service + ansible.builtin.systemd: + name: unattended-upgrades + enabled: true + state: started + +- name: Restart unattended-upgrades service + ansible.builtin.service: + name: unattended-upgrades.service + daemon_reload: true + enabled: true + state: restarted diff --git a/provision/ansible/roles/ubuntu/tasks/user.yml b/provision/ansible/roles/ubuntu/tasks/user.yml new file mode 100644 index 00000000..9b5766ff --- /dev/null +++ b/provision/ansible/roles/ubuntu/tasks/user.yml @@ -0,0 +1,16 @@ +--- +- name: Add user to sudoers + ansible.builtin.copy: + content: "{{ ansible_user }} ALL=(ALL:ALL) NOPASSWD:ALL" + dest: "/etc/sudoers.d/{{ ansible_user }}_nopasswd" + mode: "0440" + +- name: Add additional user SSH public keys + ansible.posix.authorized_key: + user: "{{ ansible_user }}" + key: "{{ item }}" + loop: "{{ ssh_authorized_keys }}" + when: + - ssh_authorized_keys is defined + - ssh_authorized_keys is iterable + - ssh_authorized_keys | length > 0 diff --git a/provision/ansible/templates/kube-vip-daemonset.yaml.j2 b/provision/ansible/templates/kube-vip-daemonset.yaml.j2 new file mode 100644 index 00000000..330f450b --- /dev/null +++ b/provision/ansible/templates/kube-vip-daemonset.yaml.j2 @@ -0,0 +1,57 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-vip + namespace: kube-system + labels: + app.kubernetes.io/instance: kube-vip + app.kubernetes.io/name: kube-vip +spec: + selector: + matchLabels: + app.kubernetes.io/instance: kube-vip + app.kubernetes.io/name: kube-vip + template: + metadata: + labels: + app.kubernetes.io/instance: kube-vip + app.kubernetes.io/name: kube-vip + spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.3.7 + imagePullPolicy: IfNotPresent + args: + - manager + env: + - name: vip_arp + value: "true" + - name: vip_interface + value: "{{ kubevip_interface }}" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "false" + - name: vip_address + value: "{{ k3s_registration_address }}" + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_TIME + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "true" + serviceAccountName: kube-vip + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule diff --git a/provision/ansible/templates/kube-vip-rbac.yaml.j2 b/provision/ansible/templates/kube-vip-rbac.yaml.j2 new file mode 100644 index 00000000..72f6fda8 --- /dev/null +++ b/provision/ansible/templates/kube-vip-rbac.yaml.j2 @@ -0,0 +1,33 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services", "services/status", "nodes"] + verbs: ["list", "get", "watch", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system diff --git a/provision/terraform/cloudflare/main.tf b/provision/terraform/cloudflare/main.tf new file mode 100644 index 00000000..fa2c7bef --- /dev/null +++ b/provision/terraform/cloudflare/main.tf @@ -0,0 +1,110 @@ +terraform { + + required_providers { + cloudflare = { + source = "cloudflare/cloudflare" + version = "3.1.0" + } + http = { + source = "hashicorp/http" + version = "2.1.0" + } + sops = { + source = "carlpett/sops" + version = "0.6.3" + } + } +} + +data "sops_file" "cloudflare_secrets" { + source_file = "secret.sops.yaml" +} + +provider "cloudflare" { + email = data.sops_file.cloudflare_secrets.data["cloudflare_email"] + api_key = data.sops_file.cloudflare_secrets.data["cloudflare_apikey"] +} + +data "cloudflare_zones" "domain" { + filter { + name = data.sops_file.cloudflare_secrets.data["cloudflare_domain"] + } +} + +resource "cloudflare_zone_settings_override" "cloudflare_settings" { + zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id") + settings { + # /ssl-tls + ssl = "strict" + # /ssl-tls/edge-certificates + always_use_https = "on" + min_tls_version = "1.0" + opportunistic_encryption = "on" + tls_1_3 = "zrt" + automatic_https_rewrites = "on" + universal_ssl = "on" + # /firewall/settings + browser_check = "on" + challenge_ttl = 1800 + privacy_pass = "on" + security_level = "medium" + # /speed/optimization + brotli = "on" + minify { + css = "on" + js = "on" + html = "on" + } + rocket_loader = "on" + # /caching/configuration + always_online = "off" + development_mode = "off" + # /network + http3 = "on" + zero_rtt = "on" + ipv6 = "on" + websockets = "on" + opportunistic_onion = "on" + pseudo_ipv4 = "off" + ip_geolocation = "on" + # /content-protection + email_obfuscation = "on" + server_side_exclude = "on" + hotlink_protection = "off" + # /workers + security_header { + enabled = false + } + } +} + +data "http" "ipv4" { + url = "http://ipv4.icanhazip.com" +} + +resource "cloudflare_record" "ipv4" { + name = "ipv4" + zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id") + value = chomp(data.http.ipv4.body) + proxied = true + type = "A" + ttl = 1 +} + +resource "cloudflare_record" "root" { + name = data.sops_file.cloudflare_secrets.data["cloudflare_domain"] + zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id") + value = "ipv4.${data.sops_file.cloudflare_secrets.data["cloudflare_domain"]}" + proxied = true + type = "CNAME" + ttl = 1 +} + +resource "cloudflare_record" "hajimari" { + name = "hajimari" + zone_id = lookup(data.cloudflare_zones.domain.zones[0], "id") + value = "ipv4.${data.sops_file.cloudflare_secrets.data["cloudflare_domain"]}" + proxied = true + type = "CNAME" + ttl = 1 +} diff --git a/tmpl/.sops.yaml b/tmpl/.sops.yaml index 4447bf44..d7f4d9bb 100644 --- a/tmpl/.sops.yaml +++ b/tmpl/.sops.yaml @@ -1,6 +1,11 @@ --- creation_rules: - - encrypted_regex: "^(data|stringData)$" + - path_regex: provision/.*\.sops\.ya?ml + unencrypted_regex: "^(kind)$" pgp: >- - ${FLUX_KEY_FP}, - ${PERSONAL_KEY_FP} + ${BOOTSTRAP_PERSONAL_KEY_FP} + - path_regex: cluster/.*\.ya?ml + encrypted_regex: "^(data|stringData)$" + pgp: >- + ${BOOTSTRAP_FLUX_KEY_FP}, + ${BOOTSTRAP_PERSONAL_KEY_FP} diff --git a/tmpl/ansible/kube-vip.yml b/tmpl/ansible/kube-vip.yml new file mode 100644 index 00000000..6e901889 --- /dev/null +++ b/tmpl/ansible/kube-vip.yml @@ -0,0 +1,5 @@ +--- +# (string) The interface on the host kube-vip should attach to +kubevip_interface: "${BOOTSTRAP_ANSIBLE_KUBE_VIP_INTERFACE}" +# (string) The ARP address kube-vip broadcasts +kubevip_address: "${BOOTSTRAP_ANSIBLE_KUBE_VIP_ADDRESS}" diff --git a/tmpl/secret.sops.yaml b/tmpl/cluster/cert-manager-secret.sops.yaml similarity index 53% rename from tmpl/secret.sops.yaml rename to tmpl/cluster/cert-manager-secret.sops.yaml index 11e96053..0c4de075 100644 --- a/tmpl/secret.sops.yaml +++ b/tmpl/cluster/cert-manager-secret.sops.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: cloudflare-token-secret + name: cloudflare-api-key namespace: cert-manager stringData: - cloudflare-token: ${BOOTSTRAP_CLOUDFLARE_TOKEN} + api-key: ${BOOTSTRAP_CLOUDFLARE_APIKEY} diff --git a/tmpl/cluster-secrets.sops.yaml b/tmpl/cluster/cluster-secrets.sops.yaml similarity index 78% rename from tmpl/cluster-secrets.sops.yaml rename to tmpl/cluster/cluster-secrets.sops.yaml index 6380718b..24a68a4a 100644 --- a/tmpl/cluster-secrets.sops.yaml +++ b/tmpl/cluster/cluster-secrets.sops.yaml @@ -5,5 +5,5 @@ metadata: name: cluster-secrets namespace: flux-system stringData: - SECRET_DOMAIN: ${BOOTSTRAP_DOMAIN} + SECRET_DOMAIN: ${BOOTSTRAP_CLOUDFLARE_DOMAIN} SECRET_CLOUDFLARE_EMAIL: ${BOOTSTRAP_CLOUDFLARE_EMAIL} diff --git a/tmpl/cluster-settings.yaml b/tmpl/cluster/cluster-settings.yaml similarity index 72% rename from tmpl/cluster-settings.yaml rename to tmpl/cluster/cluster-settings.yaml index 0081404c..bf9595de 100644 --- a/tmpl/cluster-settings.yaml +++ b/tmpl/cluster/cluster-settings.yaml @@ -6,4 +6,4 @@ metadata: namespace: flux-system data: METALLB_LB_RANGE: ${BOOTSTRAP_METALLB_LB_RANGE} - SVC_TRAEFIK_ADDR: ${BOOTSTRAP_SVC_TRAEFIK_ADDR} + METALLB_TRAEFIK_ADDR: ${BOOTSTRAP_METALLB_TRAEFIK_ADDR} diff --git a/tmpl/gotk-sync.yaml b/tmpl/cluster/gotk-sync.yaml similarity index 93% rename from tmpl/gotk-sync.yaml rename to tmpl/cluster/gotk-sync.yaml index 275b7f77..d10e5039 100644 --- a/tmpl/gotk-sync.yaml +++ b/tmpl/cluster/gotk-sync.yaml @@ -8,7 +8,7 @@ spec: interval: 1m0s ref: branch: main - url: ${BOOTSTRAP_GITHUB_REPOSITORY} + url: ${BOOTSTRAP_GIT_REPOSITORY} --- apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 kind: Kustomization diff --git a/tmpl/terraform/secret.sops.yaml b/tmpl/terraform/secret.sops.yaml new file mode 100644 index 00000000..0becc70f --- /dev/null +++ b/tmpl/terraform/secret.sops.yaml @@ -0,0 +1,4 @@ +kind: Secret +cloudflare_email: ${BOOTSTRAP_CLOUDFLARE_EMAIL} +cloudflare_apikey: ${BOOTSTRAP_CLOUDFLARE_APIKEY} +cloudflare_domain: ${BOOTSTRAP_CLOUDFLARE_DOMAIN}