diff --git a/.github/tests/config-k0s.yaml b/.github/tests/config-k0s.yaml index 1860a823..6b487d06 100644 --- a/.github/tests/config-k0s.yaml +++ b/.github/tests/config-k0s.yaml @@ -4,10 +4,9 @@ skip_tests: true bootstrap_distribution: k0s bootstrap_github_username: onedr0p bootstrap_github_repository_name: flux-cluster-template -bootstrap_github_repository_branch: main bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY bootstrap_timezone: Etc/UTC -bootstrap_acme_email: fake +bootstrap_acme_email: fake@example.com bootstrap_acme_production_enabled: false bootstrap_flux_github_webhook_token: fake bootstrap_cloudflare_domain: fake diff --git a/.github/tests/config-k3s-ipv4.yaml b/.github/tests/config-k3s-ipv4.yaml index d804ad85..9416dd49 100644 --- a/.github/tests/config-k3s-ipv4.yaml +++ b/.github/tests/config-k3s-ipv4.yaml @@ -4,10 +4,9 @@ skip_tests: true bootstrap_distribution: k3s bootstrap_github_username: onedr0p bootstrap_github_repository_name: flux-cluster-template -bootstrap_github_repository_branch: main bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY bootstrap_timezone: Etc/UTC -bootstrap_acme_email: fake +bootstrap_acme_email: fake@example.com bootstrap_acme_production_enabled: false bootstrap_flux_github_webhook_token: fake bootstrap_cloudflare_domain: fake diff --git a/.github/tests/config-k3s-ipv6.yaml b/.github/tests/config-k3s-ipv6.yaml index c937cdcb..5a7ff117 100644 --- a/.github/tests/config-k3s-ipv6.yaml +++ b/.github/tests/config-k3s-ipv6.yaml @@ -4,10 +4,9 @@ skip_tests: true bootstrap_distribution: k3s bootstrap_github_username: onedr0p bootstrap_github_repository_name: flux-cluster-template -bootstrap_github_repository_branch: main bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY bootstrap_timezone: Etc/UTC -bootstrap_acme_email: fake +bootstrap_acme_email: fake@example.com bootstrap_acme_production_enabled: false bootstrap_flux_github_webhook_token: fake bootstrap_cloudflare_domain: fake diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml index 518a2c03..a47603fd 100644 --- a/.github/tests/config-talos.yaml +++ b/.github/tests/config-talos.yaml @@ -4,10 +4,9 @@ skip_tests: true bootstrap_distribution: talos bootstrap_github_username: onedr0p bootstrap_github_repository_name: flux-cluster-template -bootstrap_github_repository_branch: main bootstrap_age_public_key: $BOOTSTRAP_AGE_PUBLIC_KEY bootstrap_timezone: Etc/UTC -bootstrap_acme_email: fake +bootstrap_acme_email: fake@example.com bootstrap_acme_production_enabled: false bootstrap_flux_github_webhook_token: fake bootstrap_cloudflare_domain: fake diff --git a/.taskfiles/Ansible/Taskfile.yaml b/.taskfiles/Ansible/Taskfile.yaml index 34184091..a3ca0d64 100644 --- a/.taskfiles/Ansible/Taskfile.yaml +++ b/.taskfiles/Ansible/Taskfile.yaml @@ -21,8 +21,8 @@ tasks: deps: desc: Set up Ansible dependencies + deps: [":setup-virtual-env"] cmds: - - task: :setup-virtual-env - .venv/bin/python3 -m pip install --upgrade --requirement "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}" - .venv/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}} preconditions: diff --git a/README.md b/README.md index c8b651c3..645ba6d4 100644 --- a/README.md +++ b/README.md @@ -284,8 +284,17 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g task configure ``` -8. Continue on to ⚡ [**Stage 4**](#-stage-4-prepare-your-nodes-for-kubernetes) +8. Push you changes to git + 📍 **Verify** all the `*.sops.yaml` and `*.sops.yaml` files under the `./ansible`, and `./kubernetes` directories are **encrypted** with SOPS + + ```sh + git add -A + git commit -m "Initial commit :rocket:" + git push + ``` + +9. Continue on to ⚡ [**Stage 4**](#-stage-4-prepare-your-nodes-for-kubernetes) ### ⚡ Stage 4: Prepare your nodes for Kubernetes @@ -396,17 +405,7 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g # ✔ prerequisites checks passed ``` -2. Push you changes to git - - 📍 **Verify** all the `*.sops.yaml` and `*.sops.yaml` files under the `./ansible`, and `./kubernetes` directories are **encrypted** with SOPS - - ```sh - git add -A - git commit -m "Initial commit :rocket:" - git push - ``` - -3. Install Flux and sync the cluster to the Git repository +2. Install Flux and sync the cluster to the Git repository ```sh task flux:bootstrap @@ -415,7 +414,7 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g # ... ``` -4. Verify Flux components are running in the cluster +3. Verify Flux components are running in the cluster ```sh kubectl -n flux-system get pods -o wide diff --git a/Taskfile.yaml b/Taskfile.yaml index ea7e7f5a..286cb3b0 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -38,8 +38,8 @@ tasks: init: desc: Initialize virtual env and configuration files + deps: ["setup-virtual-env"] cmds: - - task: setup-virtual-env - mkdir -p {{.PRIVATE_DIR}} - cp -n {{.BOOTSTRAP_ADDONS_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_ADDONS_FILE}} - cp -n {{.BOOTSTRAP_CONFIG_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_CONFIG_FILE}} @@ -53,28 +53,13 @@ tasks: silent: true configure: - desc: Configure repository from Ansible vars + desc: Configure repository from bootstrap vars prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue? + deps: ["setup-virtual-env"] cmds: - task: .template - - task: .post-validate - - .template: - internal: true - cmds: - - ./.venv/bin/makejinja - task: sops:encrypt - preconditions: - - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" } - - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" } - - { msg: "Missing Makejinja plugin file", sh: "test -f {{.BOOTSTRAP_DIR}}/scripts/plugin.py" } - - { msg: "Missing bootstrap addons file", sh: "test -f {{.BOOTSTRAP_ADDONS_FILE}}" } - - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" } - - .post-validate: - internal: true - cmds: - - task: kubernetes:kubeconform + - task: .validate setup-virtual-env: desc: Set up virtual environment @@ -88,3 +73,19 @@ tasks: - "{{.ROOT_DIR}}/.venv/pyvenv.cfg" preconditions: - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" } + + .template: + internal: true + cmds: + - ./.venv/bin/makejinja + preconditions: + - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" } + - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" } + - { msg: "Missing Makejinja plugin file", sh: "test -f {{.BOOTSTRAP_DIR}}/scripts/plugin.py" } + - { msg: "Missing bootstrap addons file", sh: "test -f {{.BOOTSTRAP_ADDONS_FILE}}" } + - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" } + + .validate: + internal: true + cmds: + - task: kubernetes:kubeconform diff --git a/bootstrap/scripts/plugin.py b/bootstrap/scripts/plugin.py index a86a7911..0bba6211 100644 --- a/bootstrap/scripts/plugin.py +++ b/bootstrap/scripts/plugin.py @@ -20,9 +20,6 @@ def encrypt(value: str) -> str: class Plugin(makejinja.plugin.Plugin): def __init__(self, data: dict[str, Any]): - if data.get("skip_tests", False): - return - validation.validate(data) def filters(self) -> makejinja.plugin.Filters: diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py index 4a322a7b..b4272f2d 100644 --- a/bootstrap/scripts/validation.py +++ b/bootstrap/scripts/validation.py @@ -100,14 +100,17 @@ def validate_distribution(distribution: str, **_) -> None: _validate_distribution(distribution) -@required("bootstrap_github_username", "bootstrap_github_repository_name", "bootstrap_github_repository_branch") -def validate_github(username: str, repository: str, branch: str, **_) -> None: +@required("bootstrap_github_username", "bootstrap_github_repository_name", "bootstrap_advanced_flags") +def validate_github(username: str, repository: str, advanced_flags: dict, **_) -> None: try: - request = requests.get(f"https://api.github.com/repos/{username}/{repository}/branches/{branch}") + request = requests.get( + f"https://api.github.com/repos/{username}/{repository}/branches/{advanced_flags.get('github_repository_branch', 'main')}") if request.status_code != 200: - raise ValueError(f"GitHub repository {username}/{repository} branch {branch} not found") + raise ValueError( + f"GitHub repository {username}/{repository} branch {advanced_flags.get('github_repository_branch', 'main')} not found") except requests.exceptions.RequestException as e: - raise ValueError(f"GitHub repository {username}/{repository} branch {branch} not found") from e + raise ValueError( + f"GitHub repository {username}/{repository} branch {advanced_flags.get('github_repository_branch', 'main')} not found") from e @required("bootstrap_age_public_key") @@ -240,18 +243,27 @@ def validate_nodes(node_cidr: str, nodes: dict[list], distribution: str, **_) -> _validate_node(node, node_cidr, distribution) +def massage(data: dict) -> dict: + data["bootstrap_advanced_flags"] = data.get("bootstrap_advanced_flags", {}) + return data + def validate(data: dict) -> None: + user_data = massage(data) + validate_python_version() - validate_cli_tools(data) - validate_distribution(data) - if not data.get("bootstrap_private_github_repo"): - validate_github(data) - validate_age(data) - validate_timezone(data) - validate_acme_email(data) - validate_flux_github_webhook_token(data) - validate_cloudflare(data) - validate_host_network(data) - validate_bootstrap_dns_server(data) - validate_cluster_cidrs(data) - validate_nodes(data) + validate_cli_tools(user_data) + validate_distribution(user_data) + validate_age(user_data) + validate_timezone(user_data) + validate_bootstrap_dns_server(user_data) + validate_cluster_cidrs(user_data) + validate_flux_github_webhook_token(user_data) + validate_host_network(user_data) + validate_acme_email(user_data) + + if not user_data.get("bootstrap_private_github_repo"): + validate_github(user_data) + + if not user_data.get("skip_tests", False): + validate_cloudflare(user_data) + validate_nodes(user_data) diff --git a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 index f052701c..1a26992a 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-prepare.yaml.j2 @@ -62,7 +62,7 @@ ff02::2 ip6-allrouters dest: /etc/hosts mode: preserve - #% if bootstrap_advanced_flags.update_resolv_conf %# + #% if bootstrap_advanced_flags.update_resolv_conf|default(true) %# # https://github.com/onedr0p/flux-cluster-template/discussions/635 - name: Network Configuration | Remove immutable flag from /etc/resolv.conf ansible.builtin.file: diff --git a/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 b/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 index 65945f1d..14731981 100644 --- a/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 @@ -7,7 +7,7 @@ metadata: spec: interval: 30m ref: - branch: "#{ bootstrap_github_repository_branch }#" + branch: "#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#" #% if bootstrap_private_github_repo %# secretRef: name: github-deploy-key diff --git a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 index 58a15226..c345e4a9 100644 --- a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 +++ b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 @@ -14,37 +14,15 @@ spec: installFlags: - --disable-components=metrics-server - --no-taints - #% if loop.first %# - files: &files - - name: system-config - src: resources/hooks/ - dstDir: /home/#{ item.username }#/k0s/hooks/ - perm: 0644 - user: root - group: root - - name: containerd-config - src: resources/containerd/ - dstDir: /home/#{ item.username }#/k0s/containerd/ - perm: 0644 - user: root - group: root - #% else %# - files: *files - #% endif %# - #% if loop.first %# - hooks: &hooks + hooks: apply: before: - - sudo bash /home/#{ item.username }#/k0s/hooks/apply-system.sh - - sudo mv /home/#{ item.username }#/k0s/containerd/default.toml /etc/k0s/containerd.d/default.toml + - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-system.sh | INSTALL_K0S_HOSTNAME="#{ item.name }#" bash -s - reset: before: - - sudo bash /home/#{ item.username }#/k0s/hooks/reset-cilium.sh + - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-cilium.sh | bash -s - after: - - sudo bash /home/#{ item.username }#/k0s/hooks/reset-system.sh - #% else %# - hooks: *hooks - #% endif %# + - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s - #% endfor %# #% if bootstrap_nodes.worker | length > 0 %# #% for item in bootstrap_nodes.worker %# @@ -52,8 +30,15 @@ spec: ssh: address: "#{ item.address }#" user: "#{ item.username }#" - files: *files - hooks: *hooks + hooks: + apply: + before: + - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/apply-system.sh | INSTALL_K0S_HOSTNAME="#{ item.name }#" bash -s - + reset: + before: + - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-cilium.sh | bash -s - + after: + - curl -sfL https://raw.githubusercontent.com/#{ bootstrap_github_username }#/#{ bootstrap_github_repository_name }#/#{ bootstrap_advanced_flags.github_repository_branch|default('main', true) }#/kubernetes/k0s/resources/reset-system.sh | bash -s - #% endfor %# #% endif %# k0s: diff --git a/bootstrap/templates/kubernetes/k0s/resources/hooks/apply-system.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/apply-system.sh.j2 similarity index 58% rename from bootstrap/templates/kubernetes/k0s/resources/hooks/apply-system.sh.j2 rename to bootstrap/templates/kubernetes/k0s/resources/apply-system.sh.j2 index 243e3b32..2643da63 100755 --- a/bootstrap/templates/kubernetes/k0s/resources/hooks/apply-system.sh.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/apply-system.sh.j2 @@ -1,11 +1,25 @@ #% if bootstrap_distribution == 'k0s' %# #!/usr/bin/env bash +set -e +set -o noglob [ $(id -u) -eq 0 ] || exec sudo $0 $@ # Prepare sudo apt-get update -y +# Hostname +if [ -n "${INSTALL_K0S_HOSTNAME}" ] && [ "$(hostnamectl hostname)" != "${INSTALL_K0S_HOSTNAME}" ]; then + hostnamectl set-hostname "${INSTALL_K0S_HOSTNAME}" +tee /etc/hosts > /dev/null < /etc/sysctl.d/99-kubernetes.conf -fs.inotify.max_queued_events: 65536 -fs.inotify.max_user_watches: 524288 -fs.inotify.max_user_instances: 8192 +fs.inotify.max_queued_events = 65536 +fs.inotify.max_user_watches = 524288 +fs.inotify.max_user_instances = 8192 EOF sysctl -p /etc/sysctl.d/99-kubernetes.conf @@ -57,6 +71,17 @@ neofetch --config none EOF chmod 755 /etc/profile.d/neofetch.sh -# Ensure k0s containerd directory exists +# Create containerd config mkdir -p /etc/k0s/containerd.d +cat < /etc/k0s/containerd.d/default.toml +[plugins."io.containerd.grpc.v1.cri"] + enable_unprivileged_ports = true + enable_unprivileged_icmp = true +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/var/lib/k0s/containerd/certs.d" +[plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + discard_unpacked_layers = false +EOF #% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/resources/containerd/default.toml.j2 b/bootstrap/templates/kubernetes/k0s/resources/containerd/default.toml.j2 deleted file mode 100644 index 8a5391a1..00000000 --- a/bootstrap/templates/kubernetes/k0s/resources/containerd/default.toml.j2 +++ /dev/null @@ -1,11 +0,0 @@ -#% if bootstrap_distribution == 'k0s' %# -[plugins."io.containerd.grpc.v1.cri"] - enable_unprivileged_ports = true - enable_unprivileged_icmp = true -[plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/var/lib/k0s/containerd/certs.d" -[plugins."io.containerd.grpc.v1.cri".containerd] - discard_unpacked_layers = false -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - discard_unpacked_layers = false -#% endif %# diff --git a/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-cilium.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/reset-cilium.sh.j2 similarity index 97% rename from bootstrap/templates/kubernetes/k0s/resources/hooks/reset-cilium.sh.j2 rename to bootstrap/templates/kubernetes/k0s/resources/reset-cilium.sh.j2 index ef69479d..4e8d17d3 100755 --- a/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-cilium.sh.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/reset-cilium.sh.j2 @@ -1,5 +1,7 @@ #% if bootstrap_distribution == 'k0s' %# #!/usr/bin/env bash +set -e +set -o noglob [ $(id -u) -eq 0 ] || exec sudo $0 $@ diff --git a/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-system.sh.j2 b/bootstrap/templates/kubernetes/k0s/resources/reset-system.sh.j2 similarity index 78% rename from bootstrap/templates/kubernetes/k0s/resources/hooks/reset-system.sh.j2 rename to bootstrap/templates/kubernetes/k0s/resources/reset-system.sh.j2 index ee63be36..05e8c928 100755 --- a/bootstrap/templates/kubernetes/k0s/resources/hooks/reset-system.sh.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/reset-system.sh.j2 @@ -1,8 +1,13 @@ #% if bootstrap_distribution == 'k0s' %# #!/usr/bin/env bash +set -e +set -o noglob [ $(id -u) -eq 0 ] || exec sudo $0 $@ +# Remove containerd config +rm -rf /etc/k0s/containerd.d + # Remove local storage data rm -rf "#{ bootstrap_advanced_flags.local_storage_path|default('/var/openebs/local', true) }#" diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index 13a004c7..3f9344d5 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -10,8 +10,6 @@ bootstrap_distribution: k3s bootstrap_github_username: # Github repository (e.g. flux-cluster-template) bootstrap_github_repository_name: -# Github repository branch (e.g. main) -bootstrap_github_repository_branch: main # Age Public Key (e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta) bootstrap_age_public_key: # Choose your timezone (e.g. America/New_York) @@ -90,6 +88,8 @@ bootstrap_nodes: # # bootstrap_advanced_flags: +# # Github repository branch +# github_repository_branch: main # # Disable to prevent changes to nodes /etc/resolv.conf # update_resolv_conf: true # # Cilium load balancer mode, choose either 'dsr' or 'snat'