fix: later summer cleanup and reorganization of stuff

.github/labeler.yaml

@@ -1,7 +1,7 @@
 ---
 area/ansible:
   - changed-files:
-      - any-glob-to-any-file: "infrastructure/ansible/**/*"
+      - any-glob-to-any-file: "ansible/**/*"
 area/ci:
   - changed-files:
       - any-glob-to-any-file: ".github/**/*"
@@ -14,6 +14,3 @@ area/docker:
 area/kubernetes:
   - changed-files:
       - any-glob-to-any-file: "kubernetes/**/*"
-area/terraform:
-  - changed-files:
-      - any-glob-to-any-file: "infrastructure/terraform/**/*"

.github/linters/.ansible-lint

@@ -1,9 +0,0 @@
-# .ansible-lint
-skip_list:
-  - yaml[line-length]
-  - no-changed-when
-  - deprecated-command-syntax
-
-warn_list:
-  - command-instead-of-shell
-  - template-instead-of-copy

.github/scripts/cloudflareNetworks.mjs

@@ -1,6 +0,0 @@
-#!/usr/bin/env zx
-$.verbose = false
-const response = await fetch('https://api.cloudflare.com/client/v4/ips')
-const body = await response.json()
-const ips = body.result.ipv4_cidrs.concat(body.result.ipv6_cidrs);
-echo(ips.join("\\,"))

.github/scripts/extract-images.mjs

@@ -1,127 +0,0 @@
-#!/usr/bin/env zx
-$.verbose = false
-
-/**
- * * extract-images.mjs
- * * Extracts all container images from a HelmRelease and renders them as a JSON object
- * @param --helmrelease    : The source Flux HelmRelease to compare against the target
- * @param --kubernetes-dir : The directory containing your Flux manifests including the HelmRepository manifests
- */
-const HelmRelease = argv["helmrelease"]
-const KubernetesDir = argv["kubernetes-dir"]
-
-const helm = await which("helm")
-const kustomize = await which("kustomize")
-
-function extractImageValues(data) {
-  const imageValues = []
-  function extractValues(obj) {
-    for (const key in obj) {
-      if (typeof obj[key] === "object") {
-        extractValues(obj[key])
-      } else if (key === "image") {
-        imageValues.push(obj[key])
-      }
-    }
-  }
-  extractValues(data)
-  return imageValues
-}
-
-async function parseHelmRelease(releaseFile) {
-  const helmRelease = await fs.readFile(releaseFile, "utf8")
-  const doc = YAML.parseAllDocuments(helmRelease).map((item) => item.toJS())
-  const release = doc.filter(
-    (item) =>
-      item.apiVersion === "helm.toolkit.fluxcd.io/v2" &&
-      item.kind === "HelmRelease"
-  )
-  return release[0]
-}
-
-async function parseHelmRepository(kubernetesDir, releaseName) {
-  const files = await globby([`${kubernetesDir}/**/*.yaml`])
-  for await (const file of files) {
-    const contents = await fs.readFile(file, "utf8")
-    const repository = YAML.parseAllDocuments(contents).map((item) =>
-      item.toJS()
-    )
-    if (
-      repository[0] &&
-      "apiVersion" in repository[0] &&
-      repository[0].apiVersion === "source.toolkit.fluxcd.io/v1beta2" &&
-      "kind" in repository[0] &&
-      repository[0].kind === "HelmRepository" &&
-      "metadata" in repository[0] &&
-      "name" in repository[0].metadata &&
-      repository[0].metadata.name === releaseName
-    ) {
-      return repository[0]
-    }
-  }
-}
-
-async function renderKustomize(releaseBaseDir, releaseName) {
-  const build =
-    await $`${kustomize} build --load-restrictor=LoadRestrictionsNone ${releaseBaseDir}`
-  const docs = YAML.parseAllDocuments(build.stdout).map((item) => item.toJS())
-  const release = docs.filter(
-    (item) =>
-      item.apiVersion === "helm.toolkit.fluxcd.io/v2" &&
-      item.kind === "HelmRelease" &&
-      item.metadata.name === releaseName
-  )
-  return release[0]
-}
-
-async function helmTemplate(release, repository) {
-  const values = new YAML.Document()
-  values.contents = release.spec.values
-  const valuesFile = await $`mktemp`
-  await fs.writeFile(valuesFile.stdout.trim(), values.toString())
-
-  // Template out helm values into Kubernetes manifests
-  let manifests
-  if ("type" in repository.spec && repository.spec.type == "oci") {
-    manifests = await $`${helm} template --kube-version 1.28.0 --release-name ${
-      release.metadata.name
-    } --include-crds=false ${repository.spec.url}/${
-      release.spec.chart.spec.chart
-    } --version ${
-      release.spec.chart.spec.version
-    } --values ${valuesFile.stdout.trim()}`
-  } else {
-    await $`${helm} repo add ${release.spec.chart.spec.sourceRef.name} ${repository.spec.url}`
-    manifests = await $`${helm} template --kube-version 1.28.0 --release-name ${
-      release.metadata.name
-    } --include-crds=false ${release.spec.chart.spec.sourceRef.name}/${
-      release.spec.chart.spec.chart
-    } --version ${
-      release.spec.chart.spec.version
-    } --values ${valuesFile.stdout.trim()}`
-  }
-
-  let documents = YAML.parseAllDocuments(manifests.stdout.trim()).map((item) =>
-    item.toJS()
-  )
-
-  const images = []
-  documents.forEach((doc) => {
-    const docImageValues = extractImageValues(doc)
-    images.push(...docImageValues)
-  })
-  return images
-}
-
-const helmRelease = await parseHelmRelease(HelmRelease)
-const kustomizeBuild = await renderKustomize(
-  path.dirname(HelmRelease),
-  helmRelease.metadata.name
-)
-const helmRepository = await parseHelmRepository(
-  KubernetesDir,
-  kustomizeBuild.spec.chart.spec.sourceRef.name
-)
-const images = await helmTemplate(kustomizeBuild, helmRepository)
-
-echo(JSON.stringify(images))

.github/scripts/kubernetes-version.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Find current Talos version .taskfiles/sidero.yaml
-TALOS_VERSION="$(sed -n 's/.*talosVersion: \([v0-9.]*\)/\1/p' infrastructure/talos/talconfig.yaml)"
+TALOS_VERSION="$(sed -n 's/.*talosVersion: \([v0-9.]*\)/\1/p' kubernetes/talos/talconfig.yaml)"
 
 # Get file from Talos repo that includes default Kubernetes version
 SOURCE_CONSTANTS=$(curl -s https://raw.githubusercontent.com/siderolabs/talos/$TALOS_VERSION/pkg/machinery/constants/constants.go)

.github/workflows/create-issue-on-kubernetes-update.yaml

@@ -6,7 +6,7 @@ on:
     branches:
       - main
     paths:
-      - "infrastructure/talos/talconfig.yaml"
+      - "kubernetes/talos/talconfig.yaml"
 
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}

.github/workflows/create-issue-on-talos-update.yaml

@@ -6,7 +6,7 @@ on:
     branches:
       - main
     paths:
-      - "infrastructure/talos/talconfig.yaml"
+      - "kubernetes/talos/talconfig.yaml"
 
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}

.github/workflows/scan-supported-k8s-version.yaml

@@ -7,7 +7,7 @@ on: # yamllint disable-line rule:truthy
     branches:
       - main
     paths:
-      - "infrastructure/talos/talconfig.yaml"
+      - "kubernetes/talos/talconfig.yaml"
 
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}

.gitignore

@@ -14,4 +14,3 @@ talosconfig*
 *.iso
 .task/checksum
 **/Brewfile.lock.json
-docker/**/*.env

.minijinja.toml

@@ -0,0 +1,5 @@
+autoescape = "none"
+newline = true
+trim-blocks = true
+lstrip-blocks = true
+env = true

.pre-commit-config.yaml

@@ -1,46 +0,0 @@
----
-fail_fast: false
-repos:
-  - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.37.1
-    hooks:
-      - id: yamllint
-        args:
-          - -c
-          - ".github/linters/.yamllint.yaml"
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: mixed-line-ending
-      - id: check-executables-have-shebangs
-      - id: fix-byte-order-marker
-      - id: check-added-large-files
-        args: [--maxkb=2048]
-      - id: check-merge-conflict
-
-  - repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.5.5
-    hooks:
-      - id: remove-crlf
-      - id: remove-tabs
-
-  - repo: https://github.com/sirosen/fix-smartquotes
-    rev: 0.2.0
-    hooks:
-      - id: fix-smartquotes
-
-  - repo: https://github.com/zricethezav/gitleaks
-    rev: v8.28.0
-    hooks:
-      - id: gitleaks
-
-  - repo: https://github.com/jumanjihouse/pre-commit-hooks
-    rev: 3.0.0
-    hooks:
-      - id: shellcheck
-        language: script
-        args: [--severity=error]
-        additional_dependencies: []

.renovate/regexManagers.json5

@@ -13,7 +13,7 @@
     {
       customType: "regex",
       description: "Process Kubernetes versions",
-      fileMatch: ["infrastructure/talos/talconfig.yaml"],
+      fileMatch: ["kubernetes/talos/talconfig.yaml"],
       matchStrings: ["kubernetesVersion: (?<currentValue>.*)"],
       depNameTemplate: "ghcr.io/siderolabs/kubelet",
       datasourceTemplate: "docker",
@@ -22,7 +22,7 @@
       customType: "regex",
       description: "Process Talos versions",
       fileMatch: [
-        "infrastructure/talos/talconfig.yaml",
+        "kubernetes/talos/talconfig.yaml",
         ".taskfiles/talos/taskfile.yaml",
       ],
       matchStrings: [
@@ -46,8 +46,7 @@
       customType: "regex",
       description: "Process GitHub release URLs",
       fileMatch: [
-        "kubernetes/.+\\.ya?ml(?:\\.j2)?$",
-        "infrastructure/.+\\.ya?ml(?:\\.j2)?$"
+        "kubernetes/.+\\.ya?ml(?:\\.j2)?$"
       ],
       matchStrings: [
         "https:\\/\\/github.com\\/(?<depName>[\\w\\d\\-_]+\\/[\\w\\d\\-_]+)\\/releases\\/download\\/v(?<currentValue>[\\w\\d\\.\\-_]+)\\/.*",

.sourceignore

@@ -2,12 +2,11 @@
 /.github/
 /.taskfiles/
 /.vscode/
-/infrastructure/
+/ansible/
 /docs/
 /.env
 /.gitattributes
 /.gitignore
-/.pre-commit-config.yaml
 /LICENSE
 /README.md
 /Taskfile.yaml

.taskfiles/bootstrap/taskfile.yaml

@@ -2,12 +2,48 @@
 # yaml-language-server: $schema=https://taskfile.dev/schema.json
 version: "3"
 
+vars:
+  CONTEXT:
+    sh: talosctl config info --output json | jq --raw-output '.context'
+  CONTROLLER:
+    sh: talosctl config info --output json | jq --raw-output '.endpoints[]' | shuf -n 1
+
 tasks:
-  cluster:
-    desc: Bootstrap Talos and K8s cluster
+  talos:
+    desc: Bootstrap Talos
+    preconditions:
+      - talosctl config info
+      - talosctl --nodes {{.CONTROLLER}} get machineconfig
+      - which jq talosctl
     cmds:
-      - task: :tools:brew
-      - task: :tools:krew
-      - task: :talos:bootstrap
-      - task: :rook:wipe
-      - task: :flux:bootstrap
+      - until talosctl --nodes {{.CONTROLLER}} bootstrap; do sleep 5; done
+      - task: kubeconfig
+
+  kubeconfig:
+    desc: Generate kubeconfig
+    cmds:
+      - talosctl kubeconfig --nodes {{.CONTROLLER}} --force --force-context-name {{.CONTEXT}} {{.KUBECONFIG}}
+      - kubectl config set-cluster {{.CONTEXT}} --server https://{{.CONTROLLER}}:6443
+    vars:
+      CONTROLLER:
+        sh: talosctl config info --output yaml | yq --exit-status '.endpoints[0]'
+    preconditions:
+      - talosctl --nodes {{.CONTROLLER}} get machineconfig
+      - which talosctl yq
+
+  apps:
+    desc: Bootstrap Kubernetes Apps
+    preconditions:
+      - which helmfile yq jq kubectl op talosctl
+      - test -f {{.K8S_DIR}}/bootstrap/helmfile.d/00-crds.yaml
+      - test -f {{.K8S_DIR}}/bootstrap/helmfile.d/01-apps.yaml
+      - test -f {{.K8S_DIR}}/bootstrap/resources.yaml.j2
+      - op user get --me
+      - talosctl config info
+      - talosctl --nodes {{.CONTROLLER}} get machineconfig
+    cmds:
+      - task: kubeconfig
+      - op run --env-file {{.K8S_DIR}}/bootstrap/resources.env --no-masking --
+        minijinja-cli {{.K8S_DIR}}/bootstrap/resources.yaml.j2 | kubectl apply --server-side --filename -
+      - helmfile --quiet --file {{.K8S_DIR}}/bootstrap/helmfile.d/00-crds.yaml template | kubectl apply --server-side --filename -
+      - helmfile --file {{.K8S_DIR}}/bootstrap/helmfile.d/01-apps.yaml sync --hide-notes

.taskfiles/kubernetes/resources/buoy-pod.tmpl.yaml

@@ -1,45 +0,0 @@
----
-apiVersion: v1
-spec:
-  containers:
-    - name: buoy
-      image: ghcr.io/ahinko/buoy:1.4.14
-      command: ["/bin/sh"]
-      stdin: true
-      stdinOnce: true
-      tty: true
-      securityContext:
-        allowPrivilegeEscalation: true
-        capabilities:
-          add:
-            - ALL
-        privileged: true
-        fsGroup: 0
-      volumeMounts:
-        - mountPath: /rootfs
-          name: rootfs
-        - name: dev
-          mountPath: /dev
-        - mountPath: /sys/firmware/efi/efivars
-          name: efivars
-        - mountPath: /run/containerd
-          name: containerd
-  dnsPolicy: ClusterFirstWithHostNet
-  hostIPC: true
-  hostNetwork: true
-  hostPID: true
-  nodeName: ${node}
-  restartPolicy: Never
-  volumes:
-    - name: dev
-      hostPath:
-        path: /dev
-    - name: rootfs
-      hostPath:
-        path: /
-    - name: efivars
-      hostPath:
-        path: /sys/firmware/efi/efivars
-    - name: containerd
-      hostPath:
-        path: /run/containerd

.taskfiles/kubernetes/taskfile.yaml

@@ -2,15 +2,16 @@
 # yaml-language-server: $schema=https://taskfile.dev/schema.json
 version: 3
 
-vars:
-  KUBERNETES_RESOURCES_DIR: "{{.ROOT_DIR}}/.taskfiles/kubernetes/resources"
-
 tasks:
-  delete-failed-succeeded-pods:
-    desc: Deletes pods with Failed and Succeeded phase
+  delete-pods:
+    desc: Cleanse pods with a Failed/Pending/Succeeded phase
     cmds:
-      - kubectl delete pods --field-selector status.phase=Failed -A --ignore-not-found=true
-      - kubectl delete pods --field-selector status.phase=Succeeded -A --ignore-not-found=true
+      - for:
+          matrix:
+            PHASE: [Failed, Pending, Succeeded]
+        cmd: kubectl delete pods --all-namespaces --field-selector status.phase={{.ITEM.PHASE}} --ignore-not-found=true
+    preconditions:
+      - which kubectl
 
   list-not-running-pods:
     desc: Lists pods that are not running
@@ -28,12 +29,13 @@ tasks:
       - kubectl node-shell --version
       - which kubectl
 
-  buoy:
-    desc: Run a buoy shell (privileged pod)
-    cmd: |
-      kubectl run buoy-{{.node}} -i --rm --image=null \
-        --overrides="$(yq {{.KUBERNETES_RESOURCES_DIR}}/buoy-pod.tmpl.yaml -o=json | envsubst)"
-    env:
-      node: "{{.node}}"
+  upgrade-arc:
+    desc: Upgrade the ARC
+    cmds:
+      - helm -n actions-runner-system uninstall arc-homelab
+      - helm -n actions-runner-system uninstall actions-runner-controller
+      - sleep 30
+      - flux -n actions-runner-system reconcile hr actions-runner-controller
+      - flux -n actions-runner-system reconcile hr arc-homelab
     preconditions:
-      - test -f {{.KUBERNETES_RESOURCES_DIR}}/buoy-pod.tmpl.yaml
+      - which flux helm

.taskfiles/rook/taskfile.yaml

@@ -1,15 +0,0 @@
----
-version: "3"
-
-tasks:
-  dashboard-password:
-    desc: Get password for Rook/Ceph dashboard
-    cmds:
-      - kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo
-    silent: true
-
-  wipe:
-    desc: Wipe rook disks
-    prompt: This will wipe all disks used by Rook/Ceph... continue?
-    cmds:
-      - kubectl apply -f kubernetes/tools/rook/wipe-job.yaml

.taskfiles/talos/taskfile.yaml

@@ -3,56 +3,10 @@
 version: "3"
 
 vars:
-  TALOS_CONTROLLER:
+  CONTROLLER:
     sh: talosctl config info --output json | jq --raw-output '.endpoints[]' | shuf -n 1
 
 tasks:
-  generate:
-    desc: Generate Talos machine configurations
-    dir: "{{.TALOS_DIR}}"
-    env:
-      VAULT: "Homelab"
-    cmds:
-      - op run --env-file="./op.env" -- talhelper genconfig
-    sources:
-      - talconfig.yaml
-      - talsecret.yaml
-    generates:
-      - clusterconfig/*.yaml
-      - clusterconfig/talosconfig
-
-  apply-config:
-    desc: Apply Talos config on a node
-    dir: "{{.TALOS_DIR}}"
-    cmds:
-      - task: generate
-      - task: wait_for_health
-        vars: { TIMEOUT: 30s }
-      - talhelper gencommand apply -n {{.node}} | bash
-    requires:
-      vars: ["node"]
-
-  upgrade-talos:
-    desc: Upgrade Talos on a node
-    dir: "{{.TALOS_DIR}}"
-    cmds:
-      - task: generate
-      - task: wait_for_health
-        vars: { TIMEOUT: 30s }
-      - talhelper gencommand upgrade -n {{.node}} --extra-flags=--stage | bash
-    requires:
-      vars: ["node"]
-
-  upgrade-k8s:
-    desc: Upgrade Kubernetes
-    dir: "{{.TALOS_DIR}}"
-    cmds:
-      - task: generate
-      - task: wait_for_health
-        vars: { TIMEOUT: 30s }
-      - talosctl -n {{.TALOS_CONTROLLER}} etcd snapshot etcd.backup
-      - talhelper gencommand upgrade-k8s | bash
-
   wait_for_health:
     internal: True
     desc: Wait for services in cluster to be healthy
@@ -62,60 +16,72 @@ tasks:
       # Ensure CloudNative-PG cluster has 3 ready instances
       - kubectl -n databases wait --for jsonpath='{.status.readyInstances}'='3' --timeout {{ .TIMEOUT | default "30s" }} cluster postgres17
 
-  bootstrap:
-    desc: Bootstrap Talos
-
-    cmds:
-      - task: :talos:generate
-      - task: :talos:bootstrap-apply-config
-      - task: :talos:bootstrap-etcd
-      - task: :talos:kubeconfig
-      - task: :talos:bootstrap-core-apps
-
-  bootstrap-apply-config:
-    desc: Apply Talos config on all nodes
-    dir: "{{.TALOS_DIR}}"
-    cmds:
-      - talhelper gencommand apply --extra-flags=--insecure | bash
-
-  bootstrap-etcd:
-    desc: Bootstrap etcd
-    cmds:
-      - until talosctl --nodes {{.TALOS_CONTROLLER}} bootstrap; do sleep 10; done
-
   talosconfig:
     desc: Get an updated version of talosconfig
     cmds:
-      - cp infrastructure/talos/clusterconfig/talosconfig ~/.talos/config
+      - cp {{ .TALOS_DIR }}/clusterconfig/talosconfig {{ .TALOSCONFIG }}
 
-  kubeconfig:
-    desc: Get an updated version of kubeconfig with updated/rotated certificates
+  generate:
+    desc: Generate Talos machine configurations
+    dir: "{{.TALOS_DIR}}"
     cmds:
-      - talosctl kubeconfig ~/.kube/configs/mainframe -n {{.TALOS_CONTROLLER}}
-      - kubectl config rename-context admin@mainframe mainframe
+      - op run --env-file="./talsecret.env" -- talhelper genconfig
+    sources:
+      - talconfig.yaml
+      - talsecret.yaml
+    generates:
+      - clusterconfig/*.yaml
+      - clusterconfig/talosconfig
 
-  bootstrap-core-apps:
-    desc: Bootstrap core helm apps
+  apply:
+    desc: Apply Talos config on a node, NODE=required
+    requires:
+      vars:
+        - NODE
+    dir: "{{.TALOS_DIR}}"
+    cmds:
+      - task: generate
+      - task: wait_for_health
+        vars: { TIMEOUT: 30s }
+      - talhelper gencommand apply -n {{.NODE}} | bash
+
+  upgrade:
+    desc: Upgrade Talos on a node, NODE=required
+    requires:
+      vars:
+        - NODE
+    dir: "{{.TALOS_DIR}}"
+    cmds:
+      - task: generate
+      - task: wait_for_health
+        vars: { TIMEOUT: 30s }
+      - talhelper gencommand upgrade -n {{.NODE}} --extra-flags=--stage | bash
+    preconditions:
+      - talosctl --nodes {{.NODE}} get machineconfig
+      - which talosctl yq
+
+  upgrade:k8s:
+    desc: Upgrade Kubernetes
+    dir: "{{.TALOS_DIR}}"
+    cmds:
+      - task: generate
+      - task: wait_for_health
+        vars: { TIMEOUT: 30s }
+      - talosctl -n {{.CONTROLLER}} etcd snapshot etcd.backup
+      - talhelper gencommand upgrade-k8s | bash
+
+  write-to-usb:
+    desc: Write Talos image to USB drive, use configuration to determine architecture and schematic, NODE=required
+    dir: "{{.TALOS_DIR}}"
+    requires:
+      vars:
+        - NODE
     vars:
-      BOOTSTRAP_TEMPLATES:
-        sh: ls {{.CLUSTER_DIR}}/bootstrap/secrets/*.j2
-    env:
-      VAULT: "Homelab"
+      IMAGE_URL:
+        sh: talhelper genurl image -n {{.NODE}} -c {{.TALOS_DIR}}/talconfig.yaml
     cmds:
-      - until kubectl wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done
-      - for: { var: BOOTSTRAP_TEMPLATES }
-        cmd: >
-          op run --env-file {{.CLUSTER_DIR}}/bootstrap/op.env --no-masking --
-          minijinja-cli --env --trim-blocks --lstrip-blocks --autoescape=none {{.ITEM}}
-          | kubectl apply --server-side --filename -
-      - helmfile --file {{.CLUSTER_DIR}}/bootstrap/helmfile.yaml apply --skip-diff-on-install --suppress-diff
-      - until kubectl wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done
-
-  write-talos-amd64-to-usb:
-    desc: Write Talos image to USB drive to be used with amd64 machines
-    silent: true
-    cmds:
-      - "curl -LOC - https://factory.talos.dev/image/07fc545562cc6c5d76cf282c30a95d10b86286cd345bac2fa963c786397475cd/v1.10.7/metal-amd64.raw.xz && xz -d metal-amd64.raw.xz"
+      - echo "Downloading image from {{.IMAGE_URL}}"
+      - curl -L -o {{.ROOT_DIR}}/talos-{{.NODE}}.raw.xz {{.IMAGE_URL}} && xz -d {{.ROOT_DIR}}/talos-{{.NODE}}.raw.xz
       - "diskutil list"
       - |
         echo "Path to USB drive:"
@@ -123,20 +89,5 @@ tasks:
         diskutil unmount ${path} || true
         diskutil unmountDisk ${path} || true
         echo "Writing image to: ${path}";
-        sudo dd if=metal-amd64.raw of=${path} bs=4m && sync
-      - "rm metal-amd64.raw"
-
-  write-talos-arm64-to-usb:
-    desc: Write Talos image to USB drive to be used with Raspberry Pi 4
-    silent: true
-    cmds:
-      - "curl -LO https://factory.talos.dev/image/7688f6b5647f1a548661fc91e8bec62277dc0e3028bbace8547a21998563b4b0/v1.10.7/metal-arm64.raw.xz && xz -d metal-arm64.raw.xz"
-      - "diskutil list"
-      - |
-        echo "Path to USB drive:"
-        read path;
-        diskutil unmount ${path} || true
-        diskutil unmountDisk ${path} || true
-        echo "Writing image to: ${path}";
-        sudo dd if=metal-arm64.raw of=${path} conv=fsync bs=4M
-      - "rm metal-arm64.raw"
+        sudo dd if={{.ROOT_DIR}}/talos-{{.NODE}}.raw of=${path} conv=fsync bs=4M
+      - "rm {{.ROOT_DIR}}/talos-{{.NODE}}.raw"

.taskfiles/tools/taskfile.yaml

@@ -1,38 +0,0 @@
-# yaml-language-server: $schema=https://taskfile.dev/schema.json
----
-version: 3
-
-env:
-  BREWFILE: "{{.ROOT_DIR}}/.taskfiles/tools/resources/Brewfile"
-  KREW: |-
-    cnpg rook-ceph browse-pvc view-secret node-shell
-
-tasks:
-  brew:
-    desc: Set up Homebrew tools
-    cmd: brew bundle --file {{.BREWFILE}}
-    generates:
-      - "{{.BREWFILE}}.lock.json"
-    preconditions:
-      - command -v brew
-      - test -f {{.BREWFILE}}
-
-  pre-commit:
-    desc: Set up pre-commit hooks
-    cmd: pre-commit install-hooks
-
-  krew:
-    desc: Install/update all required dependencies
-    silent: true
-    cmds:
-      # Install krew plugins
-      - kubectl krew install {{.KREW}}
-      - kubectl krew update
-      - kubectl krew upgrade
-
-  ansible:
-    desc: Intall/update Ansible roles
-    silent: true
-    cmds:
-      - ansible-galaxy install -r {{.ROOT_DIR}}/infrastructure/ansible/requirements.yaml --force
-      - ansible-galaxy collection install -r {{.ROOT_DIR}}/infrastructure/ansible/requirements.yaml --force

.taskfiles/workstation/resources/Brewfile

@@ -2,7 +2,6 @@
 tap "fluxcd/tap"
 tap "go-task/tap"
 tap "siderolabs/tap"
-tap "controlplaneio-fluxcd/tap"
 
 # Terminal
 brew "fish"
@@ -11,7 +10,6 @@ brew "viddy"
 # Command line tools
 brew "bash"
 brew "ansible"
-brew "pre-commit"
 brew "go-task/tap/go-task"
 brew "git"
 brew "nano"
@@ -33,7 +31,7 @@ brew "talhelper"
 brew "siderolabs/tap/talosctl"
 brew "helmfile"
 brew "krew"
-brew "controlplaneio-fluxcd/tap/flux-operator-mcp"
+brew "minijinja-cli"
 
 # git signed commit
 brew "gpg2"

.taskfiles/workstation/taskfile.yaml

@@ -0,0 +1,25 @@
+# yaml-language-server: $schema=https://taskfile.dev/schema.json
+---
+version: 3
+
+env:
+  WORKSTATION_RESOURCES_DIR: "{{.ROOT_DIR}}/.taskfiles/workstation/resources"
+
+tasks:
+  brew:
+    desc: Set up Homebrew tools
+    cmd: brew bundle --file {{.WORKSTATION_RESOURCES_DIR}}/Brewfile
+    generates:
+      - "{{.WORKSTATION_RESOURCES_DIR}}/Brewfile.lock.json"
+    preconditions:
+      - command -v brew
+      - test -f {{.WORKSTATION_RESOURCES_DIR}}/Brewfile
+
+  krew:
+    desc: Install/update all required dependencies
+    silent: true
+    cmds:
+      - kubectl krew install cnpg rook-ceph browse-pvc view-secret node-shell
+    preconditions:
+      - kubectl krew version
+      - which kubectl

.vscode/settings.json

@@ -1,17 +1,18 @@
 {
   "files.associations": {
     "*.json5": "json5",
-    "**/infrastructure/ansible/**/*.yaml": "ansible",
-    "**/infrastructure/**/inventory/**/*.yaml": "yaml",
+    "**/ansible/**/*.yaml": "ansible",
+    "**/**/inventory/**/*.yaml": "yaml",
   },
   "material-icon-theme.folders.associations": {
-    // top level
+    "*.gotmpl": "smarty",
     ".github/workflows": "ci",
     ".private": "archive",
     ".renovate": "robot",
     "bootstrap": "seeders",
     "flux": "pipe",
     "talos": "linux",
+    "bootstrap/helmfile.d": "helm",
     // namespaces
     "actions-runner-system": "github",
     "cert-manager": "guard",

.yamllint.yaml

@@ -2,7 +2,7 @@
 ignore: |
   .github/
   crds.yaml
-  infrastructure/talos/
+  kubernetes/talos/
 
 extends: default
 

Taskfile.yml

@@ -2,20 +2,30 @@
 # yaml-language-server: $schema=https://taskfile.dev/schema.json
 version: "3"
 
+set:
+  - pipefail
+shopt:
+  - globstar
+
 vars:
-  TALOS_DIR: "{{.ROOT_DIR}}/infrastructure/talos"
-  CLUSTER_DIR: "{{.ROOT_DIR}}/kubernetes"
+  K8S_DIR: "{{.ROOT_DIR}}/kubernetes"
+  MINIJINJA_CONFIG_FILE: "{{.ROOT_DIR}}/.minijinja.toml"
+  TALOS_DIR: "{{.K8S_DIR}}/talos"
+
+env:
+  KUBECONFIG: "~/.kube/configs/mainframe"
+  MINIJINJA_CONFIG_FILE: "{{.ROOT_DIR}}/.minijinja.toml"
+  TALOSCONFIG: "~/.talos/config"
+
+includes:
+  bootstrap: .taskfiles/bootstrap/taskfile.yaml
+  flux: .taskfiles/flux/taskfile.yaml
+  k8s: .taskfiles/kubernetes/taskfile.yaml
+  talos: .taskfiles/talos/taskfile.yaml
+  workstation: .taskfiles/workstation/taskfile.yaml
 
 tasks:
   default:
     silent: true
     cmds:
       - task --list
-
-includes:
-  bootstrap: .taskfiles/bootstrap/taskfile.yaml
-  flux: .taskfiles/flux/taskfile.yaml
-  k8s: .taskfiles/kubernetes/taskfile.yaml
-  rook: .taskfiles/rook/taskfile.yaml
-  talos: .taskfiles/talos/taskfile.yaml
-  tools: .taskfiles/tools/taskfile.yaml

docs/snippets.md

@@ -46,7 +46,7 @@ I ran in to an issue where I had to reset the Rook-Ceph cluster due to restructu
 * Delete all CRDs that starts with ceph*
 * Wipe disks: `kubectl apply -f kubernetes/tools/rook/wipe-job.yaml`
 * Reset nodes and reboot: `talosctl reset --system-labels-to-wipe=STATE,EPHEMERAL --reboot --graceful=true -n <IP>`
-  * Apply config again: `talosctl apply-config -n <IP> -f infrastructure/talos/clusterconfig/<CONFIG FILE>.yaml --insecure`
+  * Apply config again: `talosctl apply-config -n <IP> -f kubernetes/talos/clusterconfig/<CONFIG FILE>.yaml --insecure`
 
 ## Upgrade Tube's Zigbee Gateway firmware
 

infrastructure/ansible/inventory/host_vars/kronos/sshkeys.yaml

@@ -1,6 +0,0 @@
----
-ssh_authorized_keys:
-  - owner: peter
-    key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5Yw1MlnY1gs9/fX9zz8qgTjTcdJfjXFXg58Icb7KLE/CPc86qYSg3j+3Fy/5FNDj31KhPHL0JenEWaLPMruGwwn4or2By5GQZGOJggx9qjyyZcgQCPuGO7V4GPlDKRX+SyZ+FAY4/aZPUL2Ig2vZz0bt/nnovg1EOwaXvnTaYAAXspgn/6YoLj4SOeIz1p7VVFxQNmiyNrjfq0cK8YFfL1Z2+kTsa0jJxyFjTD22IBX7aUIbhjSGeFpwLYo52/WKhVOycNaHu0Evz6C9pu9DzSADTCsw5TdQxcsGQDtZ8M1tvDpGLPFABTmflMznYmYlXvCWSn+KTUolSsiEbFQEt john@localhost.dev"
-  - owner: root
-    key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDIHNGNGlbvILqxJvYhwBxveAOFJt/R9IQW6v3y03r4gdnpWepGNbfgGMj6Ud/mOfGHywMUqT0t4Lj9mcKbLwjf8oFy9L5yRYehm19ogcmjhwzd2toal9QOSRCRCDUlVe1Y5S0+C8b2I4KHoKvJ0Qbqe4jGIeC9PjFrCCdBh6aoZI3Bd/fILkFQIrK8crIoq/E/I349z2GmrOfEBZa9GahVF2ONFR7qTJBaPA5U03NPL4Gt6Y+f5Z2Y17EQnysTh40zXkiO6PSpYLEhhR+/spRjxe1OEV2cqKR9m5glJ52ltRaoKJoOFB2uFOMh4QaDoPS2/4IoQgQXaKWmBoGCWhkKVsUoD7JPQrPHuwSjUaXgJjKsyCWTYLnPQOmMU+Clx/xOheBGqUZJ3i3/wEJMCbOlAvKj9hYXrhQSemV5djBzafTO90sv203BoM6WNd1DJbBkycXnXQcxJfsXIQv8Ww6dDsfnSRq6WGr1bAeggn1zZYg4dNoQ/5YBJwargPKYkbOYeD1Cu/vptlxnht4d/Vq0iIHi4AQmEYAynTDRvMCfkxBwIUj4p2bKbEyCWIYeImCedtivbT7EyWQ3OfzEv9ds4lmsX/g2J+zx11mxfoKue7AiK1qWnFR15a3QnOtnkRjJ+tGb9QXyT51EIdPT58F1Mu3i0cUlEcLuRUnqcHuxRw== peter@localhost"

infrastructure/talos/op.env

@@ -1,14 +0,0 @@
-TALOS_CLUSTER_ID="op://$VAULT/talos/TALOS_CLUSTER_ID"
-TALOS_CLUSTER_SECRET="op://$VAULT/talos/TALOS_CLUSTER_SECRET"
-TALOS_SECRETS_BOOTSTRAP_TOKEN="op://$VAULT/talos/TALOS_SECRETS_BOOTSTRAP_TOKEN"
-TALOS_SECRETS_SECRETBOX_ENCRYPTION_SECRET="op://$VAULT/talos/TALOS_SECRETS_SECRETBOX_ENCRYPTION_SECRET"
-TALOS_TRUSTD_INFO_TOKEN="op://$VAULT/talos/TALOS_TRUSTD_INFO_TOKEN"
-TALOS_CERTS_ETCD_CRT="op://$VAULT/talos/TALOS_CERTS_ETCD_CRT"
-TALOS_CERTS_ETCD_KEY="op://$VAULT/talos/TALOS_CERTS_ETCD_KEY"
-TALOS_CERTS_K8S_CRT="op://$VAULT/talos/TALOS_CERTS_K8S_CRT"
-TALOS_CERTS_K8S_KEY="op://$VAULT/talos/TALOS_CERTS_K8S_KEY"
-TALOS_CERTS_K8SAGGREGATOR_CRT="op://$VAULT/talos/TALOS_CERTS_K8SAGGREGATOR_CRT"
-TALOS_CERTS_K8SAGGREGATOR_KEY="op://$VAULT/talos/TALOS_CERTS_K8SAGGREGATOR_KEY"
-TALOS_CERTS_K8SSERVICEACCOUNT_KEY="op://$VAULT/talos/TALOS_CERTS_K8SSERVICEACCOUNT_KEY"
-TALOS_CERTS_OS_CRT="op://$VAULT/talos/TALOS_CERTS_OS_CRT"
-TALOS_CERTS_OS_KEY="op://$VAULT/talos/TALOS_CERTS_OS_KEY"

kubernetes/apps/cert-manager/cert-manager/app/helm-release.yaml

@@ -16,6 +16,13 @@ spec:
     remediation:
       strategy: rollback
       retries: 3
-  valuesFrom:
-    - kind: ConfigMap
-      name: cert-manager-helm-values
+  values:
+    crds:
+      enabled: true
+    replicaCount: 2
+    dns01RecursiveNameservers: https://1.1.1.1:443/dns-query,https://1.0.0.1:443/dns-query
+    dns01RecursiveNameserversOnly: true
+    prometheus:
+      enabled: true
+      servicemonitor:
+        enabled: true

kubernetes/apps/cert-manager/cert-manager/app/helm-values.yaml

@@ -1,10 +0,0 @@
----
-crds:
-  enabled: true
-replicaCount: 2
-dns01RecursiveNameservers: https://1.1.1.1:443/dns-query,https://1.0.0.1:443/dns-query
-dns01RecursiveNameserversOnly: true
-prometheus:
-  enabled: true
-  servicemonitor:
-    enabled: true

kubernetes/apps/cert-manager/cert-manager/app/kustomization.yaml

@@ -5,9 +5,3 @@ resources:
   - ./oci-repository.yaml
   - ./helm-release.yaml
   - ./prometheus-rules.yaml
-configMapGenerator:
-  - name: cert-manager-helm-values
-    files:
-      - values.yaml=./helm-values.yaml
-generatorOptions:
-  disableNameSuffixHash: true

kubernetes/apps/flux-system/flux-instance/app/helm-release.yaml

@@ -0,0 +1,126 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: flux-instance
+spec:
+  interval: 1h
+  chartRef:
+    kind: OCIRepository
+    name: flux-instance
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      strategy: rollback
+      retries: 3
+  values:
+    instance:
+      distribution:
+        # renovate: datasource=github-releases depName=controlplaneio-fluxcd/distribution
+        version: 2.6.4
+      cluster:
+        networkPolicy: false
+      components:
+        - source-controller
+        - kustomize-controller
+        - helm-controller
+        - notification-controller
+      sync:
+        kind: GitRepository
+        url: ssh://git@github.com/ahinko/homelab
+        ref: refs/heads/main
+        path: kubernetes/flux/cluster
+        pullSecret: homelab-flux-secret
+      commonMetadata:
+        labels:
+          app.kubernetes.io/name: flux
+      kustomize:
+        patches:
+          # Allow flux components to run on control plane nodes
+          - patch: |
+              apiVersion: apps/v1
+              kind: Deployment
+              metadata:
+                name: not-used
+              spec:
+                template:
+                  spec:
+                    tolerations:
+                      - key: node-role.kubernetes.io/control-plane
+                        operator: Exists
+                        effect: NoSchedule
+            target:
+              kind: Deployment
+          - # Increase the number of workers
+            patch: |
+              - op: add
+                path: /spec/template/spec/containers/0/args/-
+                value: --concurrent=10
+              - op: add
+                path: /spec/template/spec/containers/0/args/-
+                value: --requeue-dependency=5s
+            target:
+              kind: Deployment
+              name: (kustomize-controller|helm-controller|source-controller)
+          - # Increase the memory limits
+            patch: |
+              apiVersion: apps/v1
+              kind: Deployment
+              metadata:
+                name: all
+              spec:
+                template:
+                  spec:
+                    containers:
+                      - name: manager
+                        resources:
+                          limits:
+                            memory: 2Gi
+            target:
+              kind: Deployment
+              name: (kustomize-controller|helm-controller|source-controller)
+          - # Enable in-memory kustomize builds
+            patch: |
+              - op: add
+                path: /spec/template/spec/containers/0/args/-
+                value: --concurrent=20
+              - op: replace
+                path: /spec/template/spec/volumes/0
+                value:
+                  name: temp
+                  emptyDir:
+                    medium: Memory
+            target:
+              kind: Deployment
+              name: kustomize-controller
+          - # Enable Helm repositories caching
+            patch: |
+              - op: add
+                path: /spec/template/spec/containers/0/args/-
+                value: --helm-cache-max-size=10
+              - op: add
+                path: /spec/template/spec/containers/0/args/-
+                value: --helm-cache-ttl=60m
+              - op: add
+                path: /spec/template/spec/containers/0/args/-
+                value: --helm-cache-purge-interval=5m
+            target:
+              kind: Deployment
+              name: source-controller
+          - # Flux near OOM detection for Helm
+            patch: |
+              - op: add
+                path: /spec/template/spec/containers/0/args/-
+                value: --feature-gates=OOMWatch=true
+              - op: add
+                path: /spec/template/spec/containers/0/args/-
+                value: --oom-watch-memory-threshold=95
+              - op: add
+                path: /spec/template/spec/containers/0/args/-
+                value: --oom-watch-interval=500ms
+            target:
+              kind: Deployment
+              name: helm-controller

kubernetes/apps/flux-system/flux-instance/app/kustomization.yaml

@@ -6,9 +6,3 @@ resources:
   - ./helm-release.yaml
   - ./prometheus-rule.yaml
   - ./webhook
-configMapGenerator:
-  - name: flux-instance-helm-values
-    files:
-      - values.yaml=./helm/values.yaml
-generatorOptions:
-  disableNameSuffixHash: true

kubernetes/apps/flux-system/flux-instance/install.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app flux-instance
+  namespace: &namespace flux-system
+spec:
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  dependsOn:
+    - name: flux-operator
+      namespace: *namespace
+  interval: 1h
+  path: ./kubernetes/apps/flux-system/flux-instance/app
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: *namespace
+  timeout: 5m
+  wait: true

kubernetes/apps/flux-system/flux-operator/app/helm-release.yaml

@@ -8,6 +8,10 @@ spec:
   chartRef:
     kind: OCIRepository
     name: flux-operator
-  valuesFrom:
-    - kind: ConfigMap
-      name: flux-operator-helm-values
+  values:
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+    serviceMonitor:
+      create: true

kubernetes/apps/flux-system/flux-operator/app/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./oci-repository.yaml
+  - ./helm-release.yaml

kubernetes/apps/flux-system/flux-operator/install.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app flux-operator
+  namespace: &namespace flux-system
+spec:
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  interval: 1h
+  path: ./kubernetes/apps/flux-system/flux-operator/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: *namespace
+  timeout: 5m
+  wait: true

kubernetes/apps/flux-system/kustomization.yaml

@@ -3,7 +3,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: flux-system
 resources:
-  - ./operator/install.yaml
+  - ./flux-operator/install.yaml
+  - ./flux-instance/install.yaml
 components:
   - ../../components/namespace
   - ../../components/alerts