From 016dc52f8e232a7d98cb2e8a85b5634eeb1e8432 Mon Sep 17 00:00:00 2001 From: "ryan-distrust.co" Date: Wed, 2 Apr 2025 16:59:29 -0400 Subject: [PATCH 1/6] begin transition to stagex --- .dockerignore | 1 + Containerfile.tools | 44 ++++++++++++++++++++++++++++++++++++++++++++ Makefile | 26 +++++++++++++++++++++++--- 3 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 .dockerignore create mode 100644 Containerfile.tools diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..72e8ffc --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +* diff --git a/Containerfile.tools b/Containerfile.tools new file mode 100644 index 0000000..27518d9 --- /dev/null +++ b/Containerfile.tools @@ -0,0 +1,44 @@ +# Tools used for managing the stagex stack + +FROM quay.io/stagex/core-busybox AS busybox +FROM quay.io/stagex/core-musl AS musl +FROM quay.io/stagex/core-ca-certificates AS ca-certificates +FROM quay.io/stagex/core-zlib AS zlib +FROM quay.io/stagex/user-gpg AS gpg +FROM quay.io/stagex/user-npth AS npth +FROM quay.io/stagex/user-libksba AS libksba +FROM quay.io/stagex/user-libgpg-error AS libgpg-error +FROM quay.io/stagex/user-libassuan AS libassuan +FROM quay.io/stagex/user-libgcrypt AS libgcrypt +FROM quay.io/stagex/user-tofu AS tofu +FROM quay.io/stagex/user-sops AS sops +FROM quay.io/stagex/user-talosctl AS talosctl +FROM quay.io/stagex/user-kubectl AS kubectl +FROM quay.io/stagex/user-kustomize AS kustomize +FROM quay.io/stagex/user-kustomize-sops AS kustomize-sops +FROM quay.io/stagex/user-helm AS helm + +FROM scratch +COPY --from=busybox . / +COPY --from=musl . / +COPY --from=ca-certificates . / +COPY --from=zlib . / +COPY --from=npth . / +COPY --from=libksba . / +COPY --from=libgpg-error . / +COPY --from=libassuan . / +COPY --from=libgcrypt . / +COPY --from=gpg . / +COPY --from=tofu . / +COPY --from=sops . / +COPY --from=talosctl . / +COPY --from=kubectl . / +COPY --from=kustomize . / +COPY --from=kustomize-sops . / +COPY --from=sops . / +COPY --from=helm . / + +RUN mkdir -p /root/.gnupg +RUN chmod 0700 /root/.gnupg + +ENTRYPOINT ["/bin/sh"] diff --git a/Makefile b/Makefile index 1407141..e38fc03 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,3 @@ -include $(PWD)/src/toolchain/Makefile -include $(PWD)/src/make/tools.mk - BACKEND_TF := $(wildcard infra/backend/*.tf) MAIN_TF := $(wildcard infra/main/*.tf) ENVIRONMENT := production @@ -133,6 +130,29 @@ config/$(ENVIRONMENT).tfbackend: | \ -state $(ENVIRONMENT).tfstate \ ' +.PHONY: build-container +build-container: + docker build -t git.distrust.co/public/stack-tools -f Containerfile.tools . + +GPG_TTY ?= $(shell tty) + +define run-container + docker run -it \ + -e GPG_TTY=$(GPG_TTY) \ + -e GPG_AGENT_INFO=/S.gpg-agent:0:1 \ + -v $(shell gpgconf --list-dirs agent-socket):/root/.gnupg-w/S.gpg-agent \ + -v $(shell gpgconf --list-dirs homedir):/root/.gnupg:ro \ + git.distrust.co/public/stack-tools +endef + +.PHONY: new-apply +new-apply: build-container + $(call run-container,'\ + echo $$GPG_AGENT_INFO; \ + ls -l /S.gpg-agent; \ + gpg --verbose --list-keys \ + ') + .PHONY: apply: \ $(TERRAFORM) \ -- 2.40.1 From 6f75bb991ee3bc1edb5baa79b7dd5815102a76cd Mon Sep 17 00:00:00 2001 From: "ryan-distrust.co" Date: Wed, 2 Apr 2025 17:28:45 -0400 Subject: [PATCH 2/6] fix issue with concat'ing objects --- infra/main/main.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infra/main/main.tf b/infra/main/main.tf index 0290012..6f8e3b5 100644 --- a/infra/main/main.tf +++ b/infra/main/main.tf @@ -125,7 +125,10 @@ locals { # `jq .database_users.value.forgejo | sops --encrypt` output "database_users" { value = { - for db_user in concat(module.digitalocean_database_cluster.database_users, module.digitalocean_mysql_database_cluster.database_users): + for db_user in concat( + values(module.digitalocean_database_cluster.database_users), + values(module.digitalocean_mysql_database_cluster.database_users), + ): db_user.name => { apiVersion = "v1", kind = "Secret", -- 2.40.1 From 46c9dbfa8e29bb388d312fc5ab1403cc9a451248 Mon Sep 17 00:00:00 2001 From: Anton Livaja Date: Tue, 6 May 2025 18:52:58 -0700 Subject: [PATCH 3/6] feat: update tools container deps and add update script --- Containerfile.tools | 70 ++++++++++++++++++++++----------------------- Makefile | 8 ++++-- src/make/update.sh | 69 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 37 deletions(-) create mode 100755 src/make/update.sh diff --git a/Containerfile.tools b/Containerfile.tools index 27518d9..2024e54 100644 --- a/Containerfile.tools +++ b/Containerfile.tools @@ -1,42 +1,42 @@ # Tools used for managing the stagex stack -FROM quay.io/stagex/core-busybox AS busybox -FROM quay.io/stagex/core-musl AS musl -FROM quay.io/stagex/core-ca-certificates AS ca-certificates -FROM quay.io/stagex/core-zlib AS zlib -FROM quay.io/stagex/user-gpg AS gpg -FROM quay.io/stagex/user-npth AS npth -FROM quay.io/stagex/user-libksba AS libksba -FROM quay.io/stagex/user-libgpg-error AS libgpg-error -FROM quay.io/stagex/user-libassuan AS libassuan -FROM quay.io/stagex/user-libgcrypt AS libgcrypt -FROM quay.io/stagex/user-tofu AS tofu -FROM quay.io/stagex/user-sops AS sops -FROM quay.io/stagex/user-talosctl AS talosctl -FROM quay.io/stagex/user-kubectl AS kubectl -FROM quay.io/stagex/user-kustomize AS kustomize -FROM quay.io/stagex/user-kustomize-sops AS kustomize-sops -FROM quay.io/stagex/user-helm AS helm +FROM stagex/core-busybox@sha256:cac5d773db1c69b832d022c469ccf5f52daf223b91166e6866d42d6983a3b374 AS core-busybox +FROM stagex/core-musl@sha256:d5f86324920cfc7fc34f0163502784b73161543ba0a312030a3ddff3ef8ab2f8 AS core-musl +FROM stagex/core-ca-certificates@sha256:d6fca6c0080e8e5360cd85fc1c4bd3eab71ce626f40602e38488bfd61fd3e89d AS core-ca-certificates +FROM stagex/core-zlib@sha256:b35b643642153b1620093cfe2963f5fa8e4d194fb2344a5786da5717018976c2 AS core-zlib +FROM stagex/user-gpg@sha256:92946bb4143ecbd53999cd520fbcb958aecacbac7a85bd58a758be1b57086a9c AS user-gpg +FROM stagex/user-npth@sha256:6ac9a90ca714ba01911c1f617553a5b23b96e9e37ec4a21e5ba132c4886a70e9 AS user-npth +FROM stagex/user-libksba@sha256:c165fb5b7949473cb00b0fe59add90663346b33c6c682309ca0fcccdcf78d569 AS user-libksba +FROM stagex/user-libgpg-error@sha256:6d7c09e3a7d055a6722910439c533f2babc8eda24b636bf4dfb2b29a3ed6327a AS user-libgpg-error +FROM stagex/user-libassuan@sha256:dea35799659be7b85e523312c55621007b1918ff3590631155ecf2c699ca470f AS user-libassuan +FROM stagex/user-libgcrypt@sha256:384f0e703afad6f8885ec77fb814ef182a08600a2032183d231fee5c048a7d2d AS user-libgcrypt +FROM stagex/user-opentofu@sha256:b5053a5966f7ec06ea894db315c4990b73e8bee69798889de747e9a99c32b041 AS user-opentofu +FROM stagex/user-sops@sha256:72b09ff439f422889af815f19a223b48b3b3fd0701d312a413069cbabcad7a12 AS user-sops +FROM stagex/user-talosctl@sha256:23ff2d686a0c251db4f8a8f07e9b18c81c64eaa07da97de5a75fccbea3e595c4 AS user-talosctl +FROM stagex/user-kubectl@sha256:6df028ecb71097c182276cad295f7a68a28f2c8d7fc82ea47fb22a451b11a4ff AS user-kubectl +FROM stagex/user-kustomize@sha256:9886d6c855f763398a8bf52cd16e07f78cb8dab75396903645612e9cd4094cfa AS user-kustomize +FROM stagex/user-kustomize-sops@sha256:25040e0adf7dc6806da9996a252dbf7f8f5bb4f0b9a9dd1835035eeaea3861d9 AS user-kustomize-sops +FROM stagex/user-helm@sha256:e7d2e13db8483f5356b96337308edbd5a0e602cc76c4c5ea5ed730ae6d2b2dcc AS user-helm FROM scratch -COPY --from=busybox . / -COPY --from=musl . / -COPY --from=ca-certificates . / -COPY --from=zlib . / -COPY --from=npth . / -COPY --from=libksba . / -COPY --from=libgpg-error . / -COPY --from=libassuan . / -COPY --from=libgcrypt . / -COPY --from=gpg . / -COPY --from=tofu . / -COPY --from=sops . / -COPY --from=talosctl . / -COPY --from=kubectl . / -COPY --from=kustomize . / -COPY --from=kustomize-sops . / -COPY --from=sops . / -COPY --from=helm . / +COPY --from=core-busybox . / +COPY --from=core-musl . / +COPY --from=core-ca-certificates . / +COPY --from=core-zlib . / +COPY --from=user-npth . / +COPY --from=user-libksba . / +COPY --from=user-libgpg-error . / +COPY --from=user-libassuan . / +COPY --from=user-libgcrypt . / +COPY --from=user-gpg . / +COPY --from=user-opentofu . / +COPY --from=user-sops . / +COPY --from=user-talosctl . / +COPY --from=user-kubectl . / +COPY --from=user-kustomize . / +COPY --from=user-kustomize-sops . / +COPY --from=user-sops . / +COPY --from=user-helm . / RUN mkdir -p /root/.gnupg RUN chmod 0700 /root/.gnupg diff --git a/Makefile b/Makefile index e38fc03..204c3b9 100644 --- a/Makefile +++ b/Makefile @@ -21,11 +21,15 @@ default: \ tools \ apply -.PHONY: +.PHONY: clean clean: rm -rf $(CACHE_DIR) -.PHONY: +.PHONY: update-tools +update-tools: + ./src/make/update.sh + +.PHONY: credentials credentials: \ $(CACHE_DIR)/secrets/credentials.tfvars diff --git a/src/make/update.sh b/src/make/update.sh new file mode 100755 index 0000000..0362522 --- /dev/null +++ b/src/make/update.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +TARGET="Containerfile.tools" +SOURCE="https://codeberg.org/stagex/stagex/raw/branch/main/digests" +STAGES="core user pallet bootstrap" + +TMPFILE="$(mktemp)" +DIGESTS_TMP="$(mktemp)" + +for stage in $STAGES; do + curl -fsSL "$SOURCE/$stage.txt" | while read -r digest name; do + echo "$name $digest" >> "$DIGESTS_TMP" + done +done + +while IFS= read -r line; do + case "$line" in + FROM*stagex/*) + full_image="$(printf '%s' "$line" | awk '{print $2}')" + base="${full_image%@sha256:*}" + prefix="${base%%stagex/*}" + registry="${prefix%/}" + path="stagex/${base#*stagex/}" + + rest="${path#stagex/}" + if echo "$rest" | grep -q ':'; then + name="${rest%%:*}" + tag="${rest#*:}" + else + name="$rest" + tag="" + fi + + digest="$(awk -v n="$name" '$1==n{print $2; exit}' "$DIGESTS_TMP")" + if [ -z "$digest" ]; then + for stage in $STAGES; do + staged_name="$stage-$name" + digest="$(awk -v n="$staged_name" '$1==n{print $2; exit}' "$DIGESTS_TMP")" + if [ -n "$digest" ]; then + name="$staged_name" + break + fi + done + fi + + if [ -n "$digest" ]; then + if [ -n "$registry" ]; then + image_ref="$registry/stagex/$name" + else + image_ref="stagex/$name" + fi + + if [ -n "$tag" ]; then + image_ref="$image_ref:$tag" + fi + + echo "FROM $image_ref@sha256:$digest AS $name" >> "$TMPFILE" + else + echo "$line" >> "$TMPFILE" + fi + ;; + *) + echo "$line" >> "$TMPFILE" + ;; + esac +done < "$TARGET" + +mv "$TMPFILE" "$TARGET" +rm -f "$DIGESTS_TMP" -- 2.40.1 From e899a0c11f207bf4b68b421aa998b50740002386 Mon Sep 17 00:00:00 2001 From: Anton Livaja Date: Wed, 7 May 2025 07:45:56 -0700 Subject: [PATCH 4/6] feat: update make shell command --- Makefile | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 204c3b9..d9c091b 100644 --- a/Makefile +++ b/Makefile @@ -34,15 +34,8 @@ credentials: \ $(CACHE_DIR)/secrets/credentials.tfvars .PHONY: -shell: toolchain tools - $(call toolchain," \ - HOST_OS=linux \ - HOST_ARCH=x86_64 \ - PREFIX=.local \ - XDG_CONFIG_HOME=/home/build/.config \ - make -f src/make/tools.mk tools-install \ - && PS1='build@distrust-stack\\$$ ' bash --norc \ - ",--interactive) +shell: build-container + $(call run-container, -v ./secrets:/secrets) $(KEY_DIR)/%.asc: $(call fetch_pgp_key,$(basename $(notdir $@))) @@ -141,8 +134,8 @@ build-container: GPG_TTY ?= $(shell tty) define run-container - docker run -it \ - -e GPG_TTY=$(GPG_TTY) \ + docker run -it $(1) \ + -e GPG_TTY="$(GPG_TTY)" \ -e GPG_AGENT_INFO=/S.gpg-agent:0:1 \ -v $(shell gpgconf --list-dirs agent-socket):/root/.gnupg-w/S.gpg-agent \ -v $(shell gpgconf --list-dirs homedir):/root/.gnupg:ro \ -- 2.40.1 From 6c95084aa2cb48e07610545de4f5b86c27d050ce Mon Sep 17 00:00:00 2001 From: Anton Livaja Date: Wed, 7 May 2025 07:54:51 -0700 Subject: [PATCH 5/6] feat: add k9s --- Containerfile.tools | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Containerfile.tools b/Containerfile.tools index 2024e54..e774205 100644 --- a/Containerfile.tools +++ b/Containerfile.tools @@ -17,6 +17,7 @@ FROM stagex/user-kubectl@sha256:6df028ecb71097c182276cad295f7a68a28f2c8d7fc82ea4 FROM stagex/user-kustomize@sha256:9886d6c855f763398a8bf52cd16e07f78cb8dab75396903645612e9cd4094cfa AS user-kustomize FROM stagex/user-kustomize-sops@sha256:25040e0adf7dc6806da9996a252dbf7f8f5bb4f0b9a9dd1835035eeaea3861d9 AS user-kustomize-sops FROM stagex/user-helm@sha256:e7d2e13db8483f5356b96337308edbd5a0e602cc76c4c5ea5ed730ae6d2b2dcc AS user-helm +FROM stagex/user-k9s@sha256:eff325c4d000358b2f6ed0f63d61fcea8f98c081395437d0003e7429e0c334b4 AS user-k9s FROM scratch COPY --from=core-busybox . / @@ -37,6 +38,7 @@ COPY --from=user-kustomize . / COPY --from=user-kustomize-sops . / COPY --from=user-sops . / COPY --from=user-helm . / +COPY --from=user-k9s . / RUN mkdir -p /root/.gnupg RUN chmod 0700 /root/.gnupg -- 2.40.1 From e757eb9363c6e749c5d46de9cf3c6e4dfeb45988 Mon Sep 17 00:00:00 2001 From: Anton Livaja Date: Fri, 9 May 2025 18:00:04 -0700 Subject: [PATCH 6/6] fix: gpg socket agent path --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d9c091b..6eaf557 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,8 @@ update-tools: credentials: \ $(CACHE_DIR)/secrets/credentials.tfvars +# If using QubesOS, the smart card must be connected directly to the qube, +# rather than using a 'vault' qube. .PHONY: shell: build-container $(call run-container, -v ./secrets:/secrets) @@ -136,9 +138,8 @@ GPG_TTY ?= $(shell tty) define run-container docker run -it $(1) \ -e GPG_TTY="$(GPG_TTY)" \ - -e GPG_AGENT_INFO=/S.gpg-agent:0:1 \ - -v $(shell gpgconf --list-dirs agent-socket):/root/.gnupg-w/S.gpg-agent \ - -v $(shell gpgconf --list-dirs homedir):/root/.gnupg:ro \ + -v $(shell gpgconf --list-dirs agent-socket):/root/.gnupg/S.gpg-agent:ro \ + -v $(shell gpgconf --list-dirs homedir):/root/.gnupg:rw \ git.distrust.co/public/stack-tools endef -- 2.40.1