initial commit

Dockerfile

@@ -0,0 +1,18 @@
+ARG DEBIAN_HASH
+FROM debian@sha256:${DEBIAN_HASH}
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    LANG=C.UTF-8 \
+    TZ=UTC \
+    PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+ARG CONFIG_DIR
+ADD ${CONFIG_DIR} /config
+
+ARG SCRIPTS_DIR
+ADD ${SCRIPTS_DIR} /usr/local/bin
+
+RUN packages-install
+
+RUN echo "/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1" \
+    > /etc/ld.so.preload

Makefile

@@ -0,0 +1,179 @@
+NAME := $(shell basename $(shell git rev-parse --show-toplevel))
+IMAGE := local/$(NAME):latest
+ARCH := x86_64
+TARGET := $(ARCH)
+USER := $(shell id -u):$(shell id -g)
+CPUS := $(shell docker run -it debian nproc)
+GIT_REF := $(shell git log -1 --format=%H config)
+GIT_AUTHOR := $(shell git log -1 --format=%an config)
+GIT_KEY := $(shell git log -1 --format=%GP config)
+GIT_EPOCH := $(shell git log -1 --format=%at config)
+GIT_DATETIME := \
+	$(shell git log -1 --format=%cd --date=format:'%Y-%m-%d %H:%M:%S' config)
+ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)
+	GIT_STATE=clean
+else
+	GIT_STATE=dirty
+endif
+VERSION := $(shell TZ=UTC0 git show --quiet --date='format-local:%Y%m%dT%H%M%SZ' --format="%cd")
+RELEASE_DIR := release/$(VERSION)
+CONFIG_DIR := config
+CACHE_DIR := cache
+SRC_DIR := src
+OUT_DIR := out
+docker = docker
+
+include $(CONFIG_DIR)/global.env
+export $(shell sed 's/=.*//' $(CONFIG_DIR)/global.env)
+
+## Use env vars from existing release if present
+ifneq (,$(wildcard $(RELEASE_DIR)/release.env))
+    include $(RELEASE_DIR)/release.env
+    export
+endif
+
+executables = $(docker) git patch
+
+.PHONY: toolchain
+toolchain: $(CACHE_DIR)/toolchain.tar $(CACHE_DIR)/toolchain.env
+
+# Launch a shell inside the toolchain container
+.PHONY: toolchain-shell
+toolchain-shell: toolchain
+	$(call toolchain,$(USER),"bash --norc")
+
+# Pin all packages in toolchain container to latest versions
+.PHONY: toolchain-update
+toolchain-update:
+	docker run \
+		--rm \
+		--env LOCAL_USER=$(USER) \
+		--platform=linux/$(ARCH) \
+		--volume $(PWD)/$(CONFIG_DIR):/config \
+		--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
+		--env ARCH=$(ARCH) \
+		--interactive \
+		--tty \
+		debian@sha256:$(DEBIAN_HASH) \
+		bash -c /usr/local/bin/packages-update
+
+.PHONY: attest
+attest:
+	rm -rf $(CACHE_DIR) $(OUT_DIR)
+	$(MAKE)
+	diff -q $(OUT_DIR)/manifest.txt release/$(VERSION)/manifest.txt;
+
+$(RELEASE_DIR):
+	mkdir -p $@
+
+$(CACHE_DIR):
+	mkdir -p $@
+
+$(OUT_DIR):
+	mkdir -p $@
+
+.ONESHELL:
+$(CACHE_DIR)/toolchain.env: $(CACHE_DIR)
+	cat <<- EOF > $@
+		HOME=/home/build
+		PS1=$(NAME)-toolchain
+		GNUPGHOME=/cache/.gnupg
+		ARCH=$(ARCH)
+		TARGET=$(ARCH)
+		GIT_REF=$(GIT_REF)
+		GIT_AUTHOR=$(GIT_AUTHOR)
+		GIT_KEY=$(GIT_KEY)
+		GIT_DATETIME=$(GIT_DATETIME)
+		GIT_EPOCH=$(GIT_EPOCH)
+		FAKETIME_FMT="%s"
+		FAKETIME="1"
+		SOURCE_DATE_EPOCH=1
+		KBUILD_BUILD_TIMESTAMP="1970-01-01 00:00:00 UTC"
+		KCONFIG_NOTIMESTAMP=1
+		KBUILD_BUILD_USER=root
+		KBUILD_BUILD_HOST=$(NAME)
+		KBUILD_BUILD_VERSION=1
+		UID=$(shell id -u)
+		GID=$(shell id -g)
+		RELEASE_DIR=release/$(VERSION)
+		CONFIG_DIR=/home/build/$(CONFIG_DIR)
+		CACHE_DIR=/home/build/$(CACHE_DIR)
+		SRC_DIR=/home/build/$(SRC_DIR)
+		OUT_DIR=/home/build/$(OUT_DIR)
+	EOF
+
+$(CACHE_DIR)/toolchain.tar:
+	mkdir -p $(CACHE_DIR)
+	DOCKER_BUILDKIT=1 \
+	docker build \
+		--tag $(IMAGE) \
+		--build-arg DEBIAN_HASH=$(DEBIAN_HASH) \
+		--build-arg CONFIG_DIR=$(CONFIG_DIR) \
+		--build-arg SCRIPTS_DIR=$(SRC_DIR)/toolchain/scripts \
+		--platform=linux/$(ARCH) \
+		-f $(SRC_DIR)/toolchain/Dockerfile \
+		.
+	docker save "$(IMAGE)" -o "$@"
+
+$(RELEASE_DIR)/release.env: \
+	$(RELEASE_DIR) \
+	$(OUT_DIR)/release.env
+	cp $(OUT_DIR)/release.env $(RELEASE_DIR)/release.env
+
+$(RELEASE_DIR)/manifest.txt: \
+	$(RELEASE_DIR) \
+	$(OUT_DIR)/manifest.txt
+	cp $(OUT_DIR)/manifest.txt $(RELEASE_DIR)/manifest.txt
+
+$(OUT_DIR)/release.env: | $(OUT_DIR)
+	echo 'VERSION=$(VERSION)'            > $(OUT_DIR)/release.env
+	echo 'GIT_REF=$(GIT_REF)'           >> $(OUT_DIR)/release.env
+	echo 'GIT_AUTHOR=$(GIT_AUTHOR)'     >> $(OUT_DIR)/release.env
+	echo 'GIT_KEY=$(GIT_KEY)'           >> $(OUT_DIR)/release.env
+	echo 'GIT_DATETIME=$(GIT_DATETIME)' >> $(OUT_DIR)/release.env
+
+$(OUT_DIR)/manifest.txt: | $(OUT_DIR)
+	find $(OUT_DIR) \
+		-type f \
+		-not -path "$(OUT_DIR)/manifest.txt" \
+		-exec openssl sha256 -r {} \; \
+	| sed -e 's/ \*/ /g' -e 's/ \.\// /g' \
+	| LC_ALL=C sort -k2 \
+	> $@
+
+check_executables := $(foreach exec,$(executables),\$(if \
+	$(shell which $(exec)),some string,$(error "No $(exec) in PATH")))
+
+define git_clone
+	[ -d $(CACHE_DIR)/$(1) ] || git clone $(2) $(CACHE_DIR)/$(1)
+	git -C $(CACHE_DIR)/$(1) checkout $(3)
+	git -C $(CACHE_DIR)/$(1) rev-parse --verify HEAD | grep -q $(3) || { \
+		echo 'Error: Git ref/branch collision.'; exit 1; \
+	};
+endef
+
+define apply_patches
+	[ -d $(2) ] && $(call toolchain,$(USER)," \
+		cd $(1); \
+		git restore .; \
+		find /$(2) -type f -iname '*.patch' -print0 \
+		| xargs -t -0 -n 1 patch -p1 --no-backup-if-mismatch -i ; \
+	")
+endef
+
+define toolchain
+	docker load -i $(CACHE_DIR)/toolchain.tar
+	docker run \
+		--rm \
+		--tty \
+		--interactive \
+		--user=$(1) \
+		--platform=linux/$(ARCH) \
+		--cpus $(CPUS) \
+		--volume $(PWD):/home/build \
+		--workdir /home/build \
+		--env-file=$(CONFIG_DIR)/global.env \
+		--env-file=$(CACHE_DIR)/toolchain.env \
+		$(IMAGE) \
+		bash -c $(2)
+endef

README.md

@@ -0,0 +1,59 @@
+# Toolchain #
+
+<https://codeberg.org/distrust/toolchain>
+
+## About ##
+
+A library of opinionated make functions targeting projects that either need
+deterministic builds, or a shared deterministic toolchain shared across all
+who use a project.
+
+A dev of a Toolchain enabled project should never need to have anything
+on their host system installed but docker, and git. Everything else will be
+provided via a Docker container.
+
+Debian currently has the highest reproducibility score of any major Linux
+distribution, and as such it is the chosen base for Toolchain.
+
+This was built for Distrust projects, and some of our clients. It is unlikely
+to meet the needs of everyone. We suggest including this in your project as
+a git subtree, so you can make your own changes, but also pull in changes from
+us as desired.
+
+## Uses ##
+ * Ensure everyone on a team is using the exact same tools
+ * Ensure all releases and artifacts build hash-for-hash identical every time
+ * Control supply chain security with only signed/reproducible dependencies
+
+## Features ##
+ * Can run a shell with all toolchain tooling in the current directory
+ * Provide make functions for common tasks
+   * Git clone, apply patches, etc.
+ * Use a global env file as configuration
+ * Hash-locking of apt dependencies from a list of top-level required packages
+ * Provides release.env file with required vars to re-create old releases
+
+## Requirements ##
+
+* docker 18+
+* GNU Make 4+
+
+## Build ##
+
+### Build a new release
+
+    ```
+    make VERSION=1.0.0rc1 release
+    ```
+
+### Reproduce an existing release
+
+    ```
+    make VERSION=1.0.0rc1 attest
+    ```
+
+### Sign an existing release
+
+    ```
+    make VERSION=1.0.0rc1 sign
+    ```

scripts/host-env

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+uid=${UID?}
+gid=${GID?}
+user=${USER:-"build"}
+export HOME="/home/${user}"
+
+groupadd -g "$gid" "${user}"
+useradd \
+	-g "$gid" \
+	-u "$uid" \
+	-md "/home/${user}" \
+	-s /bin/bash \
+	"${user}"
+
+mkdir -p "$HOME"
+chown -R "$uid:$gid" "$HOME"
+cd "$HOME"
+setpriv --reuid="$uid" --regid="$gid" --init-groups "$@"

scripts/packages-install

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -e;
+
+ARCH=$(uname -m)
+
+cp /config/toolchain/* /etc/apt/
+
+apt-get update
+apt-get install debian-archive-keyring
+
+until apt-get install --download-only --reinstall --allow-downgrades -y $(cat /etc/apt/packages-${ARCH}.list); do
+    echo "apt install failed. Likely throttled. Retrying in 10 mins...";
+    sleep 600;
+done;
+
+(
+	cd /var/cache/apt/archives \
+		&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
+		| sed 's/.\///g' \
+		| LC_ALL=C sort
+) > /etc/apt/package-hashes-${ARCH}-compare.txt
+
+diff /etc/apt/package-hashes-${ARCH}{,-compare}.txt
+
+apt-get install --allow-downgrades -y $(cat /etc/apt/packages-${ARCH}.list)
+rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*;

scripts/packages-update

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
+set -e
+
+snapshot_url="http://snapshot.debian.org/archive/debian"
+snapshot_date=$(date +"%Y%m%dT000000Z")
+cat <<-EOF > /etc/apt/sources.list
+deb http://deb.debian.org/debian bookworm main
+deb http://security.debian.org/debian-security bookworm-security main
+deb http://deb.debian.org/debian bookworm-updates main
+deb [check-valid-until=no] ${snapshot_url}/${snapshot_date} bookworm main
+deb [check-valid-until=no] ${snapshot_url}-security/${snapshot_date} bookworm-security main
+deb [check-valid-until=no] ${snapshot_url}/${snapshot_date} bookworm-updates main
+EOF
+cp /etc/apt/sources.list /config/toolchain/
+
+ARCH=$(uname -m)
+
+apt-get update
+apt-get install -y --download-only --reinstall $( \
+	dpkg-query \
+		-W \
+		-f='${db:Status-Abbrev}\t${binary:Package} - ${binary:Summary}\n' \
+	| awk -F'\t' '/^ii/ {print $2}' \
+	| awk '{print $1}' \
+)
+apt-get install -y --download-only $(cat /config/toolchain/packages-base.list)
+
+( cd /var/cache/apt/archives \
+	&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
+	| sed 's/.\///g' \
+	| LC_ALL=C sort
+) > /config/toolchain/package-hashes-${ARCH}.txt
+
+cp /dev/null /config/toolchain/packages-${ARCH}.list
+for deb in /var/cache/apt/archives/*.deb; do
+	package=$(dpkg-deb -f $deb Package);
+	version=$(dpkg --info ${deb} | grep "^ Version: " | sed 's/^ Version: //g');
+	echo "${package}=${version}" >> /config/toolchain/packages-${ARCH}.list;
+done
+
+chown -R $LOCAL_USER /config/toolchain