support path prefix

Makefile

@@ -43,25 +43,14 @@ else
 endif
 BIN_DIR := $(CACHE_DIR_ROOT)/bin
 SRC_DIR := src
-KEY_DIR := keys
+KEY_DIR := fetch/keys
 OUT_DIR := out
 docker = docker
 
-include $(CONFIG_DIR)/toolchain.env
-export $(shell sed 's/=.*//' $(CONFIG_DIR)/toolchain.env)
 export
 
-AUTOBUILD_TOOLCHAIN := true
+include $(CONFIG_DIR)/make.env
-ifeq ($(AUTOBUILD_TOOLCHAIN),true)
+export $(shell sed 's/=.*//' $(CONFIG_DIR)/make.env)
-ifeq ("$(wildcard $(CACHE_DIR_ROOT)/make.env)","")
-	echo := $(info $(shell echo "Initializing toolchain."))
-	build_env := $(shell $(MAKE) AUTOBUILD_TOOLCHAIN=false toolchain )
-endif
-endif
-ifneq (,$(wildcard $(CACHE_DIR_ROOT)/make.env))
-	include $(CACHE_DIR_ROOT)/make.env
-	export $(shell sed 's/=.*//' $(CACHE_DIR_ROOT)/make.env)
-endif
 
 ## Use env vars from existing release if present
 ifneq (,$(wildcard $(DIST_DIR)/release.env))
@@ -78,8 +67,7 @@ toolchain: \
 	$(BIN_DIR) \
 	$(OUT_DIR) \
 	$(CACHE_DIR_ROOT)/toolchain.state \
-	$(CACHE_DIR_ROOT)/container.env \
+	$(CACHE_DIR_ROOT)/container.env
-	$(CACHE_DIR_ROOT)/make.env
 
 # Launch a shell inside the toolchain container
 .PHONY: toolchain-shell
@@ -88,17 +76,21 @@ toolchain-shell: toolchain
 
 .PHONY: toolchain-update
 toolchain-update:
-	rm \
+	rm -rf \
 		$(CONFIG_DIR)/apt-pins-x86_64.list \
 		$(CONFIG_DIR)/apt-sources-x86_64.list \
-		$(CONFIG_DIR)/apt-hashes-x86_64.list
+		$(CONFIG_DIR)/apt-hashes-x86_64.list \
-	$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list \
+		$(FETCH_DIR)/apt
+	$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list
+
+$(CONFIG_DIR)/apt-base.list:
+	touch $(CONFIG_DIR)/apt-base.list
 
 # Regenerate toolchain dependency packages to latest versions
-$(CONFIG_DIR)/apt-base.list \
 $(CONFIG_DIR)/apt-pins-x86_64.list \
 $(CONFIG_DIR)/apt-sources-x86_64.list \
-$(CONFIG_DIR)/apt-hashes-x86_64.list:
+$(CONFIG_DIR)/apt-hashes-x86_64.list: \
+$(CONFIG_DIR)/apt-base.list
 	mkdir -p $(FETCH_DIR)/apt \
 	&& docker run \
 		--rm \
@@ -114,7 +106,7 @@ $(CONFIG_DIR)/apt-hashes-x86_64.list:
 		/usr/local/bin/packages-update
 
 # Pin all packages in toolchain container to latest versions
-$(FETCH_DIR)/apt/Packages.gz:
+$(FETCH_DIR)/apt/Packages.bz2: $(CONFIG_DIR)/apt-hashes-x86_64.list
 	docker run \
 		--rm \
 		--tty \
@@ -169,9 +161,8 @@ $(FETCH_DIR):
 $(OUT_DIR):
 	mkdir -p $@
 
-$(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
+$(CACHE_DIR_ROOT)/container.env: \
-	$(CONFIG_DIR)/global.env \
+	$(CONFIG_DIR)/make.env \
-	$(CONFIG_DIR)/toolchain.env \
 	$(CACHE_DIR_ROOT)/toolchain.state
 	docker run \
         --rm \
@@ -195,6 +186,7 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
 		--env DIST_DIR="$(DIST_DIR)" \
 		--env FETCH_DIR="$(FETCH_DIR)" \
 		--env KEY_DIR="$(KEY_DIR)" \
+		--env BIN_DIR="$(BIN_DIR)" \
 		--env OUT_DIR="$(OUT_DIR)" \
 		--env SRC_DIR="$(SRC_DIR)" \
 		--env CACHE_DIR="$(CACHE_DIR)" \
@@ -206,16 +198,16 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
         --volume $(TOOLCHAIN_VOLUME) \
         --workdir $(TOOLCHAIN_WORKDIR) \
         $(shell cat cache/toolchain.state 2> /dev/null) \
-        $(SRC_DIR)/toolchain/scripts/environment $(CACHE_DIR_ROOT)
+        $(SRC_DIR)/toolchain/scripts/environment > $@
 
 $(CACHE_DIR_ROOT)/toolchain.tar: \
-	$(CONFIG_DIR)/toolchain.env \
+	$(CONFIG_DIR)/make.env \
 	$(SRC_DIR)/toolchain/Dockerfile \
 	$(CONFIG_DIR)/apt-base.list \
 	$(CONFIG_DIR)/apt-sources-$(ARCH).list \
 	$(CONFIG_DIR)/apt-pins-$(ARCH).list \
 	$(CONFIG_DIR)/apt-hashes-$(ARCH).list \
-	$(FETCH_DIR)/apt/Packages.gz
+	$(FETCH_DIR)/apt/Packages.bz2
 	mkdir -p $(CACHE_DIR)
 	DOCKER_BUILDKIT=1 \
 	docker build \
@@ -260,18 +252,33 @@ define fetch_file
 	"
 endef
 
+define git_archive
+		$(call git_clone,$(CACHE_DIR)/$(notdir $@),$(1),$(2)) \
+		&& tar \
+				-C $(CACHE_DIR)/$(notdir $@) \
+				--sort=name \
+				--mtime='@0' \
+				--owner=0 \
+				--group=0 \
+				--numeric-owner \
+				-cvf - \
+				. \
+			| gzip -n > $@ \
+		&& rm -rf $(CACHE_DIR)/$(notdir $@)
+endef
+
 define git_clone
 	[ -d $(1) ] || \
-		mkdir -p $(FETCH_DIR) && \
+		mkdir -p $(1).tmp && \
-		mkdir $(1).tmp && \
 		git -C $(1).tmp init && \
 		git -C $(1).tmp remote add origin $(2) && \
 		git -C $(1).tmp fetch origin $(3) && \
 		git -C $(1).tmp -c advice.detachedHead=false checkout $(3) && \
+		git -C $(1).tmp submodule update --init && \
 		git -C $(1).tmp rev-parse --verify HEAD | grep -q $(3) || { \
 			echo 'Error: Git ref/branch collision.'; exit 1; \
 		} && \
-		mv $(1).tmp $(1);
+		mv $(1).tmp $(1)
 endef
 
 define apply_patches
@@ -300,7 +307,7 @@ define fetch_pgp_key
         	                --recv-keys "$(1)" \
         	        && break; \
         	done; \
-            gpg --export -a $(1) > $(KEY_DIR)/$(1).asc; \
+            gpg --export -a $(1) > $@; \
         ')
 endef
 
@@ -311,6 +318,7 @@ define toolchain
                 $(2) \
                 --env UID=$(UID) \
                 --env GID=$(GID) \
+				--env PATH_PREFIX=$(PATH_PREFIX) \
                 --platform=linux/$(ARCH) \
                 --privileged \
                 --cpus $(CPUS) \

README.md

@@ -55,8 +55,8 @@ us as desired.
 3. Define any build/dev dependencies for toolchain container
 
     ```
-    echo "libfaketime" >> config/toolchain/packages-base.txt
+    echo "libfaketime" >> config/apt-base.list
-    echo "build-essential" >> config/toolchain/packages-base.txt
+    echo "build-essential" >> config/apt-base.list
     ```
 
 4. Lock a base Debian container image hash
@@ -97,7 +97,7 @@ make reproduce
 ### Add and lock a new container dependency
 
 ```
-echo "vim-nox" >> config/toolchain/packages-base.txt
+echo "vim-nox" >> config/apt-base.list
 make toolchain-update
 ```
 

scripts/environment

@@ -1,12 +1,11 @@
 #!/bin/sh
 
-CACHE_DIR_ROOT=${1?}
 HOME=/home/build
 CONFIG_DIR=/home/build/config
 
-cat ${CONFIG_DIR}/toolchain.env > ${CACHE_DIR_ROOT}/container.env
+cat ${CONFIG_DIR}/make.env
 
-cat <<- EOF >> ${CACHE_DIR_ROOT}/container.env
+cat <<- EOF
 	HOME=${HOME}
 	PATH=${HOME}/${BIN_DIR}:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 	TZ=UTC
@@ -39,6 +38,6 @@ cat <<- EOF >> ${CACHE_DIR_ROOT}/container.env
 	FETCH_DIR=${HOME}/${FETCH_DIR}
 EOF
 
-envsubst < ${CONFIG_DIR}/global.env > ${CACHE_DIR_ROOT}/make.env
+if [ -f "${CONFIG_DIR}/container.env" ]; then
-
+    envsubst < ${CONFIG_DIR}/container.env
-cat ${CACHE_DIR_ROOT}/make.env >> ${CACHE_DIR_ROOT}/container.env
+fi

scripts/host-env

@@ -5,6 +5,8 @@ uid=${UID?}
 gid=${GID?}
 user=${USER:-"build"}
 export HOME="/home/${user}"
+[ ! -z "$PATH_PREFIX" ] && \
+    export PATH="${PATH_PREFIX}:${PATH}"
 
 # If running user is not root, pivot to custom user/group
 if [ "$uid" != "0" ]; then

scripts/packages-fetch

@@ -5,7 +5,10 @@ set -e
 
 ARCH=$(uname -m)
 
-cp /config/* /etc/apt/
+cp /config/apt-sources-x86_64.list /etc/apt/sources.list
+cp /config/apt-hashes-x86_64.list /etc/apt/
+cp /config/apt-pins-x86_64.list /etc/apt/
+rm /etc/apt/sources.list.d/*
 apt update -o Acquire::Check-Valid-Until=false
 
 until apt-get install \
@@ -31,4 +34,4 @@ mkdir -p /fetch/apt
 
 mv /var/cache/apt/archives/*.deb /fetch/apt/
 apt-get install -y dpkg-dev
-env -C /fetch dpkg-scanpackages apt | gzip > /fetch/apt/Packages.gz
+env -C /fetch dpkg-scanpackages apt | bzip2 > /fetch/apt/Packages.bz2

scripts/packages-install

@@ -3,7 +3,7 @@ set -e;
 
 ARCH=$(uname -m)
 
-cp /config/* /etc/apt/
+cp -R /config/* /etc/apt/
 
 cat <<-EOF > /etc/apt/sources.list
 deb [trusted=yes] file:///fetch apt/