cache debian artifacts in fetch directory

2023-06-15 23:23:10 -07:00 · 2023-06-15 23:23:10 -07:00 · 5bbf26be78
parent 388334ef6a
commit 5bbf26be78
5 changed files with 99 additions and 44 deletions
--- a/6
+++ b/6
@ -1,5 +1,5 @@
 ARG DEBIAN_HASH
-FROM debian@sha256:${DEBIAN_HASH}
+FROM debian@sha256:${DEBIAN_HASH} as build-base

 ARG CONFIG_DIR
 ADD ${CONFIG_DIR} /config
@ -7,4 +7,6 @@ ADD ${CONFIG_DIR} /config
 ARG SCRIPTS_DIR
 ADD ${SCRIPTS_DIR} /usr/local/bin

-RUN packages-install
+ARG FETCH_DIR
+RUN --mount=type=bind,source=fetch,target=/fetch,rw \
+    packages-install
--- a/43
+++ b/43
@ -86,10 +86,21 @@ toolchain: \
 toolchain-shell: toolchain
 	$(call toolchain,bash --norc,--interactive)

-# Pin all packages in toolchain container to latest versions
 .PHONY: toolchain-update
 toolchain-update:
-	docker run \
+	rm \
+		$(CONFIG_DIR)/apt-pins-x86_64.list \
+		$(CONFIG_DIR)/apt-sources-x86_64.list \
+		$(CONFIG_DIR)/apt-hashes-x86_64.list
+	$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list \
+
+# Regenerate toolchain dependency packages to latest versions
+$(CONFIG_DIR)/apt-base.list \
+$(CONFIG_DIR)/apt-pins-x86_64.list \
+$(CONFIG_DIR)/apt-sources-x86_64.list \
+$(CONFIG_DIR)/apt-hashes-x86_64.list:
+	mkdir -p $(FETCH_DIR)/apt \
+	&& docker run \
 		--rm \
 		--tty \
 		--platform=linux/$(ARCH) \
@ -102,6 +113,24 @@ toolchain-update:
 		debian@sha256:$(DEBIAN_HASH) \
 		/usr/local/bin/packages-update

+# Pin all packages in toolchain container to latest versions
+$(FETCH_DIR)/apt/Packages.gz:
+	docker run \
+		--rm \
+		--tty \
+		--platform=linux/$(ARCH) \
+		--env LOCAL_USER=$(UID):$(GID) \
+		--env FETCH_DIR="$(FETCH_DIR)" \
+		--env PACKAGES_LATEST=$(PACKAGES_LATEST) \
+		--volume $(PWD)/$(CONFIG_DIR):/config \
+		--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
+		--volume $(PWD)/$(FETCH_DIR):/fetch \
+		--cpus $(CPUS) \
+		--volume $(TOOLCHAIN_VOLUME) \
+		--workdir $(TOOLCHAIN_WORKDIR) \
+		debian@sha256:$(DEBIAN_HASH) \
+		/usr/local/bin/packages-fetch
+
 .PHONY: toolchain-clean
 toolchain-clean:
 	if [ -d "$(CACHE_DIR_ROOT)" ]; then \
@ -182,16 +211,18 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
 $(CACHE_DIR_ROOT)/toolchain.tar: \
 	$(CONFIG_DIR)/toolchain.env \
 	$(SRC_DIR)/toolchain/Dockerfile \
-	$(CONFIG_DIR)/toolchain/package-hashes-$(ARCH).txt \
-	$(CONFIG_DIR)/toolchain/packages-base.list \
-	$(CONFIG_DIR)/toolchain/packages-$(ARCH).list \
-	$(CONFIG_DIR)/toolchain/sources.list
+	$(CONFIG_DIR)/apt-base.list \
+	$(CONFIG_DIR)/apt-sources-$(ARCH).list \
+	$(CONFIG_DIR)/apt-pins-$(ARCH).list \
+	$(CONFIG_DIR)/apt-hashes-$(ARCH).list \
+	$(FETCH_DIR)/apt/Packages.gz
 	mkdir -p $(CACHE_DIR)
 	DOCKER_BUILDKIT=1 \
 	docker build \
 		--tag $(IMAGE) \
 		--build-arg DEBIAN_HASH=$(DEBIAN_HASH) \
 		--build-arg CONFIG_DIR=$(CONFIG_DIR) \
+		--build-arg FETCH_DIR=$(PWD)/$(FETCH_DIR) \
 		--build-arg SCRIPTS_DIR=$(SRC_DIR)/toolchain/scripts \
 		--platform=linux/$(ARCH) \
 		-f $(SRC_DIR)/toolchain/Dockerfile \
--- a/scripts/packages-fetch
+++ b/scripts/packages-fetch
@ -0,0 +1,34 @@
+#!/bin/bash
+
+[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
+set -e
+
+ARCH=$(uname -m)
+
+cp /config/* /etc/apt/
+apt update -o Acquire::Check-Valid-Until=false
+
+until apt-get install \
+    --download-only \
+    --allow-downgrades \
+    -o Acquire::Check-Valid-Until=false \
+    -y $(cat /etc/apt/apt-pins-${ARCH}.list);
+do
+    echo "apt install failed. Likely throttled. Retrying in 10 mins...";
+    sleep 600;
+done;
+
+(
+	cd /var/cache/apt/archives \
+		&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
+		| sed 's/.\///g' \
+		| LC_ALL=C sort
+) > /etc/apt/apt-hashes-${ARCH}-compare.list
+
+diff /etc/apt/apt-hashes-${ARCH}{,-compare}.list
+
+mkdir -p /fetch/apt
+
+mv /var/cache/apt/archives/*.deb /fetch/apt/
+apt-get install -y dpkg-dev
+env -C /fetch dpkg-scanpackages apt | gzip > /fetch/apt/Packages.gz
--- a/scripts/packages-install
+++ b/scripts/packages-install
@ -3,34 +3,19 @@ set -e;

 ARCH=$(uname -m)

-cp /config/toolchain/* /etc/apt/
+cp /config/* /etc/apt/

-apt-get update -o Acquire::Check-Valid-Until=false
-apt-get install debian-archive-keyring
+cat <<-EOF > /etc/apt/sources.list
+deb [trusted=yes] file:///fetch apt/
+EOF
+rm /etc/apt/sources.list.d/*

-until apt-get install \
-    --download-only \
-    --reinstall \
-    --allow-downgrades \
-    -o Acquire::Check-Valid-Until=false \
-    -y $(cat /etc/apt/packages-${ARCH}.list);
-do
-    echo "apt install failed. Likely throttled. Retrying in 10 mins...";
-    sleep 600;
-done;
-
-(
-	cd /var/cache/apt/archives \
-		&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
-		| sed 's/.\///g' \
-		| LC_ALL=C sort
-) > /etc/apt/package-hashes-${ARCH}-compare.txt
-
-diff /etc/apt/package-hashes-${ARCH}{,-compare}.txt
+apt update -o Acquire::Check-Valid-Until=false

 apt-get install \
    --allow-downgrades \
-    -y $(cat /etc/apt/packages-${ARCH}.list)
+    -y $(cat /etc/apt/apt-pins-${ARCH}.list)
+
 rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*;

 echo "%sudo ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers
--- a/scripts/packages-update
+++ b/scripts/packages-update
@ -3,20 +3,17 @@
 [ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
 set -e

-snapshot_url="http://snapshot.debian.org/archive/debian"
-snapshot_date=$(date +"%Y%m%dT000000Z")
 cat <<-EOF > /etc/apt/sources.list
-deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
-deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
-deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
+deb http://deb.debian.org/debian bookworm main
+deb http://security.debian.org/debian-security bookworm-security main
+deb http://deb.debian.org/debian bookworm-updates main
 EOF
-cp /etc/apt/sources.list /config/toolchain/
 rm /etc/apt/sources.list.d/*

 ARCH=$(uname -m)

 apt-get update
-apt-get install -y --download-only --reinstall $( \
+apt-get install -y --download-only $( \
 	dpkg-query \
 		-W \
 		-f='${db:Status-Abbrev}\t${binary:Package} - ${binary:Summary}\n' \
@ -26,20 +23,26 @@ apt-get install -y --download-only --reinstall $( \
 apt-get install \
 	-y \
 	--download-only \
-	sudo gettext \
-	$(cat /config/toolchain/packages-base.list)
+	sudo gettext dpkg-dev \
+	$(cat /config/apt-base.list)

 ( cd /var/cache/apt/archives \
 	&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
 	| sed 's/.\///g' \
 	| LC_ALL=C sort
-) > /config/toolchain/package-hashes-${ARCH}.txt
+) > /config/apt-hashes-${ARCH}.list

-cp /dev/null /config/toolchain/packages-${ARCH}.list
 for deb in /var/cache/apt/archives/*.deb; do
 	package=$(dpkg-deb -f $deb Package);
 	version=$(dpkg --info ${deb} | grep "^ Version: " | sed 's/^ Version: //g');
-	echo "${package}=${version}" >> /config/toolchain/packages-${ARCH}.list;
+	echo "${package}=${version}" >> /config/apt-pins-${ARCH}.list;
 done

-chown -R $LOCAL_USER /config/toolchain
+snapshot_url="http://snapshot.debian.org/archive/debian"
+snapshot_date=$(date +"%Y%m%dT000000Z")
+cat <<-EOF > /config/apt-sources-x86_64.list
+deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
+deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
+deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
+EOF
+chown -R $LOCAL_USER /config/