From 5bbf26be78a05fe504c5aeb989d16bfa44383253 Mon Sep 17 00:00:00 2001 From: "Lance R. Vick" Date: Thu, 15 Jun 2023 23:23:10 -0700 Subject: [PATCH] cache debian artifacts in fetch directory --- Dockerfile | 6 ++++-- Makefile | 43 ++++++++++++++++++++++++++++++++++------ scripts/packages-fetch | 34 +++++++++++++++++++++++++++++++ scripts/packages-install | 31 ++++++++--------------------- scripts/packages-update | 29 +++++++++++++++------------ 5 files changed, 99 insertions(+), 44 deletions(-) create mode 100755 scripts/packages-fetch diff --git a/Dockerfile b/Dockerfile index 021ea14..bbe3e4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG DEBIAN_HASH -FROM debian@sha256:${DEBIAN_HASH} +FROM debian@sha256:${DEBIAN_HASH} as build-base ARG CONFIG_DIR ADD ${CONFIG_DIR} /config @@ -7,4 +7,6 @@ ADD ${CONFIG_DIR} /config ARG SCRIPTS_DIR ADD ${SCRIPTS_DIR} /usr/local/bin -RUN packages-install +ARG FETCH_DIR +RUN --mount=type=bind,source=fetch,target=/fetch,rw \ + packages-install diff --git a/Makefile b/Makefile index f5a9db5..6c8740d 100644 --- a/Makefile +++ b/Makefile @@ -86,10 +86,21 @@ toolchain: \ toolchain-shell: toolchain $(call toolchain,bash --norc,--interactive) -# Pin all packages in toolchain container to latest versions .PHONY: toolchain-update toolchain-update: - docker run \ + rm \ + $(CONFIG_DIR)/apt-pins-x86_64.list \ + $(CONFIG_DIR)/apt-sources-x86_64.list \ + $(CONFIG_DIR)/apt-hashes-x86_64.list + $(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list \ + +# Regenerate toolchain dependency packages to latest versions +$(CONFIG_DIR)/apt-base.list \ +$(CONFIG_DIR)/apt-pins-x86_64.list \ +$(CONFIG_DIR)/apt-sources-x86_64.list \ +$(CONFIG_DIR)/apt-hashes-x86_64.list: + mkdir -p $(FETCH_DIR)/apt \ + && docker run \ --rm \ --tty \ --platform=linux/$(ARCH) \ @@ -102,6 +113,24 @@ toolchain-update: debian@sha256:$(DEBIAN_HASH) \ /usr/local/bin/packages-update +# Pin all packages in toolchain container to latest versions +$(FETCH_DIR)/apt/Packages.gz: + docker run \ + --rm \ + --tty \ + --platform=linux/$(ARCH) \ + --env LOCAL_USER=$(UID):$(GID) \ + --env FETCH_DIR="$(FETCH_DIR)" \ + --env PACKAGES_LATEST=$(PACKAGES_LATEST) \ + --volume $(PWD)/$(CONFIG_DIR):/config \ + --volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \ + --volume $(PWD)/$(FETCH_DIR):/fetch \ + --cpus $(CPUS) \ + --volume $(TOOLCHAIN_VOLUME) \ + --workdir $(TOOLCHAIN_WORKDIR) \ + debian@sha256:$(DEBIAN_HASH) \ + /usr/local/bin/packages-fetch + .PHONY: toolchain-clean toolchain-clean: if [ -d "$(CACHE_DIR_ROOT)" ]; then \ @@ -182,16 +211,18 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \ $(CACHE_DIR_ROOT)/toolchain.tar: \ $(CONFIG_DIR)/toolchain.env \ $(SRC_DIR)/toolchain/Dockerfile \ - $(CONFIG_DIR)/toolchain/package-hashes-$(ARCH).txt \ - $(CONFIG_DIR)/toolchain/packages-base.list \ - $(CONFIG_DIR)/toolchain/packages-$(ARCH).list \ - $(CONFIG_DIR)/toolchain/sources.list + $(CONFIG_DIR)/apt-base.list \ + $(CONFIG_DIR)/apt-sources-$(ARCH).list \ + $(CONFIG_DIR)/apt-pins-$(ARCH).list \ + $(CONFIG_DIR)/apt-hashes-$(ARCH).list \ + $(FETCH_DIR)/apt/Packages.gz mkdir -p $(CACHE_DIR) DOCKER_BUILDKIT=1 \ docker build \ --tag $(IMAGE) \ --build-arg DEBIAN_HASH=$(DEBIAN_HASH) \ --build-arg CONFIG_DIR=$(CONFIG_DIR) \ + --build-arg FETCH_DIR=$(PWD)/$(FETCH_DIR) \ --build-arg SCRIPTS_DIR=$(SRC_DIR)/toolchain/scripts \ --platform=linux/$(ARCH) \ -f $(SRC_DIR)/toolchain/Dockerfile \ diff --git a/scripts/packages-fetch b/scripts/packages-fetch new file mode 100755 index 0000000..9e15744 --- /dev/null +++ b/scripts/packages-fetch @@ -0,0 +1,34 @@ +#!/bin/bash + +[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; } +set -e + +ARCH=$(uname -m) + +cp /config/* /etc/apt/ +apt update -o Acquire::Check-Valid-Until=false + +until apt-get install \ + --download-only \ + --allow-downgrades \ + -o Acquire::Check-Valid-Until=false \ + -y $(cat /etc/apt/apt-pins-${ARCH}.list); +do + echo "apt install failed. Likely throttled. Retrying in 10 mins..."; + sleep 600; +done; + +( + cd /var/cache/apt/archives \ + && find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \ + | sed 's/.\///g' \ + | LC_ALL=C sort +) > /etc/apt/apt-hashes-${ARCH}-compare.list + +diff /etc/apt/apt-hashes-${ARCH}{,-compare}.list + +mkdir -p /fetch/apt + +mv /var/cache/apt/archives/*.deb /fetch/apt/ +apt-get install -y dpkg-dev +env -C /fetch dpkg-scanpackages apt | gzip > /fetch/apt/Packages.gz diff --git a/scripts/packages-install b/scripts/packages-install index f09a1cd..bc1365f 100755 --- a/scripts/packages-install +++ b/scripts/packages-install @@ -3,34 +3,19 @@ set -e; ARCH=$(uname -m) -cp /config/toolchain/* /etc/apt/ +cp /config/* /etc/apt/ -apt-get update -o Acquire::Check-Valid-Until=false -apt-get install debian-archive-keyring +cat <<-EOF > /etc/apt/sources.list +deb [trusted=yes] file:///fetch apt/ +EOF +rm /etc/apt/sources.list.d/* -until apt-get install \ - --download-only \ - --reinstall \ - --allow-downgrades \ - -o Acquire::Check-Valid-Until=false \ - -y $(cat /etc/apt/packages-${ARCH}.list); -do - echo "apt install failed. Likely throttled. Retrying in 10 mins..."; - sleep 600; -done; - -( - cd /var/cache/apt/archives \ - && find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \ - | sed 's/.\///g' \ - | LC_ALL=C sort -) > /etc/apt/package-hashes-${ARCH}-compare.txt - -diff /etc/apt/package-hashes-${ARCH}{,-compare}.txt +apt update -o Acquire::Check-Valid-Until=false apt-get install \ --allow-downgrades \ - -y $(cat /etc/apt/packages-${ARCH}.list) + -y $(cat /etc/apt/apt-pins-${ARCH}.list) + rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*; echo "%sudo ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers diff --git a/scripts/packages-update b/scripts/packages-update index 6516a25..5d85b20 100755 --- a/scripts/packages-update +++ b/scripts/packages-update @@ -3,20 +3,17 @@ [ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; } set -e -snapshot_url="http://snapshot.debian.org/archive/debian" -snapshot_date=$(date +"%Y%m%dT000000Z") cat <<-EOF > /etc/apt/sources.list -deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main -deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main -deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main +deb http://deb.debian.org/debian bookworm main +deb http://security.debian.org/debian-security bookworm-security main +deb http://deb.debian.org/debian bookworm-updates main EOF -cp /etc/apt/sources.list /config/toolchain/ rm /etc/apt/sources.list.d/* ARCH=$(uname -m) apt-get update -apt-get install -y --download-only --reinstall $( \ +apt-get install -y --download-only $( \ dpkg-query \ -W \ -f='${db:Status-Abbrev}\t${binary:Package} - ${binary:Summary}\n' \ @@ -26,20 +23,26 @@ apt-get install -y --download-only --reinstall $( \ apt-get install \ -y \ --download-only \ - sudo gettext \ - $(cat /config/toolchain/packages-base.list) + sudo gettext dpkg-dev \ + $(cat /config/apt-base.list) ( cd /var/cache/apt/archives \ && find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \ | sed 's/.\///g' \ | LC_ALL=C sort -) > /config/toolchain/package-hashes-${ARCH}.txt +) > /config/apt-hashes-${ARCH}.list -cp /dev/null /config/toolchain/packages-${ARCH}.list for deb in /var/cache/apt/archives/*.deb; do package=$(dpkg-deb -f $deb Package); version=$(dpkg --info ${deb} | grep "^ Version: " | sed 's/^ Version: //g'); - echo "${package}=${version}" >> /config/toolchain/packages-${ARCH}.list; + echo "${package}=${version}" >> /config/apt-pins-${ARCH}.list; done -chown -R $LOCAL_USER /config/toolchain +snapshot_url="http://snapshot.debian.org/archive/debian" +snapshot_date=$(date +"%Y%m%dT000000Z") +cat <<-EOF > /config/apt-sources-x86_64.list +deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main +deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main +deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main +EOF +chown -R $LOCAL_USER /config/