cache debian artifacts in fetch directory

This commit is contained in:
Lance Vick 2023-06-15 23:23:10 -07:00
parent 388334ef6a
commit 5bbf26be78
Signed by: lrvick
GPG Key ID: 8E47A1EC35A1551D
5 changed files with 99 additions and 44 deletions

View File

@ -1,5 +1,5 @@
ARG DEBIAN_HASH
FROM debian@sha256:${DEBIAN_HASH}
FROM debian@sha256:${DEBIAN_HASH} as build-base
ARG CONFIG_DIR
ADD ${CONFIG_DIR} /config
@ -7,4 +7,6 @@ ADD ${CONFIG_DIR} /config
ARG SCRIPTS_DIR
ADD ${SCRIPTS_DIR} /usr/local/bin
RUN packages-install
ARG FETCH_DIR
RUN --mount=type=bind,source=fetch,target=/fetch,rw \
packages-install

View File

@ -86,10 +86,21 @@ toolchain: \
toolchain-shell: toolchain
$(call toolchain,bash --norc,--interactive)
# Pin all packages in toolchain container to latest versions
.PHONY: toolchain-update
toolchain-update:
docker run \
rm \
$(CONFIG_DIR)/apt-pins-x86_64.list \
$(CONFIG_DIR)/apt-sources-x86_64.list \
$(CONFIG_DIR)/apt-hashes-x86_64.list
$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list \
# Regenerate toolchain dependency packages to latest versions
$(CONFIG_DIR)/apt-base.list \
$(CONFIG_DIR)/apt-pins-x86_64.list \
$(CONFIG_DIR)/apt-sources-x86_64.list \
$(CONFIG_DIR)/apt-hashes-x86_64.list:
mkdir -p $(FETCH_DIR)/apt \
&& docker run \
--rm \
--tty \
--platform=linux/$(ARCH) \
@ -102,6 +113,24 @@ toolchain-update:
debian@sha256:$(DEBIAN_HASH) \
/usr/local/bin/packages-update
# Pin all packages in toolchain container to latest versions
$(FETCH_DIR)/apt/Packages.gz:
docker run \
--rm \
--tty \
--platform=linux/$(ARCH) \
--env LOCAL_USER=$(UID):$(GID) \
--env FETCH_DIR="$(FETCH_DIR)" \
--env PACKAGES_LATEST=$(PACKAGES_LATEST) \
--volume $(PWD)/$(CONFIG_DIR):/config \
--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
--volume $(PWD)/$(FETCH_DIR):/fetch \
--cpus $(CPUS) \
--volume $(TOOLCHAIN_VOLUME) \
--workdir $(TOOLCHAIN_WORKDIR) \
debian@sha256:$(DEBIAN_HASH) \
/usr/local/bin/packages-fetch
.PHONY: toolchain-clean
toolchain-clean:
if [ -d "$(CACHE_DIR_ROOT)" ]; then \
@ -182,16 +211,18 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
$(CACHE_DIR_ROOT)/toolchain.tar: \
$(CONFIG_DIR)/toolchain.env \
$(SRC_DIR)/toolchain/Dockerfile \
$(CONFIG_DIR)/toolchain/package-hashes-$(ARCH).txt \
$(CONFIG_DIR)/toolchain/packages-base.list \
$(CONFIG_DIR)/toolchain/packages-$(ARCH).list \
$(CONFIG_DIR)/toolchain/sources.list
$(CONFIG_DIR)/apt-base.list \
$(CONFIG_DIR)/apt-sources-$(ARCH).list \
$(CONFIG_DIR)/apt-pins-$(ARCH).list \
$(CONFIG_DIR)/apt-hashes-$(ARCH).list \
$(FETCH_DIR)/apt/Packages.gz
mkdir -p $(CACHE_DIR)
DOCKER_BUILDKIT=1 \
docker build \
--tag $(IMAGE) \
--build-arg DEBIAN_HASH=$(DEBIAN_HASH) \
--build-arg CONFIG_DIR=$(CONFIG_DIR) \
--build-arg FETCH_DIR=$(PWD)/$(FETCH_DIR) \
--build-arg SCRIPTS_DIR=$(SRC_DIR)/toolchain/scripts \
--platform=linux/$(ARCH) \
-f $(SRC_DIR)/toolchain/Dockerfile \

34
scripts/packages-fetch Executable file
View File

@ -0,0 +1,34 @@
#!/bin/bash
[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
set -e
ARCH=$(uname -m)
cp /config/* /etc/apt/
apt update -o Acquire::Check-Valid-Until=false
until apt-get install \
--download-only \
--allow-downgrades \
-o Acquire::Check-Valid-Until=false \
-y $(cat /etc/apt/apt-pins-${ARCH}.list);
do
echo "apt install failed. Likely throttled. Retrying in 10 mins...";
sleep 600;
done;
(
cd /var/cache/apt/archives \
&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
| sed 's/.\///g' \
| LC_ALL=C sort
) > /etc/apt/apt-hashes-${ARCH}-compare.list
diff /etc/apt/apt-hashes-${ARCH}{,-compare}.list
mkdir -p /fetch/apt
mv /var/cache/apt/archives/*.deb /fetch/apt/
apt-get install -y dpkg-dev
env -C /fetch dpkg-scanpackages apt | gzip > /fetch/apt/Packages.gz

View File

@ -3,34 +3,19 @@ set -e;
ARCH=$(uname -m)
cp /config/toolchain/* /etc/apt/
cp /config/* /etc/apt/
apt-get update -o Acquire::Check-Valid-Until=false
apt-get install debian-archive-keyring
cat <<-EOF > /etc/apt/sources.list
deb [trusted=yes] file:///fetch apt/
EOF
rm /etc/apt/sources.list.d/*
until apt-get install \
--download-only \
--reinstall \
--allow-downgrades \
-o Acquire::Check-Valid-Until=false \
-y $(cat /etc/apt/packages-${ARCH}.list);
do
echo "apt install failed. Likely throttled. Retrying in 10 mins...";
sleep 600;
done;
(
cd /var/cache/apt/archives \
&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
| sed 's/.\///g' \
| LC_ALL=C sort
) > /etc/apt/package-hashes-${ARCH}-compare.txt
diff /etc/apt/package-hashes-${ARCH}{,-compare}.txt
apt update -o Acquire::Check-Valid-Until=false
apt-get install \
--allow-downgrades \
-y $(cat /etc/apt/packages-${ARCH}.list)
-y $(cat /etc/apt/apt-pins-${ARCH}.list)
rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*;
echo "%sudo ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers

View File

@ -3,20 +3,17 @@
[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
set -e
snapshot_url="http://snapshot.debian.org/archive/debian"
snapshot_date=$(date +"%Y%m%dT000000Z")
cat <<-EOF > /etc/apt/sources.list
deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
deb http://deb.debian.org/debian bookworm main
deb http://security.debian.org/debian-security bookworm-security main
deb http://deb.debian.org/debian bookworm-updates main
EOF
cp /etc/apt/sources.list /config/toolchain/
rm /etc/apt/sources.list.d/*
ARCH=$(uname -m)
apt-get update
apt-get install -y --download-only --reinstall $( \
apt-get install -y --download-only $( \
dpkg-query \
-W \
-f='${db:Status-Abbrev}\t${binary:Package} - ${binary:Summary}\n' \
@ -26,20 +23,26 @@ apt-get install -y --download-only --reinstall $( \
apt-get install \
-y \
--download-only \
sudo gettext \
$(cat /config/toolchain/packages-base.list)
sudo gettext dpkg-dev \
$(cat /config/apt-base.list)
( cd /var/cache/apt/archives \
&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
| sed 's/.\///g' \
| LC_ALL=C sort
) > /config/toolchain/package-hashes-${ARCH}.txt
) > /config/apt-hashes-${ARCH}.list
cp /dev/null /config/toolchain/packages-${ARCH}.list
for deb in /var/cache/apt/archives/*.deb; do
package=$(dpkg-deb -f $deb Package);
version=$(dpkg --info ${deb} | grep "^ Version: " | sed 's/^ Version: //g');
echo "${package}=${version}" >> /config/toolchain/packages-${ARCH}.list;
echo "${package}=${version}" >> /config/apt-pins-${ARCH}.list;
done
chown -R $LOCAL_USER /config/toolchain
snapshot_url="http://snapshot.debian.org/archive/debian"
snapshot_date=$(date +"%Y%m%dT000000Z")
cat <<-EOF > /config/apt-sources-x86_64.list
deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
EOF
chown -R $LOCAL_USER /config/