cache debian artifacts in fetch directory
This commit is contained in:
parent
388334ef6a
commit
5bbf26be78
|
@ -1,5 +1,5 @@
|
|||
ARG DEBIAN_HASH
|
||||
FROM debian@sha256:${DEBIAN_HASH}
|
||||
FROM debian@sha256:${DEBIAN_HASH} as build-base
|
||||
|
||||
ARG CONFIG_DIR
|
||||
ADD ${CONFIG_DIR} /config
|
||||
|
@ -7,4 +7,6 @@ ADD ${CONFIG_DIR} /config
|
|||
ARG SCRIPTS_DIR
|
||||
ADD ${SCRIPTS_DIR} /usr/local/bin
|
||||
|
||||
RUN packages-install
|
||||
ARG FETCH_DIR
|
||||
RUN --mount=type=bind,source=fetch,target=/fetch,rw \
|
||||
packages-install
|
||||
|
|
43
Makefile
43
Makefile
|
@ -86,10 +86,21 @@ toolchain: \
|
|||
toolchain-shell: toolchain
|
||||
$(call toolchain,bash --norc,--interactive)
|
||||
|
||||
# Pin all packages in toolchain container to latest versions
|
||||
.PHONY: toolchain-update
|
||||
toolchain-update:
|
||||
docker run \
|
||||
rm \
|
||||
$(CONFIG_DIR)/apt-pins-x86_64.list \
|
||||
$(CONFIG_DIR)/apt-sources-x86_64.list \
|
||||
$(CONFIG_DIR)/apt-hashes-x86_64.list
|
||||
$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list \
|
||||
|
||||
# Regenerate toolchain dependency packages to latest versions
|
||||
$(CONFIG_DIR)/apt-base.list \
|
||||
$(CONFIG_DIR)/apt-pins-x86_64.list \
|
||||
$(CONFIG_DIR)/apt-sources-x86_64.list \
|
||||
$(CONFIG_DIR)/apt-hashes-x86_64.list:
|
||||
mkdir -p $(FETCH_DIR)/apt \
|
||||
&& docker run \
|
||||
--rm \
|
||||
--tty \
|
||||
--platform=linux/$(ARCH) \
|
||||
|
@ -102,6 +113,24 @@ toolchain-update:
|
|||
debian@sha256:$(DEBIAN_HASH) \
|
||||
/usr/local/bin/packages-update
|
||||
|
||||
# Pin all packages in toolchain container to latest versions
|
||||
$(FETCH_DIR)/apt/Packages.gz:
|
||||
docker run \
|
||||
--rm \
|
||||
--tty \
|
||||
--platform=linux/$(ARCH) \
|
||||
--env LOCAL_USER=$(UID):$(GID) \
|
||||
--env FETCH_DIR="$(FETCH_DIR)" \
|
||||
--env PACKAGES_LATEST=$(PACKAGES_LATEST) \
|
||||
--volume $(PWD)/$(CONFIG_DIR):/config \
|
||||
--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
|
||||
--volume $(PWD)/$(FETCH_DIR):/fetch \
|
||||
--cpus $(CPUS) \
|
||||
--volume $(TOOLCHAIN_VOLUME) \
|
||||
--workdir $(TOOLCHAIN_WORKDIR) \
|
||||
debian@sha256:$(DEBIAN_HASH) \
|
||||
/usr/local/bin/packages-fetch
|
||||
|
||||
.PHONY: toolchain-clean
|
||||
toolchain-clean:
|
||||
if [ -d "$(CACHE_DIR_ROOT)" ]; then \
|
||||
|
@ -182,16 +211,18 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
|
|||
$(CACHE_DIR_ROOT)/toolchain.tar: \
|
||||
$(CONFIG_DIR)/toolchain.env \
|
||||
$(SRC_DIR)/toolchain/Dockerfile \
|
||||
$(CONFIG_DIR)/toolchain/package-hashes-$(ARCH).txt \
|
||||
$(CONFIG_DIR)/toolchain/packages-base.list \
|
||||
$(CONFIG_DIR)/toolchain/packages-$(ARCH).list \
|
||||
$(CONFIG_DIR)/toolchain/sources.list
|
||||
$(CONFIG_DIR)/apt-base.list \
|
||||
$(CONFIG_DIR)/apt-sources-$(ARCH).list \
|
||||
$(CONFIG_DIR)/apt-pins-$(ARCH).list \
|
||||
$(CONFIG_DIR)/apt-hashes-$(ARCH).list \
|
||||
$(FETCH_DIR)/apt/Packages.gz
|
||||
mkdir -p $(CACHE_DIR)
|
||||
DOCKER_BUILDKIT=1 \
|
||||
docker build \
|
||||
--tag $(IMAGE) \
|
||||
--build-arg DEBIAN_HASH=$(DEBIAN_HASH) \
|
||||
--build-arg CONFIG_DIR=$(CONFIG_DIR) \
|
||||
--build-arg FETCH_DIR=$(PWD)/$(FETCH_DIR) \
|
||||
--build-arg SCRIPTS_DIR=$(SRC_DIR)/toolchain/scripts \
|
||||
--platform=linux/$(ARCH) \
|
||||
-f $(SRC_DIR)/toolchain/Dockerfile \
|
||||
|
|
|
@ -0,0 +1,34 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
|
||||
set -e
|
||||
|
||||
ARCH=$(uname -m)
|
||||
|
||||
cp /config/* /etc/apt/
|
||||
apt update -o Acquire::Check-Valid-Until=false
|
||||
|
||||
until apt-get install \
|
||||
--download-only \
|
||||
--allow-downgrades \
|
||||
-o Acquire::Check-Valid-Until=false \
|
||||
-y $(cat /etc/apt/apt-pins-${ARCH}.list);
|
||||
do
|
||||
echo "apt install failed. Likely throttled. Retrying in 10 mins...";
|
||||
sleep 600;
|
||||
done;
|
||||
|
||||
(
|
||||
cd /var/cache/apt/archives \
|
||||
&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
|
||||
| sed 's/.\///g' \
|
||||
| LC_ALL=C sort
|
||||
) > /etc/apt/apt-hashes-${ARCH}-compare.list
|
||||
|
||||
diff /etc/apt/apt-hashes-${ARCH}{,-compare}.list
|
||||
|
||||
mkdir -p /fetch/apt
|
||||
|
||||
mv /var/cache/apt/archives/*.deb /fetch/apt/
|
||||
apt-get install -y dpkg-dev
|
||||
env -C /fetch dpkg-scanpackages apt | gzip > /fetch/apt/Packages.gz
|
|
@ -3,34 +3,19 @@ set -e;
|
|||
|
||||
ARCH=$(uname -m)
|
||||
|
||||
cp /config/toolchain/* /etc/apt/
|
||||
cp /config/* /etc/apt/
|
||||
|
||||
apt-get update -o Acquire::Check-Valid-Until=false
|
||||
apt-get install debian-archive-keyring
|
||||
cat <<-EOF > /etc/apt/sources.list
|
||||
deb [trusted=yes] file:///fetch apt/
|
||||
EOF
|
||||
rm /etc/apt/sources.list.d/*
|
||||
|
||||
until apt-get install \
|
||||
--download-only \
|
||||
--reinstall \
|
||||
--allow-downgrades \
|
||||
-o Acquire::Check-Valid-Until=false \
|
||||
-y $(cat /etc/apt/packages-${ARCH}.list);
|
||||
do
|
||||
echo "apt install failed. Likely throttled. Retrying in 10 mins...";
|
||||
sleep 600;
|
||||
done;
|
||||
|
||||
(
|
||||
cd /var/cache/apt/archives \
|
||||
&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
|
||||
| sed 's/.\///g' \
|
||||
| LC_ALL=C sort
|
||||
) > /etc/apt/package-hashes-${ARCH}-compare.txt
|
||||
|
||||
diff /etc/apt/package-hashes-${ARCH}{,-compare}.txt
|
||||
apt update -o Acquire::Check-Valid-Until=false
|
||||
|
||||
apt-get install \
|
||||
--allow-downgrades \
|
||||
-y $(cat /etc/apt/packages-${ARCH}.list)
|
||||
-y $(cat /etc/apt/apt-pins-${ARCH}.list)
|
||||
|
||||
rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*;
|
||||
|
||||
echo "%sudo ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers
|
||||
|
|
|
@ -3,20 +3,17 @@
|
|||
[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
|
||||
set -e
|
||||
|
||||
snapshot_url="http://snapshot.debian.org/archive/debian"
|
||||
snapshot_date=$(date +"%Y%m%dT000000Z")
|
||||
cat <<-EOF > /etc/apt/sources.list
|
||||
deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
|
||||
deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
|
||||
deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
|
||||
deb http://deb.debian.org/debian bookworm main
|
||||
deb http://security.debian.org/debian-security bookworm-security main
|
||||
deb http://deb.debian.org/debian bookworm-updates main
|
||||
EOF
|
||||
cp /etc/apt/sources.list /config/toolchain/
|
||||
rm /etc/apt/sources.list.d/*
|
||||
|
||||
ARCH=$(uname -m)
|
||||
|
||||
apt-get update
|
||||
apt-get install -y --download-only --reinstall $( \
|
||||
apt-get install -y --download-only $( \
|
||||
dpkg-query \
|
||||
-W \
|
||||
-f='${db:Status-Abbrev}\t${binary:Package} - ${binary:Summary}\n' \
|
||||
|
@ -26,20 +23,26 @@ apt-get install -y --download-only --reinstall $( \
|
|||
apt-get install \
|
||||
-y \
|
||||
--download-only \
|
||||
sudo gettext \
|
||||
$(cat /config/toolchain/packages-base.list)
|
||||
sudo gettext dpkg-dev \
|
||||
$(cat /config/apt-base.list)
|
||||
|
||||
( cd /var/cache/apt/archives \
|
||||
&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
|
||||
| sed 's/.\///g' \
|
||||
| LC_ALL=C sort
|
||||
) > /config/toolchain/package-hashes-${ARCH}.txt
|
||||
) > /config/apt-hashes-${ARCH}.list
|
||||
|
||||
cp /dev/null /config/toolchain/packages-${ARCH}.list
|
||||
for deb in /var/cache/apt/archives/*.deb; do
|
||||
package=$(dpkg-deb -f $deb Package);
|
||||
version=$(dpkg --info ${deb} | grep "^ Version: " | sed 's/^ Version: //g');
|
||||
echo "${package}=${version}" >> /config/toolchain/packages-${ARCH}.list;
|
||||
echo "${package}=${version}" >> /config/apt-pins-${ARCH}.list;
|
||||
done
|
||||
|
||||
chown -R $LOCAL_USER /config/toolchain
|
||||
snapshot_url="http://snapshot.debian.org/archive/debian"
|
||||
snapshot_date=$(date +"%Y%m%dT000000Z")
|
||||
cat <<-EOF > /config/apt-sources-x86_64.list
|
||||
deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
|
||||
deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
|
||||
deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
|
||||
EOF
|
||||
chown -R $LOCAL_USER /config/
|
||||
|
|
Loading…
Reference in New Issue