initial commit
This commit is contained in:
commit
5191085576
GPG Key ID:
8E47A1EC35A1551D
|
|
@ -0,0 +1,18 @@
|
|||
ARG DEBIAN_HASH
|
||||
FROM debian@sha256:${DEBIAN_HASH}
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
LANG=C.UTF-8 \
|
||||
TZ=UTC \
|
||||
PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
ARG CONFIG_DIR
|
||||
ADD ${CONFIG_DIR} /config
|
||||
|
||||
ARG SCRIPTS_DIR
|
||||
ADD ${SCRIPTS_DIR} /usr/local/bin
|
||||
|
||||
RUN packages-install
|
||||
|
||||
RUN echo "/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1" \
|
||||
> /etc/ld.so.preload
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
NAME := $(shell basename $(shell git rev-parse --show-toplevel))
|
||||
IMAGE := local/$(NAME):latest
|
||||
ARCH := x86_64
|
||||
TARGET := $(ARCH)
|
||||
USER := $(shell id -u):$(shell id -g)
|
||||
CPUS := $(shell docker run -it debian nproc)
|
||||
GIT_REF := $(shell git log -1 --format=%H config)
|
||||
GIT_AUTHOR := $(shell git log -1 --format=%an config)
|
||||
GIT_KEY := $(shell git log -1 --format=%GP config)
|
||||
GIT_EPOCH := $(shell git log -1 --format=%at config)
|
||||
GIT_DATETIME := \
|
||||
$(shell git log -1 --format=%cd --date=format:'%Y-%m-%d %H:%M:%S' config)
|
||||
ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)
|
||||
GIT_STATE=clean
|
||||
else
|
||||
GIT_STATE=dirty
|
||||
endif
|
||||
VERSION := $(shell TZ=UTC0 git show --quiet --date='format-local:%Y%m%dT%H%M%SZ' --format="%cd")
|
||||
RELEASE_DIR := release/$(VERSION)
|
||||
CONFIG_DIR := config
|
||||
CACHE_DIR := cache
|
||||
SRC_DIR := src
|
||||
OUT_DIR := out
|
||||
docker = docker
|
||||
|
||||
include $(CONFIG_DIR)/global.env
|
||||
export $(shell sed 's/=.*//' $(CONFIG_DIR)/global.env)
|
||||
|
||||
## Use env vars from existing release if present
|
||||
ifneq (,$(wildcard $(RELEASE_DIR)/release.env))
|
||||
include $(RELEASE_DIR)/release.env
|
||||
export
|
||||
endif
|
||||
|
||||
executables = $(docker) git patch
|
||||
|
||||
.PHONY: toolchain
|
||||
toolchain: $(CACHE_DIR)/toolchain.tar $(CACHE_DIR)/toolchain.env
|
||||
|
||||
# Launch a shell inside the toolchain container
|
||||
.PHONY: toolchain-shell
|
||||
toolchain-shell: toolchain
|
||||
$(call toolchain,$(USER),"bash --norc")
|
||||
|
||||
# Pin all packages in toolchain container to latest versions
|
||||
.PHONY: toolchain-update
|
||||
toolchain-update:
|
||||
docker run \
|
||||
--rm \
|
||||
--env LOCAL_USER=$(USER) \
|
||||
--platform=linux/$(ARCH) \
|
||||
--volume $(PWD)/$(CONFIG_DIR):/config \
|
||||
--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
|
||||
--env ARCH=$(ARCH) \
|
||||
--interactive \
|
||||
--tty \
|
||||
debian@sha256:$(DEBIAN_HASH) \
|
||||
bash -c /usr/local/bin/packages-update
|
||||
|
||||
.PHONY: attest
|
||||
attest:
|
||||
rm -rf $(CACHE_DIR) $(OUT_DIR)
|
||||
$(MAKE)
|
||||
diff -q $(OUT_DIR)/manifest.txt release/$(VERSION)/manifest.txt;
|
||||
|
||||
$(RELEASE_DIR):
|
||||
mkdir -p $@
|
||||
|
||||
$(CACHE_DIR):
|
||||
mkdir -p $@
|
||||
|
||||
$(OUT_DIR):
|
||||
mkdir -p $@
|
||||
|
||||
.ONESHELL:
|
||||
$(CACHE_DIR)/toolchain.env: $(CACHE_DIR)
|
||||
cat <<- EOF > $@
|
||||
HOME=/home/build
|
||||
PS1=$(NAME)-toolchain
|
||||
GNUPGHOME=/cache/.gnupg
|
||||
ARCH=$(ARCH)
|
||||
TARGET=$(ARCH)
|
||||
GIT_REF=$(GIT_REF)
|
||||
GIT_AUTHOR=$(GIT_AUTHOR)
|
||||
GIT_KEY=$(GIT_KEY)
|
||||
GIT_DATETIME=$(GIT_DATETIME)
|
||||
GIT_EPOCH=$(GIT_EPOCH)
|
||||
FAKETIME_FMT="%s"
|
||||
FAKETIME="1"
|
||||
SOURCE_DATE_EPOCH=1
|
||||
KBUILD_BUILD_TIMESTAMP="1970-01-01 00:00:00 UTC"
|
||||
KCONFIG_NOTIMESTAMP=1
|
||||
KBUILD_BUILD_USER=root
|
||||
KBUILD_BUILD_HOST=$(NAME)
|
||||
KBUILD_BUILD_VERSION=1
|
||||
UID=$(shell id -u)
|
||||
GID=$(shell id -g)
|
||||
RELEASE_DIR=release/$(VERSION)
|
||||
CONFIG_DIR=/home/build/$(CONFIG_DIR)
|
||||
CACHE_DIR=/home/build/$(CACHE_DIR)
|
||||
SRC_DIR=/home/build/$(SRC_DIR)
|
||||
OUT_DIR=/home/build/$(OUT_DIR)
|
||||
EOF
|
||||
|
||||
$(CACHE_DIR)/toolchain.tar:
|
||||
mkdir -p $(CACHE_DIR)
|
||||
DOCKER_BUILDKIT=1 \
|
||||
docker build \
|
||||
--tag $(IMAGE) \
|
||||
--build-arg DEBIAN_HASH=$(DEBIAN_HASH) \
|
||||
--build-arg CONFIG_DIR=$(CONFIG_DIR) \
|
||||
--build-arg SCRIPTS_DIR=$(SRC_DIR)/toolchain/scripts \
|
||||
--platform=linux/$(ARCH) \
|
||||
-f $(SRC_DIR)/toolchain/Dockerfile \
|
||||
.
|
||||
docker save "$(IMAGE)" -o "$@"
|
||||
|
||||
$(RELEASE_DIR)/release.env: \
|
||||
$(RELEASE_DIR) \
|
||||
$(OUT_DIR)/release.env
|
||||
cp $(OUT_DIR)/release.env $(RELEASE_DIR)/release.env
|
||||
|
||||
$(RELEASE_DIR)/manifest.txt: \
|
||||
$(RELEASE_DIR) \
|
||||
$(OUT_DIR)/manifest.txt
|
||||
cp $(OUT_DIR)/manifest.txt $(RELEASE_DIR)/manifest.txt
|
||||
|
||||
$(OUT_DIR)/release.env: | $(OUT_DIR)
|
||||
echo 'VERSION=$(VERSION)' > $(OUT_DIR)/release.env
|
||||
echo 'GIT_REF=$(GIT_REF)' >> $(OUT_DIR)/release.env
|
||||
echo 'GIT_AUTHOR=$(GIT_AUTHOR)' >> $(OUT_DIR)/release.env
|
||||
echo 'GIT_KEY=$(GIT_KEY)' >> $(OUT_DIR)/release.env
|
||||
echo 'GIT_DATETIME=$(GIT_DATETIME)' >> $(OUT_DIR)/release.env
|
||||
|
||||
$(OUT_DIR)/manifest.txt: | $(OUT_DIR)
|
||||
find $(OUT_DIR) \
|
||||
-type f \
|
||||
-not -path "$(OUT_DIR)/manifest.txt" \
|
||||
-exec openssl sha256 -r {} \; \
|
||||
| sed -e 's/ \*/ /g' -e 's/ \.\// /g' \
|
||||
| LC_ALL=C sort -k2 \
|
||||
> $@
|
||||
|
||||
check_executables := $(foreach exec,$(executables),\$(if \
|
||||
$(shell which $(exec)),some string,$(error "No $(exec) in PATH")))
|
||||
|
||||
define git_clone
|
||||
[ -d $(CACHE_DIR)/$(1) ] || git clone $(2) $(CACHE_DIR)/$(1)
|
||||
git -C $(CACHE_DIR)/$(1) checkout $(3)
|
||||
git -C $(CACHE_DIR)/$(1) rev-parse --verify HEAD | grep -q $(3) || { \
|
||||
echo 'Error: Git ref/branch collision.'; exit 1; \
|
||||
};
|
||||
endef
|
||||
|
||||
define apply_patches
|
||||
[ -d $(2) ] && $(call toolchain,$(USER)," \
|
||||
cd $(1); \
|
||||
git restore .; \
|
||||
find /$(2) -type f -iname '*.patch' -print0 \
|
||||
| xargs -t -0 -n 1 patch -p1 --no-backup-if-mismatch -i ; \
|
||||
")
|
||||
endef
|
||||
|
||||
define toolchain
|
||||
docker load -i $(CACHE_DIR)/toolchain.tar
|
||||
docker run \
|
||||
--rm \
|
||||
--tty \
|
||||
--interactive \
|
||||
--user=$(1) \
|
||||
--platform=linux/$(ARCH) \
|
||||
--cpus $(CPUS) \
|
||||
--volume $(PWD):/home/build \
|
||||
--workdir /home/build \
|
||||
--env-file=$(CONFIG_DIR)/global.env \
|
||||
--env-file=$(CACHE_DIR)/toolchain.env \
|
||||
$(IMAGE) \
|
||||
bash -c $(2)
|
||||
endef
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
# Toolchain #
|
||||
|
||||
<https://codeberg.org/distrust/toolchain>
|
||||
|
||||
## About ##
|
||||
|
||||
A library of opinionated make functions targeting projects that either need
|
||||
deterministic builds, or a shared deterministic toolchain shared across all
|
||||
who use a project.
|
||||
|
||||
A dev of a Toolchain enabled project should never need to have anything
|
||||
on their host system installed but docker, and git. Everything else will be
|
||||
provided via a Docker container.
|
||||
|
||||
Debian currently has the highest reproducibility score of any major Linux
|
||||
distribution, and as such it is the chosen base for Toolchain.
|
||||
|
||||
This was built for Distrust projects, and some of our clients. It is unlikely
|
||||
to meet the needs of everyone. We suggest including this in your project as
|
||||
a git subtree, so you can make your own changes, but also pull in changes from
|
||||
us as desired.
|
||||
|
||||
## Uses ##
|
||||
* Ensure everyone on a team is using the exact same tools
|
||||
* Ensure all releases and artifacts build hash-for-hash identical every time
|
||||
* Control supply chain security with only signed/reproducible dependencies
|
||||
|
||||
## Features ##
|
||||
* Can run a shell with all toolchain tooling in the current directory
|
||||
* Provide make functions for common tasks
|
||||
* Git clone, apply patches, etc.
|
||||
* Use a global env file as configuration
|
||||
* Hash-locking of apt dependencies from a list of top-level required packages
|
||||
* Provides release.env file with required vars to re-create old releases
|
||||
|
||||
## Requirements ##
|
||||
|
||||
* docker 18+
|
||||
* GNU Make 4+
|
||||
|
||||
## Build ##
|
||||
|
||||
### Build a new release
|
||||
|
||||
```
|
||||
make VERSION=1.0.0rc1 release
|
||||
```
|
||||
|
||||
### Reproduce an existing release
|
||||
|
||||
```
|
||||
make VERSION=1.0.0rc1 attest
|
||||
```
|
||||
|
||||
### Sign an existing release
|
||||
|
||||
```
|
||||
make VERSION=1.0.0rc1 sign
|
||||
```
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
uid=${UID?}
|
||||
gid=${GID?}
|
||||
user=${USER:-"build"}
|
||||
export HOME="/home/${user}"
|
||||
|
||||
groupadd -g "$gid" "${user}"
|
||||
useradd \
|
||||
-g "$gid" \
|
||||
-u "$uid" \
|
||||
-md "/home/${user}" \
|
||||
-s /bin/bash \
|
||||
"${user}"
|
||||
|
||||
mkdir -p "$HOME"
|
||||
chown -R "$uid:$gid" "$HOME"
|
||||
cd "$HOME"
|
||||
setpriv --reuid="$uid" --regid="$gid" --init-groups "$@"
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e;
|
||||
|
||||
ARCH=$(uname -m)
|
||||
|
||||
cp /config/toolchain/* /etc/apt/
|
||||
|
||||
apt-get update
|
||||
apt-get install debian-archive-keyring
|
||||
|
||||
until apt-get install --download-only --reinstall --allow-downgrades -y $(cat /etc/apt/packages-${ARCH}.list); do
|
||||
echo "apt install failed. Likely throttled. Retrying in 10 mins...";
|
||||
sleep 600;
|
||||
done;
|
||||
|
||||
(
|
||||
cd /var/cache/apt/archives \
|
||||
&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
|
||||
| sed 's/.\///g' \
|
||||
| LC_ALL=C sort
|
||||
) > /etc/apt/package-hashes-${ARCH}-compare.txt
|
||||
|
||||
diff /etc/apt/package-hashes-${ARCH}{,-compare}.txt
|
||||
|
||||
apt-get install --allow-downgrades -y $(cat /etc/apt/packages-${ARCH}.list)
|
||||
rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*;
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
|
||||
set -e
|
||||
|
||||
snapshot_url="http://snapshot.debian.org/archive/debian"
|
||||
snapshot_date=$(date +"%Y%m%dT000000Z")
|
||||
cat <<-EOF > /etc/apt/sources.list
|
||||
deb http://deb.debian.org/debian bookworm main
|
||||
deb http://security.debian.org/debian-security bookworm-security main
|
||||
deb http://deb.debian.org/debian bookworm-updates main
|
||||
deb [check-valid-until=no] ${snapshot_url}/${snapshot_date} bookworm main
|
||||
deb [check-valid-until=no] ${snapshot_url}-security/${snapshot_date} bookworm-security main
|
||||
deb [check-valid-until=no] ${snapshot_url}/${snapshot_date} bookworm-updates main
|
||||
EOF
|
||||
cp /etc/apt/sources.list /config/toolchain/
|
||||
|
||||
ARCH=$(uname -m)
|
||||
|
||||
apt-get update
|
||||
apt-get install -y --download-only --reinstall $( \
|
||||
dpkg-query \
|
||||
-W \
|
||||
-f='${db:Status-Abbrev}\t${binary:Package} - ${binary:Summary}\n' \
|
||||
| awk -F'\t' '/^ii/ {print $2}' \
|
||||
| awk '{print $1}' \
|
||||
)
|
||||
apt-get install -y --download-only $(cat /config/toolchain/packages-base.list)
|
||||
|
||||
( cd /var/cache/apt/archives \
|
||||
&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
|
||||
| sed 's/.\///g' \
|
||||
| LC_ALL=C sort
|
||||
) > /config/toolchain/package-hashes-${ARCH}.txt
|
||||
|
||||
cp /dev/null /config/toolchain/packages-${ARCH}.list
|
||||
for deb in /var/cache/apt/archives/*.deb; do
|
||||
package=$(dpkg-deb -f $deb Package);
|
||||
version=$(dpkg --info ${deb} | grep "^ Version: " | sed 's/^ Version: //g');
|
||||
echo "${package}=${version}" >> /config/toolchain/packages-${ARCH}.list;
|
||||
done
|
||||
|
||||
chown -R $LOCAL_USER /config/toolchain
|
||||
Reference in New Issue