From ee915227d502daccc4824d92623525973cec90d9 Mon Sep 17 00:00:00 2001 From: "Lance R. Vick" Date: Thu, 1 Jun 2023 18:04:41 -0700 Subject: [PATCH 1/3] separate toolchain.env and global.env. Allow env vars in global.env --- Makefile | 24 +++++++++++++++++------- scripts/environment | 6 ++++++ scripts/packages-update | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 96e1487..777c64a 100644 --- a/Makefile +++ b/Makefile @@ -50,8 +50,8 @@ export docker = docker -include $(CONFIG_DIR)/global.env -export $(shell sed 's/=.*//' $(CONFIG_DIR)/global.env) +include $(CONFIG_DIR)/toolchain.env +export $(shell sed 's/=.*//' $(CONFIG_DIR)/toolchain.env) ## Use env vars from existing release if present ifneq (,$(wildcard $(DIST_DIR)/release.env)) @@ -130,11 +130,22 @@ $(OUT_DIR): mkdir -p $@ $(CACHE_DIR_ROOT)/toolchain.env: \ - $(CACHE_DIR) \ - $(SRC_DIR)/toolchain/scripts/environment - $(SRC_DIR)/toolchain/scripts/environment > $@ + $(CACHE_DIR_ROOT)/toolchain.state + env > $(CACHE_DIR)/bootstrap.env + docker run \ + --rm \ + --env UID=$(UID) \ + --env GID=$(GID) \ + --env-file $(CACHE_DIR)/bootstrap.env \ + --platform=linux/$(ARCH) \ + --volume $(TOOLCHAIN_VOLUME) \ + --workdir $(TOOLCHAIN_WORKDIR) \ + $(shell cat cache/toolchain.state 2> /dev/null) \ + $(SRC_DIR)/toolchain/scripts/environment > $@ + rm $(CACHE_DIR)/bootstrap.env $(CACHE_DIR_ROOT)/toolchain.tar: \ + $(CONFIG_DIR)/toolchain.env \ $(SRC_DIR)/toolchain/Dockerfile \ $(CONFIG_DIR)/toolchain/package-hashes-$(ARCH).txt \ $(CONFIG_DIR)/toolchain/packages-base.list \ @@ -222,7 +233,7 @@ endef define toolchain docker run \ - --rm \ + --rm \ --tty \ $(2) \ --env UID=$(UID) \ @@ -232,7 +243,6 @@ define toolchain --cpus $(CPUS) \ --volume $(TOOLCHAIN_VOLUME) \ --workdir $(TOOLCHAIN_WORKDIR) \ - --env-file=$(CONFIG_DIR)/global.env \ --env-file=$(CACHE_DIR_ROOT)/toolchain.env \ $(shell cat cache/toolchain.state 2> /dev/null) \ $(SRC_DIR)/toolchain/scripts/host-env bash -c $(1) diff --git a/scripts/environment b/scripts/environment index 2bb6cd2..ed939ce 100755 --- a/scripts/environment +++ b/scripts/environment @@ -1,6 +1,10 @@ #!/bin/sh HOME=/home/build +CONFIG_DIR=/home/build/config + +cat ${CONFIG_DIR}/toolchain.env + cat <<- EOF HOME=${HOME} PATH=${HOME}/${BIN_DIR}:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin @@ -33,3 +37,5 @@ cat <<- EOF BIN_DIR=${HOME}/${BIN_DIR} FETCH_DIR=${HOME}/${FETCH_DIR} EOF + +envsubst < ${CONFIG_DIR}/global.env diff --git a/scripts/packages-update b/scripts/packages-update index fa0b4f4..8e7bac8 100755 --- a/scripts/packages-update +++ b/scripts/packages-update @@ -28,7 +28,7 @@ apt-get install -y --download-only --reinstall $( \ apt-get install \ -y \ --download-only \ - sudo \ + sudo gettext \ $(cat /config/toolchain/packages-base.list) ( cd /var/cache/apt/archives \ From a8c0099576720fcf3c17a56238d87b21fa0cf089 Mon Sep 17 00:00:00 2001 From: "Lance R. Vick" Date: Thu, 8 Jun 2023 05:37:19 -0700 Subject: [PATCH 2/3] separate generated env files between container and make uses --- Makefile | 29 ++++++++++++++++++++--------- scripts/environment | 9 ++++++--- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 5869bab..cc5e180 100644 --- a/Makefile +++ b/Makefile @@ -45,18 +45,28 @@ BIN_DIR := $(CACHE_DIR_ROOT)/bin SRC_DIR := src KEY_DIR := keys OUT_DIR := out - -export - docker = docker include $(CONFIG_DIR)/toolchain.env export $(shell sed 's/=.*//' $(CONFIG_DIR)/toolchain.env) +export + +AUTOBUILD_TOOLCHAIN := true +ifeq ($(AUTOBUILD_TOOLCHAIN),true) +ifeq ("$(wildcard $(CACHE_DIR_ROOT)/make.env)","") + echo := $(info $(shell echo "Initializing toolchain.")) + build_env := $(shell $(MAKE) AUTOBUILD_TOOLCHAIN=false toolchain ) +endif +endif +ifneq (,$(wildcard $(CACHE_DIR_ROOT)/make.env)) + include $(CACHE_DIR_ROOT)/make.env + export $(shell sed 's/=.*//' $(CACHE_DIR_ROOT)/make.env) +endif ## Use env vars from existing release if present ifneq (,$(wildcard $(DIST_DIR)/release.env)) - include $(DIST_DIR)/release.env - export + include $(DIST_DIR)/release.env + export endif executables = $(docker) git git-lfs patch @@ -68,7 +78,8 @@ toolchain: \ $(BIN_DIR) \ $(OUT_DIR) \ $(CACHE_DIR_ROOT)/toolchain.state \ - $(CACHE_DIR_ROOT)/toolchain.env + $(CACHE_DIR_ROOT)/container.env \ + $(CACHE_DIR_ROOT)/make.env # Launch a shell inside the toolchain container .PHONY: toolchain-shell @@ -129,7 +140,7 @@ $(FETCH_DIR): $(OUT_DIR): mkdir -p $@ -$(CACHE_DIR_ROOT)/toolchain.env: \ +$(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \ $(CACHE_DIR_ROOT)/toolchain.state env > $(CACHE_DIR)/bootstrap.env docker run \ @@ -141,7 +152,7 @@ $(CACHE_DIR_ROOT)/toolchain.env: \ --volume $(TOOLCHAIN_VOLUME) \ --workdir $(TOOLCHAIN_WORKDIR) \ $(shell cat cache/toolchain.state 2> /dev/null) \ - $(SRC_DIR)/toolchain/scripts/environment > $@ + $(SRC_DIR)/toolchain/scripts/environment $(CACHE_DIR_ROOT) rm $(CACHE_DIR)/bootstrap.env $(CACHE_DIR_ROOT)/toolchain.tar: \ @@ -234,7 +245,7 @@ define toolchain --cpus $(CPUS) \ --volume $(TOOLCHAIN_VOLUME) \ --workdir $(TOOLCHAIN_WORKDIR) \ - --env-file=$(CACHE_DIR_ROOT)/toolchain.env \ + --env-file=$(CACHE_DIR_ROOT)/container.env \ $(shell cat cache/toolchain.state 2> /dev/null) \ $(SRC_DIR)/toolchain/scripts/host-env bash -c $(1) endef diff --git a/scripts/environment b/scripts/environment index ed939ce..5f41ccc 100755 --- a/scripts/environment +++ b/scripts/environment @@ -1,11 +1,12 @@ #!/bin/sh +CACHE_DIR_ROOT=${1?} HOME=/home/build CONFIG_DIR=/home/build/config -cat ${CONFIG_DIR}/toolchain.env +cat ${CONFIG_DIR}/toolchain.env > ${CACHE_DIR_ROOT}/container.env -cat <<- EOF +cat <<- EOF >> ${CACHE_DIR_ROOT}/container.env HOME=${HOME} PATH=${HOME}/${BIN_DIR}:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin TZ=UTC @@ -38,4 +39,6 @@ cat <<- EOF FETCH_DIR=${HOME}/${FETCH_DIR} EOF -envsubst < ${CONFIG_DIR}/global.env +envsubst < ${CONFIG_DIR}/global.env > ${CACHE_DIR_ROOT}/make.env + +cat ${CACHE_DIR_ROOT}/make.env >> ${CACHE_DIR_ROOT}/container.env From 42f1d5850ce40fb6a86477f913a0fd03e555d00f Mon Sep 17 00:00:00 2001 From: "Lance R. Vick" Date: Fri, 9 Jun 2023 16:08:03 -0700 Subject: [PATCH 3/3] fetch file support --- Makefile | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cc5e180..8f9333b 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ VERSION := $(shell TZ=UTC0 git show --quiet --date='format-local:%Y.%m.%d' --for DIST_DIR := dist CONFIG_DIR := config CACHE_DIR_ROOT := cache -FETCH_DIR := $(CACHE_DIR_ROOT)/fetch +FETCH_DIR := fetch ifeq ($(TARGET),$(ARCH)) CACHE_DIR := $(CACHE_DIR_ROOT)/$(TARGET) else @@ -141,6 +141,8 @@ $(OUT_DIR): mkdir -p $@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \ + $(CONFIG_DIR)/global.env \ + $(CONFIG_DIR)/toolchain.env \ $(CACHE_DIR_ROOT)/toolchain.state env > $(CACHE_DIR)/bootstrap.env docker run \ @@ -189,6 +191,22 @@ $(OUT_DIR)/release.env: | $(OUT_DIR) check_executables := $(foreach exec,$(executables),\$(if \ $(shell which $(exec)),some string,$(error "No $(exec) in PATH"))) +define sha256_file +$$(openssl sha256 $(1) | awk '{ print $$2}') +endef + +define fetch_file + bash -c " \ + echo \"Fetching $(1)\" \ + && curl \ + --location $(1) \ + --output $(CACHE_DIR)/$(notdir $@) \ + && [[ "\""$(call sha256_file,$(CACHE_DIR)/$(notdir $@))"\"" == "\""$(2)"\"" ]] \ + || { echo 'Error: Hash check failed'; exit 1; } \ + && mv $(CACHE_DIR)/$(notdir $@) $@; \ + " +endef + define git_clone [ -d $(1) ] || \ mkdir -p $(FETCH_DIR) && \