on: [push, pull_request]
-env:
- APT_SET_CONF: |
- echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
- echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
- echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
-
jobs:
cache-sources:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
fetch-depth: 0
- name: depends sources cache
id: cache
- uses: actions/cache@v3
+ uses: actions/cache@v4
with:
path: contrib/depends/sources
key: sources-${{ hashFiles('contrib/depends/packages/*') }}
if: steps.cache.outputs.cache-hit != 'true'
run: make -C contrib/depends download
- cache-guix:
- runs-on: ubuntu-latest
- needs: [cache-sources]
- steps:
- - uses: actions/checkout@v3
- with:
- fetch-depth: 0
- - name: guix cache
- id: cache
- uses: actions/cache@v3
- with:
- path: |
- gnu/store
- guix_db.sqlite
- key: guix-${{ hashFiles('contrib/guix/manifest.scm') }}
- - name: move guix store
- if: steps.cache.outputs.cache-hit != 'true'
- run: |
- if [[ -e "guix_db.sqlite" ]]; then
- sudo mkdir -p /var/guix/db
- sudo mv gnu /gnu
- sudo cp guix_db.sqlite /var/guix/db/db.sqlite
-
- sudo chmod 1775 /gnu/store
- sudo chown 0644 /var/guix/db/db.sqlite
- sudo chown -R root:root /gnu/store /var/guix/db/db.sqlite
- fi
- - name: depends sources cache
- if: steps.cache.outputs.cache-hit != 'true'
- uses: actions/cache/restore@v3
- with:
- path: contrib/depends/sources
- key: sources-${{ hashFiles('contrib/depends/packages/*') }}
- - name: set apt conf
- if: steps.cache.outputs.cache-hit != 'true'
- run: ${{env.APT_SET_CONF}}
- - name: install dependencies
- if: steps.cache.outputs.cache-hit != 'true'
- run: sudo apt update; sudo apt -y install guix git ca-certificates
- - name: dry run
- if: steps.cache.outputs.cache-hit != 'true'
- run: DRY_RUN=1 SUBSTITUTE_URLS='http://ci.guix.gnu.org' JOBS=2 ./contrib/guix/guix-build
- - name: prepare guix store for caching
- if: steps.cache.outputs.cache-hit != 'true'
- run: |
- sudo systemctl stop guix-daemon
- sudo mv /gnu gnu
- sudo mv /var/guix/db/db.sqlite guix_db.sqlite
-
build-guix:
runs-on: ubuntu-latest
needs: [cache-guix]
fail-fast: false
matrix:
toolchain:
- - name: "x86_64-linux-gnu"
- host: "x86_64-linux-gnu"
- - name: "x86_64-linux-gnu.no-tor-bundle"
- host: "x86_64-linux-gnu.no-tor-bundle"
- - name: "x86_64-linux-gnu.pack"
- host: "x86_64-linux-gnu.pack"
- - name: "aarch64-linux-gnu"
- host: "aarch64-linux-gnu"
- - name: "arm-linux-gnueabihf"
- host: "arm-linux-gnueabihf"
- - name: "riscv64-linux-gnu"
- host: "riscv64-linux-gnu"
- - name: "i686-linux-gnu"
- host: "i686-linux-gnu"
- - name: "x86_64-w64-mingw32"
- host: "x86_64-w64-mingw32"
- - name: "x86_64-w64-mingw32.installer"
- host: "x86_64-w64-mingw32.installer"
- - name: "x86_64-apple-darwin"
- host: "x86_64-apple-darwin"
- - name: "arm64-apple-darwin"
- host: "arm64-apple-darwin"
- name: ${{ matrix.toolchain.name }}
+ - target: "x86_64-linux-gnu"
+ - target: "x86_64-linux-gnu.no-tor-bundle"
+ - target: "x86_64-linux-gnu.pack"
+ - target: "aarch64-linux-gnu"
+ - target: "arm-linux-gnueabihf"
+ - target: "riscv64-linux-gnu"
+ - target: "i686-linux-gnu"
+ - target: "x86_64-w64-mingw32"
+ - target: "x86_64-w64-mingw32.installer"
+ - target: "x86_64-apple-darwin"
+ - target: "arm64-apple-darwin"
+
+ name: ${{ matrix.toolchain.target }}
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
- - name: guix cache
- uses: actions/cache/restore@v3
- with:
- path: |
- gnu/store
- guix_db.sqlite
- key: guix-${{ hashFiles('contrib/guix/manifest.scm') }}
- - name: move guix store
- run: |
- if [[ -e "guix_db.sqlite" ]]; then
- sudo mkdir -p /var/guix/db
- sudo mv gnu /gnu
- sudo cp guix_db.sqlite /var/guix/db/db.sqlite
-
- sudo chmod 1775 /gnu/store
- sudo chown 0644 /var/guix/db/db.sqlite
- sudo chown -R root:root /gnu/store /var/guix/db/db.sqlite
- fi
- - name: depends cache
- uses: actions/cache@v3
- with:
- path: contrib/depends/built
- key: depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }}
+ - name: remove bundled packages
+ run: sudo rm -rf /usr/local
- name: depends sources cache
- uses: actions/cache/restore@v3
+ uses: actions/cache/restore@v4
with:
path: contrib/depends/sources
key: sources-${{ hashFiles('contrib/depends/packages/*') }}
- - name: set apt conf
- run: ${{env.APT_SET_CONF}}
- name: install dependencies
run: sudo apt update; sudo apt -y install guix git ca-certificates
- name: build
- run: SUBSTITUTE_URLS='http://ci.guix.gnu.org' HOSTS="${{ matrix.toolchain.host }}" JOBS=2 ./contrib/guix/guix-build
- - uses: actions/upload-artifact@v3
+ run: SUBSTITUTE_URLS='http://ci.guix.gnu.org' HOSTS="${{ matrix.toolchain.target }}" ./contrib/guix/guix-build
+ - uses: actions/upload-artifact@v4
with:
- name: ${{ matrix.toolchain.name }}
+ name: ${{ matrix.toolchain.target }}
path: |
- guix/guix-build-*/output/${{ matrix.toolchain.host }}/*
+ guix/guix-build-*/output/${{ matrix.toolchain.target }}/*
+ guix/guix-build-*/logs/${{ matrix.toolchain.target }}/*
+
+ bundle-logs:
+ runs-on: ubuntu-latest
+ needs: [build-guix]
+ steps:
+ - uses: actions/download-artifact@v4
+ with:
+ merge-multiple: true
+ - uses: actions/upload-artifact@v4
+ with:
+ name: "logs"
+ path: '**/logs/**'
# Pattern rule to print variables, e.g. make print-top_srcdir
print-%: FORCE
- @echo '$*'='$($*)'
+ @echo '$($*)'
# When invoking a sub-make, keep only the command line variable definitions
# matching the pattern in the filter function.
include builders/default.mk
include packages/packages.mk
-# Previously, we directly invoked the well-known programs using $(shell ...)
-# to construct build_id_string. However, that was problematic because:
-#
-# 1. When invoking a shell, GNU Make special-cases exit code 127 (command not
-# found) by not capturing the output but instead passing it through. This is
-# not done for any other exit code.
-#
-# 2. Characters like '#' (from these programs' output) would end up in make
-# variables like build_id_string, which would be wrongly interpreted by make
-# when these variables were used.
-#
-# Therefore, we should avoid having arbitrary strings in make variables where
-# possible. The gen_id script used here hashes the output to construct a
-# "make-safe" id.
-#
-# Also note that these lines need to be:
-#
-# 1. After including {hosts,builders}/*.mk, since they rely on the tool
-# variables (e.g. build_CC, host_STRIP, etc.) to be set.
-#
-# 2. Before including packages/*.mk (excluding packages/packages.mk), since
-# they rely on the build_id variables
-#
-build_id:=$(shell env SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
-$(host_arch)_$(host_os)_id:=$(shell env SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
-ifneq ($(OUTDIR),)
- $(file >> $(OUTDIR)/build-hashes.txt,$(host_arch)_$(host_os)_id=$($(host_arch)_$(host_os)_id))
-endif
+build_id_string:=$(realpath $(GUIX_ENVIRONMENT))
+$(host_arch)_$(host_os)_id_string:=$(realpath $(GUIX_ENVIRONMENT))
qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_)
tor_packages_$(NO_TOR) = $(tor_packages) $(tor_$(host_os)_packages)
toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
-final_build_id_long+=$(shell $(build_SHA256SUM) toolchain.cmake.in)
+final_build_id_long+=:[sha256sum]:$(shell $(build_SHA256SUM) toolchain.cmake.in)
final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
rm -rf $(@D)
define int_get_build_recipe_hash
$(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $(PATCHES_PATH)/$(1)/,$($(1)_patches)) | cut -d" " -f1))
-$(if $(OUTDIR), $(file >> $(OUTDIR)/build-hashes.txt,$(1)_all_file_checksums=$($(1)_all_file_checksums)))
+final_build_id_long+=:[$(1)_all_file_checksums]$(foreach checksum,$($(1)_all_file_checksums),$(shell echo ":$(checksum)")):
$(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM) | cut -d" " -f1))
endef
$(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies))
$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies)))
$(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash)))
-$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id))
-$(if $(OUTDIR), $(file >> $(OUTDIR)/build-hashes.txt,$(1)_build_id_long=$($(1)_build_id_long)))
+$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string))
$(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)))
-final_build_id_long+=$($(package)_build_id_long)
+final_build_id_long+=:[recipe]:$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type):[deps]$(foreach dep,$($(1)_build_id_deps),$(shell echo ":$(dep)")):[$($(1)_type)_id]:$($($(1)_type)_id_string):
#compute package-specific paths
$(1)_build_subdir?=.
+++ /dev/null
-#!/usr/bin/env bash
-
-# Usage: env [ CC=... ] [ C_STANDARD=...] [ CXX=... ] [CXX_STANDARD=...] \
-# [ AR=... ] [ RANLIB=... ] [ STRIP=... ] [ DEBUG=... ] \
-# [ LTO=... ] ./build-id [ID_SALT]...
-#
-# Prints to stdout a SHA256 hash representing the current toolset, used by
-# depends/Makefile as a build id for caching purposes (detecting when the
-# toolset has changed and the cache needs to be invalidated).
-#
-# If the DEBUG environment variable is non-empty and the system has `tee`
-# available in its $PATH, the pre-image to the SHA256 hash will be printed to
-# stderr. This is to help developers debug caching issues in depends.
-
-# This script explicitly does not `set -e` because id determination is mostly
-# opportunistic: it is fine that things fail, as long as they fail consistently.
-
-# Command variables (CC/CXX/AR) which can be blank are invoked with `bash -c`,
-# because the "command not found" error message printed by shells often include
-# the line number, like so:
-#
-# ./depends/gen_id: line 43: --version: command not found
-#
-# By invoking with `bash -c`, we ensure that the line number is always 1
-
-(
- # Redirect stderr to stdout
- exec 2>&1
-
- echo "BEGIN ALL"
-
- # Include any ID salts supplied via command line
- echo "BEGIN ID SALT"
- echo "$@"
- echo "END ID SALT"
-
- echo "BEGIN /gnu/store"
- bash -c "ls -1 /gnu/store | sort"
- echo "END /gnu/store"
-
- # LINES=\|COLUMNS=\|\|HOSTTYPE=\|OSTYPE=\|MACHTYPE=\|HOSTNAME=
- echo "BEGIN environment"
- bash -c "printenv | sort | grep -v '^\(BASE_CACHE=\|DISTNAME=\|DISTSRC=\|OUTDIR=\|SOURCES_PATH=\|JOBS=\|OPTIONS=\)'"
- echo "END environment"
-
- echo "END ALL"
-) | if [ -n "$DEBUG_GENID" ] && command -v tee > /dev/null 2>&1; then
- # When debugging and `tee` is available, output the preimage to stderr
- # in addition to passing through stdin to stdout
- tee >(cat 1>&2)
- else
- # Otherwise, passthrough stdin to stdout
- cat
- fi | ${SHA256SUM} - | cut -d' ' -f1
# We should be able to find at least one output
################
-echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}"
+echo "Looking for build output SHA256SUMS fragments in ${LOGDIR_BASE}"
shopt -s nullglob
-sha256sum_fragments=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories...
+sha256sum_fragments=( "$LOGDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories...
shopt -u nullglob
noncodesigned_fragments=()
if (( ${#sha256sum_fragments[@]} )); then
echo "Found build output SHA256SUMS fragments:"
- for outdir in "${sha256sum_fragments[@]}"; do
- echo " '$outdir'"
- case "$outdir" in
- "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part)
- codesigned_fragments+=("$outdir")
+ for logdir in "${sha256sum_fragments[@]}"; do
+ echo " '$logdir'"
+ case "$logdir" in
+ "$LOGDIR_BASE"/*-codesigned/SHA256SUMS.part)
+ codesigned_fragments+=("$logdir")
;;
*)
- noncodesigned_fragments+=("$outdir")
+ noncodesigned_fragments+=("$logdir")
;;
esac
done
echo
else
- echo "ERR: Could not find any build output SHA256SUMS fragments in ${OUTDIR_BASE}"
+ echo "ERR: Could not find any build output SHA256SUMS fragments in ${LOGDIR_BASE}"
exit 1
fi
## Attest ##
##############
-# Usage: out_name $outdir
+# Usage: out_name $logdir
#
# HOST: The output directory being attested
#
# HOST: The current platform triple we're building for
#
distsrc_for_host() {
- echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}"
+ echo "${DISTSRC_BASE}/build/distsrc-${VERSION}-${1}"
}
# Accumulate a list of build directories that already exist...
# Precious directories are those which should not be cleaned between successive
# guix builds
depends_precious_dir_names='SOURCES_PATH BASE_CACHE'
-precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE"
+precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE LOGDIR_BASE PROFILES_BASE"
# Usage: contains IFS-SEPARATED-LIST ITEM
contains() {
# shellcheck disable=SC2046,SC2086
{
# Get depends precious dir definitions from depends
- make -C "${PWD}/contrib/depends" \
- --no-print-directory \
- -- $(printf "print-%s\n" $depends_precious_dir_names)
+ for precious_dir_name in $depends_precious_dir_names; do
+ precious_dir_path="$(make -C "${PWD}/contrib/depends" --no-print-directory print-${precious_dir_name})"
+ echo "${precious_dir_name}=${precious_dir_path}"
+ done
# Get remaining precious dir definitions from the environment
for precious_dir_name in $precious_dir_names; do
done
} > "${VAR_BASE}/precious_dirs"
-# Make sure an output directory exists for our builds
+# Make sure an output and logs directory exists for our builds
OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
mkdir -p "$OUTDIR_BASE"
+LOGDIR_BASE="${LOGDIR_BASE:-${VERSION_BASE}/logs}"
+mkdir -p "$LOGDIR_BASE"
# Download the depends sources now as we won't have internet access in the build
# container
echo "${OUTDIR_BASE}/${1}${2:+-${2}}"
}
+# Usage: logdir_for_host HOST SUFFIX
+#
+# HOST: The current platform triple we're building for
+#
+logdir_for_host() {
+ echo "${LOGDIR_BASE}/${1}${2:+-${2}}"
+}
+
+
# Usage: profiledir_for_host HOST SUFFIX
#
# HOST: The current platform triple we're building for
echo "${PROFILES_BASE}/${1}${2:+-${2}}"
}
-
#########
# BUILD #
#########
--pure \
--no-cwd \
${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
- -- env HOST="$HOST"
+ -- echo "$HOST"
if [[ -v DRY_RUN ]]; then
echo "Dry run, exiting.."
--share="$PWD"=/feather \
--share="$DISTSRC_BASE"=/distsrc-base \
--share="$OUTDIR_BASE"=/outdir-base \
+ --share="$LOGDIR_BASE"=/logdir-base \
--expose="$(git rev-parse --git-common-dir)" \
${SOURCES_PATH:+--share="$SOURCES_PATH"} \
${BASE_CACHE:+--share="$BASE_CACHE"} \
${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \
DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$host")" \
OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$host")" \
+ LOGDIR="$(LOGDIR_BASE=/logdir-base && logdir_for_host "$host")" \
DIST_ARCHIVE_BASE=/outdir-base/dist-archive \
OPTIONS="${host_split[1]}" \
bash -c "cd /feather && bash contrib/guix/libexec/build.sh"
JOBS: ${JOBS:?not set}
DISTSRC: ${DISTSRC:?not set}
OUTDIR: ${OUTDIR:?not set}
+ LOGDIR: ${LOGDIR:?not set}
OPTIONS: ${OPTIONS}
EOF
# Environment Setup #
#####################
+# Collect some information about the build environment to help debug potential reproducibility issues
+mkdir -p "${LOGDIR}"
+ls -1 /gnu/store | sort > ${LOGDIR}/guix-hashes.txt
+printenv | sort | grep -v '^\(BASE_CACHE=\|DISTNAME=\|DISTSRC=\|OUTDIR=\|LOGDIR=\|SOURCES_PATH=\|JOBS=\|OPTIONS=\|DEPENDS_ONLY=\)' > ${LOGDIR}/guix-env.txt
+
# The depends folder also serves as a base-prefix for depends packages for
# $HOSTs after successfully building.
BASEPREFIX="${PWD}/contrib/depends"
mkdir -p "$OUTDIR"
+# Log the depends build ids
+make -C contrib/depends --no-print-directory HOST="$HOST" print-final_build_id_long | tr ':' '\n' > ${LOGDIR}/depends-hashes.txt
+
# Build the depends tree, overriding variables that assume multilib gcc
make -C contrib/depends --jobs="$JOBS" HOST="$HOST" \
${V:+V=1} \
${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \
${SDK_PATH+SDK_PATH="$SDK_PATH"} \
OUTDIR="$OUTDIR" \
+ LOGDIR="$LOGDIR" \
x86_64_linux_CC=x86_64-linux-gnu-gcc \
x86_64_linux_CXX=x86_64-linux-gnu-g++ \
x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \
x86_64_linux_STRIP=x86_64-linux-gnu-strip \
guix_ldflags="$HOST_LDFLAGS"
+# Log the depends package hashes
+DEPENDS_PACKAGES="$(make -C contrib/depends --no-print-directory HOST="$HOST" print-all_packages)"
+DEPENDS_CACHE="$(make -C contrib/depends --no-print-directory ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} print-BASE_CACHE)"
+
+{
+ for package in ${DEPENDS_PACKAGES}; do
+ cat "${DEPENDS_CACHE}/${HOST}/${package}"/*.hash
+ done
+} | sort -k2 > "${LOGDIR}/depends-packages.txt"
###########################
# Source Tarball Building #
###########################
# CFLAGS
-HOST_CFLAGS="-O2 -g"
+HOST_CFLAGS="-O2"
HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;)
case "$HOST" in
*linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;;
} | xargs realpath --relative-base="$PWD" \
| xargs sha256sum \
| sort -k2 \
- | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part
+ | sponge "$LOGDIR"/SHA256SUMS.part
)
OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
+LOGDIR_BASE="${LOGDIR_BASE:-${VERSION_BASE}/logs}"
+
var_base_basename="var"
VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}"
projects / gamesguru / feather.git / commitdiff