commit ebd9f162ea1c2dea66096e90273aaa48e97d1169
parent 71b44db8d598f3c1d8b182f0c321803e79b2a4d2
Author: ccx <ccx@te2000.cz>
Date: Thu, 17 Oct 2024 04:01:05 +0000
Move package manager code into submodules
Diffstat:
25 files changed, 17 insertions(+), 1885 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -1,8 +1,3 @@
*.sw[op]
-*.pyfmt
-downloads
-files/by-sha256
-sources/by-commit
work
-make
-namedenv
+cache
diff --git a/.gitmodules b/.gitmodules
@@ -95,3 +95,9 @@
[submodule "sources/easyseccomp"]
path = sources/easyseccomp
url = https://github.com/giuseppe/easyseccomp
+[submodule "pthbs"]
+ path = pthbs
+ url = ./pthbs
+[submodule "pthbs_genpkgpy"]
+ path = pthbs_genpkgpy
+ url = ./pthbs_genpkgpy
diff --git a/Makefile b/Makefile
@@ -1,35 +1,14 @@
-# default to homedir if not root
-VERSIONS:=$(patsubst /root/%,/%,$(HOME)/versions)
+pthbs:=./pthbs
+pthbs_genpkgpy:=./pthbs_genpkgpy
-export PATH:=$(CURDIR)/command:$(PATH)
-export pthbs_versions:=$(VERSIONS)
+packages:=packages
+work:=work
+cache:=cache
+templates:=templates
default: default.environment userspace.environment containers.environment mdevd.environment
-$(VERSIONS)/environment.%/.env:
- @echo mkdir "$$(dirname '$@')"
- @echo touch '$@'
-
-
-%/.exists:
- mkdir -p '$*'
- touch '$@'
-
-make/package.%.mk: packages/% make/.exists command/pthbs-makegen
- pthbs-makegen 'packages/$*' >'$@.new'
- mv '$@.new' '$@'
-
-make/downloads.%.mk: downloadlist.% make/.exists command/pthbs-makegen-downloads
- pthbs-makegen-downloads '$*' 'downloadlist.$*' >'$@.new'
- mv '$@.new' '$@'
-
-include make/downloads.sha256.mk
-
-pkg_files=$(wildcard packages/*)
-#env_files=$(wildcard environments/*)
-mk_files=$(patsubst packages/%,make/package.%.mk,$(pkg_files))
-include $(mk_files)
-
+include $(pthbs)/pthbs.mk
ifneq (,$(filter py%,$(MAKECMDGOALS)))
-include python.mk
+include $(pthbs_genpkgpy)/genpkgpy.mk
endif
diff --git a/command/pthbs-build b/command/pthbs-build
@@ -1,316 +0,0 @@
-#!/bin/sh
-bsh=$(sha256sum $1) || exit $?
-bsh=${bsh%% *}
-basedir=$(dirname "$(dirname "$(realpath "$0")")")
-if ! test -d "$basedir"; then
- printf '%s\n' "Error: could not determine basedir"
- exit 1
-fi
-mkdir -p "$basedir/work/builddir.$$" || exit $?
-workdir=$(realpath "$basedir/work/builddir.$$")
-if ! test -d "$workdir"; then
- printf '%s\n' "Error: could not determine workdir"
- exit 1
-fi
-script=$(realpath "$1")
-if ! test -f "$script"; then
- printf '%s\n' "Error: could not determine script path"
- exit 1
-fi
-if test -z "$pthbs_versions"; then
- case $(id -u) in
- (0) pthbs_versions=/versions ;;
- (*) pthbs_versions=$HOME/versions ;;
- esac
- export pthbs_versions
-fi
-
-pthbs_package=${1##*/}
-pthbs_package=${pthbs_package%%:*}.$bsh
-
-printf "BUILDING %s => %s => %s\n" "$1" "$workdir" "$pthbs_package"
-
-eval "$(
- awk '
-BEGIN {
- settings["sandbox"] = 1
- settings["set_path"] = 1
- FS=":"
-}
-/^#@pragma:/ {
- if($2 == "nosandbox") {
- settings["sandbox"] = 0
- } else if($2 == "nopath") {
- settings["set_path"] = 0
- } else {
- fatal("unrecognized @pragma:")
- }
-}
-END {
- print "setting_sandbox=" (settings["sandbox"]?"true":"false")
- print "setting_set_path=" (settings["set_path"]?"true":"false")
-}
-' "$1" )"
-
-if test -f "make/package.sha256.${bsh}.env"; then
- envfile="make/package.sha256.${bsh}.env"
- envhash=$(pthbs-getenvhash "$envfile") || exit $?
- export pthbs_build_environment=/versions/env.$envhash
- if $setting_set_path; then
- if ! test -x "$pthbs_build_environment/command/pthbs-enter"; then
- printf >&2 "Error: %s does not exist!" "$pthbs_build_environment/command/pthbs-enter"
- exit 1
- fi
- fi
-fi
-
-mkdir -p "$workdir"
-case $(id -u) in
- (0)
- sandbox_mode=root
- export pthbs_uid=$(id -u pthbs) || exit $?
- export pthbs_gid=$(id -g pthbs) || exit $?
- export pthbs_install_uid=0 || exit $?
- export pthbs_install_gid=$pthbs_gid || exit $?
- ;;
- (*)
- sandbox_mode=userns
- ;;
-esac
-
-env \
- basedir="$basedir" \
- workdir="$workdir" \
- script="$script" \
- envdir="$pthbs_build_environment" \
- awk -v single_quote="'" -v sandbox_mode="$sandbox_mode" >"$workdir/pthbs-setup" '
-BEGIN {
- settings["sandbox"] = 1
- settings["set_path"] = 1
- FS=":"
- print "#!/bin/sh -e"
- print "if test -n \"$pthbs_xtrace\"; then pthbs_xtrace=-x; set -x; fi"
- print "cd "q(ENVIRON["workdir"])
- if(ENVIRON["pthbs_uid"]) {
- print "export pthbs_uid=" ENVIRON["pthbs_uid"]
- }
- if(ENVIRON["pthbs_gid"]) {
- print "export pthbs_gid=" ENVIRON["pthbs_gid"]
- }
-}
-
-function q(s) { # quote string for sh
- gsub(single_quote, single_quote "\\" single_quote single_quote, s)
- return single_quote s single_quote
-}
-
-function dirname(s) { # strip the last path component
- sub("/[^/]*$", "", s)
- return s
-}
-
-function basename(s) { # strip the last path component
- sub("^.*/$", "", s)
- return s
-}
-
-function fatal(msg) {
- printf "FATAL: pthbs-build: %s %s:%d: \"%s\"\n", msg, FILENAME, FNR, $0 >"/dev/stderr"
- exit 1
-}
-
-function at_git(commit_id, dstdir){
- print "mkdir -p "q(dstdir)
- print "(cd "q(ENVIRON["basedir"]"/sources/by-commit/"commit_id)" && git archive --format=tar "q(commit_id)" ) | tar --no-same-owner --no-same-permissions -xC "q(dstdir)
-}
-
-function at_untar(extra_opts, hash_type, file_hash, dstdir){
- print "mkdir -p "q(dstdir)
- print "tar -x "extra_opts" -C "q(dstdir)" -f "q(ENVIRON["basedir"]"/files/by-"hash_type"/"file_hash)
-}
-
-function at_filehash(hash_type, file_hash, dst, dstdir){
- if(dst ~ /\//) {
- dstdir = dst
- sub("/[^/]*$", "", dstdir)
- print "mkdir -p "q(dstdir)
- }
- print "cp -Lp "q(ENVIRON["basedir"]"/files/by-"hash_type"/"file_hash)" "q(dst)
-}
-
-/^#!/ { next }
-/^#\+/ {
- if($0 == "#+*") {
- settings["sandbox"] = 0
- } else if(!length(ENVIRON["envdir"])) {
- fatal("dependency specified but no envfile:")
- }
- next
-}
-/^#@/ {
- if($1 == "#@git") {
- if(match($0, "^#@git:[0-9a-f]+:") == 0) {
- fatal("invalid syntax for @git:")
- }
- at_git($2, substr($0, RLENGTH+1))
- } else if($1 == "#@sha256") {
- if(match($0, "^#@sha256:[0-9a-f]+:") == 0) {
- fatal("invalid syntax for @sha256:")
- }
- at_filehash("sha256", $2, substr($0, RLENGTH+1))
- } else if($1 == "#@untar") {
- if(match($0, "^#@untar:[^:]*:sha256:[0-9a-f]+:") == 0) {
- fatal("invalid syntax for @untar:")
- }
- at_untar($2, $3, $4, substr($0, RLENGTH+1))
- } else if($1 == "#@pragma") {
- if($2 == "nosandbox") {
- settings["sandbox"] = 0
- } else if($2 == "nopath") {
- settings["set_path"] = 0
- } else {
- fatal("unrecognized @pragma:")
- }
- } else {
- fatal("unrecognized @command:")
- }
- next
-}
-/^$/ {
- if(settings["sandbox"]) {
- if(length(ENVIRON["PTHBS_SYD"])) {
- sandbox_cmd=" SYD_NO_SYSLOG=1 SYD_LOG_FD=3 3>syd.log"
- sandbox_cmd=sandbox_cmd " " ENVIRON["PTHBS_SYD"] " -m sandbox/stat:off -m sandbox/exec:off"
- sandbox_cmd=sandbox_cmd " -m " q("allow/read+"ENVIRON["workdir"]"/***")
- sandbox_cmd=sandbox_cmd " -m " q("allow/write+"ENVIRON["workdir"]"/***")
- sandbox_cmd=sandbox_cmd " -m " q("allow/read+/proc/loadavg")
- sandbox_cmd=sandbox_cmd " -m " q("allow/read+/etc/passwd")
- sandbox_cmd=sandbox_cmd " -m " q("allow/read+/etc/group")
- sandbox_cmd=sandbox_cmd " -m " q("allow/read+/tmp/***")
- sandbox_cmd=sandbox_cmd " -m " q("allow/write+/tmp/***")
- sandbox_cmd=sandbox_cmd " -m " q("allow/read+/dev/***")
- sandbox_cmd=sandbox_cmd " -m " q("allow/write+/dev/***")
- sandbox_cmd=sandbox_cmd " -m " q("allow/read+"ENVIRON["script"])
- sandbox_cmd=sandbox_cmd " -m " q("allow/read+"dirname(ENVIRON["envdir"])"/***")
- sandbox_cmd=sandbox_cmd " -munshare/user:1"
- sandbox_cmd=sandbox_cmd " -munshare/mount:1"
- sandbox_cmd=sandbox_cmd " -mbind+" q(ENVIRON["basedir"]"/work/bin:/bin:ro,nosuid,nodev")
- sandbox_cmd=sandbox_cmd " -m " q("allow/read+/bin/***")
- sandbox_cmd=sandbox_cmd " -munshare/net:1 -munshare/ipc:1"
- } else if(sandbox_mode == "userns") {
- sandbox_cmd=" "q(ENVIRON["basedir"]"/ns_sandbox.py")" --mode=userns"
- sandbox_cmd=sandbox_cmd" --vars="q(ENVIRON["basedir"]"/vars.yaml")
- sandbox_cmd=sandbox_cmd" --extra-mount=tmpfs:"q(ENVIRON["basedir"]"/work")
- sandbox_cmd=sandbox_cmd" --extra-mount=ro_bind:"q(ENVIRON["basedir"]"/packages:"ENVIRON["basedir"]"/packages")
- sandbox_cmd=sandbox_cmd" --extra-mount=rw_bind:"q(ENVIRON["workdir"]":"ENVIRON["workdir"])
- sandbox_cmd=sandbox_cmd" --extra-mount=rw_bind:"q(ENVIRON["workdir"]"/.tmp:/tmp")
- sandbox_cmd=sandbox_cmd" -- "q(ENVIRON["basedir"]"/work/root")
- printf "%s\n", "mkdir -p "q(ENVIRON["workdir"]"/.tmp")
- } else if(sandbox_mode == "root") {
- sandbox_cmd=" "q(ENVIRON["basedir"]"/work/venv/bin/python")" "q(ENVIRON["basedir"]"/ns_sandbox.py")" --mode=root"
- sandbox_cmd=sandbox_cmd" --untar="q(ENVIRON["basedir"]"/root.tar")
- sandbox_cmd=sandbox_cmd" --vars="q(ENVIRON["basedir"]"/vars.yaml")
- sandbox_cmd=sandbox_cmd" --chdir="q(ENVIRON["workdir"])
- sandbox_cmd=sandbox_cmd" --extra-mount=tmpfs:"q(ENVIRON["basedir"]"/work")
- sandbox_cmd=sandbox_cmd" --extra-mount=ro_bind:"q(ENVIRON["basedir"]"/packages:"ENVIRON["basedir"]"/packages")
- sandbox_cmd=sandbox_cmd" --extra-mount=rw_bind:"q(ENVIRON["workdir"]":"ENVIRON["workdir"])
- sandbox_cmd=sandbox_cmd" --extra-mount=rw_bind:"q(ENVIRON["workdir"]"/.tmp:/tmp")
- sandbox_cmd=sandbox_cmd" -- "q(ENVIRON["basedir"]"/work/root")
- printf "%s\n", "mkdir -p "q(ENVIRON["workdir"]"/.tmp")
- } else {
- fatal("unrecognized sanbox_mode " sandbox_mode)
- }
- } else if(ENVIRON["pthbs_uid"]){
- sandbox_cmd="busybox chpst -u \"$pthbs_uid:$pthbs_gid\" --"
- } else {
- sandbox_cmd=""
- }
- if(ENVIRON["pthbs_uid"]) {
- printf "%s\n", "chown -R \"$pthbs_uid:$pthbs_gid\" "q(ENVIRON["workdir"])
- }
- if(length(ENVIRON["envdir"])){
- cmd="env pthbs_build_environment="q(ENVIRON["envdir"])
- cmd=cmd" "sandbox_cmd
- if(settings["set_path"]) {
- cmd=cmd" "q(ENVIRON["envdir"]"/command/pthbs-enter")
- }
- cmd=cmd" sh -xe "q(ENVIRON["script"])
- } else {
- cmd=sandbox_cmd" sh -xe "q(ENVIRON["script"])
- }
- print "exec >build.log 2>&1 " cmd
- exit 0
-}
-{
- fatal("unexpected line")
-}
-' "$1" || exit $?
-
-if test -z "$JOBS"; then
- JOBS=$(nproc)
- if test -z "$JOBS"; then
- JOBS=$(grep -ce '^processor' /proc/cpuinfo)
- if test -z "$JOBS"; then
- JOBS=1
- fi
- fi
-fi
-
-logdir=work/logs/$(date '+%Y-%m-%d-%H%M%S')-$pthbs_package
-ret=0
-if test -n "$pthbs_xtrace"; then pthbs_xtrace=-x; set -x; fi
-trap 'trap - INT' INT
-if env -i \
- PATH="$PATH" \
- JOBS="$JOBS" \
- pthbs_script="$script" \
- pthbs_destdir="$workdir/destdir" \
- pthbs_package="$pthbs_package" \
- sh $pthbs_xtrace -e "$workdir/pthbs-setup" </dev/null; then
- trap - INT
- printf "BUILD SUCCESFUL :: %s\n" "$pthbs_package"
- if ! pthbs-install "$workdir/destdir" "$pthbs_package"; then
- ret=1
- echo "INSTALL FAILED"
- fi
-else
- ret=$?
- trap - INT
- if test -n "$pthbs_fail_log_cmd"; then
- $pthbs_fail_log_cmd "$workdir/build.log"
- else
- printf "Errors found in log:\n"
- grep -C 1 -Eie '(error|fatal)[: ]' "$workdir/build.log"
- fi
- printf "BUILD FAILED: exitcode %s :: %s :: %s :: %s\n" "$ret" "$1" "$workdir" "$logdir"
- mkdir -p "$logdir"
- rm -v "$(dirname "$logdir")/last_failed_build"
- ln -v -s -f "$(basename "$logdir")" "$(dirname "$logdir")/last_failed_build"
- find "$workdir" -name config.log -exec cp -v --backup=numbered '{}' "$logdir/config.log" \;
- pthbs-digest-tree >"$logdir/workdir-digest" "$workdir"
-fi
-
-mkdir -p "$logdir"
-
-if test -e "$workdir/build.log"; then
- mv "$workdir/build.log" "$logdir/log"
- bzip2 "$logdir/log"
-fi
-
-if test -e "$workdir/syd.log"; then
- mv "$workdir/syd.log" "$logdir/syd.log"
- bzip2 "$logdir/syd.log"
-fi
-
-if test -e "$workdir/pthbs-setup"; then
- mv "$workdir/pthbs-setup" "$logdir/"
-fi
-
-if test -n "$pthbs_build_environment"; then
- ln -s "$pthbs_build_environment" "$logdir/env"
-fi
-
-if test -z "$pthbs_skip_cleanup"; then
- rm -rf "$workdir" || ret=$?
-fi
-exit $ret
diff --git a/command/pthbs-digest-files b/command/pthbs-digest-files
@@ -1,7 +0,0 @@
-#!/bin/sh
-for fname in "$@"; do
- busybox stat -c "# %F %a %u:%g %sB %N" "$fname" || exit $?
- if test -f "$fname"; then
- sha256sum "$fname" || exit $?
- fi
-done
diff --git a/command/pthbs-digest-tree b/command/pthbs-digest-tree
@@ -1,5 +0,0 @@
-#!/bin/sh
-if test -n "$1"; then
- cd "$1" || exit $?
-fi
-busybox find -print0 | busybox sort -z | busybox xargs -0 pthbs-digest-files
diff --git a/command/pthbs-download b/command/pthbs-download
@@ -1,54 +0,0 @@
-#!/bin/sh
-test $# -gt 3 || exit 2
-checksum_format=$1
-checksum=$2
-size=$3
-shift 3 || exit
-filename=downloads/$checksum_format/$checksum
-
-case $checksum_format in
- (md5);;
- (sha1);;
- (sha256);;
- (sha3);;
- (sha512);;
- (*)
- printf>&2 "FATAL: unrecognized checksum format '%s'\n" "$checksum_format"
- exit 1
- ;;
-esac
-
-check() {
- test -e "$filename" || return 1
- ls -l "$filename"
- find "$filename" -type f -size +"$size"c -delete
- case $(find "$filename" -type f -size "$size"c -print) in
- ($filename);;
- (*) return 1;;
- esac
- if printf '%s %s\n' "$checksum" "$filename" | ${checksum_format}sum -c -; then
- ln -sf "../../$filename" files/by-sha256 || exit 1
- return 0
- fi
- rm -v "$filename"
- exit 1
-}
-
-test -z "$pthbs_xtrace" || set -x
-
-mkdir -p "downloads/$checksum_format" || exit $?
-
-if check; then
- exit 0
-fi
-
-while test $# -gt 0; do
- wget -cqO "$filename" -- "$1"
- if check; then
- exit 0
- fi
- shift
-done
-
-printf>&2 "FATAL: could not download file\n"
-exit 1
diff --git a/command/pthbs-enter-gen b/command/pthbs-enter-gen
@@ -1,100 +0,0 @@
-#!/bin/sh -e
-
-if test $# != 1; then
- printf >2 'usage: %s\n' "pthbs-enter-gen /path/to/env.hash"
- printf >2 'Error: expected 1 argument, got%s\n' "$#"
- exit 2
-fi
-envtop=$1
-envdir=$envtop/env
-envcommand=$envtop/command
-
-case $(basename "$envtop") in
- (env.*) ;;
- (*)
- printf >&2 "Error: '%s' doesn't look like environment path!\n" "$envtop"
- exit 1;;
-esac
-
-mkdir -p "$envdir"
-
-if ! test -d "$envcommand"; then
- printf >&2 "Error: '%s' doesn't exist!\n" "$envcommand"
-fi
-printf '%s\n' "$envcommand" >"$envdir/PATH"
-
-if test -d "$envtop/zsh"; then
- printf '%s\n' "$envtop/zsh/site-functions:$envtop/zsh/functions" >"$envdir/FPATH"
-fi
-
-if test -d "$envtop/library/pkgconfig"; then
- printf '%s\n' "$envtop/library/pkgconfig" >"$envdir/PKG_CONFIG_LIBDIR"
-fi
-
-mkscript() {
- exe=$envcommand/$1
- cat >"$exe.new"
- chmod +x "$exe.new"
- mv "$exe.new" "$exe"
-}
-
-exe=$envcommand/pthbs-enter
-if test -x "$envcommand/execlineb" && test -x "$envcommand/s6-envdir"; then
- mkscript pthbs-enter <<EOF
-#!$envcommand/execlineb -S0
-"$envcommand/s6-envdir" "$envdir" "$envcommand/exec" \$@
-EOF
- "$envcommand/pthbs-enter" true # TODO: don't run executables without sandbox
- mkscript pthbs-enter-execlineb <<EOF
-#!$envcommand/execlineb -S0
-"$envcommand/s6-envdir" "$envdir" "$envcommand/execlineb" \$@
-EOF
- if test -x "$envcommand/sh"; then
- mkscript pthbs-enter-sh <<EOF
-#!$envcommand/execlineb -S0
-"$envcommand/s6-envdir" "$envdir" "$envcommand/sh" \$@
-EOF
- fi
-elif test -x "$envcommand/sh" && test -x "$envcommand/envdir"; then
- mkscript pthbs-enter <<EOF
-#!$envcommand/sh
-exec "$envcommand/envdir" "$envdir" "$envcommand/env" -- "\$@"
-EOF
- "$envcommand/pthbs-enter" true # TODO: don't run executables without sandbox
- mkscript pthbs-enter-sh <<EOF
-#!$envcommand/sh
-exec "$envcommand/envdir" "$envdir" "$envcommand/sh" "\$@"
-EOF
-fi
-
-if test -x "$envcommand/install-as-current-environment.postinstall"; then
- mkscript install-as-current-environment <<EOF
-#!$envcommand/execlineb -P
-"$envcommand/s6-envdir" "$envdir" "$envcommand/exec"
-ifelse -n { test -L /run/current } {
- if { ln -s -f -v "$envtop" /run/current }
- "$envcommand/install-as-current-environment.postinstall"
-}
-foreground {
- fdmove -c 1 2
- printf "* Swapping /run/current from %s to " "$envtop"
- readlink /run/command
-}
-getpid NS_PID
-importas -i NS_PID NS_PID
-if { mv -v /run/current /run/previous.\${NS_PID} }
-if { ln -s -f -v "$envtop" /run/current }
-ifelse {
- "$envcommand/install-as-current-environment.postinstall" /run/previous.\${NS_PID}
-} { }
-foreground {
- fdmove -c 1 2
- printf "* %s returned error, reverting to " "$envcommand/install-as-current-environment.postinstall"
- readlink /run/previous.\${NS_PID}
-}
-if { rm -v /run/current }
-if { mv -v /run/previous.\${NS_PID} /run/current }
-#foreground { /run/current/command/install-as-current-environment.postinstall }
-exit 1
-EOF
-fi
diff --git a/command/pthbs-getenvhash b/command/pthbs-getenvhash
@@ -1,2 +0,0 @@
-#!/bin/sh
-sort -u "$@" | sha256sum | sed 's/ .*//'
diff --git a/command/pthbs-install b/command/pthbs-install
@@ -1,44 +0,0 @@
-#!/bin/sh -e
-test -z "$pthbs_xtrace" || set -x
-if test -z "$pthbs_versions"; then
- echo >&2 "ERROR: pthbs_versions is undefined"
- exit 2
-elif ! test -d "$pthbs_versions"; then
- echo >&2 "ERROR: pthbs_versions '$pthbs_versions' is not directory"
- exit 1
-fi
-
-test "$#" -eq 2 || exit 2
-
-pthbs_destdir="$1"
-pthbs_package="$2"
-
-if ! test -f "$pthbs_destdir/${pthbs_versions%/}/$pthbs_package/.install-links"; then
- echo >&2 "ERROR: link file not found"
- exit 1
-fi
-
-if test -n "$pthbs_install_uid"; then
- chown -R "$pthbs_install_uid:$pthbs_install_gid" "$pthbs_destdir/${pthbs_versions%/}/$pthbs_package"
-fi
-chmod -R ugo-w "$pthbs_destdir/${pthbs_versions%/}/$pthbs_package"
-pthbs-digest-tree >"$pthbs_destdir/pthbs-digest" "$pthbs_destdir/${pthbs_versions%/}/$pthbs_package/" || exit $?
-mv "$pthbs_destdir/pthbs-digest" "$pthbs_destdir/${pthbs_versions%/}/$pthbs_package/.pthbs-digest" || exit $?
-
-if test -e "$pthbs_versions/$pthbs_package"; then
- if ! diff -u "$pthbs_versions/$pthbs_package" "$pthbs_destdir/${pthbs_versions%/}/$pthbs_package/.pthbs-digest"; then
- echo >&2 "ERROR: digest file mismatch - packages with same buildhash differ"
- exit 3
- fi
- N=1
- while test -e "$pthbs_versions/$pthbs_package.$N"; do
- N=$(1+$N)
- done
- echo >&2 "INFO: replacing previous package"
- mv "$pthbs_versions/$pthbs_package" "$pthbs_versions/$pthbs_package.$N"
- # remove dangling links
- find "$pthbs_versions" -type l ! -exec test -e {} \; -exec rm -v {} +
-fi
-
-mv "$pthbs_destdir/${pthbs_versions%/}/$pthbs_package" "$pthbs_versions/$pthbs_package"
-find "$pthbs_destdir" -type d -o -exec printf 'WARNING: stray file in destdir: %s\n' '{}' +
diff --git a/command/pthbs-link b/command/pthbs-link
@@ -1,20 +0,0 @@
-#!/bin/sh -e
-test -d "$1" || exit 1
-test -d "$2" || exit 1
-while IFS= read -r line; do
- target=${line% *}
- source=${line#* }
- target=$1/${target#./}
- source=$2/${source#./}
- if test -e "$source"; then
- printf >&2 'FATAL: link source already exists: "%s"\n' "$source"
- ls >&2 -l "$source"
- exit 1
- fi
- if ! test -e "$target"; then
- printf >&2 'FATAL: link target does not exists: "%s"\n' "$target"
- exit 1
- fi
- mkdir -p "${source%/*}"
- ln -s "$target" "$source"
-done <"$1/.install-links"
diff --git a/command/pthbs-makegen b/command/pthbs-makegen
@@ -1,158 +0,0 @@
-#!/bin/sh
-#set -x
-bsh=$(sha256sum $1) || exit $?
-bsh=${bsh%% *}
-basedir=$(dirname "$(dirname "$(realpath "$0")")")
-workdir=$basedir/work/builddir.$$
-script=$(realpath "$1")
-
-pthbs_package=${1##*/}
-pthbs_package=${pthbs_package%%:*}.$bsh
-
-exec env \
- basedir="$basedir" \
- workdir="$workdir" \
- script="$script" \
- scriptname="${script##*/}" \
- bsh="$bsh" \
- package="$pthbs_package" \
- awk -v single_quote="'" '
-BEGIN {
- while(getline <"downloadlist.sha256") {
- downloadable_hashes["sha256",$1] = 1
- }
- close("downloadlist.sha256")
-
- settings["sandbox"] = 1
- settings["set_path"] = 1
- dep_count = 0
- env_count = 0
- is_envfile = ENVIRON["script"] ~ /\.environment$/
- if(is_envfile) {
- envname = substr(ENVIRON["scriptname"], 1, length(ENVIRON["scriptname"])-12)
- }
- FS=":"
-}
-
-function q(s) { # quote string for sh
- gsub(single_quote, single_quote "\\" single_quote single_quote, s)
- return single_quote s single_quote
-}
-
-function fatal(msg) {
- printf "FATAL: pthbs-makegen: %s %s:%d: \"%s\"\n", msg, FILENAME, FNR, $0 >"/dev/stderr"
- exit 1
-}
-
-function have_file(hash_type, file_hash) {
- if(!((hash_type, file_hash) in downloadable_hashes)) {
- return
- }
- deps[++dep_count] = "make/file."hash_type"."file_hash".downloaded"
-}
-
-function have_envdep(package) {
- if(match(package, "[.][0-9a-f]{64}$") == 0) {
- fatal("invalid syntax for @sha256:")
- }
- env[++env_count] = package
- package_hash = substr(package, 64, RSTART+1)
- envdep[env_count] = "$(VERSIONS)/"package"/.install-links"
-}
-
-function make_envfile( n, envfile, envhash) {
- if(!env_count) {
- fatal("environment is empty")
- }
- envfile = "make/package.sha256."ENVIRON["bsh"]".env"
- for(n=1; n<=env_count; n++) {
- print env[n] >envfile
- }
- close(envfile)
- if(("pthbs-getenvhash "q(envfile) | getline) < 1) {
- fatal("Error getting envhash from "q(envfile))
- }
- envhash=$0
- if(match(envhash, "^[0-9a-f]{64}$") == 0) {
- fatal("received invalid envhash: "q(envhash))
- }
- deps[++dep_count] = "$(VERSIONS)/env."envhash"/.pthbs-env"
- printf "ifndef %s\n", "env_"envhash"_defined"
- printf "%s:", "$(VERSIONS)/env."envhash"/.pthbs-env"
- for(n=1; n<=env_count; n++) {
- printf " %s", envdep[n]
- }
- printf "\n\t@%s", "printf "q("Creating environment for %s => %s\\n")" "q(FILENAME)" "q("$(VERSIONS)/env."envhash)
- printf "\n\t%s", "if test -e "q("$(VERSIONS)/env."envhash)"; then rm -r "q("$(VERSIONS)/env."envhash)"; fi"
- printf "\n\t%s", "mkdir -p "q("$(VERSIONS)/env."envhash"/env")
- for(n=1; n<=env_count; n++) {
- printf "\n\t%s", "pthbs-link "q("$(VERSIONS)/"env[n])" "q("$(VERSIONS)/env."envhash)
- }
- printf "\n\t%s", "pthbs-enter-gen "q("$(VERSIONS)/env."envhash)
- printf "\n\t%s\n\n", "sort -u <"q(envfile)" >"q("$@")
- printf "%s\n", "env_"envhash"_defined="ENVIRON["scriptname"]
- printf "%s\n", "endif"
- return envhash
-}
-/^#!/ { next }
-/^#\+/ {
- if($0 == "#+*") {
- settings["sandbox"] = 0
- } else {
- have_envdep(substr($0, 3))
- }
- next
-}
-/^#@/ && !is_envfile {
- if($1 == "#@git") {
- next
- } else if($1 == "#@pragma") {
- if($2 == "nosandbox") {
- settings["sandbox"] = 0
- } else if($2 == "nopath") {
- settings["set_path"] = 0
- } else {
- fatal("unrecognized @pragma:")
- }
- next
- } else if($1 == "#@sha256") {
- if(match($0, "^#@sha256:[0-9a-f]+:") == 0) {
- fatal("invalid syntax for @sha256:")
- }
- have_file("sha256", $2)
- } else if($1 == "#@untar") {
- if(match($0, "^#@untar:[^:]*:sha256:[0-9a-f]+:") == 0) {
- fatal("invalid syntax for @untar:")
- }
- have_file("sha256", $4)
- } else {
- fatal("unrecognized @command:")
- }
- next
-}
-/^$/ && !is_envfile {
- if(env_count) {
- make_envfile()
- }
- printf "%s", "$(VERSIONS)/"ENVIRON["package"]"/.install-links:"
- for(n=1; n<=dep_count; n++) {
- printf " %s", deps[n]
- }
- printf "\n\t%s\n", "if test -f "q("$@")"; then touch "q("$@")"; else pthbs-build "q(ENVIRON["script"])"; fi"
- has_body = 1
- exit 0
-}
-{
- fatal("unexpected line")
-}
-END{
- if(!is_envfile){
- if(has_body) { exit 0 }
- fatal("no build script present")
- }
- env_installdir = "$(VERSIONS)/env." make_envfile()
- printf "%s: %s %s\n", ENVIRON["scriptname"], env_installdir"/.pthbs-env", "namedenv/.exists"
- printf "\t%s\n\n", "pthbs-namedenv "q(env_installdir)" "q(envname)
- printf ".PHONY: %s\n", ENVIRON["scriptname"]
-}
-' "$script"
diff --git a/command/pthbs-makegen-downloads b/command/pthbs-makegen-downloads
@@ -1,31 +0,0 @@
-#!/bin/sh
-format=$1
-shift || exit $?
-exec awk -v format="$format" -v single_quote="'" '
-function q(s) { # quote string for sh
- gsub(single_quote, single_quote "\\" single_quote single_quote, s)
- return single_quote s single_quote
-}
-
-function fatal(msg) {
- printf "FATAL: pthbs-makegen: %s %s:%d: \"%s\"\n", msg, FILENAME, FNR, $0 >"/dev/stderr"
- exit 1
-}
-
-/^$/ { next }
-/^#/ { next }
-
-/^[0-9a-f]{64} [0-9]+ /{
- printf "%s:\n", "make/file."format"."$1".downloaded"
- printf "\tpthbs-download sha256"
- for(n=1; n<=NF; n++){
- printf " %s", q($n)
- }
- printf "\n\t%s\n\n", "touch "q("$@")
- next
-}
-
-{
- fatal("unexpected line")
-}
-' "$@"
diff --git a/command/pthbs-namedenv b/command/pthbs-namedenv
@@ -1,55 +0,0 @@
-#!/bin/sh
-set -e
-if test -n "$pthbs_xtrace"; then set -x; fi
-basedir=$(dirname "$(dirname "$(realpath "$0")")")
-if test $# != 2; then
- printf '%s\n' >&2 'usage: pthbs-namedenv /path/to/env.hash name'
- exit 2
-fi
-env_installdir=$1
-env_name=$2
-shift
-case $env_name in
- (*/*)
- (*:*)
- printf '%s\n' >&2 "pthbs-namedenv: error: invalid name '$env_name'"
- exit 2
- ;;
-esac
-case ${env_installdir##*/} in
- (env.*) ;;
- (*)
- printf '%s\n' >&2 "pthbs-namedenv: error: invalid env installdir '$env_installdir'"
- exit 2
- ;;
-esac
-
-bd_link="$basedir/namedenv/$env_name"
-v_link_base="${env_installdir%/*}/namedenv.$env_name"
-printf "%s => %s\n" "$v_link_base" "${env_installdir##*/}"
-if test -L "$bd_link"; then
- unlink "$bd_link"
-fi
-busybox ln -sTf "$env_installdir" "$bd_link"
-if test -L "${v_link_base}" && test x"$(realpath "$bd_link")" = x"$(realpath "$v_link_base")"; then
- # already present
- exit 0
-fi
-if test -L "${v_link_base}.9"; then
- unlink "${v_link_base}.9"
-fi
-mvl() {
- if test -L "${v_link_base}$1"; then
- mv "${v_link_base}$1" "${v_link_base}$2"
- fi
-}
-mvl .8 .9
-mvl .7 .8
-mvl .6 .7
-mvl .5 .6
-mvl .4 .5
-mvl .3 .4
-mvl .2 .3
-mvl .1 .2
-mvl "" .1
-busybox ln -sTf "${env_installdir##*/}" "$v_link_base"
diff --git a/genpkg.py b/genpkg.py
@@ -1,194 +0,0 @@
-#!/usr/bin/env python3
-import hashlib
-import os.path
-import subprocess
-from pathlib import Path
-
-import jinja2
-import yaml
-
-
-class SubmoduleInfo:
- def __init__(self):
- self._current_commits = None
- self._by_commit = Path('./sources/by-commit')
-
- @property
- def current(self):
- if self._current_commits is not None:
- return self._current_commits
- out = subprocess.check_output(
- ("git", "submodule", "status", "--cached")
- ).decode('utf8')
- lines = out.strip('\n').split('\n')
- records = [[line[0]] + line[1:].split() for line in lines]
- self._current_commits = {
- r[2][8:]: r[1] for r in records if r[2].startswith("sources/")
- }
- for repo, commit in self._current_commits.items():
- if not (self._by_commit / commit).exists():
- os.symlink("../" + repo, str(self._by_commit / commit))
- return self._current_commits
-
- def commit_info(self, commit_id):
- assert '/' not in commit_id
- assert '.' not in commit_id
- assert (self._by_commit / commit_id).exists()
- out = subprocess.check_output(
- ('git', 'show', '-s', '--pretty=format:%ai by %an'),
- cwd=(self._by_commit / commit_id).as_posix(),
- ).decode('utf8')
- return out
-
-class DownloadsInfo:
- def __init__(self):
- self._basenames = {}
- with open('downloadlist.sha256', 'rt') as f:
- for line in f:
- if line[0] in '#\n':
- continue
- sha256, size, url = line.rstrip().split(maxsplit=2)
- assert len(bytes.fromhex(sha256)) == 32
- assert int(size) >= 0
- basename = os.path.basename(url)
- if basename in self._basenames:
- self._basenames[basename] = ValueError(
- 'Duplicate download name: ' + repr(basename)
- )
- else:
- self._basenames[basename] = 'sha256:' + sha256
-
- def __getitem__(self, key):
- value = self._basenames[key]
- if isinstance(value, Exception):
- raise value
- return value
-
-
-class FileInfo:
- def __init__(self):
- self._sha256_cache = {}
- self._files_dir = Path('./files')
-
- def __getitem__(self, key):
- if key in self._sha256_cache:
- return self._sha256_cache[key]
- fp = self._files_dir / key
- with fp.open('rb') as f:
- file_hash = hashlib.file_digest(f, "sha256").hexdigest()
- self._sha256_cache[key] = file_hash
- if not (self._files_dir / "by-sha256" / file_hash).exists():
- os.symlink("../" + key, str(self._files_dir / "by-sha256" / file_hash))
- return file_hash
-
-
-class Main:
- def __init__(self, out_dir="packages", template_dir="templates"):
- self.out_dir = Path(out_dir)
- self.template_dir = Path(template_dir)
- self.env = jinja2.Environment(
- loader=jinja2.FileSystemLoader(template_dir),
- undefined=jinja2.StrictUndefined,
- autoescape=False,
- )
- self.env.globals["pkg_sha256"] = self.pkg_sha256
- self.env.globals["pkg_install_name"] = self.pkg_install_name
- self.env.globals["pkg_install_dir"] = self.pkg_install_dir
- self.env.globals["submodule"] = SubmoduleInfo()
- self.env.globals["files"] = FileInfo()
- self.env.globals["downloads"] = DownloadsInfo()
- self.package_hashes = {}
- self.rendering = []
- self.deps = {}
-
- def load_vars_yaml(self, fname="vars.yaml"):
- with open(fname) as f:
- self.env.globals.update(yaml.safe_load(f))
-
- def pkg_env_sha256(self, name):
- current = self.rendering[-1]
- if current not in self.deps:
- self.deps[current] = set((name,))
- else:
- self.deps[current].add(name)
- self._pkg_sha256(name)
- envlist = ''.join(sorted('%s.%s\n' % (d, self.package_hashes[d]) for d in self.deps[name]))
- return hashlib.sha256(envlist.encode()).hexdigest()
-
- def pkg_sha256(self, name):
- current = self.rendering[-1]
- if current not in self.deps:
- self.deps[current] = set((name,))
- else:
- self.deps[current].add(name)
- return self._pkg_sha256(name)
-
- def _pkg_sha256(self, name):
- if name in self.package_hashes:
- return self.package_hashes[name]
- if name in self.rendering:
- raise RuntimeError("circular dependency: %r", self.rendering)
-
- t = self.env.get_template("pkg/" + name)
- self.rendering.append(name)
- data = bytes(t.render(name=name).encode('utf8'))
- self.package_hashes[name] = hashlib.sha256(data).hexdigest()
- lastname = self.rendering.pop()
- assert name == lastname
-
- out_path = self.out_dir / name
- old_hash = None
- if out_path.exists():
- with out_path.open('rb') as f:
- old_hash = hashlib.file_digest(f, "sha256").hexdigest()
- if old_hash is None or old_hash != self.package_hashes[name]:
- tmp_path = out_path.with_suffix('.new')
- tmp_path.write_bytes(data)
- tmp_path.replace(out_path)
-
- return self.package_hashes[name]
-
- def pkg_install_name(self, name):
- rootname = name.split(":")[0]
- if rootname.endswith('.environment'):
- return "env.%s" % (self.pkg_env_sha256(name),)
- else:
- return "%s.%s" % (name.split(":")[0], self.pkg_sha256(name))
-
- def pkg_install_dir(self, name):
- return os.path.join(
- self.env.globals["versions"],
- self.pkg_install_name(name),
- )
-
- def list_packages(self):
- for tplname in self.env.list_templates():
- if not tplname.startswith("pkg/"):
- continue
- if "/." in tplname:
- continue
- yield tplname[4:]
-
- def render_all(self):
- for pkgname in self.list_packages():
- print("%s\t%s" % (pkgname, self._pkg_sha256(pkgname)))
- for dep in sorted(self.deps.get(pkgname, ())):
- print(
- " > %s.%s"
- % (
- dep,
- self.package_hashes[dep],
- )
- )
-
-
-if __name__ == '__main__':
- from pprint import pprint as pp
-
- m = Main()
- m.load_vars_yaml()
- pp(m.env.list_templates(filter_func=lambda name: "/." not in name))
- m.render_all()
-
-# pylama:linters=pycodestyle,pyflakes:ignore=D212,D203,D100,D101,D102,D105,D107
-# vim: sts=4 ts=4 sw=4 et tw=88 efm=%A%f\:%l%\:%c\ %t%n\ %m
diff --git a/make_vars.py b/make_vars.py
@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import (
- absolute_import,
- division,
- generators,
- print_function,
- with_statement,
-)
-
-import sys
-
-try:
- import venv
-except ImportError:
- venv = None
-
-try:
- from packaging.tags import sys_tags
-
- def get_tag():
- tag = sys_tags().__next__()
- return (tag.interpreter, tag.abi, tag.platform)
-
-except ImportError:
- from wheel.pep425tags import get_abbr_impl, get_abi_tag, get_impl_ver, get_platform
-
- # sporked from wheel.bdist_wheel.get_tag
- def get_tag():
- """Get the specific implementation name in a wheel-compatible format."""
- plat_name = get_platform().replace('-', '_').replace('.', '_')
- impl_name = get_abbr_impl()
- impl_ver = get_impl_ver()
- impl = impl_name + impl_ver
- abi_tag = str(get_abi_tag()).lower()
- tag = (impl, abi_tag, plat_name)
- return tag
-
-
-PY2 = sys.version_info[0] == 2
-
-
-def main():
- tag = '%s-%s-%s' % get_tag()
- mk_filename = 'make/python_vars_%s.mk' % tag
- with open(mk_filename, 'wt') as f:
- # TODO: add proper escaping whenever I feel like staring into the abyss
- f.write('PYTHON_IMPL:=%s\n' % tag)
-
- f.write(
- "PYTHON_VENV:=%s\n"
- % (
- "virtualenv -p $(PYTHON_EXE)"
- if venv is None
- else "$(PYTHON_EXE) -m venv"
- )
- )
-
- f.write(
- "PYTHON_VENV_INSTALL:=%s\n"
- % ("'setuptools<45.0.0' 'pip<20.3' 'pip-tools<6'" if PY2 else "'pip-tools'")
- )
- print("include %s" % mk_filename)
-
-
-if __name__ == '__main__':
- main()
-
-# pylama:linters=pycodestyle,pyflakes:ignore=D212,D203,D100,D101,D102,D107
-# vim: fileencoding=utf-8 ft=python et sw=4 ts=4 sts=4 tw=79
diff --git a/ns_sandbox.py b/ns_sandbox.py
@@ -1,632 +0,0 @@
-#!/usr/bin/python3
-import argparse
-import ctypes
-import dataclasses
-import enum
-import errno
-import fcntl
-import os
-import os.path
-import pathlib
-import select
-import stat
-import subprocess
-
-libc = ctypes.CDLL(None, use_errno=True)
-CLONE_NEWNS = 0x00020000 # New mount namespace group
-CLONE_NEWCGROUP = 0x02000000 # New cgroup namespace
-CLONE_NEWUTS = 0x04000000 # New utsname namespace
-CLONE_NEWIPC = 0x08000000 # New ipc namespace
-CLONE_NEWUSER = 0x10000000 # New user namespace
-CLONE_NEWPID = 0x20000000 # New pid namespace
-CLONE_NEWNET = 0x40000000 # New network namespace
-CLONE_NEWTIME = 0x00000080 # New time namespace
-
-SYS_pivot_root = 155
-
-MNT_FORCE = 1
-MNT_DETACH = 2
-MNT_EXPIRE = 4
-UMOUNT_NOFOLLOW = 8
-
-
-class MountFlag(int, enum.Enum):
- """Mount flags."""
-
- #: Mount read-only.
- RDONLY = 1
- #: Ignore suid and sgid bits.
- NOSUID = 2
- #: Disallow access to device special files.
- NODEV = 4
- #: Disallow program execution.
- NOEXEC = 8
- #: Writes are synced at once.
- SYNCHRONOUS = 16
- #: Alter flags of a mounted FS.
- REMOUNT = 32
- #: Allow mandatory locks on an FS.
- MANDLOCK = 64
- #: Directory modifications are synchronous.
- DIRSYNC = 128
- #: Do not follow symlinks.
- NOSYMFOLLOW = 256
- #: Do not update access times.
- NOATIME = 1024
- #: Do not update directory access times.
- NODIRATIME = 2048
- #: Bind directory at different place.
- BIND = 4096
- MOVE = 8192
- REC = 16384
- SILENT = 32768
- #: VFS does not apply the umask.
- POSIXACL = 1 << 16
- #: Change to unbindable.
- UNBINDABLE = 1 << 17
- #: Change to private.
- PRIVATE = 1 << 18
- #: Change to slave.
- SLAVE = 1 << 19
- #: Change to shared.
- SHARED = 1 << 20
- #: Update atime relative to mtime/ctime.
- RELATIME = 1 << 21
- #: This is a kern_mount call.
- KERNMOUNT = 1 << 22
- #: Update inode I_version field.
- I_VERSION = 1 << 23
- #: Always perform atime updates.
- STRICTATIME = 1 << 24
- #: Update the on-disk [acm]times lazily.
- LAZYTIME = 1 << 25
- ACTIVE = 1 << 30
- NOUSER = 1 << 31
-
-
-_mount = libc.mount
-_mount.restype = ctypes.c_int
-_mount.argtypes = (
- ctypes.c_char_p,
- ctypes.c_char_p,
- ctypes.c_char_p,
- ctypes.c_ulong,
- ctypes.c_void_p,
-)
-
-_umount = libc.umount
-_umount.restype = ctypes.c_int
-_umount.argtypes = (ctypes.c_char_p,)
-
-_umount2 = libc.umount2
-_umount2.restype = ctypes.c_int
-_umount2.argtypes = (ctypes.c_char_p, ctypes.c_int)
-
-_unshare = libc.unshare
-_unshare.restype = ctypes.c_int
-_unshare.argtypes = (ctypes.c_int,)
-
-
-def c_path(path):
- if path is None:
- return path
- if isinstance(path, pathlib.PosixPath):
- path = path.as_posix()
- if isinstance(path, str):
- path = path.encode()
- return path
-
-
-def c_error():
- return OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno()))
-
-
-def unshare(flags):
- if libc.unshare(flags) != 0:
- raise c_error()
-
-
-def pivot_root(new_root, put_old):
- if libc.syscall(SYS_pivot_root, c_path(new_root), c_path(put_old)) != 0:
- raise c_error()
-
-
-def mount(
- source: str,
- target: str,
- fstype: str,
- flags: int = 0,
- data: str = None,
-):
- """Mount filesystem.
-
- :param source: Device/source to mount.
- :param target: Mountpoint.
- :param fstype: Filesystem type. Available filesystem types can be found in /proc/filesystems.
- :param flags: Mount flags.
- :param data: Mount options for specified filesystem.
- :raises OSError: If mount call failed with nonzero return code.
- """
- if (
- _mount(
- c_path(source),
- c_path(target),
- fstype.encode() if fstype is not None else fstype,
- int(flags),
- data.encode() if data is not None else data,
- )
- != 0
- ):
- raise c_error()
-
-
-def bind_mount(
- source: str,
- target: str,
- write: bool = False,
-):
- return mount(
- source,
- target,
- None,
- (
- MountFlag.BIND
- | (0 if write else MountFlag.RDONLY)
- | MountFlag.NOSUID
- | MountFlag.NODEV
- ),
- )
-
-
-def umount(target: str):
- """Unmount filesystem.
-
- :param target: Mountpoint.
- :raises OSError: If umount call failed with nonzero return code.
- """
- if _umount(c_path(target)) != 0:
- raise c_error()
-
-
-def lazy_umount(target):
- target = c_path(target)
- if _umount(target) != 0:
- if _umount2(target, MNT_DETACH) != 0:
- raise c_error()
-
-
-@dataclasses.dataclass(frozen=True)
-class MountInfo:
- id: int
- parent: int
- dev: tuple
- root: str
- mountpoint: str
-
- def __post_init__(self):
- assert isinstance(self.id, int)
- assert isinstance(self.parent, int)
- assert isinstance(self.dev, tuple)
- minor, major = self.dev
- assert isinstance(minor, int)
- assert isinstance(major, int)
- assert isinstance(self.root, str)
- assert self.root[0] == '/'
- assert isinstance(self.mountpoint, str)
- assert self.mountpoint[0] == '/'
-
- @classmethod
- def from_line(cls, line):
- rec = line.split(maxsplit=5)
- major, minor = rec[2].split(':')
- return cls(
- id=int(rec[0]),
- parent=int(rec[1]),
- dev=(int(major), int(minor)),
- root=rec[3],
- mountpoint=rec[4],
- )
-
-
-def parse_mountinfo(mountinfo_path='/proc/self/mountinfo'):
- root_id = None
- mountinfo = {}
- with open(mountinfo_path, 'rt') as f:
- for line in f:
- mi = MountInfo.from_line(line)
- if mi.mountpoint == '/':
- assert root_id is None
- root_id = mi.id
- assert mi.id not in mountinfo
- mountinfo[mi.id] = mi
- assert root_id is not None
- return (root_id, mountinfo)
-
-
-def umount_order(mount_id, mountinfo):
- for mi in mountinfo.values():
- if mi.parent == mount_id:
- yield from umount_order(mi.id, mountinfo)
- yield mountinfo[mount_id]
-
-
-def pivot_and_umount(new_root, put_old, umount_list):
- mtp_prefix = '/' + put_old.relative_to(new_root).as_posix()
- pivot_root(new_root, put_old)
- os.chdir('/') # so we don't stand in the old root
- for mtp in umount_list:
- lazy_umount(mtp_prefix + mtp)
-
-
-def nonblock_cloexec(fd):
- return fcntl.fcntl(
- fd,
- fcntl.F_SETFD,
- fcntl.fcntl(fd, fcntl.F_GETFD) | os.O_NONBLOCK | fcntl.FD_CLOEXEC,
- )
-
-
-def exit_status(status):
- sig = status & 0xFF
- ret = status >> 8
- if sig:
- raise SystemExit(128 + sig)
- if ret >= 128:
- raise SystemExit(128)
- raise SystemExit(ret)
-
-
-def exec_command(argv):
- if argv[0][0] == '/':
- os.execv(argv[0], argv)
- for d in os.environ['PATH'].split(':'):
- try:
- os.execv(os.path.join(d, argv[0]), argv)
- except FileNotFoundError:
- continue
- raise SystemExit(127)
-
-
-def map_uid_gid(orig_uid, orig_gid):
- with open('/proc/self/uid_map', 'wt') as f:
- f.write(f'{orig_uid} {orig_uid} 1\n')
-
- with open('/proc/self/setgroups', 'wt') as f:
- f.write('deny\n')
-
- with open('/proc/self/gid_map', 'wt') as f:
- f.write(f'{orig_gid} {orig_gid} 1\n')
-
- os.setuid(orig_uid)
- os.setgid(orig_gid)
-
-
-def pidns_run(unshare_flags, run_pid1=True):
- (parent_rfd, parent_wfd) = os.pipe()
- nonblock_cloexec(parent_rfd)
- nonblock_cloexec(parent_wfd)
- orig_uid = os.getuid()
- orig_gid = os.getgid()
- unshare(CLONE_NEWPID | unshare_flags)
- if unshare_flags & CLONE_NEWUSER:
- map_uid_gid(orig_uid, orig_gid)
- fork_pid = os.fork()
- if fork_pid == 0:
- # child
- assert os.getpid() == 1
- os.close(parent_wfd)
- if run_pid1:
- return pidns_pid1(parent_rfd)
- else:
- return parent_rfd
- else:
- # parent
- os.close(parent_rfd)
- (pid, status) = os.waitpid(fork_pid, 0)
- exit_status(status)
-
-
-def pidns_pid1(parent_rfd):
- fork2_pid = os.fork()
- if fork2_pid == 0:
- # child
- return
- else:
- # parent
- rlist, wlist, elist = (parent_rfd,), (), ()
- while True:
- (pid, status) = os.waitpid(0, os.WNOHANG)
- if pid == fork2_pid:
- exit_status(status)
- try:
- r, w, x = select.select(rlist, wlist, elist, 1.0)
- except select.error as e:
- code, msg = e.args
- # We might get interrupted by SIGCHLD here
- if code != errno.EINTR:
- raise
-
-
-@dataclasses.dataclass(frozen=True)
-class MountTMPFS:
- path: pathlib.PosixPath
-
- def __post_init__(self):
- assert isinstance(self.path, pathlib.PosixPath)
- assert not self.path.is_absolute()
-
- def mount(self, root):
- dst = root / self.path
- dst.mkdir(parents=True, exist_ok=True)
- mount('tmpfs', dst, 'tmpfs', MountFlag.NOSUID | MountFlag.NODEV)
-
-
-@dataclasses.dataclass(frozen=True)
-class MountBind:
- src: pathlib.PosixPath
- dst: pathlib.PosixPath
- write: bool = False
-
- def __post_init__(self):
- assert isinstance(self.src, pathlib.PosixPath)
- assert self.src.is_absolute()
- assert isinstance(self.dst, pathlib.PosixPath)
- assert not self.dst.is_absolute()
-
- def mount(self, root):
- dst = root / self.dst
- if self.src.is_dir():
- dst.mkdir(parents=True, exist_ok=True)
- bind_mount(self.src, dst, self.write)
-
-
-def relpath(s):
- p = pathlib.PosixPath(s)
- return p.relative_to('/') if p.is_absolute() else p
-
-
-def parse_mount(s):
- m_type, rest = s.split(':', maxsplit=1)
- if m_type == 'tmpfs':
- return MountTMPFS(relpath(rest))
- elif m_type in ('rw_bind', 'ro_bind'):
- write = m_type == 'rw_bind'
- src, dst = rest.split(':', maxsplit=1)
- return MountBind(pathlib.PosixPath(src), relpath(dst), write)
- raise ValueError(m_type)
-
-
-@dataclasses.dataclass(frozen=True)
-class Settings:
- versions: pathlib.PosixPath
- root: pathlib.PosixPath
- chdir: pathlib.PosixPath
- vars: dict
- command: tuple
- extra_mount: tuple
- drop_to: tuple = None
- untar: pathlib.PosixPath = None
-
- def __post_init__(self):
- assert isinstance(self.command, tuple)
- assert all(isinstance(arg, (str, bytes)) for arg in self.command)
-
- assert isinstance(self.extra_mount, tuple)
- assert all(isinstance(arg, (MountTMPFS, MountBind)) for arg in self.extra_mount)
-
- assert isinstance(self.chdir, pathlib.PosixPath)
- assert self.chdir.is_absolute()
-
- assert isinstance(self.versions, pathlib.PosixPath)
- assert self.versions.is_absolute()
- assert self.versions.is_dir()
-
- if self.drop_to is not None:
- assert isinstance(self.drop_to, tuple)
- uid, gid = self.drop_to
- assert isinstance(uid, int)
- assert isinstance(gid, int)
-
- assert isinstance(self.untar, (pathlib.PosixPath, type(None)))
-
- assert isinstance(self.root, pathlib.PosixPath)
- assert self.root.is_absolute()
- assert self.root.is_dir(), self.root
- if self.untar is None:
- self._check_root()
-
- def _check_root(self):
- assert (self.root / 'oldroot').is_dir()
- assert (self.root / 'proc').is_dir()
- assert (self.root / 'dev').is_dir()
- assert (self.root / 'bin').is_dir()
- assert (self.root / 'bin/sh').exists()
-
- @classmethod
- def from_args_and_env(cls, args, env):
- if args.vars:
- import yaml
-
- with args.vars.open('rt') as f:
- v = yaml.safe_load(f)
- else:
- v = {}
-
- return cls(
- versions=(args.versions or pathlib.PosixPath(v['versions'])),
- root=args.root_dir,
- chdir=args.chdir,
- vars=v,
- command=tuple(args.command),
- extra_mount=tuple(args.extra_mount) if args.extra_mount is not None else (),
- drop_to=(
- (int(env['pthbs_uid']), int(env['pthbs_gid']))
- if args.mode == 'root'
- else None
- ),
- untar=args.untar and pathlib.PosixPath(args.untar),
- )
-
-
-def userns_sandbox_run(settings):
- assert settings.untar is None
- assert settings.drop_to is None
- mount('proc', settings.root / 'proc', 'proc', MountFlag.NOSUID | MountFlag.NODEV)
- if not (settings.root / 'dev/null').is_char_device():
- mount(
- '/dev',
- settings.root / 'dev',
- None,
- (MountFlag.BIND | MountFlag.NOSUID | MountFlag.REC),
- )
-
- mountpoints = [
- MountTMPFS(relpath('/dev/shm')),
- ]
- mountpoints.extend(settings.extra_mount)
- mountpoints.append(MountBind(settings.versions, settings.versions.relative_to('/')))
- for m in mountpoints:
- m.mount(settings.root)
-
- os.chroot(str(settings.root))
- os.chdir(settings.chdir)
- exec_command(settings.command)
-
-
-def mkchardev(path, major, minor, mode):
- if isinstance(path, pathlib.PosixPath):
- path = path.as_posix()
- os.mknod(
- path,
- mode=mode | stat.S_IFCHR,
- device=os.makedev(major, minor),
- )
-
-
-def mkblockdev(path, major, minor, mode):
- if isinstance(path, pathlib.PosixPath):
- path = path.as_posix()
- os.mknod(
- path,
- mode=mode | stat.S_IFBLK,
- device=os.makedev(major, minor),
- )
-
-
-def mknod_dev(dev):
- mkchardev(mode=0o666, major=1, minor=3, path=dev / "null")
- mkchardev(mode=0o666, major=1, minor=7, path=dev / "full")
- mkchardev(mode=0o666, major=5, minor=2, path=dev / "ptmx")
- mkchardev(mode=0o644, major=1, minor=8, path=dev / "random")
- mkchardev(mode=0o644, major=1, minor=9, path=dev / "urandom")
- mkchardev(mode=0o666, major=1, minor=5, path=dev / "zero")
- mkchardev(mode=0o666, major=5, minor=0, path=dev / "tty")
- (dev / "fd").symlink_to("/proc/self/fd")
- (dev / "stdin").symlink_to("/proc/self/fd/0")
- (dev / "stdout").symlink_to("/proc/self/fd/1")
- (dev / "stderr").symlink_to("/proc/self/fd/2")
-
-
-def root_sandbox_setup(settings):
- uid, gid = settings.drop_to
- os.umask(0)
- to_umount = [mi.mountpoint for mi in umount_order(*parse_mountinfo())]
- r = settings.root
- if settings.untar:
- mount('sandbox_root', r, 'tmpfs', MountFlag.NOSUID)
- (r / 'oldroot').mkdir()
- subprocess.check_call(
- ('/bin/tar', 'xpf', settings.untar.absolute()),
- shell=False,
- cwd=r,
- )
- mount('proc', r / 'proc', 'proc', MountFlag.NOSUID | MountFlag.NODEV)
- if not (r / 'dev/null').is_char_device():
- mknod_dev(r / 'dev')
-
- mountpoints = [
- MountTMPFS(relpath('/dev/shm')),
- ]
- mountpoints.extend(settings.extra_mount)
- mountpoints.append(MountBind(settings.versions, settings.versions.relative_to('/')))
- for m in mountpoints:
- m.mount(r)
-
- if settings.untar:
- mount(
- 'tmpfs',
- r,
- '',
- (MountFlag.REMOUNT | MountFlag.RDONLY | MountFlag.NOSUID),
- )
- pivot_and_umount(r, r / 'oldroot', to_umount)
- os.setgid(gid)
- os.setuid(uid)
- os.chdir(settings.chdir)
-
-
-def main(args, env):
- settings = Settings.from_args_and_env(args, env)
- if args.mode == 'userns':
- pidns_run(
- CLONE_NEWUSER | CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWPID,
- )
- userns_sandbox_run(settings)
- else:
- pidns_run(
- CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWPID,
- )
- unshare(CLONE_NEWNS)
- root_sandbox_setup(settings)
- os.umask(0o022)
- exec_command(settings.command)
-
-
-argument_parser = argparse.ArgumentParser(
- description="Linux namespaces based sandbox for pthbs",
- allow_abbrev=False,
-)
-argument_parser.add_argument(
- '--mode',
- '-m',
- required=True,
- choices=('userns', 'root'),
- help="sandbox mode",
-)
-argument_parser.add_argument(
- '--vars',
- '-y',
- type=pathlib.PosixPath,
- help="vars.yaml to read configuration from",
-)
-argument_parser.add_argument(
- '--versions',
- '-V',
- type=pathlib.PosixPath,
- help="versions dir (e.g. /versions)",
-)
-argument_parser.add_argument(
- '--chdir',
- '-C',
- type=pathlib.PosixPath,
- default=pathlib.PosixPath('/'),
- help="set working directory inside sandbox",
-)
-argument_parser.add_argument(
- '--untar',
- '-f',
- type=pathlib.PosixPath,
- default=pathlib.PosixPath(os.getcwd()),
- help="initial structure for build tmpfs",
-)
-argument_parser.add_argument('--extra-mount', action='append', type=parse_mount)
-argument_parser.add_argument('root_dir', type=pathlib.PosixPath)
-argument_parser.add_argument('command', nargs='+')
-
-
-if __name__ == '__main__':
- args = argument_parser.parse_args()
- main(args, os.environ)
-
-# pylama:linters=pycodestyle,pyflakes:ignore=D212,D203,D100,D101,D102,D107
-# vim: sts=4 ts=4 sw=4 et tw=88 efm=%A%f\:%l\:%c\ %t%n\ %m
diff --git a/packages.aat b/packages.aat
@@ -1,45 +0,0 @@
-|start_package("musl-cross-make")
- awk -vPWD="$PWD" '{gsub(/@@PWD@@/,PWD); print}' musl-cross-make.config.mak >musl-cross-make/config.mak.new || exit $?
- mv musl-cross-make/config.mak.new musl-cross-make/config.mak || exit $?
- make -C musl-cross-make -j$JOBS -l$JOBS || exit $?
- make -C musl-cross-make install || exit $?
-|end()
-|start_package("skalibs")
- git checkout --force || exit $?
- git clean -fx || exit $?
- ./configure --disable-shared --enable-static --enable-slashpackage="$slashpackage" || exit $?
- make -j${jobs} || exit $?
- make install || exit $?
- make -L update || exit $?
- make -L global-links || exit $?
-|end()
-|start_package("s6")
- git checkout --force || exit $?
- git clean -fx || exit $?
- #patch -up 1 -i ../s6_clone_newpid.patch || exit $?
- patch -up 1 -i ../s6_clone3_newpid.patch || exit $?
- ./configure --disable-shared --enable-static --enable-allstatic --enable-static-libc --enable-slashpackage="$slashpackage" || exit $?
- make -j${jobs} CFLAGS=-DWANT_CLONE_NEWPID || exit $?
- make install || exit $?
- make -L update || exit $?
- make -L global-links || exit $?
-|end()
-@for p in packages
-|start
- git checkout --force || exit $?
- git clean -fx || exit $?
- ./configure --disable-shared --enable-static --enable-allstatic --enable-static-libc --enable-slashpackage="$slashpackage" || exit $?
- make -j${jobs} || exit $?
- make install || exit $?
- make -L update || exit $?
- make -L global-links || exit $?
-|end()
-@endfor
-|# (cd execline && install_generic) || exit $?
-|# (cd s6-rc && install_generic) || exit $?
-|# (cd s6-portable-utils && install_generic) || exit $?
-|# (cd s6-linux-utils && install_generic) || exit $?
-|# (cd s6-linux-init && install_generic) || exit $?
-|# (cd mdevd && install_generic) || exit $?
-|# (cd s6-dns && install_generic) || exit $?
-|# (cd s6-networking && install_generic) || exit $?
diff --git a/pthbs b/pthbs
@@ -0,0 +1 @@
+Subproject commit 1c067b19b7c431f8edbd48b38b33812f513ffac3
diff --git a/pthbs_genpkgpy b/pthbs_genpkgpy
@@ -0,0 +1 @@
+Subproject commit b15529ebde7867aaf9779ecfcd2fbbace7472edb
diff --git a/py-cp311-cp311-musllinux_1_2_x86_64-requirements.txt b/py-cp311-cp311-musllinux_1_2_x86_64-requirements.txt
@@ -1,38 +0,0 @@
-#
-# This file is autogenerated by pip-compile with Python 3.11
-# by the following command:
-#
-# pip-compile --output-file=py-cp311-cp311-musllinux_1_2_x86_64-requirements.txt.new py-requirements.in
-#
-black==24.2.0
- # via -r py-requirements.in
-click==8.1.7
- # via black
-isort==5.13.2
- # via -r py-requirements.in
-jinja2==3.1.3
- # via -r py-requirements.in
-markupsafe==2.1.5
- # via jinja2
-mccabe==0.7.0
- # via pylama
-mypy-extensions==1.0.0
- # via black
-packaging==23.2
- # via black
-pathspec==0.12.1
- # via black
-platformdirs==4.2.0
- # via black
-pycodestyle==2.11.1
- # via pylama
-pydocstyle==6.3.0
- # via pylama
-pyflakes==3.2.0
- # via pylama
-pylama==8.4.1
- # via -r py-requirements.in
-pyyaml==6.0.1
- # via -r py-requirements.in
-snowballstemmer==2.2.0
- # via pydocstyle
diff --git a/py-requirements.in b/py-requirements.in
@@ -1,5 +0,0 @@
-jinja2
-pyyaml
-black
-isort
-pylama
diff --git a/python.mk b/python.mk
@@ -1,60 +0,0 @@
-PYTHONPATH:=$(abspath .)
-PYTHON_EXE:=python3
-# re-evaluate each time because there's no sensible way to check
-# whether the python interpreter changed
-$(eval $(shell $(PYTHON_EXE) ./make_vars.py))
-
-PY_SRC:=$(wildcard *.py)
-PY_REQ:=py-$(PYTHON_IMPL)-requirements.txt
-PY_WHL:=work/wheels/$(PYTHON_IMPL)
-VENV:=work/virtualenvs/$(PYTHON_IMPL)
-PYTHON_VENV_INSTALL=pip-tools wheel
-
-pycodestyle: $(patsubst %.py,.%.pyfmt,$(PY_SRC)) $(VENV)/.done
- '$(VENV)/bin/pylama' -l 88 $(PY_SRC) || true
-
-.%.pyfmt: %.py $(VENV)/.done
- '$(VENV)/bin/isort' - <'$<' >'$<.tmp1'
- cp -a '$<' '$<.tmp2'
- '$(VENV)/bin/black' -S - <'$<.tmp1' >'$<.tmp2'
- rm '$<.tmp1'
- if cmp -s '$<.tmp2' '$<'; then rm -v '$<.tmp2'; else mv -v '$<.tmp2' '$<'; fi
- touch $@
-
-.PHONY: py-requirements py-wheels py-venv py-virtualenv py-genpkg
-py-requirements: $(PY_REQ)
-
-py-wheels: $(PY_WHL)/.done
-
-py-venv: $(VENV)/.done
-
-py-virtualenv: py-venv
-
-py-genpkg: $(VENV)/.done
- '$(VENV)/bin/python' genpkg.py
-
-# -- requirement file rules
-
-$(PY_REQ): py-requirements.in $(VENV)/bin/pip-compile
- '$(VENV)/bin/pip-compile' -v --annotate -o '$@.new' py-requirements.in
- mv '$@.new' '$@'
-
-# -- wheel building rules
-
-$(PY_WHL)/.done: $(PY_REQ)
- mkdir -p '$(PY_WHL)'
- '$(VENV)/bin/python' -m pip wheel -w '$(PY_WHL)' -r '$(PY_REQ)'
- touch '$@'
-
-# -- virtualenv rules
-
-$(VENV)/bin/pip-compile $(VENV)/bin/pip-sync:
- if test -e '$(VENV)'; then rm -r '$(VENV)'; else true; fi
- mkdir -p virtualenvs
- $(PYTHON_VENV) '$(VENV)'
- '$(VENV)/bin/pip' install -I $(PYTHON_VENV_INSTALL)
-
-$(VENV)/.done: $(PY_REQ) $(VENV)/bin/pip-sync $(PY_WHL)/.done
- $(VENV)/bin/pip-sync --no-index -f '$(PY_WHL)' '$(PY_REQ)'
- ln -sf 'virtualenvs/$(PYTHON_IMPL)' work/venv
- touch '$@'
diff --git a/script_env.txt b/script_env.txt
@@ -1,3 +0,0 @@
-DESTDIR=./destdir # where files get copied to
-PREFIX=/versions/package-XXXX # final installation dir?
-MAKE_JOBS/NPROCS
diff --git a/script_header.txt b/script_header.txt
@@ -1,11 +0,0 @@
-package dependency
-file hash
-git commit hash
-
----
-#!/usr/bin/env pthbs-build
-#+package-name
-#@git:commit-hash:dirname
-#@sha256:file-hash:filename
-<mandatory empty line>
-script start
