commit e4b7512893a63ef56469a9d60836d84402fde71e
parent e3b3b0f919535f670a5298da48060b7982b17cd3
Author: Jan Pobrislo <ccx@te2000.cz>
Date: Wed, 30 Apr 2025 20:11:46 +0000
Add tool to check package consistency and external resources
Diffstat:
4 files changed, 301 insertions(+), 0 deletions(-)
diff --git a/command/pthbs-build b/command/pthbs-build
@@ -51,6 +51,10 @@ BEGIN {
settings["set_path"] = 1
FS=":"
}
+function fatal(msg) {
+ printf "FATAL: pthbs-build (eval): %s %s:%d: \"%s\"\n", msg, FILENAME, FNR, $0 >"/dev/stderr"
+ exit 1
+}
/^#@pragma:/ {
if($2 == "nosandbox") {
settings["sandbox"] = 0
@@ -60,12 +64,19 @@ BEGIN {
fatal("unrecognized @pragma:")
}
}
+/^$/ { nextfile }
END {
print "setting_sandbox=" (settings["sandbox"]?"true":"false")
print "setting_set_path=" (settings["set_path"]?"true":"false")
+ print "setting_ok=true"
}
' "$1" )"
+if test true != "$setting_ok"; then
+ printf '%s\n' >&2 "$0: fatal: could not extract information from '$1'"
+ exit 111
+fi
+
if test -f "$pthbs_cache/make/package.sha256.${bsh}.env"; then
envfile="$pthbs_cache/make/package.sha256.${bsh}.env"
envhash=$(pthbs-getenvhash "$envfile") || exit $?
diff --git a/pthbs.mk b/pthbs.mk
@@ -17,6 +17,10 @@ export pthbs_source:=$(abspath $(pthbs))
no_default:
@echo "pthbs has no default target to build"
+.PHONY: check
+check:
+ '$(pthbs_source)/util/verify-external-resources'
+
$(versions)/environment.%/.env:
@echo mkdir "$$(dirname '$@')"
@echo touch '$@'
diff --git a/util/_verify-external-resources.awk b/util/_verify-external-resources.awk
@@ -0,0 +1,271 @@
+#!/bin/awk -f
+BEGIN {
+ load_indexes()
+
+ dep_count = 0
+ env_count = 0
+ is_envfile = ENVIRON["script"] ~ /\.environment$/
+ if(is_envfile) {
+ envname = substr(ENVIRON["scriptname"], 1, length(ENVIRON["scriptname"])-12)
+ }
+ FS="[ /]"
+}
+
+function fatal(msg) {
+ if(pline) {
+ printf "FATAL: pthbs-verifypkg: %s %s:%d\n", msg, pfile, pline >"/dev/stderr"
+ } else {
+ printf "FATAL: pthbs-verifypkg: %s %s:%d: \"%s\"\n", msg, FILENAME, FNR, $0 >"/dev/stderr"
+ }
+ exit 1
+}
+
+function q(s) {
+ return "'" s "'"
+}
+function dbgprint(msg) {
+ printf "DBG: %s\n", msg >"/dev/stderr"
+}
+function dbgprint_count(n) {
+ printf "DBG: %d entries\n", n >"/dev/stderr"
+}
+function dbgprint_dep(dep_type, dep_value) {
+ dbgprint(pfile" -> ["dep_type"]"dep_value)
+}
+function dbgprint_pkg() {
+ dbgprint(q(pfile)" => "q(pname))
+}
+
+function vprint(msg) {
+ printf "%s\n", msg
+}
+
+function load_indexes() {
+ if(!length(ENVIRON["pthbs_indexdir"])) {
+ fatal("variable 'pthbs_indexdir' not defined")
+ }
+ load_downloadlist(ENVIRON["pthbs_indexdir"]"/downloadlist.sha256")
+ load_filelist(ENVIRON["pthbs_indexdir"]"/filelist.sha256")
+ load_commitlist(ENVIRON["pthbs_indexdir"]"/commitlist.sha1")
+}
+
+function load_downloadlist(downloadlist_path, n) {
+ dbgprint("reading downloadlist: "q(downloadlist_path))
+ n = 0
+ while(getline <downloadlist_path) {
+ if(/^$/ || /^#/) {
+ continue
+ }
+ downloadable_hashes["sha256:"$1] = 0
+ hash_info["sha256:"$1] = downloadlist_path ":" FNR substr($0, length($1 + 2))
+ n += 1
+ }
+ close(downloadlist_path)
+ dbgprint_count(n)
+}
+
+function load_filelist(filelist_path, n) {
+ dbgprint("reading filelist: "q(filelist_path))
+ n = 0
+ while(getline <filelist_path) {
+ linkable_hashes["sha256:"$1] = 0
+ hash_info["sha256:"$1] = filelist_path ":" FNR substr($0, length($1 + 2))
+ n += 1
+ }
+ close(filelist_path)
+ dbgprint_count(n)
+}
+
+function load_commitlist(commitlist_path, n) {
+ dbgprint("reading commitlist: "q(commitlist_path))
+ n = 0
+ while(getline <commitlist_path) {
+ commits[$1] = 0
+ n += 1
+ }
+ close(filelist_path)
+ dbgprint_count(n)
+}
+
+/^[0-9a-f]+ .*\/[-_:.0-9a-zA-Z]+\/[-_:.0-9a-zA-Z]+/ {
+ pname = $NF
+ sub(/:.*/, "", pname)
+ if(pname ~ /\.environment$/) {
+ pname = "env"
+ }
+ pname = $(NF-1)"/"pname"."$1
+ pfile = substr($0, length($1) + 3)
+ pvariant = $(NF-1)
+
+ dbgprint_pkg()
+ package_names[pname] = 0
+ package_variants[pfile] = pvariant
+ next
+}
+
+{
+ fatal("unexpected line")
+}
+function record_failure(dep_type, dep_value, msg) {
+ if(!pline) {
+ fatal("internal error: record_failure() called without pline")
+ }
+ failed[pfile] = failed[pfile] pfile ":" pline " [" dep_type "] '" dep_value "'\t" msg "\n"
+}
+
+END {
+ delete pname
+ for(pfile in package_variants) {
+ read_package(pfile)
+ }
+ count = 0
+ unused = 0
+ for(commit_id in commits) {
+ count += 1
+ unused += commits[commit_id] ? 0 : 1
+ }
+ printf "commit ids: %4d unused: %4d\n", count, unused
+
+ count = 0
+ unused = 0
+ for(i in linkable_hashes) {
+ count += 1
+ unused += linkable_hashes[i] ? 0 : 1
+ if(!linkable_hashes[i]) {
+ vprint(i " unused local file " hash_info[i])
+ }
+ }
+ printf "local files: %4d unused: %4d\n", count, unused
+
+ count = 0
+ unused = 0
+ for(i in downloadable_hashes) {
+ count += 1
+ unused += downloadable_hashes[i] ? 0 : 1
+ if(!downloadable_hashes[i]) {
+ vprint(i " unused local file " hash_info[i])
+ }
+ }
+ printf "downloads: %4d unused: %4d\n", count, unused
+
+ failed_files = 0
+ for(pfile in failed) {
+ failed_files += 1
+ printf "errors in file %s:\n%s\n", pfile, failed[pfile]
+ }
+ exit failed_files ? 1 : 0
+}
+
+function read_package(f, line, seen_shebang) {
+ dbgprint("reading package: "q(f))
+ pvariant = package_variants[f]
+ seen_shebang = 0
+ pline = 0
+ while(getline line <f) {
+ pline += 1
+ if(line == "") {
+ break
+ }
+ if(pline == 1 && line ~ /^#!.*pthbs-build/) {
+ seen_shebang = 1
+ continue
+ }
+ if(line ~ /^#\+/) {
+ hash_plus(line)
+ continue
+ }
+ if(line ~ /^#@/) {
+ hash_at(line)
+ continue
+ }
+ fatal("unexpected line")
+ }
+ close(f)
+ if(pline == 0) {
+ fatal("error reading file: '"f"'")
+ }
+ if(!seen_shebang) {
+ fatal("did not encounter expected shebang line in package")
+ }
+ delete pline
+}
+
+function depend_git(commit_id) {
+ dbgprint_dep("git-sha1", commit_id)
+ if(commit_id in commits) {
+ commits[commit_id] += 1
+ } else {
+ #dbgprint("!!! could not resolve '"commit_id"' to repository file in "pfile)
+ record_failure("git-sha1", commit_id, "could not resolve '"commit_id"' to git repository")
+ }
+}
+
+function depend_file(hash_type, file_hash) {
+ dbgprint_dep(hash_type, file_hash)
+ if((hash_type":"file_hash) in linkable_hashes) {
+ dbgprint("local")
+ linkable_hashes[hash_type":"file_hash] += 1
+ } else if((hash_type":"file_hash) in downloadable_hashes) {
+ dbgprint("download")
+ downloadable_hashes[hash_type":"file_hash] += 1
+ } else {
+ #dbgprint("!!! could not resolve how to get file with "q(hash_type":"file_hash)" in")
+ record_failure(hash_type, file_hash, "could not resolve how to get file with "q(hash_type":"file_hash))
+ }
+}
+
+function at_git(commit_id, dstdir){
+ depend_git(commit_id)
+}
+
+function at_untar(extra_opts, hash_type, file_hash, dstdir){
+ depend_file(hash_type, file_hash)
+}
+
+function at_filehash(hash_type, file_hash, dst, dstdir){
+ depend_file(hash_type, file_hash)
+}
+
+function hash_plus(line, dep) {
+ if(line == "#+*") {
+ # settings["sandbox"] = 0
+ } else if(!length(ENVIRON["envdir"])) {
+ dep = pvariant "/" substr(line, 3)
+ if(!(dep in package_names)) {
+ fatal("could not resolve '"dep"' to package file in")
+ } else {
+ dbgprint_dep("pkg", dep)
+ package_names[dep] += 1
+ }
+ }
+}
+
+function hash_at(line, a){
+ split(line, a, ":")
+ if(a[1] == "#@git") {
+ if(match(line, "^#@git:[0-9a-f]+:") == 0) {
+ fatal("invalid syntax for @git:")
+ }
+ at_git(a[2], substr(line, RLENGTH+1))
+ } else if(a[1] == "#@sha256") {
+ if(match(line, "^#@sha256:[0-9a-f]+:") == 0) {
+ fatal("invalid syntax for @sha256:")
+ }
+ at_filehash("sha256", a[2], substr(line, RLENGTH+1))
+ } else if(a[1] == "#@untar") {
+ if(match(line, "^#@untar:[^:]*:sha256:[0-9a-f]+:") == 0) {
+ fatal("invalid syntax for @untar:")
+ }
+ at_untar(a[2], a[3], a[4], substr(line, RLENGTH+1))
+ } else if(a[1] == "#@pragma") {
+ if(a[2] == "nosandbox") {
+ # settings["sandbox"] = 0
+ } else if(a[2] == "nopath") {
+ # settings["set_path"] = 0
+ } else {
+ fatal("unrecognized @pragma:")
+ }
+ } else {
+ fatal("unrecognized @command:")
+ }
+}
diff --git a/util/verify-external-resources b/util/verify-external-resources
@@ -0,0 +1,15 @@
+#!/bin/sh
+if test -n "$pthbs_xtrace"; then
+ set -x
+ if test -n "$BB_ASH_VERSION"; then PS4="+${0##*/}"':${FUNCNAME}:${LINENO} '; fi
+fi
+if test -z "$pthbs_source"; then
+ printf '%s\n' >&2 "$0: fatal: pthbs_source env var undefined or empty"
+ exit 100
+fi
+if test -z "$pthbs_indexdir"; then
+ printf '%s\n' >&2 "$0: fatal: pthbs_indexdir env var undefined or empty"
+ exit 100
+fi
+
+find ./variants/*/ -type f -exec sha256sum '{}' + | awk -f "$pthbs_source/util/_verify-external-resources.awk"
