pthbs

_verify-external-resources.awk (7416B)

---

 #!/bin/awk -f
 BEGIN {
 	load_indexes()
 
 	dep_count = 0
 	env_count = 0
 	FS="[ /]"
 }
 
 function fatal(msg) {
 	if(pline) {
 		printf "FATAL: pthbs-verifypkg: %s %s:%d\n", msg, pfile, pline >"/dev/stderr"
 	} else {
 		printf "FATAL: pthbs-verifypkg: %s %s:%d: \"%s\"\n", msg, FILENAME, FNR, $0 >"/dev/stderr"
 	}
 	exit 1
 }
 
 function q(s) {
 	return "'" s "'"
 }
 function dbgprint(msg) {
 	if(length(ENVIRON["pthbs_debug"])) {
 		printf "DBG: %s\n", msg >"/dev/stderr"
 	}
 }
 function dbgprint_count(n) {
 	if(length(ENVIRON["pthbs_debug"])) {
 		printf "DBG: %d entries\n", n >"/dev/stderr"
 	}
 }
 function dbgprint_dep(dep_type, dep_value) {
 	dbgprint(pfile" -> ["dep_type"]"dep_value)
 }
 function dbgprint_pkg() {
 	dbgprint(q(pfile)" => "q(pname))
 }
 
 function vprint(msg) {
 	printf "%s\n", msg
 }
 
 function load_indexes() {
 	if(!length(ENVIRON["pthbs_indexdir"])) {
 		fatal("variable 'pthbs_indexdir' not defined")
 	}
 	load_downloadlist(ENVIRON["pthbs_indexdir"]"/downloadlist.sha256")
 	load_filelist(ENVIRON["pthbs_indexdir"]"/filelist.sha256")
 	load_commitlist(ENVIRON["pthbs_indexdir"]"/commitlist.sha1")
 }
 
 function load_downloadlist(downloadlist_path,    n) {
 	dbgprint("reading downloadlist: "q(downloadlist_path))
 	n = 0
 	while(getline <downloadlist_path) {
 		if(/^$/ || /^#/) {
 			continue
 		}
 		downloadable_hashes["sha256:"$1] = 0
 		hash_info["sha256:"$1] = downloadlist_path ":" FNR substr($0, length($1 + 2))
 		n += 1
 	}
 	close(downloadlist_path)
 	dbgprint_count(n)
 }
 
 function load_filelist(filelist_path,    n) {
 	dbgprint("reading filelist: "q(filelist_path))
 	n = 0
 	while(getline <filelist_path) {
 		linkable_hashes["sha256:"$1] = 0
 		hash_info["sha256:"$1] = filelist_path ":" FNR substr($0, length($1 + 2))
 		n += 1
 	}
 	close(filelist_path)
 	dbgprint_count(n)
 }
 
 function load_commitlist(commitlist_path,    n) {
 	dbgprint("reading commitlist: "q(commitlist_path))
 	n = 0
 	while(getline <commitlist_path) {
 		commits[$1] = 0
 		n += 1
 	}
 	close(filelist_path)
 	dbgprint_count(n)
 }
 
 /^[0-9a-f]+  .*\/[-_:.0-9a-zA-Z]+\/[-_:.0-9a-zA-Z]+/ {
 	pfile = substr($0, length($1) + 3)
 	pvariant = $(NF-1)
 	pname = $NF
 	sub(/:.*/, "", pname)
 	if(pname ~ /\.environment$/) {
 		namedenv_name[pfile] = pvariant"/"pname
 		pname = "env"
 	}
 	pname = $(NF-1)"/"pname"."$1
 
 	dbgprint_pkg()
 	package_names[pname] = 0
 	package_files[pname] = pfile
 	package_variants[pfile] = pvariant
 	next
 }
 
 {
 	fatal("unexpected line")
 }
 function record_failure(dep_type, dep_value, msg) {
 	if(!pline) {
 		fatal("internal error: record_failure() called without pline")
 	}
 	failed[pfile] = failed[pfile] pfile ":" pline " [" dep_type "] '" dep_value "'\t" msg "\n"
 }
 
 function find_namedenvs(pfile,  rdeps, rdep_count, n, result) {
 	if (pfile ~ /\.environment$/) {
 		return pfile
 	}
 	rdep_count = split(package_rdeps[pfile], rdeps, " ")
 	for(n=1; n<=rdep_count; n++) {
 		result = spacemerge(result, find_namedenvs(rdeps[n]))
 	}
 	return result
 }
 
 END {
 	pname=""
 	for(pfile in package_variants) {
 		read_package(pfile)
 	}
 	count = 0
 	unused = 0
 	for(commit_id in commits) {
 		count += 1
 		unused += commits[commit_id] ? 0 : 1
 	}
 	printf "commit ids:  %4d  unused: %4d\n", count, unused
 
 	count = 0
 	unused = 0
 	for(i in linkable_hashes) {
 		count += 1
 		unused += linkable_hashes[i] ? 0 : 1
 		if(!linkable_hashes[i]) {
 			vprint(i " unused local file " hash_info[i])
 		}
 	}
 	printf "local files: %4d  unused: %4d\n", count, unused
 
 	count = 0
 	unused = 0
 	for(i in downloadable_hashes) {
 		count += 1
 		unused += downloadable_hashes[i] ? 0 : 1
 		if(!downloadable_hashes[i]) {
 			vprint(i " unused download " hash_info[i])
 		}
 	}
 	printf "downloads:   %4d  unused: %4d\n", count, unused
 
 	# for(pfile in package_rdeps) {
 	for(pfile in package_variants) {
 		dbgprint("'" pfile "' <- '" package_rdeps[pfile] "'")
 		nes = find_namedenvs(pfile)
 		if(length(nes)){
 			dbgprint("'" pfile "' <= '" nes "'")
 		} else {
 			vprint("unused package: " pfile)
 		}
 	}
 
 	failed_files = 0
 	for(pfile in failed) {
 		failed_files += 1
 		printf "errors in file %s:\n%s\n", pfile, failed[pfile]
 	}
 	exit failed_files ? 1 : 0
 }
 
 function read_package(f,    line, seen_shebang) {
 	dbgprint("reading package: "q(f))
 	pvariant = package_variants[f]
 	seen_shebang = 0
 	pline = 0
 	while(getline line <f) {
 		pline += 1
 		if(line == "") {
 			break
 		}
 		if(pline == 1 && line ~ /^#!.*pthbs-build/) {
 			seen_shebang = 1
 			continue
 		}
 		if(line ~ /^#\+/) {
 			hash_plus(line)
 			continue
 		}
 		if(line ~ /^#@/) {
 			hash_at(line)
 			continue
 		}
 		fatal("unexpected line")
 	}
 	close(f)
 	if(pline == 0) {
 		fatal("error reading file: '"f"'")
 	}
 	if(!seen_shebang) {
 		fatal("did not encounter expected shebang line in package")
 	}
 	pline=""
 }
 
 function depend_git(commit_id) {
 	dbgprint_dep("git-sha1", commit_id)
 	if(commit_id in commits) {
 		commits[commit_id] += 1
 	} else {
 		#dbgprint("!!! could not resolve '"commit_id"' to repository file in "pfile)
 		record_failure("git-sha1", commit_id, "could not resolve '"commit_id"' to git repository")
 	}
 }
 
 function depend_file(hash_type, file_hash) {
 	dbgprint_dep(hash_type, file_hash)
 	if((hash_type":"file_hash) in linkable_hashes) {
 		dbgprint("local")
 		linkable_hashes[hash_type":"file_hash] += 1
 	} else if((hash_type":"file_hash) in downloadable_hashes) {
 		dbgprint("download")
 		downloadable_hashes[hash_type":"file_hash] += 1
 	} else {
 		#dbgprint("!!! could not resolve how to get file with "q(hash_type":"file_hash)" in")
 		record_failure(hash_type, file_hash, "could not resolve how to get file with "q(hash_type":"file_hash))
 	}
 }
 
 function at_git(commit_id, dstdir){
 	depend_git(commit_id)
 }
 
 function at_untar(extra_opts, hash_type, file_hash, dstdir){
 	depend_file(hash_type, file_hash)
 }
 
 function at_filehash(hash_type, file_hash, dst,    dstdir){
 	depend_file(hash_type, file_hash)
 }
 
 function spacejoin(a, b) {
 	return (length(a) ? a " " : "") b
 }
 
 function spacemerge(a, b,    arr, cnt, n, marr, merged) {
 	cnt = split(a, arr, " ")
 	for(n=1; n<=cnt; n++) {
 		marr[arr[n]]=1
 	}
 	cnt = split(b, arr, " ")
 	for(n=1; n<=cnt; n++) {
 		marr[arr[n]]=1
 	}
 	for(n in marr) {
 		merged = spacejoin(merged, n)
 	}
 	return merged
 }
 
 function hash_plus(line,    dep, depfile) {
 	if(line == "#+*") {
 		# settings["sandbox"] = 0
 	} else {
 		dep = pvariant "/" substr(line, 3)
 		if(!(dep in package_names)) {
 			fatal("could not resolve '"dep"' to package file in")
 		} else {
 			dbgprint_dep("pkg", dep)
 			package_names[dep] += 1
 			depfile = package_files[dep]
 			package_deps[pfile] = spacejoin(package_deps[pfile], depfile)
 			package_rdeps[depfile] = spacejoin(package_rdeps[depfile], pfile)
 		}
 	}
 }
 
 function hash_at(line,    a){
 	split(line, a, ":")
 	if(a[1] == "#@git") {
 		if(match(line, "^#@git:[0-9a-f]+:") == 0) {
 			fatal("invalid syntax for @git:")
 		}
 		at_git(a[2], substr(line, RLENGTH+1))
 	} else if(a[1] == "#@sha256") {
 		if(match(line, "^#@sha256:[0-9a-f]+:") == 0) {
 			fatal("invalid syntax for @sha256:")
 		}
 		at_filehash("sha256", a[2], substr(line, RLENGTH+1))
 	} else if(a[1] == "#@untar") {
 		if(match(line, "^#@untar:[^:]*:sha256:[0-9a-f]+:") == 0) {
 			fatal("invalid syntax for @untar:")
 		}
 		at_untar(a[2], a[3], a[4], substr(line, RLENGTH+1))
 	} else if(a[1] == "#@pragma") {
 		if(a[2] == "nosandbox") {
 			# settings["sandbox"] = 0
 		} else if(a[2] == "nopath") {
 			# settings["set_path"] = 0
 		} else {
 			fatal("unrecognized @pragma:")
 		}
 	} else {
 		fatal("unrecognized @command:")
 	}
 }