genpkg.py (11979B)
1 #!/usr/bin/env python3 2 import argparse 3 import hashlib 4 import operator 5 import os.path 6 import subprocess 7 from pathlib import Path 8 9 import jinja2 10 import yaml 11 12 13 class SubmoduleInfo: 14 def __init__(self, cache_dir='cache'): 15 self._current_commits = None 16 self._by_commit = Path(cache_dir) / "link" / "git-commit-sha1" 17 18 @property 19 def current(self): 20 if self._current_commits is not None: 21 return self._current_commits 22 source_type = os.environ.get('pthbs_genpkgpy_submodule_source', None) 23 if source_type == 'current': 24 cmd = ("git", "submodule", "status") 25 elif source_type == 'cached': 26 cmd = ("git", "submodule", "status", '--cached') 27 else: 28 raise RuntimeError( 29 'Environment variable pthbs_genpkgpy_submodule_source' 30 ' must be set to either "current" or "cached"' 31 ) 32 out = subprocess.check_output(cmd).decode('utf8') 33 lines = out.strip('\n').split('\n') 34 records = [[line[0]] + line[1:].split() for line in lines] 35 self._current_commits = { 36 r[2][8:]: r[1] for r in records if r[2].startswith("sources/") 37 } 38 for repo, commit in self._current_commits.items(): 39 if not (self._by_commit / commit).exists(): 40 raise RuntimeError( 41 f"commit ID does not seem to be linked:" 42 f" {commit} from {repo!r} in {self._by_commit}" 43 ) 44 return self._current_commits 45 46 def commit_info(self, commit_id): 47 assert '/' not in commit_id 48 assert '.' not in commit_id 49 assert (self._by_commit / commit_id).exists() 50 out = subprocess.check_output( 51 ('git', 'show', '-s', '--pretty=format:%ai by %an'), 52 cwd=(self._by_commit / commit_id).as_posix(), 53 ).decode('utf8') 54 return out 55 56 57 class DownloadsInfo: 58 def __init__(self, downloadlist_path): 59 assert isinstance(downloadlist_path, Path) 60 self._basenames = {} 61 self._urls = {} 62 with downloadlist_path.open('rt') as f: 63 for line in f: 64 if line[0] in '#\n': 65 continue 66 sha256, size, url = line.rstrip().split(maxsplit=2) 67 assert len(bytes.fromhex(sha256)) == 32 68 assert int(size) >= 0 69 assert url not in self._urls 70 self._urls[url] = 'sha256:' + sha256 71 basename = os.path.basename(url) 72 if basename in self._basenames: 73 self._basenames[basename] = ValueError( 74 'Duplicate download name: ' + repr(basename) 75 ) 76 else: 77 self._basenames[basename] = 'sha256:' + sha256 78 79 def __getitem__(self, key): 80 if '/' in key: 81 value = self._urls[key] 82 else: 83 value = self._basenames[key] 84 if isinstance(value, Exception): 85 raise value 86 return value 87 88 89 def parse_filelist(filelist_path): 90 assert isinstance(filelist_path, Path) 91 with filelist_path.open('rt') as f: 92 return { 93 os.path.basename(fname): fhash 94 for fhash, fname in ( 95 line.rstrip('\n').split(' ', maxsplit=1) for line in f 96 ) 97 } 98 99 100 def _assertion(value): 101 assert value 102 return value 103 104 105 def _value_error(*args, **kwargs): 106 raise ValueError(*args, **kwargs) 107 108 109 class SkipPackageException(Exception): 110 pass 111 112 113 def _skip_package(*args, **kwargs): 114 raise SkipPackageException(*args, **kwargs) 115 116 117 def to_install_name(name, pkg_hash, env_hash=None): 118 rootname = name.split(":")[0] 119 if rootname.endswith('.environment'): 120 if env_hash is None: 121 raise RuntimeError("can't derive namedenv hash without data") 122 return "env." + env_hash 123 else: 124 return "%s.%s" % (rootname, pkg_hash) 125 126 127 def parse_package_deps(script_data): 128 lines = script_data.split(b'\n') 129 if not lines[0].startswith(b"#!"): 130 raise ValueError(lines[0]) 131 for line in lines[1:]: 132 if line == b'': 133 return 134 elif line.startswith(b'#@'): 135 pass 136 elif line.startswith(b'#+'): 137 yield line[2:] 138 else: 139 raise ValueError(line) 140 141 142 def package_env_hash(script_data): 143 deps = sorted(d + b'\n' for d in parse_package_deps(script_data)) 144 if not deps: 145 return None 146 return hashlib.sha256(b''.join(deps)).hexdigest() 147 148 149 class Main: 150 argument_parser = argparse.ArgumentParser() 151 argument_parser.add_argument('-P', '--package-dir', default='packages') 152 argument_parser.add_argument('-T', '--template-dir', default='templates') 153 argument_parser.add_argument('-I', '--index-dir', default='.') 154 argument_parser.add_argument('-C', '--cache-dir', default='cache') 155 argument_parser.add_argument('-V', '--vars-file', default='vars.yaml') 156 argument_parser.add_argument('-M', '--write-mkfile') 157 158 def __init__(self, out_dir, template_dir, index_dir, cache_dir): 159 assert isinstance(out_dir, Path) 160 assert isinstance(template_dir, Path) 161 assert isinstance(index_dir, Path) 162 assert isinstance(cache_dir, Path) 163 self.out_dir = Path(out_dir) 164 self.template_dir = Path(template_dir) 165 self.env = jinja2.Environment( 166 loader=jinja2.FileSystemLoader(template_dir), 167 undefined=jinja2.StrictUndefined, 168 autoescape=False, 169 extensions=["jinja2.ext.do"], 170 ) 171 self.env.globals["pkg_sha256"] = self.pkg_sha256 172 self.env.globals["pkg_install_name"] = self.pkg_install_name 173 self.env.globals["pkg_install_dir"] = self.pkg_install_dir 174 self.env.globals["submodule"] = SubmoduleInfo(cache_dir=cache_dir) 175 self.env.globals["files"] = parse_filelist(index_dir / 'filelist.sha256') 176 self.env.globals["downloads"] = DownloadsInfo(index_dir / 'downloadlist.sha256') 177 self.env.globals["assertion"] = _assertion 178 self.env.globals["value_error"] = _value_error 179 self.env.globals["skip"] = _skip_package 180 self.env.globals["set"] = set 181 self.env.globals["setitem"] = operator.setitem 182 self.env.filters["shesc"] = lambda s: "'%s'" % s.replace("'", r"'\''") 183 self.package_hashes = {} 184 self.package_buildenv_hashes = {} 185 self.rendering = [] 186 self.deps = {} 187 188 @classmethod 189 def from_argv(cls, args=None): 190 args = cls.argument_parser.parse_args(args) 191 obj = cls( 192 out_dir=Path(args.package_dir), 193 template_dir=Path(args.template_dir), 194 index_dir=Path(args.index_dir), 195 cache_dir=Path(args.cache_dir), 196 ) 197 obj.load_vars_yaml(args.vars_file) 198 if args.write_mkfile: 199 obj.write_mkfile(args.write_mkfile) 200 return obj 201 202 def load_vars_yaml(self, fname): 203 with open(fname) as f: 204 self.env.globals.update(yaml.safe_load(f)) 205 206 def write_mkfile(self, fname): 207 t = self.env.from_string('versions:={{versions}}\npackages:={{packages}}\n') 208 with open(fname + '.new', 'wt') as f: 209 f.write(t.render(packages=self.out_dir)) 210 os.rename(fname + '.new', fname) 211 212 def pkg_env_sha256(self, name): 213 current = self.rendering[-1] 214 if current not in self.deps: 215 self.deps[current] = set((name,)) 216 else: 217 self.deps[current].add(name) 218 self._pkg_sha256(name) 219 envlist = ''.join( 220 sorted('%s.%s\n' % (d, self.package_hashes[d]) for d in self.deps[name]) 221 ) 222 return hashlib.sha256(envlist.encode()).hexdigest() 223 224 def pkg_sha256(self, name): 225 return self.pkg_sha256_env_sha256(name)[0] 226 227 def pkg_sha256_env_sha256(self, name): 228 current = self.rendering[-1] 229 if current not in self.deps: 230 self.deps[current] = set((name,)) 231 else: 232 self.deps[current].add(name) 233 return self._pkg_sha256_env_sha256(name) 234 235 def _pkg_sha256(self, name, *args, **kwargs): 236 return self._pkg_sha256_env_sha256(name, *args, **kwargs)[0] 237 238 def _pkg_sha256_env_sha256(self, name, error_on_skip=True): 239 if name in self.package_hashes: 240 return (self.package_hashes[name], self.package_buildenv_hashes[name]) 241 if name in self.rendering: 242 raise RuntimeError("circular dependency: %r", self.rendering) 243 244 out_path = self.out_dir / name 245 246 t = self.env.get_template("pkg/" + name) 247 self.rendering.append(name) 248 try: 249 data = bytes( 250 t.render( 251 name=name, 252 shortname=name.split(':')[0], 253 import_functions=set(), # for "generic" template 254 env_template={}, # for "generic" template 255 env={}, # for "generic" template 256 ).encode('utf8') 257 ) 258 except SkipPackageException as e: 259 if error_on_skip: 260 raise RuntimeError("dependency on skipped package: %s", e) 261 else: 262 if out_path.exists(): 263 out_path.unlink() 264 raise 265 self.package_hashes[name] = hashlib.sha256(data).hexdigest() 266 self.package_buildenv_hashes[name] = package_env_hash(data) 267 lastname = self.rendering.pop() 268 assert name == lastname 269 270 old_hash = None 271 if out_path.exists(): 272 with out_path.open('rb') as f: 273 old_hash = hashlib.file_digest(f, "sha256").hexdigest() 274 if old_hash is None or old_hash != self.package_hashes[name]: 275 tmp_path = out_path.with_suffix('.new') 276 tmp_path.write_bytes(data) 277 tmp_path.replace(out_path) 278 279 return (self.package_hashes[name], self.package_buildenv_hashes[name]) 280 281 def pkg_install_name(self, name): 282 return to_install_name(name, *self.pkg_sha256_env_sha256(name)) 283 284 def pkg_install_dir(self, name): 285 return os.path.join( 286 self.env.globals["versions"], 287 self.pkg_install_name(name), 288 ) 289 290 def pkg_transitive_deps(self, name): 291 # raise NotImplementedError(f'{self.__class__.__name}.pkg_transitive_deps()') 292 self._pkg_sha256(name) 293 294 def recur(name): 295 for n in self.deps[name]: 296 yield from recur(n) 297 yield self.pkg_install_name(name) 298 299 return set(recur(name)) 300 301 def list_packages(self): 302 for tplname in self.env.list_templates(): 303 if not tplname.startswith("pkg/"): 304 continue 305 if "/." in tplname: 306 continue 307 yield tplname[4:] 308 309 def render_all(self): 310 print("digraph G {") 311 for pkgname in self.list_packages(): 312 try: 313 fullname = to_install_name(pkgname, *self._pkg_sha256_env_sha256(pkgname, False)) 314 except SkipPackageException as e: 315 print("// [SKIPPED: %s] %s" % (pkgname, e)) 316 continue 317 # print("%s\t%s" % (pkgname, pkghash)) 318 print( 319 '"%s" [shape=%s]; // %s' 320 % ( 321 pkgname, 322 "note" if pkgname.endswith('.environment') else "box", 323 fullname, 324 ) 325 ) 326 for dep in sorted(self.deps.get(pkgname, ())): 327 print('"%s" -> "%s";' % (pkgname, dep)) 328 # print( 329 # " > %s.%s" 330 # % ( 331 # dep, 332 # self.package_hashes[dep], 333 # ) 334 # ) 335 print("}") 336 337 338 if __name__ == '__main__': 339 m = Main.from_argv() 340 m.render_all() 341 342 # pylama:linters=pycodestyle,pyflakes:ignore=D212,D203,D100,D101,D102,D105,D107 343 # vim: sts=4 ts=4 sw=4 et tw=88 efm=%A%f\:%l%\:%c\ %t%n\ %m