genpkg.py (8523B)
1 #!/usr/bin/env python3 2 import argparse 3 import hashlib 4 import os.path 5 import subprocess 6 from pathlib import Path 7 8 import jinja2 9 import yaml 10 11 12 class SubmoduleInfo: 13 def __init__(self, cache_dir='cache'): 14 self._current_commits = None 15 self._by_commit = Path(cache_dir) / "link" / "git-commit-sha1" 16 17 @property 18 def current(self): 19 if self._current_commits is not None: 20 return self._current_commits 21 source_type = os.environ.get('pthbs_genpkgpy_submodule_source', None) 22 if source_type == 'current': 23 cmd = ("git", "submodule", "status") 24 elif source_type == 'cached': 25 cmd = ("git", "submodule", "status", '--cached') 26 else: 27 raise RuntimeError( 28 'Environment variable pthbs_genpkgpy_submodule_source' 29 ' must be set to either "current" or "cached"' 30 ) 31 out = subprocess.check_output(cmd).decode('utf8') 32 lines = out.strip('\n').split('\n') 33 records = [[line[0]] + line[1:].split() for line in lines] 34 self._current_commits = { 35 r[2][8:]: r[1] for r in records if r[2].startswith("sources/") 36 } 37 for repo, commit in self._current_commits.items(): 38 if not (self._by_commit / commit).exists(): 39 raise RuntimeError( 40 f"commit ID does not seem to be linked:" 41 f" {commit} from {repo!r} in {self._by_commit}" 42 ) 43 return self._current_commits 44 45 def commit_info(self, commit_id): 46 assert '/' not in commit_id 47 assert '.' not in commit_id 48 assert (self._by_commit / commit_id).exists() 49 out = subprocess.check_output( 50 ('git', 'show', '-s', '--pretty=format:%ai by %an'), 51 cwd=(self._by_commit / commit_id).as_posix(), 52 ).decode('utf8') 53 return out 54 55 56 class DownloadsInfo: 57 def __init__(self, downloadlist_path): 58 assert isinstance(downloadlist_path, Path) 59 self._basenames = {} 60 with downloadlist_path.open('rt') as f: 61 for line in f: 62 if line[0] in '#\n': 63 continue 64 sha256, size, url = line.rstrip().split(maxsplit=2) 65 assert len(bytes.fromhex(sha256)) == 32 66 assert int(size) >= 0 67 basename = os.path.basename(url) 68 if basename in self._basenames: 69 self._basenames[basename] = ValueError( 70 'Duplicate download name: ' + repr(basename) 71 ) 72 else: 73 self._basenames[basename] = 'sha256:' + sha256 74 75 def __getitem__(self, key): 76 value = self._basenames[key] 77 if isinstance(value, Exception): 78 raise value 79 return value 80 81 82 def parse_filelist(filelist_path): 83 assert isinstance(filelist_path, Path) 84 with filelist_path.open('rt') as f: 85 return { 86 os.path.basename(fname): fhash 87 for fhash, fname in ( 88 line.rstrip('\n').split(' ', maxsplit=1) for line in f 89 ) 90 } 91 92 93 def _assertion(value): 94 assert value 95 return value 96 97 98 class Main: 99 argument_parser = argparse.ArgumentParser() 100 argument_parser.add_argument('-P', '--package-dir', default='packages') 101 argument_parser.add_argument('-T', '--template-dir', default='templates') 102 argument_parser.add_argument('-I', '--index-dir', default='.') 103 argument_parser.add_argument('-C', '--cache-dir', default='cache') 104 argument_parser.add_argument('-V', '--vars-file', default='vars.yaml') 105 argument_parser.add_argument('-M', '--write-mkfile') 106 107 def __init__(self, out_dir, template_dir, index_dir, cache_dir): 108 assert isinstance(out_dir, Path) 109 assert isinstance(template_dir, Path) 110 assert isinstance(index_dir, Path) 111 assert isinstance(cache_dir, Path) 112 self.out_dir = Path(out_dir) 113 self.template_dir = Path(template_dir) 114 self.env = jinja2.Environment( 115 loader=jinja2.FileSystemLoader(template_dir), 116 undefined=jinja2.StrictUndefined, 117 autoescape=False, 118 ) 119 self.env.globals["pkg_sha256"] = self.pkg_sha256 120 self.env.globals["pkg_install_name"] = self.pkg_install_name 121 self.env.globals["pkg_install_dir"] = self.pkg_install_dir 122 self.env.globals["submodule"] = SubmoduleInfo(cache_dir=cache_dir) 123 self.env.globals["files"] = parse_filelist(index_dir / 'filelist.sha256') 124 self.env.globals["downloads"] = DownloadsInfo(index_dir / 'downloadlist.sha256') 125 self.env.globals["assertion"] = _assertion 126 self.package_hashes = {} 127 self.rendering = [] 128 self.deps = {} 129 130 @classmethod 131 def from_argv(cls, args=None): 132 args = cls.argument_parser.parse_args(args) 133 obj = cls( 134 out_dir=Path(args.package_dir), 135 template_dir=Path(args.template_dir), 136 index_dir=Path(args.index_dir), 137 cache_dir=Path(args.cache_dir), 138 ) 139 obj.load_vars_yaml(args.vars_file) 140 if args.write_mkfile: 141 obj.write_mkfile(args.write_mkfile) 142 return obj 143 144 def load_vars_yaml(self, fname): 145 with open(fname) as f: 146 self.env.globals.update(yaml.safe_load(f)) 147 148 def write_mkfile(self, fname): 149 t = self.env.from_string('versions:={{versions}}\npackages:={{packages}}\n') 150 with open(fname + '.new', 'wt') as f: 151 f.write(t.render(packages=self.out_dir)) 152 os.rename(fname + '.new', fname) 153 154 def pkg_env_sha256(self, name): 155 current = self.rendering[-1] 156 if current not in self.deps: 157 self.deps[current] = set((name,)) 158 else: 159 self.deps[current].add(name) 160 self._pkg_sha256(name) 161 envlist = ''.join( 162 sorted('%s.%s\n' % (d, self.package_hashes[d]) for d in self.deps[name]) 163 ) 164 return hashlib.sha256(envlist.encode()).hexdigest() 165 166 def pkg_sha256(self, name): 167 current = self.rendering[-1] 168 if current not in self.deps: 169 self.deps[current] = set((name,)) 170 else: 171 self.deps[current].add(name) 172 return self._pkg_sha256(name) 173 174 def _pkg_sha256(self, name): 175 if name in self.package_hashes: 176 return self.package_hashes[name] 177 if name in self.rendering: 178 raise RuntimeError("circular dependency: %r", self.rendering) 179 180 t = self.env.get_template("pkg/" + name) 181 self.rendering.append(name) 182 data = bytes(t.render(name=name).encode('utf8')) 183 self.package_hashes[name] = hashlib.sha256(data).hexdigest() 184 lastname = self.rendering.pop() 185 assert name == lastname 186 187 out_path = self.out_dir / name 188 old_hash = None 189 if out_path.exists(): 190 with out_path.open('rb') as f: 191 old_hash = hashlib.file_digest(f, "sha256").hexdigest() 192 if old_hash is None or old_hash != self.package_hashes[name]: 193 tmp_path = out_path.with_suffix('.new') 194 tmp_path.write_bytes(data) 195 tmp_path.replace(out_path) 196 197 return self.package_hashes[name] 198 199 def pkg_install_name(self, name): 200 rootname = name.split(":")[0] 201 if rootname.endswith('.environment'): 202 return "env.%s" % (self.pkg_env_sha256(name),) 203 else: 204 return "%s.%s" % (name.split(":")[0], self.pkg_sha256(name)) 205 206 def pkg_install_dir(self, name): 207 return os.path.join( 208 self.env.globals["versions"], 209 self.pkg_install_name(name), 210 ) 211 212 def list_packages(self): 213 for tplname in self.env.list_templates(): 214 if not tplname.startswith("pkg/"): 215 continue 216 if "/." in tplname: 217 continue 218 yield tplname[4:] 219 220 def render_all(self): 221 for pkgname in self.list_packages(): 222 print("%s\t%s" % (pkgname, self._pkg_sha256(pkgname))) 223 for dep in sorted(self.deps.get(pkgname, ())): 224 print( 225 " > %s.%s" 226 % ( 227 dep, 228 self.package_hashes[dep], 229 ) 230 ) 231 232 233 if __name__ == '__main__': 234 m = Main.from_argv() 235 m.render_all() 236 237 # pylama:linters=pycodestyle,pyflakes:ignore=D212,D203,D100,D101,D102,D105,D107 238 # vim: sts=4 ts=4 sw=4 et tw=88 efm=%A%f\:%l%\:%c\ %t%n\ %m