genpkg.py (6734B)
1 #!/usr/bin/env python3 2 import hashlib 3 import os.path 4 import subprocess 5 from pathlib import Path 6 7 import jinja2 8 import yaml 9 10 11 class SubmoduleInfo: 12 def __init__(self): 13 self._current_commits = None 14 self._by_commit = Path('./sources/by-commit') 15 16 @property 17 def current(self): 18 if self._current_commits is not None: 19 return self._current_commits 20 out = subprocess.check_output( 21 ("git", "submodule", "status", "--cached") 22 ).decode('utf8') 23 lines = out.strip('\n').split('\n') 24 records = [[line[0]] + line[1:].split() for line in lines] 25 self._current_commits = { 26 r[2][8:]: r[1] for r in records if r[2].startswith("sources/") 27 } 28 for repo, commit in self._current_commits.items(): 29 if not (self._by_commit / commit).exists(): 30 os.symlink("../" + repo, str(self._by_commit / commit)) 31 return self._current_commits 32 33 def commit_info(self, commit_id): 34 assert '/' not in commit_id 35 assert '.' not in commit_id 36 assert (self._by_commit / commit_id).exists() 37 out = subprocess.check_output( 38 ('git', 'show', '-s', '--pretty=format:%ai by %an'), 39 cwd=(self._by_commit / commit_id).as_posix(), 40 ).decode('utf8') 41 return out 42 43 class DownloadsInfo: 44 def __init__(self): 45 self._basenames = {} 46 with open('downloadlist.sha256', 'rt') as f: 47 for line in f: 48 if line[0] in '#\n': 49 continue 50 sha256, size, url = line.rstrip().split(maxsplit=2) 51 assert len(bytes.fromhex(sha256)) == 32 52 assert int(size) >= 0 53 basename = os.path.basename(url) 54 if basename in self._basenames: 55 self._basenames[basename] = ValueError( 56 'Duplicate download name: ' + repr(basename) 57 ) 58 else: 59 self._basenames[basename] = 'sha256:' + sha256 60 61 def __getitem__(self, key): 62 value = self._basenames[key] 63 if isinstance(value, Exception): 64 raise value 65 return value 66 67 68 class FileInfo: 69 def __init__(self): 70 self._sha256_cache = {} 71 self._files_dir = Path('./files') 72 73 def __getitem__(self, key): 74 if key in self._sha256_cache: 75 return self._sha256_cache[key] 76 fp = self._files_dir / key 77 with fp.open('rb') as f: 78 file_hash = hashlib.file_digest(f, "sha256").hexdigest() 79 self._sha256_cache[key] = file_hash 80 if not (self._files_dir / "by-sha256" / file_hash).exists(): 81 os.symlink("../" + key, str(self._files_dir / "by-sha256" / file_hash)) 82 return file_hash 83 84 85 class Main: 86 def __init__(self, out_dir="packages", template_dir="templates"): 87 self.out_dir = Path(out_dir) 88 self.template_dir = Path(template_dir) 89 self.env = jinja2.Environment( 90 loader=jinja2.FileSystemLoader(template_dir), 91 undefined=jinja2.StrictUndefined, 92 autoescape=False, 93 ) 94 self.env.globals["pkg_sha256"] = self.pkg_sha256 95 self.env.globals["pkg_install_name"] = self.pkg_install_name 96 self.env.globals["pkg_install_dir"] = self.pkg_install_dir 97 self.env.globals["submodule"] = SubmoduleInfo() 98 self.env.globals["files"] = FileInfo() 99 self.env.globals["downloads"] = DownloadsInfo() 100 self.package_hashes = {} 101 self.rendering = [] 102 self.deps = {} 103 104 def load_vars_yaml(self, fname="vars.yaml"): 105 with open(fname) as f: 106 self.env.globals.update(yaml.safe_load(f)) 107 108 def pkg_env_sha256(self, name): 109 current = self.rendering[-1] 110 if current not in self.deps: 111 self.deps[current] = set((name,)) 112 else: 113 self.deps[current].add(name) 114 self._pkg_sha256(name) 115 envlist = ''.join(sorted('%s.%s\n' % (d, self.package_hashes[d]) for d in self.deps[name])) 116 return hashlib.sha256(envlist.encode()).hexdigest() 117 118 def pkg_sha256(self, name): 119 current = self.rendering[-1] 120 if current not in self.deps: 121 self.deps[current] = set((name,)) 122 else: 123 self.deps[current].add(name) 124 return self._pkg_sha256(name) 125 126 def _pkg_sha256(self, name): 127 if name in self.package_hashes: 128 return self.package_hashes[name] 129 if name in self.rendering: 130 raise RuntimeError("circular dependency: %r", self.rendering) 131 132 t = self.env.get_template("pkg/" + name) 133 self.rendering.append(name) 134 data = bytes(t.render(name=name).encode('utf8')) 135 self.package_hashes[name] = hashlib.sha256(data).hexdigest() 136 lastname = self.rendering.pop() 137 assert name == lastname 138 139 out_path = self.out_dir / name 140 old_hash = None 141 if out_path.exists(): 142 with out_path.open('rb') as f: 143 old_hash = hashlib.file_digest(f, "sha256").hexdigest() 144 if old_hash is None or old_hash != self.package_hashes[name]: 145 tmp_path = out_path.with_suffix('.new') 146 tmp_path.write_bytes(data) 147 tmp_path.replace(out_path) 148 149 return self.package_hashes[name] 150 151 def pkg_install_name(self, name): 152 rootname = name.split(":")[0] 153 if rootname.endswith('.environment'): 154 return "env.%s" % (self.pkg_env_sha256(name),) 155 else: 156 return "%s.%s" % (name.split(":")[0], self.pkg_sha256(name)) 157 158 def pkg_install_dir(self, name): 159 return os.path.join( 160 self.env.globals["versions"], 161 self.pkg_install_name(name), 162 ) 163 164 def list_packages(self): 165 for tplname in self.env.list_templates(): 166 if not tplname.startswith("pkg/"): 167 continue 168 if "/." in tplname: 169 continue 170 yield tplname[4:] 171 172 def render_all(self): 173 for pkgname in self.list_packages(): 174 print("%s\t%s" % (pkgname, self._pkg_sha256(pkgname))) 175 for dep in sorted(self.deps.get(pkgname, ())): 176 print( 177 " > %s.%s" 178 % ( 179 dep, 180 self.package_hashes[dep], 181 ) 182 ) 183 184 185 if __name__ == '__main__': 186 from pprint import pprint as pp 187 188 m = Main() 189 m.load_vars_yaml() 190 pp(m.env.list_templates(filter_func=lambda name: "/." not in name)) 191 m.render_all() 192 193 # pylama:linters=pycodestyle,pyflakes:ignore=D212,D203,D100,D101,D102,D105,D107 194 # vim: sts=4 ts=4 sw=4 et tw=88 efm=%A%f\:%l%\:%c\ %t%n\ %m