commit 737eed758312d7bc1cd3ffd57830e2dae6016a68
parent 1101d12e83256da3bbda0f1170f01e101dc3caff
Author: Jan Pobříslo <ccx@te2000.cz>
Date: Thu, 22 Feb 2024 06:52:20 +0100
Revamp python workflow, codestyle
Diffstat:
10 files changed, 272 insertions(+), 75 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,5 @@
*.sw[op]
-.pycodestyle.*
+*.pyfmt
downloads
files/by-sha256
sources/by-commit
diff --git a/Makefile b/Makefile
@@ -36,14 +36,6 @@ pkg_files=$(wildcard packages/*)
mk_files=$(patsubst packages/%,make/package.%.mk,$(pkg_files))
include $(mk_files)
-pycodestyle: .pycodestyle.genpkg.py
- pylama -l 88 ./scripts/genpkg.py
-
-.pycodestyle.%.py: %.py
- isort - <$< >$<.tmp1
- cp -a $< $<.tmp2
- black -S - <$<.tmp1 >$<.tmp2
- rm $<.tmp1
- if cmp -s $<.tmp2 $<; then mv $<.tmp2 $<; else rm $<.tmp2; fi
- touch $@
-
+ifneq (,$(filter py%,$(MAKECMDGOALS)))
+include python.mk
+endif
diff --git a/build b/build
@@ -1,5 +1,5 @@
#!/bin/sh -xe
cd "$(dirname "$0")"
mkdir -p packages
-./genpkg.py
+make py-genpkg
exec make -rs "$@"
diff --git a/genpkg.py b/genpkg.py
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
import hashlib
import os.path
-from pathlib import Path
import subprocess
+from pathlib import Path
import jinja2
import yaml
@@ -17,12 +17,13 @@ class SubmoduleInfo:
def current(self):
if self._current_commits is not None:
return self._current_commits
- out = subprocess.check_output(("git", "submodule", "status", "--cached")).decode('utf8')
+ out = subprocess.check_output(
+ ("git", "submodule", "status", "--cached")
+ ).decode('utf8')
lines = out.strip('\n').split('\n')
records = [[line[0]] + line[1:].split() for line in lines]
self._current_commits = {
- r[2][8:]: r[1]
- for r in records if r[2].startswith("sources/")
+ r[2][8:]: r[1] for r in records if r[2].startswith("sources/")
}
for repo, commit in self._current_commits.items():
if not (self._by_commit / commit).exists():
@@ -42,9 +43,11 @@ class DownloadsInfo:
assert int(size) >= 0
basename = os.path.basename(url)
if basename in self._basenames:
- self._basenames[basename] = ValueError('Duplicate download name: '+repr(basename))
+ self._basenames[basename] = ValueError(
+ 'Duplicate download name: ' + repr(basename)
+ )
else:
- self._basenames[basename] = 'sha256:'+sha256
+ self._basenames[basename] = 'sha256:' + sha256
def __getitem__(self, key):
value = self._basenames[key]
@@ -100,7 +103,7 @@ class Main:
else:
self.deps[current].add(name)
return self._pkg_sha256(name)
-
+
def _pkg_sha256(self, name):
if name in self.package_hashes:
return self.package_hashes[name]
@@ -147,7 +150,13 @@ class Main:
for pkgname in self.list_packages():
print("%s\t%s" % (pkgname, self._pkg_sha256(pkgname)))
for dep in sorted(self.deps.get(pkgname, ())):
- print(" > %s.%s" % (dep, self.package_hashes[dep],))
+ print(
+ " > %s.%s"
+ % (
+ dep,
+ self.package_hashes[dep],
+ )
+ )
if __name__ == '__main__':
@@ -157,3 +166,6 @@ if __name__ == '__main__':
m.load_vars_yaml()
pp(m.env.list_templates(filter_func=lambda name: "/." not in name))
m.render_all()
+
+# pylama:linters=pycodestyle,pyflakes:ignore=D212,D203,D100,D101,D102,D105,D107
+# vim: sts=4 ts=4 sw=4 et tw=88 efm=%A%f\:%l%\:%c\ %t%n\ %m
diff --git a/make_vars.py b/make_vars.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+from __future__ import (
+ absolute_import,
+ division,
+ generators,
+ print_function,
+ with_statement,
+)
+
+import sys
+
+try:
+ import venv
+except ImportError:
+ venv = None
+
+try:
+ from packaging.tags import sys_tags
+
+ def get_tag():
+ tag = sys_tags().__next__()
+ return (tag.interpreter, tag.abi, tag.platform)
+
+except ImportError:
+ from wheel.pep425tags import get_abbr_impl, get_abi_tag, get_impl_ver, get_platform
+
+ # sporked from wheel.bdist_wheel.get_tag
+ def get_tag():
+ """Get the specific implementation name in a wheel-compatible format."""
+ plat_name = get_platform().replace('-', '_').replace('.', '_')
+ impl_name = get_abbr_impl()
+ impl_ver = get_impl_ver()
+ impl = impl_name + impl_ver
+ abi_tag = str(get_abi_tag()).lower()
+ tag = (impl, abi_tag, plat_name)
+ return tag
+
+
+PY2 = sys.version_info[0] == 2
+
+
+def main():
+ tag = '%s-%s-%s' % get_tag()
+ mk_filename = 'python_vars_%s.mk' % tag
+ with open(mk_filename, 'wt') as f:
+ # TODO: add proper escaping whenever I feel like staring into the abyss
+ f.write('PYTHON_IMPL:=%s\n' % tag)
+
+ f.write(
+ "PYTHON_VENV:=%s\n"
+ % (
+ "virtualenv -p $(PYTHON_EXE)"
+ if venv is None
+ else "$(PYTHON_EXE) -m venv"
+ )
+ )
+
+ f.write(
+ "PYTHON_VENV_INSTALL:=%s\n"
+ % ("'setuptools<45.0.0' 'pip<20.3' 'pip-tools<6'" if PY2 else "'pip-tools'")
+ )
+ print("include %s" % mk_filename)
+
+
+if __name__ == '__main__':
+ main()
+
+# pylama:linters=pycodestyle,pyflakes:ignore=D212,D203,D100,D101,D102,D107
+# vim: fileencoding=utf-8 ft=python et sw=4 ts=4 sts=4 tw=79
diff --git a/py-cp311-cp311-musllinux_1_2_x86_64-requirements.txt b/py-cp311-cp311-musllinux_1_2_x86_64-requirements.txt
@@ -0,0 +1,38 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+# pip-compile --output-file=py-cp311-cp311-musllinux_1_2_x86_64-requirements.txt.new py-requirements.in
+#
+black==24.2.0
+ # via -r py-requirements.in
+click==8.1.7
+ # via black
+isort==5.13.2
+ # via -r py-requirements.in
+jinja2==3.1.3
+ # via -r py-requirements.in
+markupsafe==2.1.5
+ # via jinja2
+mccabe==0.7.0
+ # via pylama
+mypy-extensions==1.0.0
+ # via black
+packaging==23.2
+ # via black
+pathspec==0.12.1
+ # via black
+platformdirs==4.2.0
+ # via black
+pycodestyle==2.11.1
+ # via pylama
+pydocstyle==6.3.0
+ # via pylama
+pyflakes==3.2.0
+ # via pylama
+pylama==8.4.1
+ # via -r py-requirements.in
+pyyaml==6.0.1
+ # via -r py-requirements.in
+snowballstemmer==2.2.0
+ # via pydocstyle
diff --git a/py-requirements.in b/py-requirements.in
@@ -0,0 +1,5 @@
+jinja2
+pyyaml
+black
+isort
+pylama
diff --git a/python.mk b/python.mk
@@ -0,0 +1,59 @@
+PYTHONPATH:=$(abspath .)
+PYTHON_EXE:=python3
+# re-evaluate each time because there's no sensible way to check
+# whether the python interpreter changed
+$(eval $(shell $(PYTHON_EXE) ./make_vars.py))
+
+PY_SRC:=$(wildcard *.py)
+PY_REQ:=py-$(PYTHON_IMPL)-requirements.txt
+PY_WHL:=work/wheels/$(PYTHON_IMPL)
+VENV:=work/virtualenvs/$(PYTHON_IMPL)
+
+pycodestyle: $(patsubst %.py,.%.pyfmt,$(PY_SRC)) $(VENV)/.done
+ '$(VENV)/bin/pylama' -l 88 $(PY_SRC) || true
+
+.%.pyfmt: %.py $(VENV)/.done
+ '$(VENV)/bin/isort' - <'$<' >'$<.tmp1'
+ cp -a '$<' '$<.tmp2'
+ '$(VENV)/bin/black' -S - <'$<.tmp1' >'$<.tmp2'
+ rm '$<.tmp1'
+ if cmp -s '$<.tmp2' '$<'; then rm -v '$<.tmp2'; else mv -v '$<.tmp2' '$<'; fi
+ touch $@
+
+.PHONY: py-genpkg
+py-genpkg: $(VENV)/.done
+ '$(VENV)/bin/python' genpkg.py
+
+py-requirements: $(PY_REQ)
+
+py-wheels: $(PY_WHL)/.done
+
+py-venv: $(VENV)/.done
+ ln -sf 'virtualenvs/$(PYTHON_IMPL)' work/venv
+
+py-virtualenv: py-venv
+
+# -- requirement file rules
+
+$(PY_REQ): py-requirements.in
+ '$(PYTHON_EXE)' -m piptools compile -v --annotate -o '$@.new' py-requirements.in
+ mv '$@.new' '$@'
+
+# -- wheel building rules
+
+$(PY_WHL)/.done: $(PY_REQ)
+ mkdir -p '$(PY_WHL)'
+ '$(PYTHON_EXE)' -m pip wheel -w '$(PY_WHL)' -r '$(PY_REQ)'
+ touch '$@'
+
+# -- virtualenv rules
+
+$(VENV)/bin/pip-sync:
+ if test -e '$(VENV)'; then rm -r '$(VENV)'; else true; fi
+ mkdir -p virtualenvs
+ $(PYTHON_VENV) '$(VENV)'
+ '$(VENV)/bin/pip' install -I $(PYTHON_VENV_INSTALL)
+
+$(VENV)/.done: $(PY_REQ) $(VENV)/bin/pip-sync $(PY_WHL)/.done
+ $(VENV)/bin/pip-sync --no-index -f '$(PY_WHL)' '$(PY_REQ)'
+ touch '$@'
diff --git a/python_vars_cp311-cp311-musllinux_1_2_x86_64.mk b/python_vars_cp311-cp311-musllinux_1_2_x86_64.mk
@@ -0,0 +1,3 @@
+PYTHON_IMPL:=cp311-cp311-musllinux_1_2_x86_64
+PYTHON_VENV:=$(PYTHON_EXE) -m venv
+PYTHON_VENV_INSTALL:='pip-tools'
diff --git a/userns_sandbox.py b/userns_sandbox.py
@@ -1,28 +1,28 @@
#!/usr/bin/python3
-import sys
-import os
-import os.path
-import ctypes
-import fcntl
-import select
-import errno
import argparse
+import ctypes
+import dataclasses
import enum
+import errno
+import fcntl
+import os
+import os.path
import pathlib
-import dataclasses
+import select
libc = ctypes.CDLL(None, use_errno=True)
-CLONE_NEWNS = 0x00020000 # New mount namespace group
-CLONE_NEWCGROUP = 0x02000000 # New cgroup namespace
-CLONE_NEWUTS = 0x04000000 # New utsname namespace
-CLONE_NEWIPC = 0x08000000 # New ipc namespace
-CLONE_NEWUSER = 0x10000000 # New user namespace
-CLONE_NEWPID = 0x20000000 # New pid namespace
-CLONE_NEWNET = 0x40000000 # New network namespace
-CLONE_NEWTIME = 0x00000080 # New time namespace
+CLONE_NEWNS = 0x00020000 # New mount namespace group
+CLONE_NEWCGROUP = 0x02000000 # New cgroup namespace
+CLONE_NEWUTS = 0x04000000 # New utsname namespace
+CLONE_NEWIPC = 0x08000000 # New ipc namespace
+CLONE_NEWUSER = 0x10000000 # New user namespace
+CLONE_NEWPID = 0x20000000 # New pid namespace
+CLONE_NEWNET = 0x40000000 # New network namespace
+CLONE_NEWTIME = 0x00000080 # New time namespace
SYS_pivot_root = 155
+
class MountFlag(int, enum.Enum):
"""Mount flags."""
@@ -76,27 +76,34 @@ class MountFlag(int, enum.Enum):
ACTIVE = 1 << 30
NOUSER = 1 << 31
+
_mount = libc.mount
-_mount.restype = c_int
-_mount.argtypes = (c_char_p, c_char_p, c_char_p, c_ulong, c_void_p)
+_mount.restype = ctypes.c_int
+_mount.argtypes = (
+ ctypes.c_char_p,
+ ctypes.c_char_p,
+ ctypes.c_char_p,
+ ctypes.c_ulong,
+ ctypes.c_void_p,
+)
_umount = libc.umount
-_umount.restype = c_int
-_umount.argtypes = (c_char_p)
-
-_chroot = libc.chroot
-_chroot.restype = c_int
-_chroot.argtypes = (c_char_p)
+_umount.restype = ctypes.c_int
+_umount.argtypes = ctypes.c_char_p
def c_path(path):
- if isinstance(path, PosixPath):
+ if isinstance(path, pathlib.PosixPath):
path = path.as_posix()
if isinstance(path, str):
path = path.encode()
return path
+def c_error():
+ return OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno()))
+
+
def mount(
source: str,
target: str,
@@ -123,7 +130,7 @@ def mount(
)
!= 0
):
- raise OSError(get_errno(), strerror(get_errno()))
+ raise c_error()
def bind_mount(
@@ -131,22 +138,27 @@ def bind_mount(
target: str,
write: bool = False,
):
- return mount(source, target, "", (
- MountFlag.BIND
- | (0 if write else MountFlag.RDONLY)
- | MountFlag.NOSUID
- | MountFlag.NODEV
- ))
+ return mount(
+ source,
+ target,
+ "",
+ (
+ MountFlag.BIND
+ | (0 if write else MountFlag.RDONLY)
+ | MountFlag.NOSUID
+ | MountFlag.NODEV
+ ),
+ )
-def umount(target: str)
+def umount(target: str):
"""Unmount filesystem.
:param target: Mountpoint.
:raises OSError: If umount call failed with nonzero return code.
"""
- if (_umount(c_path(target)) != 0):
- raise OSError(get_errno(), strerror(get_errno()))
+ if _umount(c_path(target)) != 0:
+ raise c_error()
def parse_mountinfo(mountinfo_path='/proc/self/mountinfo'):
@@ -166,7 +178,7 @@ def nonblock_cloexec(fd):
def exit_status(status):
- sig = status & 0xff
+ sig = status & 0xFF
ret = status >> 8
if sig:
raise SystemExit(128 + sig)
@@ -191,7 +203,7 @@ def pidns_run(unshare_flags, continuation, *args, **kwargs):
nonblock_cloexec(parent_rfd)
nonblock_cloexec(parent_wfd)
if libc.unshare(CLONE_NEWPID | unshare_flags) != 0:
- raise OSError(ctypes.get_errno())
+ raise c_error()
fork_pid = os.fork()
if fork_pid == 0:
# child
@@ -222,7 +234,7 @@ def pidns_run(unshare_flags, continuation, *args, **kwargs):
exit_status(status)
-@dataclasses.dataclass(frozen=True, slots=True):
+@dataclasses.dataclass(frozen=True, slots=True)
class MountTMPFS:
path: pathlib.PosixPath
@@ -236,7 +248,7 @@ class MountTMPFS:
mount('tmpfs', dst, 'tmpfs', MountFlag.NOSUID | MountFlag.NODEV)
-@dataclasses.dataclass(frozen=True, slots=True):
+@dataclasses.dataclass(frozen=True, slots=True)
class MountBind:
src: pathlib.PosixPath
dst: pathlib.PosixPath
@@ -271,7 +283,7 @@ def parse_mount(s):
raise ValueError(m_type)
-@dataclasses.dataclass(frozen=True, slots=True):
+@dataclasses.dataclass(frozen=True, slots=True)
class Settings:
versions: pathlib.PosixPath
root: pathlib.PosixPath
@@ -281,11 +293,11 @@ class Settings:
extra_mount: tuple
def __post_init__(self):
- assert isinstance(command, tuple)
- assert all(isinstance(arg, (str, bytes)) for arg in command)
+ assert isinstance(self.command, tuple)
+ assert all(isinstance(arg, (str, bytes)) for arg in self.command)
- assert isinstance(extra_mount, tuple)
- assert all(isinstance(arg, (MountTMPFS, MountBind)) for arg in command)
+ assert isinstance(self.extra_mount, tuple)
+ assert all(isinstance(arg, (MountTMPFS, MountBind)) for arg in self.extra_mount)
assert isinstance(self.versions, pathlib.PosixPath)
assert self.versions.is_absolute()
@@ -309,7 +321,8 @@ class Settings:
def from_args(cls, args):
if args.vars:
import yaml
- with args.vars.open('rt'):
+
+ with args.vars.open('rt') as f:
v = yaml.safe_load(f)
else:
v = {}
@@ -339,8 +352,8 @@ def sandbox_run(settings, command):
def main(args):
pidns_run(
CLONE_NEWUSER | CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWPID,
- sandbox_run
- Settings.from_args(args)
+ sandbox_run,
+ Settings.from_args(args),
args.command,
)
@@ -349,16 +362,18 @@ argument_parser = argparse.ArgumentParser(
description="User namespaces based sandbox for pthbs"
)
argument_parser.add_argument(
- '--vars', '-y', type=pathlib.PosixPath,
- description="vars.yaml to read configuration from"
+ '--vars',
+ '-y',
+ type=pathlib.PosixPath,
+ description="vars.yaml to read configuration from",
)
argument_parser.add_argument(
- '--versions', '-V', type=pathlib.PosixPath,
- description="versions dir (e.g. /versions)"
-)
-argument_parser.add_argument(
- '--extra-mount', action='append', type=parse_mount
+ '--versions',
+ '-V',
+ type=pathlib.PosixPath,
+ description="versions dir (e.g. /versions)",
)
+argument_parser.add_argument('--extra-mount', action='append', type=parse_mount)
argument_parser.add_argument('root_dir', type=pathlib.PosixPath)
argument_parser.add_argument('environment')
argument_parser.add_argument('command', nargs='+')
@@ -367,3 +382,6 @@ argument_parser.add_argument('command', nargs='+')
if __name__ == '__main__':
args = argument_parser.parse_args()
main(args)
+
+# pylama:linters=pycodestyle,pyflakes:ignore=D212,D203,D100,D101,D102,D107
+# vim: sts=4 ts=4 sw=4 et tw=88 efm=%A%f\:%l%\:%c\ %t%n\ %m
