From 67aca9de443e5596575bf235f9bffdb8bcdf2af6 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 15 Feb 2020 19:01:21 +0100 Subject: [PATCH] No need for background in cmd output --- shlax/actions/base.py | 13 +++++++- shlax/actions/packages.py | 30 ++++++++--------- shlax/actions/pip.py | 31 +++++++++--------- shlax/output.py | 2 +- shlax/targets/buildah.py | 69 +++++++++++++++++++++++++++------------ 5 files changed, 91 insertions(+), 54 deletions(-) diff --git a/shlax/actions/base.py b/shlax/actions/base.py index c1d36c6..4b90331 100644 --- a/shlax/actions/base.py +++ b/shlax/actions/base.py @@ -105,7 +105,10 @@ class Action: self.output_fail(e) self.status = 'fail' proc = getattr(e, 'proc', None) - result = proc.rc if proc else 1 + if proc: + result = proc.rc + else: + raise else: self.output_success() if self.status == 'running': @@ -155,3 +158,11 @@ class Action: def kwargs_output(self): return self.kwargs + + def action(self, action, *args, **kwargs): + p = action(*args, **kwargs) + for parent in self.parents(): + if hasattr(parent, 'actions'): + break + p.parent = parent + return p diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 6d37117..81fb4f5 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -53,12 +53,10 @@ class Packages(Action): def __init__(self, *packages, **kwargs): self.packages = [] - for package in packages: line = dedent(package).strip().replace('\n', ' ') self.packages += line.split(' ') - - self.mgr = kwargs.pop('mgr') if 'mgr' in kwargs else None + super().__init__(*packages, **kwargs) @property def cache_root(self): @@ -109,7 +107,7 @@ class Packages(Action): if cached: self.mgr = cached else: - mgr = await self.which(*self.mgrs.values()) + mgr = await self.which(*self.mgrs.keys()) if mgr: self.mgr = mgr.split('/')[-1] @@ -120,20 +118,18 @@ class Packages(Action): if not getattr(self, '_packages_upgraded', None): await self.update() await self.rexec(self.cmds['upgrade']) - - # first run on container means inject visitor packages - packages = [] - for sibbling in self.sibblings: - pp = getattr(sibbling, 'packages', None) - if pp: - if isinstance(pp, list): - packages += pp - elif self.mgr in pp: - packages += pp[self.mgr] - self._packages_upgraded = True - else: - packages = self.packages + + packages = [] + for package in self.packages: + if ',' in package: + parts = package.split(',') + package = parts[0] + if self.mgr in parts[1:]: + # include apt on apt + packages.append(package) + else: + packages.append(package) await self.rexec(*self.cmds['install'].split(' ') + packages) diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py index df41ec4..8fa1342 100644 --- a/shlax/actions/pip.py +++ b/shlax/actions/pip.py @@ -17,35 +17,36 @@ class Pip(Action): async def call(self, *args, **kwargs): self.pip = await self.which('pip3', 'pip', 'pip2') if not self.pip: - raise Exception('Could not find pip command') + from .packages import Packages + action = self.action(Packages, 'python3,apk', 'python3-pip,apt', args=args, kwargs=kwargs) + await action(*args, **kwargs) + + self.pip = await self.which('pip3', 'pip', 'pip2') + if not self.pip: + raise Exception('Could not install a pip command') if 'CACHE_DIR' in os.environ: cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') else: cache = os.path.join(os.getenv('HOME'), '.cache', 'pip') - await script.mount(cache, '/root/.cache/pip') - await script.crexec(f'{self.pip} install --upgrade pip') + if getattr(self, 'mount', None): + # we are in a target which shares a mount command + await self.mount(cache, '/root/.cache/pip') + await self.exec(f'{self.pip} install --upgrade pip') # https://github.com/pypa/pip/issues/5599 self.pip = 'python3 -m pip' - pip_packages = [] - for visitor in script.container.visitors: - pp = getattr(visitor, 'pip_packages', None) - if not pp: - continue - pip_packages += pip_packages - - source = [p for p in pip_packages if p.startswith('/')] + source = [p for p in self.pip_packages if p.startswith('/')] if source: - await script.crexec( + await self.exec( f'{self.pip} install --upgrade --editable {" ".join(source)}' ) - nonsource = [p for p in pip_packages if not p.startswith('/')] + nonsource = [p for p in self.pip_packages if not p.startswith('/')] if nonsource: - await script.crexec(f'{self.pip} install --upgrade {" ".join(nonsource)}') + await self.exec(f'{self.pip} install --upgrade {" ".join(nonsource)}') if self.requirements: - await script.crexec(f'{self.pip} install --upgrade -r {self.requirements}') + await self.exec(f'{self.pip} install --upgrade -r {self.requirements}') diff --git a/shlax/output.py b/shlax/output.py index 1654283..6ab1adb 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -59,7 +59,7 @@ class Output: def cmd(self, line): self( self.colorize(251, '+') - + '\x1b[1;38;5;15;48;5;244m' + + '\x1b[1;38;5;15m' + ' ' + self.highlight(line, 'bash') + self.colors['reset'], diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 11b74fd..ee6e4cf 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -18,15 +18,16 @@ class Buildah(Localhost): The build script iterates over visitors and runs the build functions, it also provides wrappers around the buildah command. """ - contextualize = Localhost.contextualize + ['mnt', 'ctr'] + contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] - def __init__(self, base, *args, commit=None, **kwargs): + def __init__(self, base, *args, commit=None, push=False, **kwargs): super().__init__(*args, **kwargs) self.base = base self.mounts = dict() self.ctr = None self.mnt = None - self.commit = commit + self.image = Image(commit) if commit else None + self.push = push or os.getenv('CI') def shargs(self, *args, user=None, buildah=True, **kwargs): if not buildah: @@ -68,20 +69,10 @@ class Buildah(Localhost): async def mount(self, src, dst): """Mount a host directory into the container.""" target = self.mnt / str(dst)[1:] - await self.exec(f'mkdir -p {src} {target}') - await self.exec(f'mount -o bind {src} {target}') + await self.exec(f'mkdir -p {src} {target}', buildah=False) + await self.exec(f'mount -o bind {src} {target}', buildah=False) self.mounts[src] = dst - async def umounts(self): - """Unmount all mounted directories from the container.""" - for src, dst in self.mounts.items(): - await super().exec('umount', self.mnt / str(dst)[1:]) - - async def umount(self): - """Unmount the buildah container with buildah unmount.""" - if self.ctr: - await super().exec(f'buildah unmount {self.ctr}') - async def which(self, *cmd): """ Return the first path to the cmd in the container. @@ -95,9 +86,6 @@ class Buildah(Localhost): if os.path.exists(p): return p[len(str(self.mnt)):] - def __repr__(self): - return f'Build' - @property def _compatible(self): return Proc.test or os.getuid() == 0 or getattr(self.parent, 'parent', None) @@ -106,8 +94,8 @@ class Buildah(Localhost): if self._compatible: self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out self.mnt = Path((await self.exec('buildah', 'mount', self.ctr, buildah=False)).out) - - return await super().call(*args, **kwargs) + result = await super().call(*args, **kwargs) + return result from shlax.cli import cli debug = kwargs.get('debug', False) @@ -135,8 +123,49 @@ class Buildah(Localhost): await proc.communicate() cli.exit_code = await proc.wait() + async def commit(self): + self.sha = (await self.exec( + 'buildah', + 'commit', + '--format=' + self.image.format, + self.ctr, + buildah=False, + )).out + + if self.image.tags: + tags = ' '.join([f'{self.image.repository}:{tag}' for tag in self.image.tags]) + await self.exec('buildah', 'tag', self.sha, self.image.repository, tags, buildah=False) + + if self.push: + user = os.getenv('DOCKER_USER') + passwd = os.getenv('DOCKER_PASS') + if user and passwd and os.getenv('CI') and self.registry: + await self.exec( + 'podman', + 'login', + '-u', + user, + '-p', + passwd, + self.registry, + buildah=False, + ) + + for tag in self.image.tags: + await self.exec('podman', 'push', f'{self.image.repository}:{tag}', buildah=False) + async def clean(self, *args, **kwargs): + if not self._compatible: + return + + for src, dst in self.mounts.items(): + await self.exec('umount', self.mnt / str(dst)[1:], buildah=False) + + if self.status == 'success': + await self.commit() + if self.mnt is not None: await self.exec('buildah', 'umount', self.ctr, buildah=False) + if self.ctr is not None: await self.exec('buildah', 'rm', self.ctr, buildah=False)