From 600043ae645edd313e4d35c504fb36c8136a2a74 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 30 May 2020 23:54:10 +0200 Subject: [PATCH] Adding Copy/User/Pip actions again --- shlax/actions/copy.py | 19 +++++++++++++ shlax/actions/packages.py | 2 +- shlax/actions/pip.py | 59 +++++++++++++++++++++++++++++++++++++++ shlax/actions/user.py | 32 +++++++++++++++++++++ shlax/shortcuts.py | 6 ++++ shlax/targets/base.py | 32 ++++++++++++++++++++- shlax/targets/buildah.py | 24 ++++++++++++---- shlaxfile.py | 19 +++++++++---- 8 files changed, 180 insertions(+), 13 deletions(-) create mode 100644 shlax/actions/copy.py create mode 100644 shlax/actions/pip.py create mode 100644 shlax/actions/user.py diff --git a/shlax/actions/copy.py b/shlax/actions/copy.py new file mode 100644 index 0000000..7209aa6 --- /dev/null +++ b/shlax/actions/copy.py @@ -0,0 +1,19 @@ +class Copy: + def __init__(self, *args): + self.src = args[:-1] + self.dst = args[-1] + + @property + def files(self): + for root, dirs, files in os.walk(self.dst): + pass + + + async def __call__(self, target): + await target.copy(*self.args) + + def __str__(self): + return f'Copy(*{self.src}, {self.dst})' + + def cachehash(self): + return str(self) diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index cbdb385..814da6a 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -161,5 +161,5 @@ class Packages: async def pacman_setup(self, target): return self.cache_root + '/pacman' - def __repr__(self): + def __str__(self): return f'Packages({self.packages}, upgrade={self.upgrade})' diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py new file mode 100644 index 0000000..3ef9887 --- /dev/null +++ b/shlax/actions/pip.py @@ -0,0 +1,59 @@ +from glob import glob +import os + +from .base import Action + + +class Pip(Action): + """Pip abstraction layer.""" + + def __init__(self, *pip_packages, pip=None, requirements=None): + self.requirements = requirements + super().__init__(*pip_packages, pip=pip, requirements=requirements) + + async def call(self, *args, **kwargs): + pip = self.kwargs.get('pip', None) + if not pip: + pip = await self.which('pip3', 'pip', 'pip2') + if pip: + pip = pip[0] + else: + from .packages import Packages + action = self.action( + Packages, + 'python3,apk', 'python3-pip,apt', + args=args, kwargs=kwargs + ) + await action(*args, **kwargs) + pip = await self.which('pip3', 'pip', 'pip2') + if not pip: + raise Exception('Could not install a pip command') + else: + pip = pip[0] + + if 'CACHE_DIR' in os.environ: + cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') + else: + cache = os.path.join(os.getenv('HOME'), '.cache', 'pip') + + if getattr(self, 'mount', None): + # we are in a target which shares a mount command + await self.mount(cache, '/root/.cache/pip') + await self.exec(f'{pip} install --upgrade pip') + + # https://github.com/pypa/pip/issues/5599 + if 'pip' not in self.kwargs: + pip = 'python3 -m pip' + + source = [p for p in self.args if p.startswith('/') or p.startswith('.')] + if source: + await self.exec( + f'{pip} install --upgrade --editable {" ".join(source)}' + ) + + nonsource = [p for p in self.args if not p.startswith('/')] + if nonsource: + await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}') + + if self.requirements: + await self.exec(f'{pip} install --upgrade -r {self.requirements}') diff --git a/shlax/actions/user.py b/shlax/actions/user.py new file mode 100644 index 0000000..f919852 --- /dev/null +++ b/shlax/actions/user.py @@ -0,0 +1,32 @@ +import os +import re + +from .packages import Packages + + +class User: + def __init__(self, username, home, uid): + self.username = username + self.home = home + self.uid = uid + + def __str__(self): + return f'User({self.username}, {self.home}, {self.uid})' + + async def __call__(self, target): + result = await target.rexec('id', self.uid) + if result.rc == 0: + old = re.match('.*\(([^)]*)\).*', result.out).group(1) + await target.rexec( + 'usermod', + '-d', self.home, + '-l', self.username, + old + ) + else: + await target.rexec( + 'useradd', + '-d', self.home, + '-u', self.uid, + self.username + ) diff --git a/shlax/shortcuts.py b/shlax/shortcuts.py index efa02f4..9d77cc9 100644 --- a/shlax/shortcuts.py +++ b/shlax/shortcuts.py @@ -3,10 +3,16 @@ from .targets.buildah import Buildah from .targets.localhost import Localhost from .targets.stub import Stub +from .actions.copy import Copy from .actions.packages import Packages from .actions.run import Run from .actions.pip import Pip from .actions.parallel import Parallel +from .actions.user import User + +from .cli import Command, Group from .container import Container from .pod import Pod + +from os import getenv, environ diff --git a/shlax/targets/base.py b/shlax/targets/base.py index e1b919f..110f0b7 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -1,4 +1,6 @@ import copy +from pathlib import Path +import os import re from ..output import Output @@ -7,11 +9,12 @@ from ..result import Result, Results class Target: - def __init__(self, *actions): + def __init__(self, *actions, root=None): self.actions = actions self.results = [] self.output = Output() self.parent = None + self.root = root or os.getcwd() @property def parent(self): @@ -54,6 +57,8 @@ class Target: # nested call, re-raise raise else: + import traceback + traceback.print_exception(type(e), e, None) return True else: self.output.success(action) @@ -114,3 +119,28 @@ class Target: if kwargs.get('wait', True): await proc.wait() return proc + + @property + def root(self): + return self._root + + @root.setter + def root(self, value): + self._root = Path(value or os.getcwd()) + + def path(self, path): + if str(path).startswith('/'): + path = str(path)[1:] + return self.root / path + + async def mkdir(self, path): + return await self.exec('mkdir -p ' + str(path)) + + async def copy(self, *args): + src = args[:-1] + dst = self.path(args[-1]) + await self.mkdir(dst) + return await self.exec( + *('cp', '-av') + src, + dst + ) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index fa78596..211027b 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -22,7 +22,7 @@ class Buildah(Target): self.image = Image(commit) if commit else None self.ctr = None - self.mnt = None + self.root = None self.mounts = dict() self.config = dict( @@ -66,7 +66,7 @@ class Buildah(Target): return self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out - self.mnt = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) + self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) await super().__call__(*actions) async def images(self): @@ -96,7 +96,11 @@ class Buildah(Target): prefix = self.image_previous.tags[0] else: prefix = self.base - key = prefix + repr(action) + if hasattr(action, 'cachehash'): + action_key = action.cachehash() + else: + action_key = str(action) + key = prefix + action_key sha1 = hashlib.sha1(key.encode('ascii')) return self.image.layer(sha1.hexdigest()) @@ -109,7 +113,7 @@ class Buildah(Target): async def clean(self, target): for src, dst in self.mounts.items(): - await self.parent.exec('umount', self.mnt / str(dst)[1:]) + await self.parent.exec('umount', self.root / str(dst)[1:]) else: return @@ -118,7 +122,7 @@ class Buildah(Target): if os.getenv('BUILDAH_PUSH'): await self.image.push(target) - if self.mnt is not None: + if self.root is not None: await self.parent.exec('buildah', 'umount', self.ctr) if self.ctr is not None: @@ -126,7 +130,7 @@ class Buildah(Target): async def mount(self, src, dst): """Mount a host directory into the container.""" - target = self.mnt / str(dst)[1:] + target = self.root / str(dst)[1:] await self.parent.exec(f'mkdir -p {src} {target}') await self.parent.exec(f'mount -o bind {src} {target}') self.mounts[src] = dst @@ -163,3 +167,11 @@ class Buildah(Target): for tag in tags: await self.parent.exec('buildah', 'tag', self.sha, tag) + + async def mkdir(self, path): + return await self.parent.mkdir(self.root / path) + + async def copy(self, *args): + args = list(args) + args[-1] = self.path(args[-1]) + return await self.parent.copy(*args) diff --git a/shlaxfile.py b/shlaxfile.py index dfb13b7..ee28c5a 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -1,12 +1,21 @@ -#!/usr/bin/env shlax +#!/usr/bin/env python """ Shlaxfile for shlax itself. """ from shlax.shortcuts import * -build = Buildah( - Run('echo hi'), - Packages('python38'), - base='quay.io/podman/stable', +shlax = Container( + build=Buildah( + User('app', '/app', getenv('_CONTAINERS_ROOTLESS_UID')), + Packages('python38', 'buildah', 'unzip', 'findutils'), + Copy('setup.py', 'shlax', '/app'), + #Pip('/app', pip='pip3.8'), + base='quay.io/podman/stable', + commit='shlax', + ), ) + + +if __name__ == '__main__': + print(Group(doc=__doc__).load(shlax).entry_point())