From b2797603749ec79501b3014407a5aeb76f63684c Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 20:14:20 +0100 Subject: [PATCH 01/90] Complete core rewrite, with documentation Still missing documentation about Output core component And actual Action/Targets etc ... in the process of migrating to the new engine --- README.md | 228 +++++++++++++++++++++++++++++++++++ shlax/__init__.py | 7 -- shlax/actions/__init__.py | 6 - shlax/actions/base.py | 211 +------------------------------- shlax/actions/copy.py | 6 - shlax/actions/packages.py | 49 ++++---- shlax/actions/parallel.py | 11 ++ shlax/actions/pip.py | 57 --------- shlax/actions/run.py | 23 ++-- shlax/actions/service.py | 16 --- shlax/cli.py | 181 +++++++-------------------- shlax/contrib/gitlab.py | 37 ------ shlax/exceptions.py | 22 ---- shlax/image.py | 1 + shlax/output.py | 79 +++++++++--- shlax/proc.py | 24 +++- shlax/result.py | 13 ++ shlax/shlaxfile.py | 27 ----- shlax/shortcuts.py | 7 ++ shlax/strategies/__init__.py | 3 - shlax/strategies/asyn.py | 11 -- shlax/strategies/pod.py | 27 ----- shlax/strategies/script.py | 34 ------ shlax/targets/__init__.py | 4 - shlax/targets/base.py | 80 ++++++++++++ shlax/targets/buildah.py | 168 +++++++++----------------- shlax/targets/docker.py | 76 ------------ shlax/targets/localhost.py | 49 ++------ shlax/targets/ssh.py | 17 --- shlax/targets/stub.py | 23 ++++ shlaxfile.py | 57 ++------- tests/actions/test_base.py | 53 -------- tests/test_image.py | 2 +- tests/test_output.py | 2 +- tests/test_proc.py | 65 ---------- tests/test_target.py | 101 ++++++++++++++++ 36 files changed, 693 insertions(+), 1084 deletions(-) create mode 100644 README.md delete mode 100644 shlax/__init__.py delete mode 100644 shlax/actions/__init__.py delete mode 100644 shlax/actions/copy.py create mode 100644 shlax/actions/parallel.py delete mode 100644 shlax/actions/pip.py delete mode 100644 shlax/actions/service.py delete mode 100644 shlax/contrib/gitlab.py delete mode 100644 shlax/exceptions.py create mode 100644 shlax/result.py delete mode 100644 shlax/shlaxfile.py create mode 100644 shlax/shortcuts.py delete mode 100644 shlax/strategies/__init__.py delete mode 100644 shlax/strategies/asyn.py delete mode 100644 shlax/strategies/pod.py delete mode 100644 shlax/strategies/script.py delete mode 100644 shlax/targets/__init__.py create mode 100644 shlax/targets/base.py delete mode 100644 shlax/targets/docker.py delete mode 100644 shlax/targets/ssh.py create mode 100644 shlax/targets/stub.py delete mode 100644 tests/actions/test_base.py delete mode 100644 tests/test_proc.py create mode 100644 tests/test_target.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..605d61f --- /dev/null +++ b/README.md @@ -0,0 +1,228 @@ +# Shlax: Pythonic automation tool + +Shlax is a Python framework for system automation, initially with the purpose +of replacing docker, docker-compose and ansible with a single tool, with the +purpose of code-reuse. It may be viewed as "async fabric rewrite by a +megalomanic Django fanboy". + +The pattern resolves around two moving parts: Actions and Targets. + +## Action + +An action is a function that takes a target argument, it may execute nested +actions by passing over the target argument which collects the results. + +Example: + +```python +async def hello_world(target): + """Bunch of silly commands to demonstrate action programming.""" + await target.mkdir('foo') + python = await target.which('python3', 'python') + await target.exec(f'{python} --version > foo/test') + version = target.exec('cat foo/test').output + print('version') +``` + +### Recursion + +An action may call other actions recursively. There are two ways: + +```python +async def something(target): + # just run the other action code + hello_world(target) + + # or delegate the call to target + target(hello_world) +``` + +In the first case, the resulting count of ran actions will remain 1: +"something" action. + +In the second case, the resulting count of ran actions will be 2: "something" +and "hello_world". + +### Callable classes + +Actually in practice, Actions are basic callable Python classes, here's a basic +example to run a command: + +```python +class Run: + def __init__(self, cmd): + self.cmd = cmd + + async def __call__(self, target): + return await target.exec(self.cmd) +``` + +This allows to create callable objects which may be called just like functions +and as such be appropriate actions, instead of: + +```python +async def one(target): + target.exec('one') + +async def two(target): + target.exec('two') +``` + +You can do: + +```python +one = Run('one') +two = Run('two') +``` + +### Parallel execution + +Actions may be executed in parallel with an action named ... Parallel. This +defines an action that will execute three actions in parallel: + +```python +action = Parallel( + hello_world, + something, + Run('echo hi'), +) +``` + +In this case, all actions must succeed for the parallel action to be considered +a success. + +### Methods + +An action may also be a method, as long as it just takes a target argument, for +example: + +```python +class Thing: + def start(self, target): + """Starts thing""" + + def stop(self, target): + """Stops thing""" + +action = Thing().start +``` + +### Cleaning + +If an action defines a `clean` method, it will always be called wether or not +the action succeeded. Example: + +```python +class Thing: + def __call__(self, target): + """Do some thing""" + + def clean(self, target): + """Clean-up target after __call__""" +``` + +### Colorful actions + +If an action defines a `colorize` method, it will be called with the colorset +as argument for every output, this allows to code custom output rendering. + +## Target + +A Target is mainly an object providing an abstraction layer over the system we +want to automate with actions. It defines functions to execute a command, mount +a directory, copy a file, manage environment variables and so on. + +### Pre-configuration + +A Target can be pre-configured with a list of Actions in which case calling the +target without argument will execute its Actions until one fails by raising an +Exception: + +```python +say_hello = Localhost( + hello_world, + Run('echo hi'), +) +await say_hello() +``` + +### Results + +Every time a target execute an action, it will set the "status" attribute on it +to "success" or "failure", and add it to the "results" attribute: + +```python +say_hello = Localhost(Run('echo hi')) +await say_hello() +say_hello.results # contains the action with status="success" +``` + +## Targets as Actions: the nesting story + +We've seen that any callable taking a target argument is good to be considered +an action, and that targets are callables. + +To make a Target runnable like any action, all we had to do is add the target +keyword argument to `Target.__call__`. + +But `target()` fills `self.results`, so nested action results would not +propagate to the parent target. + +That's why if Target receives a non-None target argument, it will has to set +`self.parent` with it. + +This allows nested targets to traverse parents and get to the root Target +with `target.caller`, where it can then attach results to. + +This opens the nice side effect that a target implementation may call the +parent target if any, you could write a Docker target as such: + +```python +class Docker(Target): + def __init__(self, *actions, name): + self.name = name + super().__init__(*actions) + + async def exec(self, *args): + return await self.parent.exec(*['docker', 'exec', self.name] + args) +``` + +Don't worry about `self.parent` being set, it is enforced to `Localhost` if +unset so that we always have something that actually spawns a process in the +chain ;) + +The result of that design is that the following use cases are open for +business: + +```python +# This action installs my favorite package on any distro +action = Packages('python3') + +# Run it right here: apt install python3 +Localhost()(action) + +# Or remotely: ssh yourhost apt install python3 +Ssh(host='yourhost')(action) + +# Let's make a container build receipe with that action +build = Buildah(package) + +# Run it locally: buildah exec apt install python3 +Localhost()(build) + +# Or on a server: ssh yourhost build exec apt install python3 +Ssh(host='yourhost')(build) + +# Or on a server behingh a bastion: +# ssh yourbastion ssh yourhost build exec apt install python3 +Ssh(host='bastion')(Ssh(host='yourhost')(build)) + +# That's going to do the same +Ssh( + Ssh( + build, + host='yourhost' + ), + host='bastion' +)() +``` diff --git a/shlax/__init__.py b/shlax/__init__.py deleted file mode 100644 index 0021973..0000000 --- a/shlax/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .actions import * -from .image import Image -from .strategies import * -from .output import Output -from .proc import Proc -from .targets import * -from .shlaxfile import Shlaxfile diff --git a/shlax/actions/__init__.py b/shlax/actions/__init__.py deleted file mode 100644 index 44accfa..0000000 --- a/shlax/actions/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .copy import Copy -from .packages import Packages # noqa -from .base import Action # noqa -from .run import Run # noqa -from .pip import Pip -from .service import Service diff --git a/shlax/actions/base.py b/shlax/actions/base.py index d2e0e76..890abe0 100644 --- a/shlax/actions/base.py +++ b/shlax/actions/base.py @@ -1,211 +1,2 @@ -import functools -import inspect -import importlib -import sys - -from ..output import Output -from ..exceptions import WrongResult - - class Action: - parent = None - contextualize = [] - regexps = { - r'([\w]+):': '{cyan}\\1{gray}:{reset}', - r'(^|\n)( *)\- ': '\\1\\2{red}-{reset} ', - } - options = dict( - debug=dict( - alias='d', - default='visit', - help=''' - Display debug output. Supports values (combinable): cmd,out,visit - '''.strip(), - immediate=True, - ), - ) - - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - - @property - def context(self): - if not self.parent: - if '_context' not in self.__dict__: - self._context = dict() - return self._context - else: - return self.parent.context - - def actions_filter(self, results, f=None, **filters): - if f: - def ff(a): - try: - return f(a) - except: - return False - results = [*filter(ff, results)] - - for k, v in filters.items(): - if k == 'type': - results = [*filter( - lambda s: type(s).__name__.lower() == str(v).lower(), - results - )] - else: - results = [*filter( - lambda s: getattr(s, k, None) == v, - results - )] - - return results - - - def sibblings(self, f=None, **filters): - if not self.parent: - return [] - return self.actions_filter( - [a for a in self.parent.actions if a is not self], - f, **filters - ) - - def parents(self, f=None, **filters): - if self.parent: - return self.actions_filter( - [self.parent] + self.parent.parents(), - f, **filters - ) - return [] - - def children(self, f=None, **filters): - children = [] - def add(parent): - if parent != self: - children.append(parent) - if 'actions' not in parent.__dict__: - return - - for action in parent.actions: - add(action) - add(self) - return self.actions_filter(children, f, **filters) - - def __getattr__(self, name): - for a in self.parents() + self.sibblings() + self.children(): - if name in a.contextualize: - return getattr(a, name) - raise AttributeError(f'{type(self).__name__} has no {name}') - - async def call(self, *args, **kwargs): - print(f'{self}.call(*args, **kwargs) not implemented') - sys.exit(1) - - def output_factory(self, *args, **kwargs): - kwargs.setdefault('regexps', self.regexps) - return Output(**kwargs) - - async def __call__(self, *args, **kwargs): - self.call_args = args - self.call_kwargs = kwargs - self.output = self.output_factory(*args, **kwargs) - self.output_start() - self.status = 'running' - try: - result = await self.call(*args, **kwargs) - except Exception as e: - self.output_fail(e) - self.status = 'fail' - proc = getattr(e, 'proc', None) - if proc: - result = proc.rc - else: - raise - else: - self.output_success() - if self.status == 'running': - self.status = 'success' - finally: - clean = getattr(self, 'clean', None) - if clean: - self.output.clean(self) - await clean(*args, **kwargs) - return result - - def output_start(self): - if self.kwargs.get('quiet', False): - return - self.output.start(self) - - def output_fail(self, exception=None): - if self.kwargs.get('quiet', False): - return - self.output.fail(self, exception) - - def output_success(self): - if self.kwargs.get('quiet', False): - return - self.output.success(self) - - def __repr__(self): - return ' '.join([type(self).__name__] + list(self.args) + [ - f'{k}={v}' - for k, v in self.kwargs.items() - ]) - - def colorized(self): - return ' '.join([ - self.output.colors['pink1'] - + type(self).__name__ - + self.output.colors['yellow'] - ] + list(self.args) + [ - f'{self.output.colors["blue"]}{k}{self.output.colors["gray"]}={self.output.colors["green2"]}{v}' - for k, v in self.kwargs_output().items() - ] + [self.output.colors['reset']]) - - def callable(self): - from ..targets import Localhost - async def cb(*a, **k): - from shlax.cli import cli - script = Localhost(self, quiet=True) - result = await script(*a, **k) - - success = functools.reduce( - lambda a, b: a + b, - [1 for c in script.children() if c.status == 'success'] or [0]) - if success: - script.output.success(f'{success} PASS') - - failures = functools.reduce( - lambda a, b: a + b, - [1 for c in script.children() if c.status == 'fail'] or [0]) - if failures: - script.output.fail(f'{failures} FAIL') - cli.exit_code = failures - return result - return cb - - def kwargs_output(self): - return self.kwargs - - def action(self, action, *args, **kwargs): - if isinstance(action, str): - import cli2 - a = cli2.Callable.factory(action).target - if not a: - a = cli2.Callable.factory( - '.'.join(['shlax', action]) - ).target - if a: - action = a - - p = action(*args, **kwargs) - for parent in self.parents(): - if hasattr(parent, 'actions'): - break - p.parent = parent - if 'actions' not in self.__dict__: - # "mutate" to Strategy - from ..strategies.script import Actions - self.actions = Actions(self, [p]) - return p + pass diff --git a/shlax/actions/copy.py b/shlax/actions/copy.py deleted file mode 100644 index c325f03..0000000 --- a/shlax/actions/copy.py +++ /dev/null @@ -1,6 +0,0 @@ -from .base import Action - - -class Copy(Action): - async def call(self, *args, **kwargs): - await self.copy(*self.args) diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index fc11109..2294f80 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -7,19 +7,16 @@ import os import subprocess from textwrap import dedent -from .base import Action - -class Packages(Action): +class Packages: """ - The Packages visitor wraps around the container's package manager. + Package manager abstract layer with caching. It's a central piece of the build process, and does iterate over other container visitors in order to pick up packages. For example, the Pip visitor will declare ``self.packages = dict(apt=['python3-pip'])``, and the Packages visitor will pick it up. """ - contextualize = ['mgr'] regexps = { #r'Installing ([\w\d-]+)': '{cyan}\\1', r'Installing': '{cyan}lol', @@ -55,12 +52,11 @@ class Packages(Action): installed = [] - def __init__(self, *packages, **kwargs): + def __init__(self, *packages): self.packages = [] for package in packages: line = dedent(package).strip().replace('\n', ' ') self.packages += line.split(' ') - super().__init__(*packages, **kwargs) @property def cache_root(self): @@ -69,9 +65,9 @@ class Packages(Action): else: return os.path.join(os.getenv('HOME'), '.cache') - async def update(self): + async def update(self, target): # run pkgmgr_setup functions ie. apk_setup - cachedir = await getattr(self, self.mgr + '_setup')() + cachedir = await getattr(self, self.mgr + '_setup')(target) lastupdate = None if os.path.exists(cachedir + '/lastupdate'): @@ -95,7 +91,7 @@ class Packages(Action): f.write(str(os.getpid())) try: - await self.rexec(self.cmds['update']) + await target.rexec(self.cmds['update']) finally: os.unlink(lockfile) @@ -103,15 +99,15 @@ class Packages(Action): f.write(str(now)) else: while os.path.exists(lockfile): - print(f'{self.container.name} | Waiting for update ...') + print(f'{self.target} | Waiting for {lockfile} ...') await asyncio.sleep(1) - async def call(self, *args, **kwargs): - cached = getattr(self, '_pagkages_mgr', None) + async def __call__(self, target): + cached = getattr(target, 'pkgmgr', None) if cached: self.mgr = cached else: - mgr = await self.which(*self.mgrs.keys()) + mgr = await target.which(*self.mgrs.keys()) if mgr: self.mgr = mgr[0].split('/')[-1] @@ -119,11 +115,8 @@ class Packages(Action): raise Exception('Packages does not yet support this distro') self.cmds = self.mgrs[self.mgr] - if not getattr(self, '_packages_upgraded', None): - await self.update() - if self.kwargs.get('upgrade', True): - await self.rexec(self.cmds['upgrade']) - self._packages_upgraded = True + await self.update(target) + await target.rexec(self.cmds['upgrade']) packages = [] for package in self.packages: @@ -136,22 +129,22 @@ class Packages(Action): else: packages.append(package) - await self.rexec(*self.cmds['install'].split(' ') + packages) + await target.rexec(*self.cmds['install'].split(' ') + packages) - async def apk_setup(self): + async def apk_setup(self, target): cachedir = os.path.join(self.cache_root, self.mgr) - await self.mount(cachedir, '/var/cache/apk') + await target.mount(cachedir, '/var/cache/apk') # special step to enable apk cache - await self.rexec('ln -sf /var/cache/apk /etc/apk/cache') + await target.rexec('ln -sf /var/cache/apk /etc/apk/cache') return cachedir - async def dnf_setup(self): + async def dnf_setup(self, target): cachedir = os.path.join(self.cache_root, self.mgr) - await self.mount(cachedir, f'/var/cache/{self.mgr}') - await self.rexec('echo keepcache=True >> /etc/dnf/dnf.conf') + await target.mount(cachedir, f'/var/cache/{self.mgr}') + await target.rexec('echo keepcache=True >> /etc/dnf/dnf.conf') return cachedir - async def apt_setup(self): + async def apt_setup(self, target): codename = (await self.rexec( f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME' )).out @@ -163,7 +156,7 @@ class Packages(Action): await self.mount(cache_lists, f'/var/lib/apt/lists') return cachedir - async def pacman_setup(self): + async def pacman_setup(self, target): return self.cache_root + '/pacman' def __repr__(self): diff --git a/shlax/actions/parallel.py b/shlax/actions/parallel.py new file mode 100644 index 0000000..ed14a53 --- /dev/null +++ b/shlax/actions/parallel.py @@ -0,0 +1,11 @@ +import asyncio + + +class Parallel: + def __init__(self, *actions): + self.actions = actions + + async def __call__(self, target): + return await asyncio.gather(*[ + target(action) for action in self.actions + ]) diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py deleted file mode 100644 index c74815c..0000000 --- a/shlax/actions/pip.py +++ /dev/null @@ -1,57 +0,0 @@ -from glob import glob -import os - -from .base import Action - - -class Pip(Action): - def __init__(self, *pip_packages, pip=None, requirements=None): - self.requirements = requirements - super().__init__(*pip_packages, pip=pip, requirements=requirements) - - async def call(self, *args, **kwargs): - pip = self.kwargs.get('pip', None) - if not pip: - pip = await self.which('pip3', 'pip', 'pip2') - if pip: - pip = pip[0] - else: - from .packages import Packages - action = self.action( - Packages, - 'python3,apk', 'python3-pip,apt', - args=args, kwargs=kwargs - ) - await action(*args, **kwargs) - pip = await self.which('pip3', 'pip', 'pip2') - if not pip: - raise Exception('Could not install a pip command') - else: - pip = pip[0] - - if 'CACHE_DIR' in os.environ: - cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') - else: - cache = os.path.join(os.getenv('HOME'), '.cache', 'pip') - - if getattr(self, 'mount', None): - # we are in a target which shares a mount command - await self.mount(cache, '/root/.cache/pip') - await self.exec(f'{pip} install --upgrade pip') - - # https://github.com/pypa/pip/issues/5599 - if 'pip' not in self.kwargs: - pip = 'python3 -m pip' - - source = [p for p in self.args if p.startswith('/') or p.startswith('.')] - if source: - await self.exec( - f'{pip} install --upgrade --editable {" ".join(source)}' - ) - - nonsource = [p for p in self.args if not p.startswith('/')] - if nonsource: - await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}') - - if self.requirements: - await self.exec(f'{pip} install --upgrade -r {self.requirements}') diff --git a/shlax/actions/run.py b/shlax/actions/run.py index e3f8730..4bb5036 100644 --- a/shlax/actions/run.py +++ b/shlax/actions/run.py @@ -1,18 +1,11 @@ -from ..targets.buildah import Buildah -from ..targets.docker import Docker - -from .base import Action -class Run(Action): - async def call(self, *args, **kwargs): - image = self.kwargs.get('image', None) - if not image: - return await self.exec(*self.args, **self.kwargs) - if isinstance(image, Buildah): - breakpoint() - result = await self.action(image, *args, **kwargs) +class Run: + def __init__(self, cmd): + self.cmd = cmd - return await Docker( - image=image, - ).exec(*args, **kwargs) + async def __call__(self, target): + self.proc = await target.exec(self.cmd) + + def __str__(self): + return f'Run({self.cmd})' diff --git a/shlax/actions/service.py b/shlax/actions/service.py deleted file mode 100644 index 3561c64..0000000 --- a/shlax/actions/service.py +++ /dev/null @@ -1,16 +0,0 @@ -import asyncio - -from .base import Action - - -class Service(Action): - def __init__(self, *names, state=None): - self.state = state or 'started' - self.names = names - super().__init__() - - async def call(self, *args, **kwargs): - return asyncio.gather(*[ - self.exec('systemctl', 'start', name, user='root') - for name in self.names - ]) diff --git a/shlax/cli.py b/shlax/cli.py index fa1767f..9f6a6f0 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -1,146 +1,55 @@ -''' -shlax is a micro-framework to orchestrate commands. - - shlax yourfile.py: to list actions you have declared. - shlax yourfile.py : to execute a given action - #!/usr/bin/env shlax: when making yourfile.py an executable. -''' - -import asyncio +""" +Shlax executes mostly in 3 ways: +- Execute actions on targets with the command line +- With your shlaxfile as first argument: offer defined Actions +- With the name of a module in shlax.repo: a community maintained shlaxfile +""" +import ast import cli2 -import copy +import glob import inspect +import importlib import os import sys -from .exceptions import * -from .shlaxfile import Shlaxfile -from .targets import Localhost - - -async def runall(*args, **kwargs): - for name, action in cli.shlaxfile.actions.items(): - await Localhost(action)(*args, **kwargs) - - -@cli2.option('debug', alias='d', help='Display debug output.') -async def test(*args, **kwargs): - """Run podctl test over a bunch of paths.""" - report = [] - - for arg in args: - candidates = [ - os.path.join(os.getcwd(), arg, 'pod.py'), - os.path.join(os.getcwd(), arg, 'pod_test.py'), - ] - for candidate in candidates: - if not os.path.exists(candidate): - continue - podfile = Podfile.factory(candidate) - - # disable push - for name, container in podfile.containers.items(): - commit = container.visitor('commit') - if commit: - commit.push = False - - output.print( - '\n\x1b[1;38;5;160;48;5;118m BUILD START \x1b[0m' - + ' ' + podfile.path + '\n' - ) - - old_exit_code = console_script.exit_code - console_script.exit_code = 0 - try: - await podfile.pod.script('build')() - except Exception as e: - report.append(('build ' + candidate, False)) - continue - - if console_script.exit_code != 0: - report.append(('build ' + candidate, False)) - continue - console_script.exit_code = old_exit_code - - for name, test in podfile.tests.items(): - name = '::'.join([podfile.path, name]) - output.print( - '\n\x1b[1;38;5;160;48;5;118m TEST START \x1b[0m' - + ' ' + name + '\n' - ) - - try: - await test(podfile.pod) - except Exception as e: - report.append((name, False)) - output.print('\x1b[1;38;5;15;48;5;196m TEST FAIL \x1b[0m' + name) - else: - report.append((name, True)) - output.print('\x1b[1;38;5;200;48;5;44m TEST SUCCESS \x1b[0m' + name) - output.print('\n') - - print('\n') - - for name, success in report: - if success: - output.print('\n\x1b[1;38;5;200;48;5;44m TEST SUCCESS \x1b[0m' + name) - else: - output.print('\n\x1b[1;38;5;15;48;5;196m TEST FAIL \x1b[0m' + name) - - print('\n') - - success = [*filter(lambda i: i[1], report)] - failures = [*filter(lambda i: not i[1], report)] - - output.print( - '\n\x1b[1;38;5;200;48;5;44m TEST TOTAL: \x1b[0m' - + str(len(report)) - ) - if success: - output.print( - '\n\x1b[1;38;5;200;48;5;44m TEST SUCCESS: \x1b[0m' - + str(len(success)) - ) - if failures: - output.print( - '\n\x1b[1;38;5;15;48;5;196m TEST FAIL: \x1b[0m' - + str(len(failures)) - ) - - if failures: - console_script.exit_code = 1 - class ConsoleScript(cli2.ConsoleScript): - def __call__(self, *args, **kwargs): - self.shlaxfile = None - shlaxfile = sys.argv.pop(1) if len(sys.argv) > 1 else '' - if os.path.exists(shlaxfile.split('::')[0]): - self.shlaxfile = Shlaxfile() - self.shlaxfile.parse(shlaxfile) - for name, action in self.shlaxfile.actions.items(): - self[name] = cli2.Callable( - name, - action.callable(), - options={ - k: cli2.Option(name=k, **v) - for k, v in action.options.items() - }, - color=getattr(action, 'color', cli2.YELLOW), - ) - return super().__call__(*args, **kwargs) + def __call__(self): + repo = os.path.join(os.path.dirname(__file__), 'repo') - def call(self, command): - kwargs = copy.copy(self.parser.funckwargs) - kwargs.update(self.parser.options) - try: - return command(*self.parser.funcargs, **kwargs) - except WrongResult as e: - print(e) - self.exit_code = e.proc.rc - except ShlaxException as e: - print(e) - self.exit_code = 1 + if len(self.argv) > 1: + repofile = os.path.join(repo, sys.argv[1] + '.py') + if os.path.isfile(self.argv[1]): + self.argv = sys.argv[1:] + self.load_shlaxfile(sys.argv[1]) + elif os.path.isfile(repofile): + self.argv = sys.argv[1:] + self.load_shlaxfile(repofile) + else: + raise Exception('File not found ' + sys.argv[1]) + else: + available = glob.glob(os.path.join(repo, '*.py')) + return super().__call__() -cli = ConsoleScript(__doc__).add_module('shlax.cli') + def load_shlaxfile(self, path): + with open(path) as f: + src = f.read() + tree = ast.parse(src) + + members = [] + for node in tree.body: + if not isinstance(node, ast.Assign): + continue + if not isinstance(node.value, ast.Call): + continue + members.append(node.targets[0].id) + + spec = importlib.util.spec_from_file_location('shlaxfile', sys.argv[1]) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + for member in members: + from shlax.targets.localhost import Localhost + self[member] = cli2.Callable(member, Localhost(getattr(mod, member))) + +cli = ConsoleScript(__doc__) diff --git a/shlax/contrib/gitlab.py b/shlax/contrib/gitlab.py deleted file mode 100644 index 000efc0..0000000 --- a/shlax/contrib/gitlab.py +++ /dev/null @@ -1,37 +0,0 @@ -from copy import deepcopy -import yaml - -from shlax import * - - -class GitLabCI(Script): - async def call(self, *args, write=True, **kwargs): - output = dict() - for key, value in self.kwargs.items(): - if isinstance(value, dict): - output[key] = deepcopy(value) - image = output[key].get('image', 'alpine') - if hasattr(image, 'image'): - output[key]['image'] = image.image.repository + ':$CI_COMMIT_SHORT_SHA' - else: - output[key] = value - - output = yaml.dump(output) - if kwargs['debug'] is True: - self.output(output) - if write: - with open('.gitlab-ci.yml', 'w+') as f: - f.write(output) - - from shlax.cli import cli - for arg in args: - job = self.kwargs[arg] - _args = [] - if not isinstance(job['image'], str): - image = str(job['image'].image) - else: - image = job['image'] - await self.action('Docker', Run(job['script']), image=image)(*_args, **kwargs) - - def colorized(self): - return type(self).__name__ diff --git a/shlax/exceptions.py b/shlax/exceptions.py deleted file mode 100644 index 0d5e9f4..0000000 --- a/shlax/exceptions.py +++ /dev/null @@ -1,22 +0,0 @@ -class ShlaxException(Exception): - pass - - -class Mistake(ShlaxException): - pass - - -class WrongResult(ShlaxException): - def __init__(self, proc): - self.proc = proc - - msg = f'FAIL exit with {proc.rc} ' + proc.args[0] - - if not proc.debug or 'cmd' not in str(proc.debug): - msg += '\n' + proc.cmd - - if not proc.debug or 'out' not in str(proc.debug): - msg += '\n' + proc.out - msg += '\n' + proc.err - - super().__init__(msg) diff --git a/shlax/image.py b/shlax/image.py index 80b4562..1a01869 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -1,6 +1,7 @@ import os import re + class Image: ENV_TAGS = ( # gitlab diff --git a/shlax/output.py b/shlax/output.py index 9a8955f..19adeb8 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -25,7 +25,23 @@ class Output: def colorize(self, code, content): return self.color(code) + content + self.color() - def __init__(self, prefix=None, regexps=None, debug=False, write=None, flush=None, **kwargs): + def colorized(self): + if hasattr(self.subject, 'colorized'): + return self.subject.colorized(self.colors) + else: + return str(self.subject) + + def __init__( + self, + subject=None, + prefix=None, + regexps=None, + debug='cmd,visit,out', + write=None, + flush=None, + **kwargs + ): + self.subject = subject self.prefix = prefix self.debug = debug self.prefix_length = 0 @@ -98,52 +114,77 @@ class Output: return line - def test(self, action): - if self.debug is True: - self(''.join([ - self.colors['purplebold'], - '! TEST ', - self.colors['reset'], - action.colorized(), - '\n', - ])) + def test(self): + self(''.join([ + self.colors['purplebold'], + '! TEST ', + self.colors['reset'], + self.colorized(), + '\n', + ])) - def clean(self, action): - if self.debug is True: + def clean(self): + if self.debug: self(''.join([ self.colors['bluebold'], '+ CLEAN ', self.colors['reset'], - action.colorized(), + self.colorized(), '\n', ])) - def start(self, action): + def start(self): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['orangebold'], '⚠ START ', self.colors['reset'], - action.colorized(), + self.colorized(), '\n', ])) - def success(self, action): + def success(self): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['greenbold'], '✔ SUCCESS ', self.colors['reset'], - action.colorized() if hasattr(action, 'colorized') else str(action), + self.colorized(), '\n', ])) - def fail(self, action, exception=None): + def fail(self, exception=None): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['redbold'], '✘ FAIL ', self.colors['reset'], - action.colorized() if hasattr(action, 'colorized') else str(action), + self.colorized(), + '\n', + ])) + + def results(self): + success = 0 + fail = 0 + for result in self.subject.results: + if result.status == 'success': + success += 1 + if result.status == 'failure': + fail += 1 + + self(''.join([ + self.colors['greenbold'], + '✔ SUCCESS REPORT: ', + self.colors['reset'], + str(success), + '\n', + ])) + + if fail: + self(''.join([ + self.colors['redbold'], + '✘ FAIL REPORT: ', + self.colors['reset'], + str(fail), '\n', ])) diff --git a/shlax/proc.py b/shlax/proc.py index d50c0f8..292bb37 100644 --- a/shlax/proc.py +++ b/shlax/proc.py @@ -7,10 +7,25 @@ import os import shlex import sys -from .exceptions import WrongResult from .output import Output +class ProcFailure(Exception): + def __init__(self, proc): + self.proc = proc + + msg = f'FAIL exit with {proc.rc} ' + proc.args[0] + + if not proc.output.debug or 'cmd' not in str(proc.output.debug): + msg += '\n' + proc.cmd + + if not proc.output.debug or 'out' not in str(proc.output.debug): + msg += '\n' + proc.out + msg += '\n' + proc.err + + super().__init__(msg) + + class PrefixStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): """ Internal subprocess stream protocol to add a prefix in front of output to @@ -54,8 +69,7 @@ class Proc: """ test = False - def __init__(self, *args, prefix=None, raises=True, debug=None, output=None): - self.debug = debug if not self.test else False + def __init__(self, *args, prefix=None, raises=True, output=None): self.output = output or Output() self.cmd = ' '.join(args) self.args = args @@ -87,7 +101,7 @@ class Proc: if self.called: raise Exception('Already called: ' + self.cmd) - if self.debug is True or 'cmd' in str(self.debug): + if 'cmd' in str(self.output.debug): self.output.cmd(self.cmd) if self.test: @@ -123,7 +137,7 @@ class Proc: if not self.communicated: await self.communicate() if self.raises and self.proc.returncode: - raise WrongResult(self) + raise ProcFailure(self) return self @property diff --git a/shlax/result.py b/shlax/result.py new file mode 100644 index 0000000..7f60be4 --- /dev/null +++ b/shlax/result.py @@ -0,0 +1,13 @@ +class Result: + def __init__(self, target, action): + self.target = target + self.action = action + self.status = 'pending' + self.exception = None + + +class Results(list): + def new(self, target, action): + result = Result(target, action) + self.append(result) + return result diff --git a/shlax/shlaxfile.py b/shlax/shlaxfile.py deleted file mode 100644 index c1039a5..0000000 --- a/shlax/shlaxfile.py +++ /dev/null @@ -1,27 +0,0 @@ -import importlib -import os - -from .actions.base import Action - - -class Shlaxfile: - def __init__(self, actions=None, tests=None): - self.actions = actions or {} - self.tests = tests or {} - self.paths = [] - - def parse(self, path): - spec = importlib.util.spec_from_file_location('shlaxfile', path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - for name, value in mod.__dict__.items(): - if isinstance(value, Action): - value.name = name - self.actions[name] = value - elif callable(value) and getattr(value, '__name__', '').startswith('test_'): - self.tests[value.__name__] = value - self.paths.append(path) - - @property - def path(self): - return self.paths[0] diff --git a/shlax/shortcuts.py b/shlax/shortcuts.py new file mode 100644 index 0000000..82ff556 --- /dev/null +++ b/shlax/shortcuts.py @@ -0,0 +1,7 @@ +from .targets.base import Target +from .targets.buildah import Buildah +from .targets.localhost import Localhost +from .targets.stub import Stub + +from .actions.packages import Packages +from .actions.run import Run diff --git a/shlax/strategies/__init__.py b/shlax/strategies/__init__.py deleted file mode 100644 index 1712bfe..0000000 --- a/shlax/strategies/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .asyn import Async -from .script import Script -from .pod import Pod, Container diff --git a/shlax/strategies/asyn.py b/shlax/strategies/asyn.py deleted file mode 100644 index 06213e0..0000000 --- a/shlax/strategies/asyn.py +++ /dev/null @@ -1,11 +0,0 @@ -import asyncio - -from .script import Script - - -class Async(Script): - async def call(self, *args, **kwargs): - return await asyncio.gather(*[ - action(*args, **kwargs) - for action in self.actions - ]) diff --git a/shlax/strategies/pod.py b/shlax/strategies/pod.py deleted file mode 100644 index 97da07c..0000000 --- a/shlax/strategies/pod.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -from .script import Script - - -class Container(Script): - async def call(self, *args, **kwargs): - if not args or 'build' in args: - await self.kwargs['build'](**kwargs) - self.image = self.kwargs['build'].image - - if not args or 'test' in args: - self.output.test(self) - await self.action('Docker', - *self.kwargs['test'].actions, - image=self.image, - mount={'.': '/app'}, - workdir='/app', - )(**kwargs) - - if not args or 'push' in args: - await self.image.push(action=self) - - #name = kwargs.get('name', os.getcwd()).split('/')[-1] - - -class Pod(Script): - pass diff --git a/shlax/strategies/script.py b/shlax/strategies/script.py deleted file mode 100644 index bd8b0d7..0000000 --- a/shlax/strategies/script.py +++ /dev/null @@ -1,34 +0,0 @@ -import copy -import os - -from ..exceptions import WrongResult -from ..actions.base import Action -from ..proc import Proc - - -class Actions(list): - def __init__(self, owner, actions): - self.owner = owner - super().__init__() - for action in actions: - self.append(action) - - def append(self, value): - value = copy.deepcopy(value) - value.parent = self.owner - value.status = 'pending' - super().append(value) - - -class Script(Action): - contextualize = ['shargs', 'exec', 'rexec', 'env', 'which', 'copy'] - - def __init__(self, *actions, **kwargs): - super().__init__(**kwargs) - self.actions = Actions(self, actions) - - async def call(self, *args, **kwargs): - for action in self.actions: - result = await action(*args, **kwargs) - if action.status != 'success': - break diff --git a/shlax/targets/__init__.py b/shlax/targets/__init__.py deleted file mode 100644 index 3f7353f..0000000 --- a/shlax/targets/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .buildah import Buildah -from .docker import Docker -from .localhost import Localhost -from .ssh import Ssh diff --git a/shlax/targets/base.py b/shlax/targets/base.py new file mode 100644 index 0000000..d737a15 --- /dev/null +++ b/shlax/targets/base.py @@ -0,0 +1,80 @@ +import copy +import re + +from ..output import Output +from ..proc import Proc +from ..result import Result, Results + + +class Target: + def __init__(self, *actions, **options): + self.actions = actions + self.options = options + self.results = [] + self.output = Output(self, **self.options) + self.parent = None + + @property + def caller(self): + """Traverse parents and return the top-levels Target.""" + if not self.parent: + return self + caller = self.parent + while caller.parent: + caller = caller.parent + return caller + + async def __call__(self, *actions, target=None): + if target: + # that's going to be used by other target methods, to access + # the calling target + self.parent = target + + for action in actions or self.actions: + result = Result(self, action) + + self.output = Output(action, **self.options) + self.output.start() + try: + await action(target=self) + except Exception as e: + self.output.fail(e) + result.status = 'failure' + result.exception = e + if actions: + # nested call, re-raise + raise + else: + break + else: + self.output.success() + result.status = 'success' + finally: + self.caller.results.append(result) + + clean = getattr(action, 'clean', None) + if clean: + action.result = result + self.output.clean() + await clean(self) + + async def rexec(self, *args, **kwargs): + kwargs['user'] = 'root' + return await self.exec(*args, **kwargs) + + async def which(self, *cmd): + """ + Return the first path to the cmd in the container. + + If cmd argument is a list then it will try all commands. + """ + proc = await self.exec('type ' + ' '.join(cmd), raises=False) + result = [] + for res in proc.out.split('\n'): + match = re.match('([^ ]+) is ([^ ]+)$', res.strip()) + if match: + result.append(match.group(1)) + return result + + async def exec(self, *args, **kwargs): + raise NotImplemented() diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index bf5e28c..d7cfb78 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -1,127 +1,89 @@ -import asyncio import os -import asyncio -from pathlib import Path -import signal -import shlex -import subprocess import sys -import textwrap +from pathlib import Path + +from .base import Target -from ..actions.base import Action -from ..exceptions import Mistake -from ..proc import Proc from ..image import Image -from .localhost import Localhost +from ..proc import Proc -class Buildah(Localhost): - """ - The build script iterates over visitors and runs the build functions, it - also provides wrappers around the buildah command. - """ - contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount', 'image'] +class Buildah(Target): + def __init__(self, + *actions, + base=None, commit=None, + cmd=None, + **options): + self.base = base or 'alpine' + self.image = Image(commit) if commit else None - def __init__(self, base, *args, commit=None, push=False, cmd=None, **kwargs): - if isinstance(base, Action): - args = [base] + list(args) - base = 'alpine' # default selection in case of mistake - super().__init__(*args, **kwargs) - self.base = base - self.mounts = dict() self.ctr = None self.mnt = None - self.image = Image(commit) if commit else None - self.config= dict( + self.mounts = dict() + + self.config = dict( cmd=cmd or 'sh', ) - def shargs(self, *args, user=None, buildah=True, **kwargs): - if not buildah or args[0].startswith('buildah'): - return super().shargs(*args, user=user, **kwargs) + # Always consider localhost as parent for now + self.parent = Target() - _args = ['buildah', 'run'] - if user: - _args += ['--user', user] - _args += [self.ctr, '--', 'sh', '-euc'] - return super().shargs( - *( - _args - + [' '.join([str(a) for a in args])] - ), - **kwargs - ) + super().__init__(*actions, **options) - def __repr__(self): - return f'Base({self.base})' + def is_runnable(self): + return Proc.test or os.getuid() == 0 - async def config(self, line): - """Run buildah config.""" - return await self.exec(f'buildah config {line} {self.ctr}', buildah=False) + def __str__(self): + if not self.is_runnable(): + return 'Replacing with: buildah unshare ' + ' '.join(sys.argv) + return 'Buildah image builder' - async def copy(self, *args): - """Run buildah copy to copy a file from host into container.""" - src = args[:-1] - dst = args[-1] - await self.mkdir(dst) + async def __call__(self, *actions, target=None): + self.parent = target + if not self.is_runnable(): + os.execvp('buildah', ['buildah', 'unshare'] + sys.argv) + # program has been replaced - procs = [] - for s in src: - if Path(s).is_dir(): - target = self.mnt / s - if not target.exists(): - await self.mkdir(target) - args = ['buildah', 'copy', self.ctr, s, Path(dst) / s] - else: - args = ['buildah', 'copy', self.ctr, s, dst] - procs.append(self.exec(*args, buildah=False)) + self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out + self.mnt = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) + await super().__call__() - return await asyncio.gather(*procs) + async def clean(self, target): + for src, dst in self.mounts.items(): + await self.parent.exec('umount', self.mnt / str(dst)[1:]) + + if self.result.status == 'success': + await self.commit() + if os.getenv('BUILDAH_PUSH'): + await self.image.push(target) + + if self.mnt is not None: + await self.parent.exec('buildah', 'umount', self.ctr) + + if self.ctr is not None: + await self.parent.exec('buildah', 'rm', self.ctr) async def mount(self, src, dst): """Mount a host directory into the container.""" target = self.mnt / str(dst)[1:] - await self.exec(f'mkdir -p {src} {target}', buildah=False) - await self.exec(f'mount -o bind {src} {target}', buildah=False) + await self.parent.exec(f'mkdir -p {src} {target}') + await self.parent.exec(f'mount -o bind {src} {target}') self.mounts[src] = dst - def is_runnable(self): - return ( - Proc.test - or os.getuid() == 0 - ) - - async def call(self, *args, **kwargs): - if self.is_runnable(): - self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out - self.mnt = Path((await self.exec('buildah', 'mount', self.ctr, buildah=False)).out) - result = await super().call(*args, **kwargs) - return result - - from shlax.cli import cli - debug = kwargs.get('debug', False) - # restart under buildah unshare environment - argv = [ - 'buildah', 'unshare', - sys.argv[0], # current script location - cli.shlaxfile.path, # current shlaxfile location - ] - if debug is True: - argv.append('-d') - elif isinstance(debug, str) and debug: - argv.append('-d=' + debug) - argv += [ - cli.parser.command.name, # script name ? - ] - - await self.exec(*argv) + async def exec(self, *args, user=None, **kwargs): + _args = ['buildah', 'run'] + if user: + _args += ['--user', user] + _args += [self.ctr, '--', 'sh', '-euc'] + _args += [' '.join([str(a) for a in args])] + return await self.parent.exec(*_args, **kwargs) async def commit(self): if not self.image: return for key, value in self.config.items(): - await self.exec(f'buildah config --{key} "{value}" {self.ctr}') + await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') self.sha = (await self.exec( 'buildah', @@ -137,20 +99,4 @@ class Buildah(Localhost): tags = [self.image.repository] for tag in tags: - await self.exec('buildah', 'tag', self.sha, tag, buildah=False) - - async def clean(self, *args, **kwargs): - if self.is_runnable(): - for src, dst in self.mounts.items(): - await self.exec('umount', self.mnt / str(dst)[1:], buildah=False) - - if self.status == 'success': - await self.commit() - if 'push' in args: - await self.image.push(action=self) - - if self.mnt is not None: - await self.exec('buildah', 'umount', self.ctr, buildah=False) - - if self.ctr is not None: - await self.exec('buildah', 'rm', self.ctr, buildah=False) + await self.parent.exec('buildah', 'tag', self.sha, tag) diff --git a/shlax/targets/docker.py b/shlax/targets/docker.py deleted file mode 100644 index 331c8b9..0000000 --- a/shlax/targets/docker.py +++ /dev/null @@ -1,76 +0,0 @@ -import asyncio -from pathlib import Path -import os - -from ..image import Image -from .localhost import Localhost - - -class Docker(Localhost): - contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] - - def __init__(self, *args, **kwargs): - self.image = kwargs.get('image', 'alpine') - if not isinstance(self.image, Image): - self.image = Image(self.image) - super().__init__(*args, **kwargs) - self.context['ctr'] = None - - def shargs(self, *args, daemon=False, **kwargs): - if args[0] == 'docker': - return args, kwargs - - extra = [] - if 'user' in kwargs: - extra += ['--user', kwargs.pop('user')] - - args, kwargs = super().shargs(*args, **kwargs) - - if self.context['ctr']: - executor = 'exec' - extra = [self.context['ctr']] - return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + list(args), kwargs - - executor = 'run' - cwd = os.getcwd() - if daemon: - extra += ['-d'] - extra = extra + ['-v', f'{cwd}:{cwd}', '-w', f'{cwd}'] - return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + [str(self.image)] + list(args), kwargs - - async def call(self, *args, **kwargs): - name = kwargs.get('name', os.getcwd()).split('/')[-1] - self.context['ctr'] = ( - await self.exec( - 'docker', 'ps', '-aq', '--filter', - 'name=' + name, - raises=False - ) - ).out.split('\n')[0] - - if 'recreate' in args and self.context['ctr']: - await self.exec('docker', 'rm', '-f', self.context['ctr']) - self.context['ctr'] = None - - if self.context['ctr']: - self.context['ctr'] = (await self.exec('docker', 'start', name)).out - return await super().call(*args, **kwargs) - - async def copy(self, *args): - src = args[:-1] - dst = args[-1] - await self.mkdir(dst) - - procs = [] - for s in src: - ''' - if Path(s).is_dir(): - await self.mkdir(s) - args = ['docker', 'copy', self.ctr, s, Path(dst) / s] - else: - args = ['docker', 'copy', self.ctr, s, dst] - ''' - args = ['docker', 'cp', s, self.context['ctr'] + ':' + dst] - procs.append(self.exec(*args)) - - return await asyncio.gather(*procs) diff --git a/shlax/targets/localhost.py b/shlax/targets/localhost.py index d424179..03bcec8 100644 --- a/shlax/targets/localhost.py +++ b/shlax/targets/localhost.py @@ -1,14 +1,14 @@ -import os +import copy import re -from shlax.proc import Proc +from ..output import Output +from ..proc import Proc +from ..result import Result, Results -from ..strategies.script import Script +from .base import Target -class Localhost(Script): - root = '/' - +class Localhost(Target): def shargs(self, *args, **kwargs): user = kwargs.pop('user', None) args = [str(arg) for arg in args if args is not None] @@ -24,48 +24,17 @@ class Localhost(Script): elif user: args = ['sudo', '-u', user] + args + return args, kwargs + if self.parent: return self.parent.shargs(*args, **kwargs) else: return args, kwargs async def exec(self, *args, **kwargs): - if 'debug' not in kwargs: - kwargs['debug'] = getattr(self, 'call_kwargs', {}).get('debug', False) - kwargs.setdefault('output', self.output) + kwargs['output'] = self.output args, kwargs = self.shargs(*args, **kwargs) proc = await Proc(*args, **kwargs)() if kwargs.get('wait', True): await proc.wait() return proc - - async def rexec(self, *args, **kwargs): - kwargs['user'] = 'root' - return await self.exec(*args, **kwargs) - - async def env(self, name): - return (await self.exec('echo $' + name)).out - - async def which(self, *cmd): - """ - Return the first path to the cmd in the container. - - If cmd argument is a list then it will try all commands. - """ - proc = await self.exec('type ' + ' '.join(cmd), raises=False) - result = [] - for res in proc.out.split('\n'): - match = re.match('([^ ]+) is ([^ ]+)$', res.strip()) - if match: - result.append(match.group(1)) - return result - - async def copy(self, *args): - args = ['cp', '-ra'] + list(args) - return await self.exec(*args) - - async def mount(self, *dirs): - pass - - async def mkdir(self, *dirs): - return await self.exec(*['mkdir', '-p'] + list(dirs)) diff --git a/shlax/targets/ssh.py b/shlax/targets/ssh.py deleted file mode 100644 index 9bd22f8..0000000 --- a/shlax/targets/ssh.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -from shlax.proc import Proc - -from .localhost import Localhost - - -class Ssh(Localhost): - root = '/' - - def __init__(self, host, *args, **kwargs): - self.host = host - super().__init__(*args, **kwargs) - - def shargs(self, *args, **kwargs): - args, kwargs = super().shargs(*args, **kwargs) - return (['ssh', self.host] + list(args)), kwargs diff --git a/shlax/targets/stub.py b/shlax/targets/stub.py new file mode 100644 index 0000000..b5ab487 --- /dev/null +++ b/shlax/targets/stub.py @@ -0,0 +1,23 @@ +from ..proc import Proc + +from .base import Target + + +class ProcStub(Proc): + async def __call__(self, wait=True): + return self + + async def communicate(self): + self.communicated = True + return self + + async def wait(self): + return self + + +class Stub(Target): + async def exec(self, *args, **kwargs): + proc = await ProcStub(*args, **kwargs)() + if kwargs.get('wait', True): + await proc.wait() + return proc diff --git a/shlaxfile.py b/shlaxfile.py index 3921ed5..dfb13b7 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -1,55 +1,12 @@ #!/usr/bin/env shlax -from shlax.contrib.gitlab import * +""" +Shlaxfile for shlax itself. +""" -PYTEST = 'py.test -svv tests' - -test = Script( - Pip('.[test]'), - Run(PYTEST), -) +from shlax.shortcuts import * build = Buildah( - 'quay.io/podman/stable', - Packages('python38', 'buildah', 'unzip', 'findutils', 'python3-yaml', upgrade=False), - Async( - # python3.8 on centos with pip dance ... - Run(''' - curl -o setuptools.zip https://files.pythonhosted.org/packages/42/3e/2464120172859e5d103e5500315fb5555b1e908c0dacc73d80d35a9480ca/setuptools-45.1.0.zip - unzip setuptools.zip - mkdir -p /usr/local/lib/python3.8/site-packages/ - sh -c "cd setuptools-* && python3.8 setup.py install" - easy_install-3.8 pip - echo python3.8 -m pip > /usr/bin/pip - chmod +x /usr/bin/pip - '''), - Copy('shlax/', 'setup.py', '/app'), - ), - Pip('/app[full]'), - commit='docker.io/yourlabs/shlax', - workdir='/app', -) - -shlax = Container( - build=build, - test=Script(Run('./shlaxfile.py -d test')), -) - -gitlabci = GitLabCI( - test=dict( - stage='build', - script='pip install -U --user -e .[test] && ' + PYTEST, - image='yourlabs/python', - ), - build=dict( - stage='build', - image='yourlabs/shlax', - script='pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push', - cache=dict(paths=['.cache'], key='cache'), - ), - pypi=dict( - stage='deploy', - only=['tags'], - image='yourlabs/python', - script='pypi-release', - ), + Run('echo hi'), + Packages('python38'), + base='quay.io/podman/stable', ) diff --git a/tests/actions/test_base.py b/tests/actions/test_base.py deleted file mode 100644 index eedca9d..0000000 --- a/tests/actions/test_base.py +++ /dev/null @@ -1,53 +0,0 @@ -from shlax import * - -inner = Run() -other = Run('ls') -middle = Buildah('alpine', inner, other) -outer = Localhost(middle) -middle = outer.actions[0] -other = middle.actions[1] -inner = middle.actions[0] - - -def test_action_init_args(): - assert other.args == ('ls',) - - -def test_action_parent_autoset(): - assert list(outer.actions) == [middle] - assert middle.parent == outer - assert inner.parent == middle - assert other.parent == middle - - -def test_action_context(): - assert outer.context is inner.context - assert middle.context is inner.context - assert middle.context is outer.context - assert other.context is outer.context - - -def test_action_sibblings(): - assert inner.sibblings() == [other] - assert inner.sibblings(lambda s: s.args[0] == 'ls') == [other] - assert inner.sibblings(lambda s: s.args[0] == 'foo') == [] - assert inner.sibblings(type='run') == [other] - assert inner.sibblings(args=('ls',)) == [other] - - -def test_actions_parents(): - assert other.parents() == [middle, outer] - assert other.parents(lambda p: p.base == 'alpine') == [middle] - assert inner.parents(type='localhost') == [outer] - assert inner.parents(type='buildah') == [middle] - - -def test_action_childrens(): - assert middle.children() == [inner, other] - assert middle.children(lambda a: a.args[0] == 'ls') == [other] - assert outer.children() == [middle, inner, other] - - -def test_action_getattr(): - assert other.exec == middle.exec - assert other.shargs == middle.shargs diff --git a/tests/test_image.py b/tests/test_image.py index 4edfc33..257f591 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -1,7 +1,7 @@ import pytest import os -from shlax import Image +from shlax.image import Image tests = { diff --git a/tests/test_output.py b/tests/test_output.py index 5da9588..e98d5a4 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -1,5 +1,5 @@ import pytest -from shlax import Output +from shlax.output import Output class Write: diff --git a/tests/test_proc.py b/tests/test_proc.py deleted file mode 100644 index 4276263..0000000 --- a/tests/test_proc.py +++ /dev/null @@ -1,65 +0,0 @@ -import pytest - -from unittest.mock import patch - -from shlax import * -from shlax import proc - - -test_args_params = [ - ( - Localhost(Run('echo hi')), - [('sh', '-euc', 'echo hi')] - ), - ( - Localhost(Run('echo hi', user='jimi')), - [('sudo', '-u', 'jimi', 'sh', '-euc', 'echo hi')] - ), - ( - Localhost(Run('echo hi', user='root')), - [('sudo', 'sh', '-euc', 'echo hi')] - ), - ( - Ssh('host', Run('echo hi', user='root')), - [('ssh', 'host', 'sudo', 'sh', '-euc', 'echo hi')] - ), - ( - Buildah('alpine', Run('echo hi')), - [ - ('buildah', 'from', 'alpine'), - ('buildah', 'mount', ''), - ('buildah', 'run', '', '--', 'sh', '-euc', 'echo hi'), - ('buildah', 'umount', ''), - ('buildah', 'rm', ''), - ] - ), - ( - Buildah('alpine', Run('echo hi', user='root')), - [ - ('buildah', 'from', 'alpine'), - ('buildah', 'mount', ''), - ('buildah', 'run', '--user', 'root', '', '--', 'sh', '-euc', 'echo hi'), - ('buildah', 'umount', ''), - ('buildah', 'rm', ''), - ] - ), - ( - Ssh('host', Buildah('alpine', Run('echo hi', user='root'))), - [ - ('ssh', 'host', 'buildah', 'from', 'alpine'), - ('ssh', 'host', 'buildah', 'mount', ''), - ('ssh', 'host', 'buildah', 'run', '--user', 'root', '', '--', 'sh', '-euc', 'echo hi'), - ('ssh', 'host', 'buildah', 'umount', ''), - ('ssh', 'host', 'buildah', 'rm', ''), - ] - ), -] -@pytest.mark.parametrize( - 'script,commands', - test_args_params -) -@pytest.mark.asyncio -async def test_args(script, commands): - with Proc.mock(): - await script() - assert commands == Proc.test diff --git a/tests/test_target.py b/tests/test_target.py new file mode 100644 index 0000000..1fa3770 --- /dev/null +++ b/tests/test_target.py @@ -0,0 +1,101 @@ +import pytest + +from shlax.targets.stub import Stub +from shlax.actions.run import Run +from shlax.actions.parallel import Parallel +from shlax.result import Result + + +class Error: + async def __call__(self, target): + raise Exception('lol') + + +@pytest.mark.asyncio +async def test_success(): + action = Run('echo hi') + target = Stub(action) + await target() + assert target.results[0].action == action + assert target.results[0].status == 'success' + + +@pytest.mark.asyncio +async def test_error(): + action = Error() + target = Stub(action) + await target() + assert target.results[0].action == action + assert target.results[0].status == 'failure' + + +@pytest.mark.asyncio +async def test_nested(): + nested = Error() + + class Nesting: + async def __call__(self, target): + await target(nested) + nesting = Nesting() + + target = Stub(nesting) + await target() + + assert len(target.results) == 2 + assert target.results[0].status == 'failure' + assert target.results[0].action == nested + assert target.results[1].status == 'failure' + assert target.results[1].action == nesting + + +@pytest.mark.asyncio +async def test_parallel(): + winner = Run('echo hi') + looser = Error() + parallel = Parallel(winner, looser) + + target = Stub(parallel) + await target() + assert len(target.results) == 3 + assert target.results[0].status == 'success' + assert target.results[0].action == winner + assert target.results[1].status == 'failure' + assert target.results[1].action == looser + assert target.results[2].status == 'failure' + assert target.results[2].action == parallel + + +@pytest.mark.asyncio +async def test_function(): + async def hello(target): + target.exec('hello') + await Stub()(hello) + + +@pytest.mark.asyncio +async def test_method(): + class Example: + def __init__(self): + self.was_called = False + async def test(self, target): + self.was_called = True + + example = Example() + action = example.test + target = Stub() + await target(action) + assert example.was_called + + +@pytest.mark.asyncio +async def test_target_action(): + child = Stub(Run('echo hi')) + parent = Stub(child) + + grandpa = Stub() + await grandpa(parent) + assert len(grandpa.results) == 3 + + grandpa = Stub(parent) + await grandpa() + assert len(grandpa.results) == 3 From 4601ead8f84cf7195836d36c95d9b54477827686 Mon Sep 17 00:00:00 2001 From: jpic Date: Tue, 21 Apr 2020 21:50:45 +0200 Subject: [PATCH 02/90] Enable gitlab-ci --- .gitlab-ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8fb8b4..9ddc781 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,13 +3,16 @@ build: key: cache paths: [.cache] image: yourlabs/shlax - script: pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d - shlax build push + script: pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py build stage: build + +test: + image: yourlabs/python + stage: build + script: pip install -U --user -e . && py.test -sv tests + pypi: image: yourlabs/python only: [tags] script: pypi-release stage: deploy -test: {image: yourlabs/python, script: 'pip install -U --user -e .[test] && py.test - -svv tests', stage: build} From 111f086bdd7b3a8c02b24d34b341ba44fb99cae7 Mon Sep 17 00:00:00 2001 From: jpic Date: Tue, 21 Apr 2020 21:54:39 +0200 Subject: [PATCH 03/90] Fix dependencies --- .gitlab-ci.yml | 4 ++-- setup.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ddc781..a0de8fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,13 +3,13 @@ build: key: cache paths: [.cache] image: yourlabs/shlax - script: pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py build + script: pip install -U --user -e .[cli] && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py build stage: build test: image: yourlabs/python stage: build - script: pip install -U --user -e . && py.test -sv tests + script: pip install -U --user -e .[test] && py.test -sv tests pypi: image: yourlabs/python diff --git a/setup.py b/setup.py index b15babf..1579178 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,9 @@ setup( name='shlax', versioning='dev', setup_requires='setupmeta', - install_requires=['cli2'], extras_require=dict( - full=[ - 'pyyaml', + cli=[ + 'cli2', ], test=[ 'pytest', From 29505d10042a0c8ba8334d5a4b1ad6ea73ca3452 Mon Sep 17 00:00:00 2001 From: jpic Date: Tue, 21 Apr 2020 21:57:42 +0200 Subject: [PATCH 04/90] No need for a shlax image to build the shlax image It will always build on python with shlax[cli] installed because that's part of the requirement: no extra dependency to use just the framework besides current Python stable release --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0de8fd..1b58598 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ build: cache: key: cache paths: [.cache] - image: yourlabs/shlax + image: yourlabs/python script: pip install -U --user -e .[cli] && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py build stage: build From 00bc4a74aecb9d8c43f9dd7288c394bba53aed95 Mon Sep 17 00:00:00 2001 From: jpic Date: Tue, 21 Apr 2020 22:01:55 +0200 Subject: [PATCH 05/90] Fix can't execute shlax: no such file or dir --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b58598..51e8afe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,13 +3,17 @@ build: key: cache paths: [.cache] image: yourlabs/python - script: pip install -U --user -e .[cli] && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py build + script: + - pip install -U --user -e .[cli] + - CACHE_DIR=$(pwd)/.cache ~/.local/bin/shlax ./shlaxfile.py build stage: build test: image: yourlabs/python stage: build - script: pip install -U --user -e .[test] && py.test -sv tests + script: + - pip install -U --user -e .[test] + - py.test -sv tests pypi: image: yourlabs/python From 8ab2fbcccd113cd59cbbce027b3cf3c58fe82c4f Mon Sep 17 00:00:00 2001 From: jpic Date: Wed, 22 Apr 2020 03:04:14 +0200 Subject: [PATCH 06/90] Pip for python 3.8 --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 51e8afe..6640159 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,9 +2,12 @@ build: cache: key: cache paths: [.cache] - image: yourlabs/python + image: quay.io/buildah/stable script: - - pip install -U --user -e .[cli] + - dnf install -y curl python38 + - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + - python3.8 get-pip.py + - pip3.8 install -U --user -e .[cli] - CACHE_DIR=$(pwd)/.cache ~/.local/bin/shlax ./shlaxfile.py build stage: build From b044dd015ed47599f2f9923dce583aae75983037 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 30 May 2020 18:32:00 +0200 Subject: [PATCH 07/90] Add Package.upgrade option --- README.md | 51 +++++++++++++++++++++++++++++++++------ shlax/actions/packages.py | 8 +++--- shlax/cli.py | 17 +++++++++++-- shlax/output.py | 34 ++++++++++++-------------- shlax/shortcuts.py | 5 ++++ shlax/targets/base.py | 17 ++++++------- shlax/targets/buildah.py | 9 ++++--- 7 files changed, 96 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 605d61f..5bd1a8f 100644 --- a/README.md +++ b/README.md @@ -187,12 +187,13 @@ class Docker(Target): return await self.parent.exec(*['docker', 'exec', self.name] + args) ``` -Don't worry about `self.parent` being set, it is enforced to `Localhost` if -unset so that we always have something that actually spawns a process in the -chain ;) +This also means that you always need a parent with an exec implementation, +there are two: -The result of that design is that the following use cases are open for -business: +- Localhost, executes on localhost +- Stub, for testing + +The result of that design is that the following use cases are available: ```python # This action installs my favorite package on any distro @@ -215,14 +216,48 @@ Ssh(host='yourhost')(build) # Or on a server behingh a bastion: # ssh yourbastion ssh yourhost build exec apt install python3 -Ssh(host='bastion')(Ssh(host='yourhost')(build)) +Localhost()(Ssh(host='bastion')(Ssh(host='yourhost')(build)) # That's going to do the same -Ssh( +Localhost(Ssh( Ssh( build, host='yourhost' ), host='bastion' -)() +))() +``` + +## CLI + +You should build your CLI with your favorite CLI framework. Nonetheless, shlax +provides a ConsoleScript built on cli2 (a personnal experiment, still pre-alpha +stage) that will expose any callable you define in a script, for example: + +```python +#!/usr/bin/env shlax + +from shlax.shortcuts import * + +webpack = Container( + build=Buildah( + Packages('npm') + ) +) + +django = Container( + build=Buildah( + Packages('python') + ) +) + +pod = Pod( + django=django, + webpack=webpack, +) +``` + +Running this file will output: + +``` ``` diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 2294f80..cbdb385 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -52,8 +52,9 @@ class Packages: installed = [] - def __init__(self, *packages): + def __init__(self, *packages, upgrade=True): self.packages = [] + self.upgrade = upgrade for package in packages: line = dedent(package).strip().replace('\n', ' ') self.packages += line.split(' ') @@ -116,7 +117,8 @@ class Packages: self.cmds = self.mgrs[self.mgr] await self.update(target) - await target.rexec(self.cmds['upgrade']) + if self.upgrade: + await target.rexec(self.cmds['upgrade']) packages = [] for package in self.packages: @@ -160,4 +162,4 @@ class Packages: return self.cache_root + '/pacman' def __repr__(self): - return f'Packages({self.packages})' + return f'Packages({self.packages}, upgrade={self.upgrade})' diff --git a/shlax/cli.py b/shlax/cli.py index 9f6a6f0..0dfa364 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -5,6 +5,7 @@ Shlax executes mostly in 3 ways: - With the name of a module in shlax.repo: a community maintained shlaxfile """ import ast +import asyncio import cli2 import glob import inspect @@ -49,7 +50,19 @@ class ConsoleScript(cli2.ConsoleScript): mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) for member in members: - from shlax.targets.localhost import Localhost - self[member] = cli2.Callable(member, Localhost(getattr(mod, member))) + subject = getattr(mod, member) + if callable(subject): + self[member] = cli2.Callable(member, subject) + else: + importable = cli2.Importable(member, subject) + self[member] = cli2.Group(member) + for cb in importable.get_callables(): + self[member][cb.name] = cb + + def call(self, command): + if command.name == 'help': + return super().call(command) + from shlax.targets.localhost import Localhost + asyncio.run(Localhost()(command.target)) cli = ConsoleScript(__doc__) diff --git a/shlax/output.py b/shlax/output.py index 19adeb8..04ef6f2 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -25,15 +25,14 @@ class Output: def colorize(self, code, content): return self.color(code) + content + self.color() - def colorized(self): - if hasattr(self.subject, 'colorized'): - return self.subject.colorized(self.colors) + def colorized(self, action): + if hasattr(action, 'colorized'): + return action.colorized(self.colors) else: - return str(self.subject) + return str(action) def __init__( self, - subject=None, prefix=None, regexps=None, debug='cmd,visit,out', @@ -41,7 +40,6 @@ class Output: flush=None, **kwargs ): - self.subject = subject self.prefix = prefix self.debug = debug self.prefix_length = 0 @@ -114,59 +112,59 @@ class Output: return line - def test(self): + def test(self, action): self(''.join([ self.colors['purplebold'], '! TEST ', self.colors['reset'], - self.colorized(), + self.colorized(action), '\n', ])) - def clean(self): + def clean(self, action): if self.debug: self(''.join([ self.colors['bluebold'], '+ CLEAN ', self.colors['reset'], - self.colorized(), + self.colorized(action), '\n', ])) - def start(self): + def start(self, action): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['orangebold'], '⚠ START ', self.colors['reset'], - self.colorized(), + self.colorized(action), '\n', ])) - def success(self): + def success(self, action): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['greenbold'], '✔ SUCCESS ', self.colors['reset'], - self.colorized(), + self.colorized(action), '\n', ])) - def fail(self, exception=None): + def fail(self, action, exception=None): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['redbold'], '✘ FAIL ', self.colors['reset'], - self.colorized(), + self.colorized(action), '\n', ])) - def results(self): + def results(self, action): success = 0 fail = 0 - for result in self.subject.results: + for result in action.results: if result.status == 'success': success += 1 if result.status == 'failure': diff --git a/shlax/shortcuts.py b/shlax/shortcuts.py index 82ff556..efa02f4 100644 --- a/shlax/shortcuts.py +++ b/shlax/shortcuts.py @@ -5,3 +5,8 @@ from .targets.stub import Stub from .actions.packages import Packages from .actions.run import Run +from .actions.pip import Pip +from .actions.parallel import Parallel + +from .container import Container +from .pod import Pod diff --git a/shlax/targets/base.py b/shlax/targets/base.py index d737a15..1ca6402 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -7,11 +7,10 @@ from ..result import Result, Results class Target: - def __init__(self, *actions, **options): + def __init__(self, *actions): self.actions = actions - self.options = options self.results = [] - self.output = Output(self, **self.options) + self.output = Output() self.parent = None @property @@ -32,13 +31,11 @@ class Target: for action in actions or self.actions: result = Result(self, action) - - self.output = Output(action, **self.options) - self.output.start() + self.output.start(action) try: await action(target=self) except Exception as e: - self.output.fail(e) + self.output.fail(action, e) result.status = 'failure' result.exception = e if actions: @@ -47,7 +44,7 @@ class Target: else: break else: - self.output.success() + self.output.success(action) result.status = 'success' finally: self.caller.results.append(result) @@ -55,7 +52,7 @@ class Target: clean = getattr(action, 'clean', None) if clean: action.result = result - self.output.clean() + self.output.clean(action) await clean(self) async def rexec(self, *args, **kwargs): @@ -77,4 +74,4 @@ class Target: return result async def exec(self, *args, **kwargs): - raise NotImplemented() + raise Exception(f'{self} should run in Localhost() or Stub()') diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index d7cfb78..179fd69 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -12,8 +12,7 @@ class Buildah(Target): def __init__(self, *actions, base=None, commit=None, - cmd=None, - **options): + cmd=None): self.base = base or 'alpine' self.image = Image(commit) if commit else None @@ -28,7 +27,7 @@ class Buildah(Target): # Always consider localhost as parent for now self.parent = Target() - super().__init__(*actions, **options) + super().__init__(*actions) def is_runnable(self): return Proc.test or os.getuid() == 0 @@ -39,7 +38,9 @@ class Buildah(Target): return 'Buildah image builder' async def __call__(self, *actions, target=None): - self.parent = target + if target: + self.parent = target + if not self.is_runnable(): os.execvp('buildah', ['buildah', 'unshare'] + sys.argv) # program has been replaced From e62d44514a48183403b9e67c0ba5db42923b11f0 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 30 May 2020 18:32:36 +0200 Subject: [PATCH 08/90] Add Proc.quiet --- shlax/proc.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/shlax/proc.py b/shlax/proc.py index 292bb37..41b58f2 100644 --- a/shlax/proc.py +++ b/shlax/proc.py @@ -32,21 +32,21 @@ class PrefixStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): make asynchronous output readable. """ - def __init__(self, output, *args, **kwargs): - self.output = output + def __init__(self, proc, *args, **kwargs): + self.proc = proc super().__init__(*args, **kwargs) def pipe_data_received(self, fd, data): - if self.output.debug is True or 'out' in str(self.output.debug): + if self.proc.output.debug is True or 'out' in str(self.proc.output.debug): if fd in (1, 2): - self.output(data) + self.proc.output(data) super().pipe_data_received(fd, data) -def protocol_factory(output): +def protocol_factory(proc): def _p(): return PrefixStreamProtocol( - output, + proc, limit=asyncio.streams._DEFAULT_LIMIT, loop=asyncio.events.get_event_loop() ) @@ -69,8 +69,11 @@ class Proc: """ test = False - def __init__(self, *args, prefix=None, raises=True, output=None): - self.output = output or Output() + def __init__(self, *args, prefix=None, raises=True, output=None, quiet=False): + if quiet: + self.output = Output(debug=False) + else: + self.output = output or Output() self.cmd = ' '.join(args) self.args = args self.prefix = prefix @@ -112,7 +115,7 @@ class Proc: loop = asyncio.events.get_event_loop() transport, protocol = await loop.subprocess_exec( - protocol_factory(self.output), *self.args) + protocol_factory(self), *self.args) self.proc = asyncio.subprocess.Process(transport, protocol, loop) self.called = True From 1d5e8ab1c8df0b134575c6bc34edacbaa7003951 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 30 May 2020 18:34:19 +0200 Subject: [PATCH 09/90] Add layer caching --- shlax/actions/packages.py | 6 ++++ shlax/targets/base.py | 50 +++++++++++++++-------------- shlax/targets/buildah.py | 67 ++++++++++++++++++++++++++++++++------- 3 files changed, 89 insertions(+), 34 deletions(-) diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index cbdb385..523bd1a 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -163,3 +163,9 @@ class Packages: def __repr__(self): return f'Packages({self.packages}, upgrade={self.upgrade})' + + def layerhasher(self, parent=None): + import hashlib + blurb = (parent or '') + repr(self) + sha1 = hashlib.sha1(blurb.encode('ascii')) + return sha1.hexdigest() diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 1ca6402..7c50a59 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -30,30 +30,34 @@ class Target: self.parent = target for action in actions or self.actions: - result = Result(self, action) - self.output.start(action) - try: - await action(target=self) - except Exception as e: - self.output.fail(action, e) - result.status = 'failure' - result.exception = e - if actions: - # nested call, re-raise - raise - else: - break - else: - self.output.success(action) - result.status = 'success' - finally: - self.caller.results.append(result) + if await self.action(action, reraise=bool(actions)): + break - clean = getattr(action, 'clean', None) - if clean: - action.result = result - self.output.clean(action) - await clean(self) + async def action(self, action, reraise=False): + result = Result(self, action) + self.output.start(action) + try: + await action(target=self) + except Exception as e: + self.output.fail(action, e) + result.status = 'failure' + result.exception = e + if reraise: + # nested call, re-raise + raise + else: + return True + else: + self.output.success(action) + result.status = 'success' + finally: + self.caller.results.append(result) + + clean = getattr(action, 'clean', None) + if clean: + action.result = result + self.output.clean(action) + await clean(self) async def rexec(self, *args, **kwargs): kwargs['user'] = 'root' diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 179fd69..cfd9a57 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -1,3 +1,5 @@ +import copy +import json import os import sys from pathlib import Path @@ -45,9 +47,50 @@ class Buildah(Target): os.execvp('buildah', ['buildah', 'unshare'] + sys.argv) # program has been replaced + actions_done = await self.cache_load(*actions) + + if actions: + actions = actions[len(actions_done):] + else: + self.actions = self.actions[len(actions_done):] + self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out self.mnt = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) - await super().__call__() + await super().__call__(*actions) + + async def cache_load(self, *actions): + actions_done = [] + result = await self.parent.exec( + 'podman image list --format json', + quiet=True, + ) + result = json.loads(result.out) + images = [item for sublist in result for item in sublist['History']] + self.hash_previous = None + for action in actions or self.actions: + hasher = getattr(action, 'layerhasher', None) + if not hasher: + break + layerhash = hasher(self.hash_previous) + layerimage = copy.deepcopy(self.image) + layerimage.tags = [layerhash] + if 'localhost/' + str(layerimage) in images: + self.base = str(layerimage) + self.hash_previous = layerhash + actions_done.append(action) + self.output.success(f'Found cached layer for {action}') + + return actions_done + + async def action(self, action, reraise=False): + result = await super().action(action, reraise) + hasher = getattr(action, 'layerhasher', None) + if hasher: + layerhash = hasher(self.hash_previous if self.hash_previous else None) + layerimage = copy.deepcopy(self.image) + layerimage.tags = [layerhash] + await self.commit(layerimage) + return result async def clean(self, target): for src, dst in self.mounts.items(): @@ -79,25 +122,27 @@ class Buildah(Target): _args += [' '.join([str(a) for a in args])] return await self.parent.exec(*_args, **kwargs) - async def commit(self): - if not self.image: + async def commit(self, image=None): + image = image or self.image + if not image: return - for key, value in self.config.items(): - await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') + if not image: + # don't go through that if layer commit + for key, value in self.config.items(): + await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') - self.sha = (await self.exec( + self.sha = (await self.parent.exec( 'buildah', 'commit', - '--format=' + self.image.format, + '--format=' + image.format, self.ctr, - buildah=False, )).out - if self.image.tags: - tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags] + if image.tags: + tags = [f'{image.repository}:{tag}' for tag in image.tags] else: - tags = [self.image.repository] + tags = [image.repository] for tag in tags: await self.parent.exec('buildah', 'tag', self.sha, tag) From 1b1a121a2415044ab19fcce19d0382852f9d9b23 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 30 May 2020 18:35:06 +0200 Subject: [PATCH 10/90] Replace Localhost with plain Target, ensure parent presence --- shlax/targets/base.py | 45 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 7c50a59..e1b919f 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -13,14 +13,22 @@ class Target: self.output = Output() self.parent = None + @property + def parent(self): + return self._parent or Target() + + @parent.setter + def parent(self, value): + self._parent = value + @property def caller(self): """Traverse parents and return the top-levels Target.""" - if not self.parent: + if not self._parent: return self - caller = self.parent - while caller.parent: - caller = caller.parent + caller = self._parent + while caller._parent: + caller = caller._parent return caller async def __call__(self, *actions, target=None): @@ -77,5 +85,32 @@ class Target: result.append(match.group(1)) return result + def shargs(self, *args, **kwargs): + user = kwargs.pop('user', None) + args = [str(arg) for arg in args if args is not None] + + if args and ' ' in args[0]: + if len(args) == 1: + args = ['sh', '-euc', args[0]] + else: + args = ['sh', '-euc'] + list(args) + + if user == 'root': + args = ['sudo'] + args + elif user: + args = ['sudo', '-u', user] + args + + return args, kwargs + + if self.parent: + return self.parent.shargs(*args, **kwargs) + else: + return args, kwargs + async def exec(self, *args, **kwargs): - raise Exception(f'{self} should run in Localhost() or Stub()') + kwargs['output'] = self.output + args, kwargs = self.shargs(*args, **kwargs) + proc = await Proc(*args, **kwargs)() + if kwargs.get('wait', True): + await proc.wait() + return proc From b6bb06054d2cedf7079085202e60c5d355dfe8d9 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 30 May 2020 19:45:56 +0200 Subject: [PATCH 11/90] fixup! Add layer caching --- shlax/actions/packages.py | 6 ---- shlax/image.py | 6 ++++ shlax/output.py | 10 +++++++ shlax/targets/buildah.py | 61 +++++++++++++++++++++++++-------------- 4 files changed, 55 insertions(+), 28 deletions(-) diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 523bd1a..cbdb385 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -163,9 +163,3 @@ class Packages: def __repr__(self): return f'Packages({self.packages}, upgrade={self.upgrade})' - - def layerhasher(self, parent=None): - import hashlib - blurb = (parent or '') + repr(self) - sha1 = hashlib.sha1(blurb.encode('ascii')) - return sha1.hexdigest() diff --git a/shlax/image.py b/shlax/image.py index 1a01869..be4460a 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -1,3 +1,4 @@ +import copy import os import re @@ -74,3 +75,8 @@ class Image: for tag in self.tags: await action.exec('buildah', 'push', f'{self.repository}:{tag}') + + def layer(self, key): + layer = copy.deepcopy(self) + layer.tags = [key] + return layer diff --git a/shlax/output.py b/shlax/output.py index 04ef6f2..2dced2f 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -141,6 +141,16 @@ class Output: '\n', ])) + def skip(self, action): + if self.debug is True or 'visit' in str(self.debug): + self(''.join([ + self.colors['yellowbold'], + '↪️ SKIP ', + self.colors['reset'], + self.colorized(action), + '\n', + ])) + def success(self, action): if self.debug is True or 'visit' in str(self.debug): self(''.join([ diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index cfd9a57..fa78596 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -1,4 +1,5 @@ import copy +import hashlib import json import os import sys @@ -11,6 +12,8 @@ from ..proc import Proc class Buildah(Target): + """Build container image with buildah""" + def __init__(self, *actions, base=None, commit=None, @@ -51,50 +54,64 @@ class Buildah(Target): if actions: actions = actions[len(actions_done):] + if not actions: + self.clean = None + self.output.success('Image up to date') + return else: self.actions = self.actions[len(actions_done):] + if not self.actions: + self.clean = None + self.output.success('Image up to date') + return self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out self.mnt = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) await super().__call__(*actions) - async def cache_load(self, *actions): - actions_done = [] + async def images(self): result = await self.parent.exec( 'podman image list --format json', quiet=True, ) result = json.loads(result.out) - images = [item for sublist in result for item in sublist['History']] - self.hash_previous = None - for action in actions or self.actions: - hasher = getattr(action, 'layerhasher', None) - if not hasher: - break - layerhash = hasher(self.hash_previous) - layerimage = copy.deepcopy(self.image) - layerimage.tags = [layerhash] - if 'localhost/' + str(layerimage) in images: - self.base = str(layerimage) - self.hash_previous = layerhash - actions_done.append(action) - self.output.success(f'Found cached layer for {action}') + return [item for sublist in result for item in sublist['History']] + async def cache_load(self, *actions): + actions_done = [] + self.image_previous = Image(self.base) + images = await self.images() + for action in actions or self.actions: + action_image = self.action_image(action) + if 'localhost/' + str(action_image) in images: + self.base = self.image_previous = action_image + actions_done.append(action) + self.output.skip(f'Found valid cached layer for {action}') + else: + break return actions_done + def action_image(self, action): + if self.image_previous: + prefix = self.image_previous.tags[0] + else: + prefix = self.base + key = prefix + repr(action) + sha1 = hashlib.sha1(key.encode('ascii')) + return self.image.layer(sha1.hexdigest()) + async def action(self, action, reraise=False): result = await super().action(action, reraise) - hasher = getattr(action, 'layerhasher', None) - if hasher: - layerhash = hasher(self.hash_previous if self.hash_previous else None) - layerimage = copy.deepcopy(self.image) - layerimage.tags = [layerhash] - await self.commit(layerimage) + action_image = self.action_image(action) + await self.commit(action_image) + self.image_previous = action_image return result async def clean(self, target): for src, dst in self.mounts.items(): await self.parent.exec('umount', self.mnt / str(dst)[1:]) + else: + return if self.result.status == 'success': await self.commit() From f374d77c21ff42ee11eabf130596981c6db07514 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 30 May 2020 19:46:11 +0200 Subject: [PATCH 12/90] Proper render method actions --- shlax/output.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shlax/output.py b/shlax/output.py index 2dced2f..0a713dc 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -1,5 +1,6 @@ import re import sys +import types from .colors import colors @@ -28,6 +29,8 @@ class Output: def colorized(self, action): if hasattr(action, 'colorized'): return action.colorized(self.colors) + elif isinstance(action, types.MethodType): + return f'{action.__self__}.{action.__name__}' else: return str(action) From b2169ff7ebdc49b45de5443500fe4ee6325207e7 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 30 May 2020 23:52:18 +0200 Subject: [PATCH 13/90] Add CLI to execute Actions on the fly --- setup.py | 2 +- shlax/cli.py | 102 +++++++++++++++++++++++++++---------------------- shlax/image.py | 4 -- 3 files changed, 58 insertions(+), 50 deletions(-) diff --git a/setup.py b/setup.py index 1579178..7feaf11 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( python_requires='>=3', entry_points={ 'console_scripts': [ - 'shlax = shlax.cli:cli', + 'shlax = shlax.cli:cli.entry_point', ], }, ) diff --git a/shlax/cli.py b/shlax/cli.py index 0dfa364..feae747 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -14,55 +14,67 @@ import os import sys -class ConsoleScript(cli2.ConsoleScript): - def __call__(self): - repo = os.path.join(os.path.dirname(__file__), 'repo') - - if len(self.argv) > 1: - repofile = os.path.join(repo, sys.argv[1] + '.py') - if os.path.isfile(self.argv[1]): - self.argv = sys.argv[1:] - self.load_shlaxfile(sys.argv[1]) - elif os.path.isfile(repofile): - self.argv = sys.argv[1:] - self.load_shlaxfile(repofile) - else: - raise Exception('File not found ' + sys.argv[1]) - else: - available = glob.glob(os.path.join(repo, '*.py')) - return super().__call__() +class Group(cli2.Group): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cmdclass = Command - def load_shlaxfile(self, path): - with open(path) as f: - src = f.read() - tree = ast.parse(src) +class Command(cli2.Command): + def call(self, *args, **kwargs): + return self.shlax_target(self.target) - members = [] - for node in tree.body: - if not isinstance(node, ast.Assign): + def __call__(self, *argv): + from shlax.targets.base import Target + self.shlax_target = Target() + result = super().__call__(*argv) + self.shlax_target.output.results(self.shlax_target) + return result + + +class ActionCommand(Command): + def call(self, *args, **kwargs): + self.target = self.target(*args, **kwargs) + return super().call(*args, **kwargs) + + +class ConsoleScript(Group): + def __call__(self, *argv): + self.load_actions() + #self.load_shlaxfiles() # wip + return super().__call__(*argv) + + def load_shlaxfiles(self): + filesdir = os.path.dirname(__file__) + '/shlaxfiles/' + for filename in os.listdir(filesdir): + filepath = filesdir + filename + if not os.path.isfile(filepath): continue - if not isinstance(node.value, ast.Call): + + with open(filepath, 'r') as f: + tree = ast.parse(f.read()) + group = self.group(filename[:-3]) + + main = Group(doc=__doc__).load(shlax) + + def load_actions(self): + actionsdir = os.path.dirname(__file__) + '/actions/' + for filename in os.listdir(actionsdir): + filepath = actionsdir + filename + if not os.path.isfile(filepath): continue - members.append(node.targets[0].id) + with open(filepath, 'r') as f: + tree = ast.parse(f.read()) + cls = [ + node + for node in tree.body + if isinstance(node, ast.ClassDef) + ] + if not cls: + continue + mod = importlib.import_module('shlax.actions.' + filename[:-3]) + cls = getattr(mod, cls[0].name) + self.add(cls, name=filename[:-3], cmdclass=ActionCommand) - spec = importlib.util.spec_from_file_location('shlaxfile', sys.argv[1]) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - for member in members: - subject = getattr(mod, member) - if callable(subject): - self[member] = cli2.Callable(member, subject) - else: - importable = cli2.Importable(member, subject) - self[member] = cli2.Group(member) - for cb in importable.get_callables(): - self[member][cb.name] = cb - def call(self, command): - if command.name == 'help': - return super().call(command) - from shlax.targets.localhost import Localhost - asyncio.run(Localhost()(command.target)) - -cli = ConsoleScript(__doc__) +cli = ConsoleScript(doc=__doc__) diff --git a/shlax/image.py b/shlax/image.py index be4460a..60fb2b7 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -58,10 +58,6 @@ class Image: if not self.tags: self.tags = ['latest'] - async def __call__(self, action, *args, **kwargs): - args = list(args) - return await action.exec(*args, **self.kwargs) - def __str__(self): return f'{self.repository}:{self.tags[-1]}' From 600043ae645edd313e4d35c504fb36c8136a2a74 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 30 May 2020 23:54:10 +0200 Subject: [PATCH 14/90] Adding Copy/User/Pip actions again --- shlax/actions/copy.py | 19 +++++++++++++ shlax/actions/packages.py | 2 +- shlax/actions/pip.py | 59 +++++++++++++++++++++++++++++++++++++++ shlax/actions/user.py | 32 +++++++++++++++++++++ shlax/shortcuts.py | 6 ++++ shlax/targets/base.py | 32 ++++++++++++++++++++- shlax/targets/buildah.py | 24 ++++++++++++---- shlaxfile.py | 19 +++++++++---- 8 files changed, 180 insertions(+), 13 deletions(-) create mode 100644 shlax/actions/copy.py create mode 100644 shlax/actions/pip.py create mode 100644 shlax/actions/user.py diff --git a/shlax/actions/copy.py b/shlax/actions/copy.py new file mode 100644 index 0000000..7209aa6 --- /dev/null +++ b/shlax/actions/copy.py @@ -0,0 +1,19 @@ +class Copy: + def __init__(self, *args): + self.src = args[:-1] + self.dst = args[-1] + + @property + def files(self): + for root, dirs, files in os.walk(self.dst): + pass + + + async def __call__(self, target): + await target.copy(*self.args) + + def __str__(self): + return f'Copy(*{self.src}, {self.dst})' + + def cachehash(self): + return str(self) diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index cbdb385..814da6a 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -161,5 +161,5 @@ class Packages: async def pacman_setup(self, target): return self.cache_root + '/pacman' - def __repr__(self): + def __str__(self): return f'Packages({self.packages}, upgrade={self.upgrade})' diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py new file mode 100644 index 0000000..3ef9887 --- /dev/null +++ b/shlax/actions/pip.py @@ -0,0 +1,59 @@ +from glob import glob +import os + +from .base import Action + + +class Pip(Action): + """Pip abstraction layer.""" + + def __init__(self, *pip_packages, pip=None, requirements=None): + self.requirements = requirements + super().__init__(*pip_packages, pip=pip, requirements=requirements) + + async def call(self, *args, **kwargs): + pip = self.kwargs.get('pip', None) + if not pip: + pip = await self.which('pip3', 'pip', 'pip2') + if pip: + pip = pip[0] + else: + from .packages import Packages + action = self.action( + Packages, + 'python3,apk', 'python3-pip,apt', + args=args, kwargs=kwargs + ) + await action(*args, **kwargs) + pip = await self.which('pip3', 'pip', 'pip2') + if not pip: + raise Exception('Could not install a pip command') + else: + pip = pip[0] + + if 'CACHE_DIR' in os.environ: + cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') + else: + cache = os.path.join(os.getenv('HOME'), '.cache', 'pip') + + if getattr(self, 'mount', None): + # we are in a target which shares a mount command + await self.mount(cache, '/root/.cache/pip') + await self.exec(f'{pip} install --upgrade pip') + + # https://github.com/pypa/pip/issues/5599 + if 'pip' not in self.kwargs: + pip = 'python3 -m pip' + + source = [p for p in self.args if p.startswith('/') or p.startswith('.')] + if source: + await self.exec( + f'{pip} install --upgrade --editable {" ".join(source)}' + ) + + nonsource = [p for p in self.args if not p.startswith('/')] + if nonsource: + await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}') + + if self.requirements: + await self.exec(f'{pip} install --upgrade -r {self.requirements}') diff --git a/shlax/actions/user.py b/shlax/actions/user.py new file mode 100644 index 0000000..f919852 --- /dev/null +++ b/shlax/actions/user.py @@ -0,0 +1,32 @@ +import os +import re + +from .packages import Packages + + +class User: + def __init__(self, username, home, uid): + self.username = username + self.home = home + self.uid = uid + + def __str__(self): + return f'User({self.username}, {self.home}, {self.uid})' + + async def __call__(self, target): + result = await target.rexec('id', self.uid) + if result.rc == 0: + old = re.match('.*\(([^)]*)\).*', result.out).group(1) + await target.rexec( + 'usermod', + '-d', self.home, + '-l', self.username, + old + ) + else: + await target.rexec( + 'useradd', + '-d', self.home, + '-u', self.uid, + self.username + ) diff --git a/shlax/shortcuts.py b/shlax/shortcuts.py index efa02f4..9d77cc9 100644 --- a/shlax/shortcuts.py +++ b/shlax/shortcuts.py @@ -3,10 +3,16 @@ from .targets.buildah import Buildah from .targets.localhost import Localhost from .targets.stub import Stub +from .actions.copy import Copy from .actions.packages import Packages from .actions.run import Run from .actions.pip import Pip from .actions.parallel import Parallel +from .actions.user import User + +from .cli import Command, Group from .container import Container from .pod import Pod + +from os import getenv, environ diff --git a/shlax/targets/base.py b/shlax/targets/base.py index e1b919f..110f0b7 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -1,4 +1,6 @@ import copy +from pathlib import Path +import os import re from ..output import Output @@ -7,11 +9,12 @@ from ..result import Result, Results class Target: - def __init__(self, *actions): + def __init__(self, *actions, root=None): self.actions = actions self.results = [] self.output = Output() self.parent = None + self.root = root or os.getcwd() @property def parent(self): @@ -54,6 +57,8 @@ class Target: # nested call, re-raise raise else: + import traceback + traceback.print_exception(type(e), e, None) return True else: self.output.success(action) @@ -114,3 +119,28 @@ class Target: if kwargs.get('wait', True): await proc.wait() return proc + + @property + def root(self): + return self._root + + @root.setter + def root(self, value): + self._root = Path(value or os.getcwd()) + + def path(self, path): + if str(path).startswith('/'): + path = str(path)[1:] + return self.root / path + + async def mkdir(self, path): + return await self.exec('mkdir -p ' + str(path)) + + async def copy(self, *args): + src = args[:-1] + dst = self.path(args[-1]) + await self.mkdir(dst) + return await self.exec( + *('cp', '-av') + src, + dst + ) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index fa78596..211027b 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -22,7 +22,7 @@ class Buildah(Target): self.image = Image(commit) if commit else None self.ctr = None - self.mnt = None + self.root = None self.mounts = dict() self.config = dict( @@ -66,7 +66,7 @@ class Buildah(Target): return self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out - self.mnt = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) + self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) await super().__call__(*actions) async def images(self): @@ -96,7 +96,11 @@ class Buildah(Target): prefix = self.image_previous.tags[0] else: prefix = self.base - key = prefix + repr(action) + if hasattr(action, 'cachehash'): + action_key = action.cachehash() + else: + action_key = str(action) + key = prefix + action_key sha1 = hashlib.sha1(key.encode('ascii')) return self.image.layer(sha1.hexdigest()) @@ -109,7 +113,7 @@ class Buildah(Target): async def clean(self, target): for src, dst in self.mounts.items(): - await self.parent.exec('umount', self.mnt / str(dst)[1:]) + await self.parent.exec('umount', self.root / str(dst)[1:]) else: return @@ -118,7 +122,7 @@ class Buildah(Target): if os.getenv('BUILDAH_PUSH'): await self.image.push(target) - if self.mnt is not None: + if self.root is not None: await self.parent.exec('buildah', 'umount', self.ctr) if self.ctr is not None: @@ -126,7 +130,7 @@ class Buildah(Target): async def mount(self, src, dst): """Mount a host directory into the container.""" - target = self.mnt / str(dst)[1:] + target = self.root / str(dst)[1:] await self.parent.exec(f'mkdir -p {src} {target}') await self.parent.exec(f'mount -o bind {src} {target}') self.mounts[src] = dst @@ -163,3 +167,11 @@ class Buildah(Target): for tag in tags: await self.parent.exec('buildah', 'tag', self.sha, tag) + + async def mkdir(self, path): + return await self.parent.mkdir(self.root / path) + + async def copy(self, *args): + args = list(args) + args[-1] = self.path(args[-1]) + return await self.parent.copy(*args) diff --git a/shlaxfile.py b/shlaxfile.py index dfb13b7..ee28c5a 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -1,12 +1,21 @@ -#!/usr/bin/env shlax +#!/usr/bin/env python """ Shlaxfile for shlax itself. """ from shlax.shortcuts import * -build = Buildah( - Run('echo hi'), - Packages('python38'), - base='quay.io/podman/stable', +shlax = Container( + build=Buildah( + User('app', '/app', getenv('_CONTAINERS_ROOTLESS_UID')), + Packages('python38', 'buildah', 'unzip', 'findutils'), + Copy('setup.py', 'shlax', '/app'), + #Pip('/app', pip='pip3.8'), + base='quay.io/podman/stable', + commit='shlax', + ), ) + + +if __name__ == '__main__': + print(Group(doc=__doc__).load(shlax).entry_point()) From 7a61b405ae41e0d4e9e36be1d51b6f6c1d3f7109 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 00:00:10 +0200 Subject: [PATCH 15/90] Work on the CLI story --- README.md | 47 +++++++++++++++++++++-------------------------- setup.py | 2 +- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 5bd1a8f..054603e 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ # Shlax: Pythonic automation tool Shlax is a Python framework for system automation, initially with the purpose -of replacing docker, docker-compose and ansible with a single tool, with the -purpose of code-reuse. It may be viewed as "async fabric rewrite by a -megalomanic Django fanboy". +of replacing docker, docker-compose and ansible with a single tool with the +purpose of code-reuse made possible by target abstraction. The pattern resolves around two moving parts: Actions and Targets. @@ -230,34 +229,30 @@ Localhost(Ssh( ## CLI -You should build your CLI with your favorite CLI framework. Nonetheless, shlax -provides a ConsoleScript built on cli2 (a personnal experiment, still pre-alpha -stage) that will expose any callable you define in a script, for example: +You can execute Shlax actions directly on the command line with the `shlax` CLI +command. + +For your own Shlaxfiles, you can build your CLI with your favorite CLI +framework. If you decide to use `cli2`, then Shlax provides a thin layer on top +of it: Group and Command objects made for Shlax objects. + +For example: ```python -#!/usr/bin/env shlax - -from shlax.shortcuts import * - -webpack = Container( +yourcontainer = Container( build=Buildah( - Packages('npm') - ) + User('app', '/app', 1000), + Packages('python', 'unzip', 'findutils'), + Copy('setup.py', 'yourdir', '/app'), + base='archlinux', + commit='yourimage', + ), ) -django = Container( - build=Buildah( - Packages('python') - ) -) -pod = Pod( - django=django, - webpack=webpack, -) +if __name__ == '__main__': + print(Group(doc=__doc__).load(yourcontainer).entry_point()) ``` -Running this file will output: - -``` -``` +The above will execute a cli2 command with each method of yourcontainer as a +sub-command. diff --git a/setup.py b/setup.py index 7feaf11..3ef0693 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( setup_requires='setupmeta', extras_require=dict( cli=[ - 'cli2', + 'cli2>=2.2.2', ], test=[ 'pytest', From 4af938fab0df31131db70c4cf4e6155a77711d52 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 02:42:59 +0200 Subject: [PATCH 16/90] Copy action: refactor, caching, filtering --- shlax/actions/copy.py | 53 ++++++++++++++++++++++++++++++++++------ shlax/targets/base.py | 15 ++++++------ shlax/targets/buildah.py | 6 ++--- 3 files changed, 54 insertions(+), 20 deletions(-) diff --git a/shlax/actions/copy.py b/shlax/actions/copy.py index 7209aa6..2fee0a1 100644 --- a/shlax/actions/copy.py +++ b/shlax/actions/copy.py @@ -1,19 +1,56 @@ +import asyncio +import binascii +import os + + class Copy: def __init__(self, *args): self.src = args[:-1] self.dst = args[-1] - @property - def files(self): - for root, dirs, files in os.walk(self.dst): - pass + def listfiles(self): + if getattr(self, '_listfiles', None): + return self._listfiles + result = [] + for src in self.src: + if os.path.isfile(src): + result.append(src) + continue + + for root, dirs, files in os.walk(src): + if '__pycache__' in root: + continue + result += [ + os.path.join(root, f) + for f in files + if not f.endswith('.pyc') + ] + self._listfiles = result + return result async def __call__(self, target): - await target.copy(*self.args) + await target.mkdir(self.dst) + + for path in self.listfiles(): + if os.path.isdir(path): + await target.mkdir(os.path.join(self.dst, path)) + elif '/' in path: + dirname = os.path.join( + self.dst, + '/'.join(path.split('/')[:-1]) + ) + await target.mkdir(dirname) + await target.copy(path, dirname) + else: + await target.copy(path, self.dst) def __str__(self): - return f'Copy(*{self.src}, {self.dst})' + return f'Copy({", ".join(self.src)}, {self.dst})' - def cachehash(self): - return str(self) + async def cachekey(self): + async def chksum(path): + with open(path, 'rb') as f: + return (path, str(binascii.crc32(f.read()))) + results = await asyncio.gather(*[chksum(f) for f in self.listfiles()]) + return {path: chks for path, chks in results} diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 110f0b7..7f6e30e 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -134,13 +134,12 @@ class Target: return self.root / path async def mkdir(self, path): - return await self.exec('mkdir -p ' + str(path)) + if '_mkdir' not in self.__dict__: + self._mkdir = [] + path = str(path) + if path not in self._mkdir: + await self.exec('mkdir', '-p', path) + self._mkdir.append(path) async def copy(self, *args): - src = args[:-1] - dst = self.path(args[-1]) - await self.mkdir(dst) - return await self.exec( - *('cp', '-av') + src, - dst - ) + return await self.exec('cp', '-a', *args) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 211027b..74bd221 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -169,9 +169,7 @@ class Buildah(Target): await self.parent.exec('buildah', 'tag', self.sha, tag) async def mkdir(self, path): - return await self.parent.mkdir(self.root / path) + return await self.parent.mkdir(self.path(path)) async def copy(self, *args): - args = list(args) - args[-1] = self.path(args[-1]) - return await self.parent.copy(*args) + return await self.parent.copy(*args[:-1], self.path(args[-1])) From c58d1618ed992dcd92919122326a7c58e9cf9d0f Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 02:44:13 +0200 Subject: [PATCH 17/90] Proper traceback prints --- shlax/targets/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 7f6e30e..ac4d938 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -2,6 +2,7 @@ import copy from pathlib import Path import os import re +import sys from ..output import Output from ..proc import Proc @@ -58,7 +59,7 @@ class Target: raise else: import traceback - traceback.print_exception(type(e), e, None) + traceback.print_exception(type(e), e, sys.exc_info()[2]) return True else: self.output.success(action) From 5caf9d92e39a0feb72311dc46e29df87e69951ea Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 02:44:46 +0200 Subject: [PATCH 18/90] Proper cache invalidation --- shlax/image.py | 2 +- shlax/output.py | 12 +++++- shlax/targets/base.py | 1 + shlax/targets/buildah.py | 81 ++++++++++++++++++++++++++-------------- 4 files changed, 67 insertions(+), 29 deletions(-) diff --git a/shlax/image.py b/shlax/image.py index 60fb2b7..4ae7567 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -74,5 +74,5 @@ class Image: def layer(self, key): layer = copy.deepcopy(self) - layer.tags = [key] + layer.tags = ['layer-' + key] return layer diff --git a/shlax/output.py b/shlax/output.py index 0a713dc..fa07783 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -144,11 +144,21 @@ class Output: '\n', ])) + def info(self, text): + if self.debug is True or 'visit' in str(self.debug): + self(''.join([ + self.colors['cyanbold'], + '➤ INFO ', + self.colors['reset'], + text, + '\n', + ])) + def skip(self, action): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['yellowbold'], - '↪️ SKIP ', + '↪️ SKIP ', self.colors['reset'], self.colorized(action), '\n', diff --git a/shlax/targets/base.py b/shlax/targets/base.py index ac4d938..e9fbe5f 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -1,3 +1,4 @@ +import asyncio import copy from pathlib import Path import os diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 74bd221..484ce35 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -1,3 +1,4 @@ +import asyncio import copy import hashlib import json @@ -50,54 +51,74 @@ class Buildah(Target): os.execvp('buildah', ['buildah', 'unshare'] + sys.argv) # program has been replaced - actions_done = await self.cache_load(*actions) + layers = await self.layers() + keep = await self.cache_setup(layers, *actions) + keepnames = [*map(lambda x: 'localhost/' + str(x), keep)] + self.invalidate = [name for name in layers if name not in keepnames] + if self.invalidate: + self.output.info('Invalidating old layers') + await self.parent.exec( + 'buildah', 'rmi', *self.invalidate, raises=False) if actions: - actions = actions[len(actions_done):] + actions = actions[len(keep):] if not actions: - self.clean = None - self.output.success('Image up to date') - return + return self.uptodate() else: - self.actions = self.actions[len(actions_done):] + self.actions = self.actions[len(keep):] if not self.actions: - self.clean = None - self.output.success('Image up to date') - return + return self.uptodate() self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) - await super().__call__(*actions) - async def images(self): - result = await self.parent.exec( - 'podman image list --format json', + return await super().__call__(*actions) + + def uptodate(self): + self.clean = None + self.output.success('Image up to date') + return + + async def layers(self): + ret = set() + results = await self.parent.exec( + 'buildah images --json', quiet=True, ) - result = json.loads(result.out) - return [item for sublist in result for item in sublist['History']] + results = json.loads(results.out) - async def cache_load(self, *actions): - actions_done = [] + prefix = 'localhost/' + self.image.repository + ':layer-' + for result in results: + if not result.get('names', None): + continue + for name in result['names']: + if name.startswith(prefix): + ret.add(name) + return ret + + async def cache_setup(self, layers, *actions): + keep = [] self.image_previous = Image(self.base) - images = await self.images() for action in actions or self.actions: - action_image = self.action_image(action) - if 'localhost/' + str(action_image) in images: + action_image = await self.action_image(action) + name = 'localhost/' + str(action_image) + if name in layers: self.base = self.image_previous = action_image - actions_done.append(action) + keep.append(action_image) self.output.skip(f'Found valid cached layer for {action}') else: break - return actions_done + return keep - def action_image(self, action): + async def action_image(self, action): if self.image_previous: prefix = self.image_previous.tags[0] else: prefix = self.base - if hasattr(action, 'cachehash'): - action_key = action.cachehash() + if hasattr(action, 'cachekey'): + action_key = action.cachekey() + if asyncio.iscoroutine(action_key): + action_key = str(await action_key) else: action_key = str(action) key = prefix + action_key @@ -106,8 +127,14 @@ class Buildah(Target): async def action(self, action, reraise=False): result = await super().action(action, reraise) - action_image = self.action_image(action) - await self.commit(action_image) + action_image = await self.action_image(action) + await self.parent.exec( + 'buildah', + 'commit', + '--format=' + action_image.format, + self.ctr, + action_image, + ) self.image_previous = action_image return result From c570f0fd6d89b60a0c3331fceae5a72b3fea883a Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 02:45:24 +0200 Subject: [PATCH 19/90] Bugfix: legacy code would prevent containers from shuting down after build --- shlax/targets/buildah.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 484ce35..b7d52ae 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -141,13 +141,9 @@ class Buildah(Target): async def clean(self, target): for src, dst in self.mounts.items(): await self.parent.exec('umount', self.root / str(dst)[1:]) - else: - return if self.result.status == 'success': await self.commit() - if os.getenv('BUILDAH_PUSH'): - await self.image.push(target) if self.root is not None: await self.parent.exec('buildah', 'umount', self.ctr) @@ -155,6 +151,10 @@ class Buildah(Target): if self.ctr is not None: await self.parent.exec('buildah', 'rm', self.ctr) + if self.result.status == 'success': + if os.getenv('BUILDAH_PUSH'): + await self.image.push(target) + async def mount(self, src, dst): """Mount a host directory into the container.""" target = self.root / str(dst)[1:] From a07d9c5e67a324cefc5faf5d39efcfcc437a27af Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 03:39:26 +0200 Subject: [PATCH 20/90] Pip action implementation --- shlax/actions/pip.py | 82 +++++++++++++++++++++++++------------------- shlaxfile.py | 4 +-- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py index 3ef9887..7d8e8c7 100644 --- a/shlax/actions/pip.py +++ b/shlax/actions/pip.py @@ -1,59 +1,69 @@ from glob import glob import os +from urllib import request from .base import Action class Pip(Action): """Pip abstraction layer.""" + def __init__(self, *pip_packages): + self.pip_packages = pip_packages - def __init__(self, *pip_packages, pip=None, requirements=None): - self.requirements = requirements - super().__init__(*pip_packages, pip=pip, requirements=requirements) + async def __call__(self, target): + # ensure python presence + results = await target.which('python3', 'python') + if results: + python = results[0] + else: + raise Exception('Could not find pip nor python') - async def call(self, *args, **kwargs): - pip = self.kwargs.get('pip', None) - if not pip: - pip = await self.which('pip3', 'pip', 'pip2') - if pip: - pip = pip[0] - else: - from .packages import Packages - action = self.action( - Packages, - 'python3,apk', 'python3-pip,apt', - args=args, kwargs=kwargs + # ensure pip module presence + result = await target.exec(python, '-m', 'pip', raises=False) + if result.rc != 0: + if not os.path.exists('get-pip.py'): + req = request.urlopen( + 'https://bootstrap.pypa.io/get-pip.py' ) - await action(*args, **kwargs) - pip = await self.which('pip3', 'pip', 'pip2') - if not pip: - raise Exception('Could not install a pip command') - else: - pip = pip[0] + content = req.read() + with open('get-pip.py', 'wb+') as f: + f.write(content) + await target.copy('get-pip.py', '.') + await target.exec(python, 'get-pip.py') + + # choose a cache directory if 'CACHE_DIR' in os.environ: cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') else: cache = os.path.join(os.getenv('HOME'), '.cache', 'pip') - if getattr(self, 'mount', None): + # and mount it + if getattr(target, 'mount', None): # we are in a target which shares a mount command - await self.mount(cache, '/root/.cache/pip') - await self.exec(f'{pip} install --upgrade pip') + await target.mount(cache, '/root/.cache/pip') - # https://github.com/pypa/pip/issues/5599 - if 'pip' not in self.kwargs: - pip = 'python3 -m pip' + source = [] + nonsource = [] + for package in self.pip_packages: + if os.path.exists(package): + source.append(package) + else: + nonsource.append(package) - source = [p for p in self.args if p.startswith('/') or p.startswith('.')] - if source: - await self.exec( - f'{pip} install --upgrade --editable {" ".join(source)}' + if nonsource: + await target.exec( + python, '-m', 'pip', + 'install', '--upgrade', + *nonsource ) - nonsource = [p for p in self.args if not p.startswith('/')] - if nonsource: - await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}') + if source: + await target.exec( + python, '-m', 'pip', + 'install', '--upgrade', '--editable', + *source + ) - if self.requirements: - await self.exec(f'{pip} install --upgrade -r {self.requirements}') + def __str__(self): + return f'Pip({", ".join(self.pip_packages)})' diff --git a/shlaxfile.py b/shlaxfile.py index ee28c5a..f3dfa7e 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -7,10 +7,10 @@ from shlax.shortcuts import * shlax = Container( build=Buildah( - User('app', '/app', getenv('_CONTAINERS_ROOTLESS_UID')), Packages('python38', 'buildah', 'unzip', 'findutils'), + User('app', '/app', getenv('_CONTAINERS_ROOTLESS_UID')), Copy('setup.py', 'shlax', '/app'), - #Pip('/app', pip='pip3.8'), + Pip('/app'), base='quay.io/podman/stable', commit='shlax', ), From 93b95a1f507ef06e395baadbb23c0e3836015af6 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 03:39:42 +0200 Subject: [PATCH 21/90] Set the action result prior to calling clean --- shlax/targets/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/targets/base.py b/shlax/targets/base.py index e9fbe5f..6bc79a0 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -66,11 +66,11 @@ class Target: self.output.success(action) result.status = 'success' finally: + action.result = result self.caller.results.append(result) clean = getattr(action, 'clean', None) if clean: - action.result = result self.output.clean(action) await clean(self) From c6d7eedde22e2ce29e4bceda85badb4e918f1c10 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 03:40:00 +0200 Subject: [PATCH 22/90] Improve Buildah clean method --- shlax/targets/buildah.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index b7d52ae..d461bdd 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -126,33 +126,33 @@ class Buildah(Target): return self.image.layer(sha1.hexdigest()) async def action(self, action, reraise=False): - result = await super().action(action, reraise) - action_image = await self.action_image(action) - await self.parent.exec( - 'buildah', - 'commit', - '--format=' + action_image.format, - self.ctr, - action_image, - ) - self.image_previous = action_image - return result + stop = await super().action(action, reraise) + if not stop: + action_image = await self.action_image(action) + await self.parent.exec( + 'buildah', + 'commit', + '--format=' + action_image.format, + self.ctr, + action_image, + ) + self.image_previous = action_image + return stop async def clean(self, target): for src, dst in self.mounts.items(): await self.parent.exec('umount', self.root / str(dst)[1:]) - if self.result.status == 'success': - await self.commit() - if self.root is not None: await self.parent.exec('buildah', 'umount', self.ctr) if self.ctr is not None: + if self.result.status == 'success': + await self.commit() + await self.parent.exec('buildah', 'rm', self.ctr) - if self.result.status == 'success': - if os.getenv('BUILDAH_PUSH'): + if self.result.status == 'success' and os.getenv('BUILDAH_PUSH'): await self.image.push(target) async def mount(self, src, dst): From 2e1bbf098a094f2b94c5f64e004a47c5f4c45333 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 03:44:51 +0200 Subject: [PATCH 23/90] Pass status directly to clean --- shlax/targets/base.py | 3 +-- shlax/targets/buildah.py | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 6bc79a0..04f5306 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -66,13 +66,12 @@ class Target: self.output.success(action) result.status = 'success' finally: - action.result = result self.caller.results.append(result) clean = getattr(action, 'clean', None) if clean: self.output.clean(action) - await clean(self) + await clean(self, result) async def rexec(self, *args, **kwargs): kwargs['user'] = 'root' diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index d461bdd..bd1af09 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -139,7 +139,7 @@ class Buildah(Target): self.image_previous = action_image return stop - async def clean(self, target): + async def clean(self, target, result): for src, dst in self.mounts.items(): await self.parent.exec('umount', self.root / str(dst)[1:]) @@ -147,12 +147,12 @@ class Buildah(Target): await self.parent.exec('buildah', 'umount', self.ctr) if self.ctr is not None: - if self.result.status == 'success': + if result.status == 'success': await self.commit() await self.parent.exec('buildah', 'rm', self.ctr) - if self.result.status == 'success' and os.getenv('BUILDAH_PUSH'): + if result.status == 'success' and os.getenv('BUILDAH_PUSH'): await self.image.push(target) async def mount(self, src, dst): From c6bc849574dbf0db616f834dcd459358fa06ae10 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 03:44:57 +0200 Subject: [PATCH 24/90] Missing await in test function --- tests/test_target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_target.py b/tests/test_target.py index 1fa3770..7827167 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -68,7 +68,7 @@ async def test_parallel(): @pytest.mark.asyncio async def test_function(): async def hello(target): - target.exec('hello') + await target.exec('hello') await Stub()(hello) From 0a983340b8fe651f8f9da3ccdf7c3a9978965cf6 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 03:51:13 +0200 Subject: [PATCH 25/90] Buildah __str__ --- shlax/targets/buildah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index bd1af09..4564f02 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -41,7 +41,7 @@ class Buildah(Target): def __str__(self): if not self.is_runnable(): return 'Replacing with: buildah unshare ' + ' '.join(sys.argv) - return 'Buildah image builder' + return f'Buildah({self.image})' async def __call__(self, *actions, target=None): if target: From 70e354dd02ee12e6fcfd806236b9561ee27bb9fe Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 03:53:53 +0200 Subject: [PATCH 26/90] Fix CI command to build --- .gitlab-ci.yml | 6 +++--- shlax/actions/user.py | 10 ++++++++++ shlax/container.py | 32 ++++++++++++++++++++++++++++++++ shlax/image.py | 18 ------------------ shlax/pod.py | 26 ++++++++++++++++++++++++++ shlax/targets/buildah.py | 32 +++++++++++++++++++++++++++----- shlaxfile.py | 3 +-- tests/test_image.py | 6 ------ 8 files changed, 99 insertions(+), 34 deletions(-) create mode 100644 shlax/container.py create mode 100644 shlax/pod.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6640159..5c62b09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,9 +6,9 @@ build: script: - dnf install -y curl python38 - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - - python3.8 get-pip.py - - pip3.8 install -U --user -e .[cli] - - CACHE_DIR=$(pwd)/.cache ~/.local/bin/shlax ./shlaxfile.py build + - python3 get-pip.py + - pip3 install -U --user -e .[cli] + - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build stage: build test: diff --git a/shlax/actions/user.py b/shlax/actions/user.py index f919852..f52c20c 100644 --- a/shlax/actions/user.py +++ b/shlax/actions/user.py @@ -5,6 +5,16 @@ from .packages import Packages class User: + """ + Create a user. + + Example: + + User('app', '/app', getenv('_CONTAINERS_ROOTLESS_UID', 1000)), + + _CONTAINERS_ROOTLESS_UID allows to get your UID during build, which happens + in buildah unshare. + """ def __init__(self, username, home, uid): self.username = username self.home = home diff --git a/shlax/container.py b/shlax/container.py new file mode 100644 index 0000000..fcc074b --- /dev/null +++ b/shlax/container.py @@ -0,0 +1,32 @@ +import os + +from .image import Image + + +class Container: + def __init__(self, build=None, image=None): + self.build = build + self.image = self.build.image + prefix = os.getcwd().split('/')[-1] + repo = self.image.repository.replace('/', '-') + if prefix == repo: + self.name = repo + else: + self.name = '-'.join([prefix, repo]) + + async def start(self, target): + """Start the container""" + await target.rexec( + 'podman', + 'run', + '--name', + self.name, + str(self.image), + ) + + async def stop(self, target): + """Start the container""" + await target.rexec('podman', 'stop', self.name) + + def __str__(self): + return f'Container(name={self.name}, image={self.image})' diff --git a/shlax/image.py b/shlax/image.py index 4ae7567..f21737b 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -4,18 +4,6 @@ import re class Image: - ENV_TAGS = ( - # gitlab - 'CI_COMMIT_SHORT_SHA', - 'CI_COMMIT_REF_NAME', - 'CI_COMMIT_TAG', - # CircleCI - 'CIRCLE_SHA1', - 'CIRCLE_TAG', - 'CIRCLE_BRANCH', - # contributions welcome here - ) - PATTERN = re.compile( '^((?P[a-z]*)://)?((?P[^/]*[.][^/]*)/)?((?P[^:]+))?(:(?P.*))?$' # noqa , re.I @@ -45,12 +33,6 @@ class Image: if self.registry == 'docker.io': self.format = 'docker' - # figure tags from CI vars - for name in self.ENV_TAGS: - value = os.getenv(name) - if value: - self.tags.append(value) - # filter out tags which resolved to None self.tags = [t for t in self.tags if t] diff --git a/shlax/pod.py b/shlax/pod.py new file mode 100644 index 0000000..82da2af --- /dev/null +++ b/shlax/pod.py @@ -0,0 +1,26 @@ +import cli2 + +from shlax.targets.base import Target +from shlax.actions.parallel import Parallel + + +class Pod: + """Help text""" + def __init__(self, **containers): + self.containers = containers + + async def _call(self, target, method, *names): + methods = [ + getattr(container, method) + for name, container in self.containers.items() + if not names or name in names + ] + await target(Parallel(*methods)) + + async def build(self, target, *names): + """Build container images""" + await self._call(target, 'build', *names) + + async def start(self, target, *names): + """Start container images""" + await self._call(target, 'start', *names) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 4564f02..927fb26 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -105,16 +105,19 @@ class Buildah(Target): if name in layers: self.base = self.image_previous = action_image keep.append(action_image) - self.output.skip(f'Found valid cached layer for {action}') + self.output.skip( + f'Found layer for {action}: {action_image.tags[0]}' + ) else: break return keep async def action_image(self, action): - if self.image_previous: - prefix = self.image_previous.tags[0] - else: - prefix = self.base + prefix = str(self.image_previous) + for tag in self.image_previous.tags: + if tag.startswith('layer-'): + prefix = tag + break if hasattr(action, 'cachekey'): action_key = action.cachekey() if asyncio.iscoroutine(action_key): @@ -129,6 +132,7 @@ class Buildah(Target): stop = await super().action(action, reraise) if not stop: action_image = await self.action_image(action) + self.output.info(f'Commiting {action_image} for {action}') await self.parent.exec( 'buildah', 'commit', @@ -187,6 +191,24 @@ class Buildah(Target): self.ctr, )).out + ENV_TAGS = ( + # gitlab + 'CI_COMMIT_SHORT_SHA', + 'CI_COMMIT_REF_NAME', + 'CI_COMMIT_TAG', + # CircleCI + 'CIRCLE_SHA1', + 'CIRCLE_TAG', + 'CIRCLE_BRANCH', + # contributions welcome here + ) + + # figure tags from CI vars + for name in ENV_TAGS: + value = os.getenv(name) + if value: + self.image.tags.append(value) + if image.tags: tags = [f'{image.repository}:{tag}' for tag in image.tags] else: diff --git a/shlaxfile.py b/shlaxfile.py index f3dfa7e..7a87b23 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -7,8 +7,7 @@ from shlax.shortcuts import * shlax = Container( build=Buildah( - Packages('python38', 'buildah', 'unzip', 'findutils'), - User('app', '/app', getenv('_CONTAINERS_ROOTLESS_UID')), + Packages('python38', 'buildah', 'unzip', 'findutils', upgrade=False), Copy('setup.py', 'shlax', '/app'), Pip('/app'), base='quay.io/podman/stable', diff --git a/tests/test_image.py b/tests/test_image.py index 257f591..df25f72 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -25,9 +25,3 @@ def test_args(arg, expected): im = Image(arg) for k, v in expected.items(): assert getattr(im, k) == v - -def test_args_env(): - os.environ['IMAGE_TEST_ARGS_ENV'] = 'foo' - Image.ENV_TAGS = ['IMAGE_TEST_ARGS_ENV'] - im = Image('re/po:x,y') - assert im.tags == ['x', 'y', 'foo'] From 0ac17644646f9376e6cda4c5eb78aa194ecf90ba Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 12:23:48 +0200 Subject: [PATCH 27/90] Try more build --- .gitlab-ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c62b09..d9fb414 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,20 @@ build: - cache: - key: cache - paths: [.cache] - image: quay.io/buildah/stable - script: - dnf install -y curl python38 - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - python3 get-pip.py + script: - pip3 install -U --user -e .[cli] - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build stage: build +build-itself: + cache: + key: cache + paths: [.cache] + image: shlax:$CI_COMMIT_SHORT_SHA + script: python3 ./shlaxfile.py build + stage: test + test: image: yourlabs/python stage: build From 8dd4fa52ac89edc460a3beb9b9e18df7bd29f4b1 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 12:28:49 +0200 Subject: [PATCH 28/90] Typo in ci --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d9fb414..bb4f630 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,8 @@ build: - - dnf install -y curl python38 - - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - - python3 get-pip.py + cache: + key: cache + paths: [.cache] + image: yourlabs/buildah script: - pip3 install -U --user -e .[cli] - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build From 0929b37b3639a5d9ae28e79583c093bc4f71a142 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 12:48:06 +0200 Subject: [PATCH 29/90] Tag/build/push refactor --- shlax/targets/buildah.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 927fb26..1451b95 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -63,22 +63,17 @@ class Buildah(Target): if actions: actions = actions[len(keep):] if not actions: - return self.uptodate() + return self.output.success('Image up to date') else: self.actions = self.actions[len(keep):] if not self.actions: - return self.uptodate() + return self.output.success('Image up to date') self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) return await super().__call__(*actions) - def uptodate(self): - self.clean = None - self.output.success('Image up to date') - return - async def layers(self): ret = set() results = await self.parent.exec( @@ -144,19 +139,15 @@ class Buildah(Target): return stop async def clean(self, target, result): - for src, dst in self.mounts.items(): - await self.parent.exec('umount', self.root / str(dst)[1:]) - - if self.root is not None: - await self.parent.exec('buildah', 'umount', self.ctr) - if self.ctr is not None: - if result.status == 'success': - await self.commit() - + for src, dst in self.mounts.items(): + await self.parent.exec('umount', self.root / str(dst)[1:]) + await self.parent.exec('buildah', 'umount', self.ctr) await self.parent.exec('buildah', 'rm', self.ctr) - if result.status == 'success' and os.getenv('BUILDAH_PUSH'): + if result.status == 'success': + await self.commit() + if os.getenv('BUILDAH_PUSH'): await self.image.push(target) async def mount(self, src, dst): @@ -184,13 +175,6 @@ class Buildah(Target): for key, value in self.config.items(): await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') - self.sha = (await self.parent.exec( - 'buildah', - 'commit', - '--format=' + image.format, - self.ctr, - )).out - ENV_TAGS = ( # gitlab 'CI_COMMIT_SHORT_SHA', @@ -214,8 +198,7 @@ class Buildah(Target): else: tags = [image.repository] - for tag in tags: - await self.parent.exec('buildah', 'tag', self.sha, tag) + await self.parent.exec('buildah', 'tag', self.image_previous, *tags) async def mkdir(self, path): return await self.parent.mkdir(self.path(path)) From 62299da7c1f4bfea9c76ab95856a9afc472bbb21 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 13:21:09 +0200 Subject: [PATCH 30/90] A bit of work on the command line --- shlax/cli.py | 31 ++++++++++++++++++++++--------- shlax/targets/base.py | 3 +++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/shlax/cli.py b/shlax/cli.py index feae747..a40b741 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -20,22 +20,35 @@ class Group(cli2.Group): self.cmdclass = Command +class TargetArgument(cli2.Argument): + """DSN of the target to execute on, localhost by default, TBI""" + + def __init__(self, cmd, param, doc=None, color=None, default=None): + from shlax.targets.base import Target + super().__init__(cmd, param, doc=self.__doc__, default=Target()) + self.alias = ['target', 't'] + + class Command(cli2.Command): - def call(self, *args, **kwargs): - return self.shlax_target(self.target) + def setargs(self): + super().setargs() + self['target'] = TargetArgument( + self, + self.sig.parameters['target'], + ) + if 'actions' in self: + del self['actions'] def __call__(self, *argv): - from shlax.targets.base import Target - self.shlax_target = Target() - result = super().__call__(*argv) - self.shlax_target.output.results(self.shlax_target) - return result + super().__call__(*argv) + self['target'].value.output.results(self['target'].value) -class ActionCommand(Command): +class ActionCommand(cli2.Command): def call(self, *args, **kwargs): self.target = self.target(*args, **kwargs) - return super().call(*args, **kwargs) + from shlax.targets.base import Target + return super().call(Target()) class ConsoleScript(Group): diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 04f5306..0b555b3 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -18,6 +18,9 @@ class Target: self.parent = None self.root = root or os.getcwd() + def __str__(self): + return 'localhost' + @property def parent(self): return self._parent or Target() From 754227d2d12ba11ee6a53d33c460f5fd2279bf7f Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 13:24:31 +0200 Subject: [PATCH 31/90] Silence pip output --- shlax/actions/pip.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py index 7d8e8c7..949442b 100644 --- a/shlax/actions/pip.py +++ b/shlax/actions/pip.py @@ -19,7 +19,10 @@ class Pip(Action): raise Exception('Could not find pip nor python') # ensure pip module presence - result = await target.exec(python, '-m', 'pip', raises=False) + result = await target.exec( + python, '-m', 'pip', + raises=False, quiet=True + ) if result.rc != 0: if not os.path.exists('get-pip.py'): req = request.urlopen( From b6fa18d82a6181bb1d3351ed7ccb2684fc7615e1 Mon Sep 17 00:00:00 2001 From: jpic Date: Mon, 1 Jun 2020 15:41:22 +0200 Subject: [PATCH 32/90] Added ssh target --- shlax/actions/packages.py | 3 +++ shlax/cli.py | 16 ++++++++++++++-- shlax/targets/base.py | 1 + shlax/targets/ssh.py | 16 ++++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 shlax/targets/ssh.py diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 814da6a..43615bd 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -67,6 +67,9 @@ class Packages: return os.path.join(os.getenv('HOME'), '.cache') async def update(self, target): + if not target.islocal: + return await target.rexec(self.cmds['update']) + # run pkgmgr_setup functions ie. apk_setup cachedir = await getattr(self, self.mgr + '_setup')(target) diff --git a/shlax/cli.py b/shlax/cli.py index a40b741..4fac0fb 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -28,6 +28,12 @@ class TargetArgument(cli2.Argument): super().__init__(cmd, param, doc=self.__doc__, default=Target()) self.alias = ['target', 't'] + def cast(self, value): + from shlax.targets.ssh import Ssh + if '@' in value: + user, host = value.split('@') + return Ssh(host=host, user=user) + class Command(cli2.Command): def setargs(self): @@ -45,10 +51,16 @@ class Command(cli2.Command): class ActionCommand(cli2.Command): + def setargs(self): + super().setargs() + self['target'] = TargetArgument( + self, + inspect.Parameter('target', inspect.Parameter.KEYWORD_ONLY), + ) + def call(self, *args, **kwargs): self.target = self.target(*args, **kwargs) - from shlax.targets.base import Target - return super().call(Target()) + return super().call(self['target'].value) class ConsoleScript(Group): diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 0b555b3..1e8a843 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -17,6 +17,7 @@ class Target: self.output = Output() self.parent = None self.root = root or os.getcwd() + self.islocal = getattr(self, 'islocal', True) def __str__(self): return 'localhost' diff --git a/shlax/targets/ssh.py b/shlax/targets/ssh.py new file mode 100644 index 0000000..d3dd707 --- /dev/null +++ b/shlax/targets/ssh.py @@ -0,0 +1,16 @@ +from .base import Target + + +class Ssh(Target): + def __init__(self, *actions, host, user=None): + self.host = host + self.user = user + self.islocal = False + super().__init__(*actions) + + async def exec(self, *args, user=None, **kwargs): + _args = ['ssh', self.host] + if user == 'root': + _args += ['sudo'] + _args += [' '.join([str(a) for a in args])] + return await self.parent.exec(*_args, **kwargs) From eafa371617cf47ac7c55ab4ed79993e16d1f4f8e Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 03:19:01 +0200 Subject: [PATCH 33/90] Package action working both on SSH host and Buildah guest now --- shlax/actions/packages.py | 74 ++++++++++++++++++++++----------------- shlax/targets/base.py | 47 ++++++++++++++++++++----- shlax/targets/buildah.py | 5 +-- shlax/targets/ssh.py | 1 - 4 files changed, 82 insertions(+), 45 deletions(-) diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 43615bd..7d0e9f2 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -37,11 +37,13 @@ class Packages: update='pacman -Sy', upgrade='pacman -Su --noconfirm', install='pacman -S --noconfirm', + lastupdate='stat -c %Y /var/lib/pacman/sync/core.db', ), dnf=dict( update='dnf makecache --assumeyes', upgrade='dnf upgrade --best --assumeyes --skip-broken', # noqa install='dnf install --setopt=install_weak_deps=False --best --assumeyes', # noqa + lastupdate='stat -c %Y /var/cache/dnf/* | head -n1', ), yum=dict( update='yum update', @@ -52,57 +54,52 @@ class Packages: installed = [] - def __init__(self, *packages, upgrade=True): + def __init__(self, *packages, upgrade=False): self.packages = [] self.upgrade = upgrade for package in packages: line = dedent(package).strip().replace('\n', ' ') self.packages += line.split(' ') - @property - def cache_root(self): + async def cache_setup(self, target): if 'CACHE_DIR' in os.environ: - return os.path.join(os.getenv('CACHE_DIR')) + self.cache_root = os.path.join(os.getenv('CACHE_DIR')) else: - return os.path.join(os.getenv('HOME'), '.cache') - - async def update(self, target): - if not target.islocal: - return await target.rexec(self.cmds['update']) + self.cache_root = os.path.join(await target.parent.getenv('HOME'), '.cache') # run pkgmgr_setup functions ie. apk_setup - cachedir = await getattr(self, self.mgr + '_setup')(target) + await getattr(self, self.mgr + '_setup')(target) + async def update(self, target): + # lastupdate = await target.exec(self.cmds['lastupdate'], raises=False) + # lastupdate = int(lastupdate.out) if lastupdate.rc == 0 else None lastupdate = None - if os.path.exists(cachedir + '/lastupdate'): - with open(cachedir + '/lastupdate', 'r') as f: - try: - lastupdate = int(f.read().strip()) - except: - pass - - if not os.path.exists(cachedir): - os.makedirs(cachedir) - now = int(datetime.now().strftime('%s')) - # cache for a week + if not lastupdate or now - lastupdate > 604800: + await target.rexec(self.cmds['update']) + + return + + # disabling with the above return call until needed again + # might have to rewrite this to not have our own lockfile + # or find a better place on the filesystem + # also make sure the lockfile is actually needed when running on + # targets that don't have isguest=True if not lastupdate or now - lastupdate > 604800: # crude lockfile implementation, should work against *most* # race-conditions ... lockfile = cachedir + '/update.lock' - if not os.path.exists(lockfile): - with open(lockfile, 'w+') as f: - f.write(str(os.getpid())) + if not await target.parent.exists(lockfile): + await target.parent.write(lockfile, str(os.getpid())) try: await target.rexec(self.cmds['update']) finally: - os.unlink(lockfile) + await target.parent.rm(lockfile) - with open(cachedir + '/lastupdate', 'w+') as f: - f.write(str(now)) + await target.parent.write(cachedir + '/lastupdate', str(now)) else: - while os.path.exists(lockfile): + while await target.parent.exists(lockfile): print(f'{self.target} | Waiting for {lockfile} ...') await asyncio.sleep(1) @@ -119,7 +116,13 @@ class Packages: raise Exception('Packages does not yet support this distro') self.cmds = self.mgrs[self.mgr] + + if target.isguest: + # we're going to mount + await self.cache_setup(target) + await self.update(target) + if self.upgrade: await target.rexec(self.cmds['upgrade']) @@ -150,19 +153,24 @@ class Packages: return cachedir async def apt_setup(self, target): - codename = (await self.rexec( - f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME' + codename = (await target.rexec( + f'source /etc/os-release; echo $VERSION_CODENAME' )).out cachedir = os.path.join(self.cache_root, self.mgr, codename) await self.rexec('rm /etc/apt/apt.conf.d/docker-clean') cache_archives = os.path.join(cachedir, 'archives') - await self.mount(cache_archives, f'/var/cache/apt/archives') + await target.mount(cache_archives, f'/var/cache/apt/archives') cache_lists = os.path.join(cachedir, 'lists') - await self.mount(cache_lists, f'/var/lib/apt/lists') + await target.mount(cache_lists, f'/var/lib/apt/lists') return cachedir async def pacman_setup(self, target): - return self.cache_root + '/pacman' + cachedir = os.path.join(self.cache_root, self.mgr) + await target.mkdir(cachedir + '/cache', cachedir + '/sync') + await target.mount(cachedir + '/sync', '/var/lib/pacman/sync') + await target.mount(cachedir + '/cache', '/var/cache/pacman') + if await target.host.exists('/etc/pacman.d/mirrorlist'): + await target.copy('/etc/pacman.d/mirrorlist', '/etc/pacman.d/mirrorlist') def __str__(self): return f'Packages({self.packages}, upgrade={self.upgrade})' diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 1e8a843..37a2856 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -11,13 +11,14 @@ from ..result import Result, Results class Target: + isguest = False + def __init__(self, *actions, root=None): self.actions = actions self.results = [] self.output = Output() self.parent = None - self.root = root or os.getcwd() - self.islocal = getattr(self, 'islocal', True) + self.root = root or '' def __str__(self): return 'localhost' @@ -131,20 +132,48 @@ class Target: @root.setter def root(self, value): - self._root = Path(value or os.getcwd()) + self._root = Path(value) if value else '' + + @property + def host(self): + current = self + while current.isguest: + current = self.parent + return current def path(self, path): + if not self.root: + return path if str(path).startswith('/'): path = str(path)[1:] - return self.root / path + return str(self.root / path) - async def mkdir(self, path): + async def mkdir(self, *paths): if '_mkdir' not in self.__dict__: self._mkdir = [] - path = str(path) - if path not in self._mkdir: - await self.exec('mkdir', '-p', path) - self._mkdir.append(path) + + make = [str(path) for path in paths if str(path) not in self._mkdir] + if make: + await self.exec('mkdir', '-p', *make) + self._mkdir += make async def copy(self, *args): return await self.exec('cp', '-a', *args) + + async def exists(self, path): + return (await self.exec('ls ' + self.path(path), raises=False)).rc == 0 + + async def read(self, path): + return (await self.exec('cat', self.path(path))).out + + async def write(self, path, content): + return await self.exec('echo ' + content + ' > ' + self.path(path)) + + async def rm(self, path): + return await self.exec('rm', self.path(path)) + + async def getenv(self, key): + return (await self.exec('echo $' + key)).out + + async def getcwd(self): + return (await self.exec('pwd')).out diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 1451b95..15a09e7 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -14,6 +14,7 @@ from ..proc import Proc class Buildah(Target): """Build container image with buildah""" + isguest = True def __init__(self, *actions, @@ -200,8 +201,8 @@ class Buildah(Target): await self.parent.exec('buildah', 'tag', self.image_previous, *tags) - async def mkdir(self, path): - return await self.parent.mkdir(self.path(path)) + async def mkdir(self, *paths): + return await self.parent.mkdir(*[self.path(path) for path in paths]) async def copy(self, *args): return await self.parent.copy(*args[:-1], self.path(args[-1])) diff --git a/shlax/targets/ssh.py b/shlax/targets/ssh.py index d3dd707..c0b9e57 100644 --- a/shlax/targets/ssh.py +++ b/shlax/targets/ssh.py @@ -5,7 +5,6 @@ class Ssh(Target): def __init__(self, *actions, host, user=None): self.host = host self.user = user - self.islocal = False super().__init__(*actions) async def exec(self, *args, user=None, **kwargs): From 224608878ecdca60ead9caf81f145f62514892ef Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 03:19:13 +0200 Subject: [PATCH 34/90] Clean on KeyboardInterrupt, proper call in shlax Command --- shlax/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shlax/cli.py b/shlax/cli.py index 4fac0fb..770e920 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -45,9 +45,13 @@ class Command(cli2.Command): if 'actions' in self: del self['actions'] + def call(self, *args, **kwargs): + self.shlax_target = self['target'].value + return self.shlax_target(self.target) + def __call__(self, *argv): super().__call__(*argv) - self['target'].value.output.results(self['target'].value) + self.shlax_target.output.results(self.shlax_target) class ActionCommand(cli2.Command): From 4f5326c81bc9e1a5fb18f5c5db724fac736df094 Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 17:37:09 +0200 Subject: [PATCH 35/90] Commiting build-itself job until i re-implement docker commiting --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb4f630..187f2d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,13 +8,13 @@ build: - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build stage: build -build-itself: - cache: - key: cache - paths: [.cache] - image: shlax:$CI_COMMIT_SHORT_SHA - script: python3 ./shlaxfile.py build - stage: test +#build-itself: +# cache: +# key: cache +# paths: [.cache] +# image: shlax:$CI_COMMIT_SHORT_SHA +# script: python3 ./shlaxfile.py build +# stage: test test: image: yourlabs/python From 63e610ca4f6617714525d291b77173ab5a307dc1 Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 17:37:30 +0200 Subject: [PATCH 36/90] Fixing container config support --- shlax/targets/buildah.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 15a09e7..0073d3c 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -144,13 +144,15 @@ class Buildah(Target): for src, dst in self.mounts.items(): await self.parent.exec('umount', self.root / str(dst)[1:]) await self.parent.exec('buildah', 'umount', self.ctr) - await self.parent.exec('buildah', 'rm', self.ctr) if result.status == 'success': await self.commit() if os.getenv('BUILDAH_PUSH'): await self.image.push(target) + if self.ctr is not None: + await self.parent.exec('buildah', 'rm', self.ctr) + async def mount(self, src, dst): """Mount a host directory into the container.""" target = self.root / str(dst)[1:] @@ -166,15 +168,13 @@ class Buildah(Target): _args += [' '.join([str(a) for a in args])] return await self.parent.exec(*_args, **kwargs) - async def commit(self, image=None): - image = image or self.image - if not image: - return + async def commit(self): + for key, value in self.config.items(): + await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') - if not image: - # don't go through that if layer commit - for key, value in self.config.items(): - await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') + await self.parent.exec( + f'buildah commit {self.ctr} {self.image.repository}:final' + ) ENV_TAGS = ( # gitlab @@ -194,12 +194,12 @@ class Buildah(Target): if value: self.image.tags.append(value) - if image.tags: - tags = [f'{image.repository}:{tag}' for tag in image.tags] + if self.image.tags: + tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags] else: - tags = [image.repository] + tags = [self.image.repository] - await self.parent.exec('buildah', 'tag', self.image_previous, *tags) + await self.parent.exec('buildah', 'tag', self.image.repository + ':final', *tags) async def mkdir(self, *paths): return await self.parent.mkdir(*[self.path(path) for path in paths]) From 6a0b60d68a0798a5e9254724d5dadf3d1b284f62 Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 17:44:02 +0200 Subject: [PATCH 37/90] Do not try to commit a container that has not been created because layers are uptodate --- shlax/targets/buildah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 0073d3c..eaf1418 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -145,7 +145,7 @@ class Buildah(Target): await self.parent.exec('umount', self.root / str(dst)[1:]) await self.parent.exec('buildah', 'umount', self.ctr) - if result.status == 'success': + if result.status == 'success' and self.ctr: await self.commit() if os.getenv('BUILDAH_PUSH'): await self.image.push(target) From ae61e1b8cf2ba71892d475492e2b35ea5629aa16 Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 18:08:59 +0200 Subject: [PATCH 38/90] Going to maintain a little news at the top of the README --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 054603e..a0849a7 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,26 @@ purpose of code-reuse made possible by target abstraction. The pattern resolves around two moving parts: Actions and Targets. +## Development status: Design state + +I got the thing to work with an ugly PoC that I basically brute-forced, I'm +currently rewriting the codebase with a proper design. + +The stories are in development in this order: + +- replacing docker build, that's in the state of polishing +- replacing docker-compose, not in use but the PoC works so far +- replacing ansible, also working in working PoC state + +This project is supposed to unblock me from adding the CI feature to the +Sentry/GitLab/Portainer implementation I'm doing in pure python on top of +Django, CRUDLFA+ and Ryzom (isomorphic components in Python to replace +templates). + +Shlax builds its container itself, so check the shlaxfile.py of this repository +to see what it currently looks like, and check the build job of the CI pipeline +to see the output. + ## Action An action is a function that takes a target argument, it may execute nested From 84b10ce143fb7c495e2555b3bff928a7abe3a478 Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 18:11:06 +0200 Subject: [PATCH 39/90] Lolnote --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a0849a7..0627713 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,8 @@ The stories are in development in this order: This project is supposed to unblock me from adding the CI feature to the Sentry/GitLab/Portainer implementation I'm doing in pure python on top of Django, CRUDLFA+ and Ryzom (isomorphic components in Python to replace -templates). +templates). So, as you can see, I'm really deep in it with a strong +determination. Shlax builds its container itself, so check the shlaxfile.py of this repository to see what it currently looks like, and check the build job of the CI pipeline From f07787d86558a304974d455894d0c818bd63a8bf Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 18:12:02 +0200 Subject: [PATCH 40/90] Fix title order --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0627713..2532feb 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ Shlax is a Python framework for system automation, initially with the purpose of replacing docker, docker-compose and ansible with a single tool with the purpose of code-reuse made possible by target abstraction. -The pattern resolves around two moving parts: Actions and Targets. - ## Development status: Design state I got the thing to work with an ugly PoC that I basically brute-forced, I'm @@ -27,6 +25,10 @@ Shlax builds its container itself, so check the shlaxfile.py of this repository to see what it currently looks like, and check the build job of the CI pipeline to see the output. +# Design + +The pattern resolves around two moving parts: Actions and Targets. + ## Action An action is a function that takes a target argument, it may execute nested From afc1823e5493edefa88f3ce0ccd8b0c990ebf138 Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 18:13:01 +0200 Subject: [PATCH 41/90] Mention command line --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2532feb..3b37689 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ The stories are in development in this order: - replacing docker build, that's in the state of polishing - replacing docker-compose, not in use but the PoC works so far -- replacing ansible, also working in working PoC state +- replacing ansible, also working in working PoC state, the shlax command line + demonstrates This project is supposed to unblock me from adding the CI feature to the Sentry/GitLab/Portainer implementation I'm doing in pure python on top of From 93301252087c9b29f4c21f8244421c25e77c933d Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 18:39:43 +0200 Subject: [PATCH 42/90] Update TargetArgument docs --- shlax/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shlax/cli.py b/shlax/cli.py index 770e920..7757975 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -21,7 +21,9 @@ class Group(cli2.Group): class TargetArgument(cli2.Argument): - """DSN of the target to execute on, localhost by default, TBI""" + """ + Target to execute on: localhost by default, target=@ssh_host for ssh. + """ def __init__(self, cmd, param, doc=None, color=None, default=None): from shlax.targets.base import Target From 47740f01f9b442f4685c4b006470e48cfb36d544 Mon Sep 17 00:00:00 2001 From: jpic Date: Mon, 6 Jul 2020 08:58:36 +0200 Subject: [PATCH 43/90] Test build-itself --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 187f2d6..bb4f630 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,13 +8,13 @@ build: - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build stage: build -#build-itself: -# cache: -# key: cache -# paths: [.cache] -# image: shlax:$CI_COMMIT_SHORT_SHA -# script: python3 ./shlaxfile.py build -# stage: test +build-itself: + cache: + key: cache + paths: [.cache] + image: shlax:$CI_COMMIT_SHORT_SHA + script: python3 ./shlaxfile.py build + stage: test test: image: yourlabs/python From 07a6c5b6281611a756aea9e6443a75203741dbe0 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 2 Aug 2020 06:50:33 +0200 Subject: [PATCH 44/90] Run(root) --- shlax/actions/run.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/shlax/actions/run.py b/shlax/actions/run.py index 4bb5036..3fc561c 100644 --- a/shlax/actions/run.py +++ b/shlax/actions/run.py @@ -1,11 +1,15 @@ class Run: - def __init__(self, cmd): + def __init__(self, cmd, root=False): self.cmd = cmd + self.root = root async def __call__(self, target): - self.proc = await target.exec(self.cmd) + if self.root: + self.proc = await target.rexec(self.cmd) + else: + self.proc = await target.exec(self.cmd) def __str__(self): return f'Run({self.cmd})' From 9235ffcd278bc81e41772d2a80e811fefd8966a6 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 2 Aug 2020 06:50:42 +0200 Subject: [PATCH 45/90] Fix user action --- shlax/actions/user.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shlax/actions/user.py b/shlax/actions/user.py index f52c20c..663c841 100644 --- a/shlax/actions/user.py +++ b/shlax/actions/user.py @@ -24,7 +24,7 @@ class User: return f'User({self.username}, {self.home}, {self.uid})' async def __call__(self, target): - result = await target.rexec('id', self.uid) + result = await target.rexec('id', self.uid, raises=False) if result.rc == 0: old = re.match('.*\(([^)]*)\).*', result.out).group(1) await target.rexec( @@ -40,3 +40,5 @@ class User: '-u', self.uid, self.username ) + await target.mkdir(self.home) + await target.rexec('chown', self.uid, self.home) From 1ce27e1cd10edad23840eb16391525a1f7f4f630 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 2 Aug 2020 06:51:07 +0200 Subject: [PATCH 46/90] Container: support not having a build job and new methods --- shlax/container.py | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/shlax/container.py b/shlax/container.py index fcc074b..6091704 100644 --- a/shlax/container.py +++ b/shlax/container.py @@ -1,12 +1,17 @@ +import copy import os from .image import Image class Container: - def __init__(self, build=None, image=None): + def __init__(self, build=None, image=None, env=None, volumes=None): self.build = build - self.image = self.build.image + self.image = image or self.build.image + if isinstance(self.image, str): + self.image = Image(self.image) + self.volumes = volumes or {} + self.env = env or {} prefix = os.getcwd().split('/')[-1] repo = self.image.repository.replace('/', '-') if prefix == repo: @@ -14,19 +19,44 @@ class Container: else: self.name = '-'.join([prefix, repo]) - async def start(self, target): - """Start the container""" - await target.rexec( + async def up(self, target, *args): + """Start the container foreground""" + cmd = [ 'podman', 'run', + ] + list(args) + + for src, dest in self.volumes.items(): + cmd += ['--volume', ':'.join([src, dest])] + + for src, dest in self.env.items(): + cmd += ['--env', '='.join([src, str(dest)])] + + cmd += [ '--name', self.name, str(self.image), - ) + ] + await target.exec(*cmd) + + async def start(self, target): + """Start the container background""" + await self.up(target, '-d') async def stop(self, target): """Start the container""" - await target.rexec('podman', 'stop', self.name) + await target.exec('podman', 'stop', self.name) + + async def down(self, target): + """Start the container""" + await target.exec('podman', 'rm', '-f', self.name, raises=False) + + async def apply(self, target): + """Start the container""" + if self.build: + await target(self.build) + await target(self.down) + await target(self.start) def __str__(self): - return f'Container(name={self.name}, image={self.image})' + return f'Container(name={self.name}, image={self.image}, volumes={self.volumes})' From 073c3713c16c36bf8d549b3f7dea867a6aa88fee Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 2 Aug 2020 06:51:25 +0200 Subject: [PATCH 47/90] Let an action mark itself as skipped --- shlax/targets/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 37a2856..7f0aa6c 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -68,7 +68,10 @@ class Target: traceback.print_exception(type(e), e, sys.exc_info()[2]) return True else: - self.output.success(action) + if getattr(action, 'skipped', False): + self.output.skip(action) + else: + self.output.success(action) result.status = 'success' finally: self.caller.results.append(result) From 7f150ae17a4ab72fdf8290144f247c37f89e2ee4 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 2 Aug 2020 06:52:03 +0200 Subject: [PATCH 48/90] New write with bash heredoc --- shlax/targets/base.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 7f0aa6c..28dbb45 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -169,8 +169,13 @@ class Target: async def read(self, path): return (await self.exec('cat', self.path(path))).out - async def write(self, path, content): - return await self.exec('echo ' + content + ' > ' + self.path(path)) + async def write(self, path, content, **kwargs): + return await self.exec( + f'cat > {self.path(path)} < Date: Sun, 2 Aug 2020 06:52:39 +0200 Subject: [PATCH 49/90] Add Buildah.Env and Buildah.Config --- shlax/targets/buildah.py | 53 +++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index eaf1418..2652386 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -16,10 +16,7 @@ class Buildah(Target): """Build container image with buildah""" isguest = True - def __init__(self, - *actions, - base=None, commit=None, - cmd=None): + def __init__(self, *actions, base=None, commit=None): self.base = base or 'alpine' self.image = Image(commit) if commit else None @@ -27,10 +24,6 @@ class Buildah(Target): self.root = None self.mounts = dict() - self.config = dict( - cmd=cmd or 'sh', - ) - # Always consider localhost as parent for now self.parent = Target() @@ -169,9 +162,6 @@ class Buildah(Target): return await self.parent.exec(*_args, **kwargs) async def commit(self): - for key, value in self.config.items(): - await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') - await self.parent.exec( f'buildah commit {self.ctr} {self.image.repository}:final' ) @@ -206,3 +196,44 @@ class Buildah(Target): async def copy(self, *args): return await self.parent.copy(*args[:-1], self.path(args[-1])) + + async def write(self, path, content): + return await self.write(path, content) + + async def write(self, path, content, **kwargs): + return await self.exec( + f'cat > {path} < Date: Sun, 2 Aug 2020 20:09:40 +0200 Subject: [PATCH 50/90] Name parallel executor --- shlax/actions/parallel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shlax/actions/parallel.py b/shlax/actions/parallel.py index ed14a53..4359dba 100644 --- a/shlax/actions/parallel.py +++ b/shlax/actions/parallel.py @@ -9,3 +9,6 @@ class Parallel: return await asyncio.gather(*[ target(action) for action in self.actions ]) + + def __str__(self): + return 'Parallel executor' From 6e7f957d49ed81c9ce6203811655780746f00b88 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 2 Aug 2020 20:10:19 +0200 Subject: [PATCH 51/90] Remove artificial centering from output labels --- shlax/output.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shlax/output.py b/shlax/output.py index fa07783..e877107 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -118,7 +118,7 @@ class Output: def test(self, action): self(''.join([ self.colors['purplebold'], - '! TEST ', + '! TEST ', self.colors['reset'], self.colorized(action), '\n', @@ -128,7 +128,7 @@ class Output: if self.debug: self(''.join([ self.colors['bluebold'], - '+ CLEAN ', + '+ CLEAN ', self.colors['reset'], self.colorized(action), '\n', @@ -138,7 +138,7 @@ class Output: if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['orangebold'], - '⚠ START ', + '⚠ START ', self.colors['reset'], self.colorized(action), '\n', @@ -148,7 +148,7 @@ class Output: if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['cyanbold'], - '➤ INFO ', + '➤ INFO ', self.colors['reset'], text, '\n', From 15c71a3bf79f666ac09f292e37c64acfe2c251c3 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 2 Aug 2020 20:10:52 +0200 Subject: [PATCH 52/90] Getting on par with docker-compose --- setup.py | 2 +- shlax/container.py | 43 +++++++++++++++++++++++++++++++++++--- shlax/pod.py | 52 +++++++++++++++++++++++++++++++++++++++++++++- shlax/podman.py | 18 ++++++++++++++++ 4 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 shlax/podman.py diff --git a/setup.py b/setup.py index 3ef0693..b89a78a 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( setup_requires='setupmeta', extras_require=dict( cli=[ - 'cli2>=2.2.2', + 'cli2>=2.3.0', ], test=[ 'pytest', diff --git a/shlax/container.py b/shlax/container.py index 6091704..79ed172 100644 --- a/shlax/container.py +++ b/shlax/container.py @@ -1,6 +1,7 @@ import copy import os +from .podman import Podman from .image import Image @@ -12,6 +13,7 @@ class Container: self.image = Image(self.image) self.volumes = volumes or {} self.env = env or {} + prefix = os.getcwd().split('/')[-1] repo = self.image.repository.replace('/', '-') if prefix == repo: @@ -19,8 +21,39 @@ class Container: else: self.name = '-'.join([prefix, repo]) + self.pod = None + + @property + def full_name(self): + if self.pod: + return '-'.join([self.pod.name, self.name]) + return self.name + async def up(self, target, *args): """Start the container foreground""" + podman = Podman(target) + if self.pod: + pod = None + for _ in await podman.pod.ps(): + if _['Name'] == self.pod.name: + pod = _ + break + if not pod: + await podman.pod.create('--name', self.pod.name) + args = list(args) + ['--pod', self.pod.name] + + # skip if already up + for result in await podman.ps('-a'): + for name in result['Names']: + if name == self.full_name: + if result['State'] == 'running': + target.output.info(f'{self.full_name} already running') + return + elif result['State'] == 'exited': + target.output.info(f'{self.full_name} starting') + await target.exec('podman', 'start', self.full_name) + return + cmd = [ 'podman', 'run', @@ -34,7 +67,7 @@ class Container: cmd += [ '--name', - self.name, + self.full_name, str(self.image), ] await target.exec(*cmd) @@ -45,11 +78,15 @@ class Container: async def stop(self, target): """Start the container""" - await target.exec('podman', 'stop', self.name) + await target.exec('podman', 'stop', self.full_name) + + async def logs(self, target): + """Start the container""" + await target.exec('podman', 'logs', self.full_name) async def down(self, target): """Start the container""" - await target.exec('podman', 'rm', '-f', self.name, raises=False) + await target.exec('podman', 'rm', '-f', self.full_name, raises=False) async def apply(self, target): """Start the container""" diff --git a/shlax/pod.py b/shlax/pod.py index 82da2af..e19c519 100644 --- a/shlax/pod.py +++ b/shlax/pod.py @@ -1,13 +1,21 @@ import cli2 +import json +import os from shlax.targets.base import Target from shlax.actions.parallel import Parallel +from .podman import Podman + class Pod: """Help text""" def __init__(self, **containers): self.containers = containers + for name, container in self.containers.items(): + container.pod = self + container.name = name + self.name = os.getcwd().split('/')[-1] async def _call(self, target, method, *names): methods = [ @@ -19,8 +27,50 @@ class Pod: async def build(self, target, *names): """Build container images""" - await self._call(target, 'build', *names) + if not Proc.test or os.getuid() == 0: + os.execvp('buildah', ['buildah', 'unshare'] + sys.argv) + else: + await self._call(target, 'build', *names) + + async def down(self, target, *names): + """Delete container images""" + await self._call(target, 'down', *names) async def start(self, target, *names): """Start container images""" await self._call(target, 'start', *names) + + async def logs(self, target, *names): + """Start container images""" + await self._call(target, 'logs', *names) + + async def ps(self, target): + """Show containers and volumes""" + containers = [] + names = [] + for container in await Podman(target).ps('-a'): + for name in container['Names']: + if name.startswith(self.name + '-'): + container['Name'] = name + containers.append(container) + names.append(name) + + for name, container in self.containers.items(): + full_name = '-'.join([self.name, container.name]) + if full_name in names: + continue + containers.append(dict( + Name=full_name, + State='not created', + )) + + cli2.Table( + ['Name', 'State'], + *[ + (container['Name'], container['State']) + for container in containers + ] + ).print() + + def __str__(self): + return f'Pod({self.name})' diff --git a/shlax/podman.py b/shlax/podman.py new file mode 100644 index 0000000..05d46e7 --- /dev/null +++ b/shlax/podman.py @@ -0,0 +1,18 @@ +import json + + +class Podman(list): + def __init__(self, target, *args): + self.target = target + super().__init__(args or ['podman']) + + def __getattr__(self, command): + if command.startswith('_'): + return super().__getattr__(command) + return Podman(self.target, *self + [command]) + + async def __call__(self, *args, **kwargs): + cmd = self + list(args) + [f'--{k}={v}' for k, v in kwargs.items()] + if 'ps' in cmd: + cmd += ['--format=json'] + return (await self.target.exec(*cmd, quiet=True)).json From d2c0694005145ba1937621ec6a1b17e0262d8407 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 2 Aug 2020 20:11:24 +0200 Subject: [PATCH 53/90] Skip report display for only one result --- shlax/output.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shlax/output.py b/shlax/output.py index e877107..910061c 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -185,6 +185,8 @@ class Output: ])) def results(self, action): + if len(action.results) < 2: + return success = 0 fail = 0 for result in action.results: From 12edb5168f845f184aaca7b5c11e03385840eb76 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 2 Aug 2020 22:19:13 +0200 Subject: [PATCH 54/90] Add exec command --- shlax/container.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/shlax/container.py b/shlax/container.py index 79ed172..acd8a4f 100644 --- a/shlax/container.py +++ b/shlax/container.py @@ -84,6 +84,28 @@ class Container: """Start the container""" await target.exec('podman', 'logs', self.full_name) + async def exec(self, target, cmd=None): + """Execute a command in the container""" + cmd = cmd or 'bash' + if cmd.endswith('sh'): + import os + os.execvp( + '/usr/bin/podman', + [ + 'podman', + 'exec', + '-it', + self.full_name, + cmd, + ] + ) + result = await target.exec( + 'podman', + 'exec', + self.full_name, + cmd, + ) + async def down(self, target): """Start the container""" await target.exec('podman', 'rm', '-f', self.full_name, raises=False) From ce931a2cc6401ac8dc1e993ddaa9e8500366ff83 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 8 Aug 2020 15:54:02 +0200 Subject: [PATCH 55/90] Fix TargetArgument implementation to stay optional --- shlax/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/shlax/cli.py b/shlax/cli.py index 7757975..390c784 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -32,9 +32,11 @@ class TargetArgument(cli2.Argument): def cast(self, value): from shlax.targets.ssh import Ssh - if '@' in value: - user, host = value.split('@') - return Ssh(host=host, user=user) + user, host = value.split('@') + return Ssh(host=host, user=user) + + def match(self, arg): + return arg if isinstance(arg, str) and '@' in arg else None class Command(cli2.Command): From d5544a6e85d03399c3f48fdbfb0c6b1c32065c65 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 8 Aug 2020 16:07:15 +0200 Subject: [PATCH 56/90] Support configured containers in start --- shlax/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/container.py b/shlax/container.py index acd8a4f..4daa2ba 100644 --- a/shlax/container.py +++ b/shlax/container.py @@ -49,7 +49,7 @@ class Container: if result['State'] == 'running': target.output.info(f'{self.full_name} already running') return - elif result['State'] == 'exited': + elif result['State'] in ('exited', 'configured'): target.output.info(f'{self.full_name} starting') await target.exec('podman', 'start', self.full_name) return From 5a20ccf2f727b361a62c81e7d3c6be4a6bdf7bae Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 8 Aug 2020 16:08:01 +0200 Subject: [PATCH 57/90] Layers refactor --- shlax/image.py | 40 +++++++++++++++++++++++++++++++++------- shlax/targets/buildah.py | 32 ++++++++------------------------ 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/shlax/image.py b/shlax/image.py index f21737b..33c80e3 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -1,21 +1,52 @@ -import copy +import json import os import re +class Layers(set): + def __init__(self, image): + self.image = image + + async def ls(self, target): + """Fetch layers from localhost""" + ret = set() + results = await target.parent.exec( + 'buildah images --json', + quiet=True, + ) + results = json.loads(results.out) + + prefix = 'localhost/' + self.image.repository + ':layer-' + for result in results: + if not result.get('names', None): + continue + for name in result['names']: + if name.startswith(prefix): + self.add(name) + return self + + async def rm(self, target, tags=None): + """Drop layers for this image""" + if tags is None: + tags = [layer for layer in await self.ls(target)] + await target.exec('podman', 'rmi', *tags) + + class Image: PATTERN = re.compile( '^((?P[a-z]*)://)?((?P[^/]*[.][^/]*)/)?((?P[^:]+))?(:(?P.*))?$' # noqa , re.I ) - def __init__(self, arg=None, format=None, backend=None, registry=None, repository=None, tags=None): + def __init__(self, arg=None, format=None, backend=None, registry=None, + repository=None, tags=None): self.arg = arg self.format = format self.backend = backend self.registry = registry self.repository = repository self.tags = tags or [] + self.layers = Layers(self) match = re.match(self.PATTERN, arg) if match: @@ -53,8 +84,3 @@ class Image: for tag in self.tags: await action.exec('buildah', 'push', f'{self.repository}:{tag}') - - def layer(self, key): - layer = copy.deepcopy(self) - layer.tags = ['layer-' + key] - return layer diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 2652386..5314a61 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -43,16 +43,15 @@ class Buildah(Target): if not self.is_runnable(): os.execvp('buildah', ['buildah', 'unshare'] + sys.argv) - # program has been replaced + return # process has been replaced - layers = await self.layers() - keep = await self.cache_setup(layers, *actions) + layers = await self.image.layers.ls(self) + keep = await self.cache_setup(self.image.layers, *actions) keepnames = [*map(lambda x: 'localhost/' + str(x), keep)] - self.invalidate = [name for name in layers if name not in keepnames] + self.invalidate = [name for name in self.image.layers if name not in keepnames] if self.invalidate: self.output.info('Invalidating old layers') - await self.parent.exec( - 'buildah', 'rmi', *self.invalidate, raises=False) + await self.image.layers.rm(self.parent, self.invalidate) if actions: actions = actions[len(keep):] @@ -68,23 +67,6 @@ class Buildah(Target): return await super().__call__(*actions) - async def layers(self): - ret = set() - results = await self.parent.exec( - 'buildah images --json', - quiet=True, - ) - results = json.loads(results.out) - - prefix = 'localhost/' + self.image.repository + ':layer-' - for result in results: - if not result.get('names', None): - continue - for name in result['names']: - if name.startswith(prefix): - ret.add(name) - return ret - async def cache_setup(self, layers, *actions): keep = [] self.image_previous = Image(self.base) @@ -115,7 +97,9 @@ class Buildah(Target): action_key = str(action) key = prefix + action_key sha1 = hashlib.sha1(key.encode('ascii')) - return self.image.layer(sha1.hexdigest()) + action_image = copy.deepcopy(self.image) + action_image.tags = ['layer-' + sha1.hexdigest()] + return action_image async def action(self, action, reraise=False): stop = await super().action(action, reraise) From 61fb39ebeaf8649054858ffc0dc9cd61a3af1670 Mon Sep 17 00:00:00 2001 From: jpic Date: Fri, 28 Aug 2020 18:37:25 +0200 Subject: [PATCH 58/90] Add container inspect --- shlax/container.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shlax/container.py b/shlax/container.py index 4daa2ba..df75e14 100644 --- a/shlax/container.py +++ b/shlax/container.py @@ -80,8 +80,12 @@ class Container: """Start the container""" await target.exec('podman', 'stop', self.full_name) + async def inspect(self, target): + """Inspect container""" + await target.exec('podman', 'inspect', self.full_name) + async def logs(self, target): - """Start the container""" + """Show container logs""" await target.exec('podman', 'logs', self.full_name) async def exec(self, target, cmd=None): From 324a8f49624111c4d22012c007905d8bb30995c9 Mon Sep 17 00:00:00 2001 From: jpic Date: Fri, 28 Aug 2020 18:38:11 +0200 Subject: [PATCH 59/90] Readability --- shlax/podman.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shlax/podman.py b/shlax/podman.py index 05d46e7..e1e26de 100644 --- a/shlax/podman.py +++ b/shlax/podman.py @@ -12,7 +12,9 @@ class Podman(list): return Podman(self.target, *self + [command]) async def __call__(self, *args, **kwargs): - cmd = self + list(args) + [f'--{k}={v}' for k, v in kwargs.items()] + cmd = self + list(args) + [ + f'--{k}={v}' for k, v in kwargs.items() + ] if 'ps' in cmd: cmd += ['--format=json'] return (await self.target.exec(*cmd, quiet=True)).json From ecabf6c9ce19cec33a1b60a00b5fff2298f7cece Mon Sep 17 00:00:00 2001 From: jpic Date: Fri, 28 Aug 2020 18:39:16 +0200 Subject: [PATCH 60/90] Add target argument only if present in sig --- shlax/cli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shlax/cli.py b/shlax/cli.py index 390c784..77a33cf 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -42,10 +42,11 @@ class TargetArgument(cli2.Argument): class Command(cli2.Command): def setargs(self): super().setargs() - self['target'] = TargetArgument( - self, - self.sig.parameters['target'], - ) + if 'target' in self.sig.parameters: + self['target'] = TargetArgument( + self, + self.sig.parameters['target'], + ) if 'actions' in self: del self['actions'] From c5af90238223786a7c3fa47a5325fd69491f1490 Mon Sep 17 00:00:00 2001 From: jpic Date: Fri, 28 Aug 2020 18:42:29 +0200 Subject: [PATCH 61/90] Fix Pod.build to unshare prior to building async --- shlax/pod.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shlax/pod.py b/shlax/pod.py index e19c519..d3c823c 100644 --- a/shlax/pod.py +++ b/shlax/pod.py @@ -1,9 +1,11 @@ import cli2 import json import os +import sys from shlax.targets.base import Target from shlax.actions.parallel import Parallel +from shlax.proc import Proc from .podman import Podman @@ -27,7 +29,7 @@ class Pod: async def build(self, target, *names): """Build container images""" - if not Proc.test or os.getuid() == 0: + if not (Proc.test or os.getuid() == 0): os.execvp('buildah', ['buildah', 'unshare'] + sys.argv) else: await self._call(target, 'build', *names) From 5c76ef3f2da35626e4e831479431661a6263e921 Mon Sep 17 00:00:00 2001 From: jpic Date: Fri, 28 Aug 2020 18:47:38 +0200 Subject: [PATCH 62/90] Forward CLI arguments --- shlax/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/cli.py b/shlax/cli.py index 77a33cf..487ab88 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -52,7 +52,7 @@ class Command(cli2.Command): def call(self, *args, **kwargs): self.shlax_target = self['target'].value - return self.shlax_target(self.target) + return self.shlax_target(*args) def __call__(self, *argv): super().__call__(*argv) From 54e3266789ef30faf579df2d3c52f33b5065a086 Mon Sep 17 00:00:00 2001 From: jpic Date: Fri, 28 Aug 2020 18:56:28 +0200 Subject: [PATCH 63/90] Prevent print traceback for failed commands --- shlax/cli.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/shlax/cli.py b/shlax/cli.py index 487ab88..13a6926 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -13,6 +13,8 @@ import importlib import os import sys +from .proc import ProcFailure + class Group(cli2.Group): def __init__(self, *args, **kwargs): @@ -55,7 +57,12 @@ class Command(cli2.Command): return self.shlax_target(*args) def __call__(self, *argv): - super().__call__(*argv) + try: + super().__call__(*argv) + except ProcFailure: + # just output the failure without TB, as command was already + # printed anyway + pass self.shlax_target.output.results(self.shlax_target) From 4ea9783c934f3db5a671741c57474e81af94ec2d Mon Sep 17 00:00:00 2001 From: jpic Date: Mon, 5 Oct 2020 18:27:49 +0200 Subject: [PATCH 64/90] Cache invalidation failures must not stop build Images may still be in use by running containers --- shlax/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/image.py b/shlax/image.py index 33c80e3..9aade3c 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -29,7 +29,7 @@ class Layers(set): """Drop layers for this image""" if tags is None: tags = [layer for layer in await self.ls(target)] - await target.exec('podman', 'rmi', *tags) + await target.exec('podman', 'rmi', *tags, raises=False) class Image: From b37e4c1aeda1b2031c1e680ef45ad9ab6f97b183 Mon Sep 17 00:00:00 2001 From: jpic Date: Mon, 5 Oct 2020 18:24:32 +0200 Subject: [PATCH 65/90] Fix shlax.cli (finally) --- shlax/cli.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/shlax/cli.py b/shlax/cli.py index 13a6926..6e99c7e 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -52,18 +52,17 @@ class Command(cli2.Command): if 'actions' in self: del self['actions'] - def call(self, *args, **kwargs): - self.shlax_target = self['target'].value - return self.shlax_target(*args) - def __call__(self, *argv): + result = None + try: - super().__call__(*argv) + result = super().__call__(*argv) except ProcFailure: # just output the failure without TB, as command was already # printed anyway pass - self.shlax_target.output.results(self.shlax_target) + self['target'].value.output.results(self['target'].value) + return result class ActionCommand(cli2.Command): From 4fcabf1aed851489d692c49daa9b6dbe6bd8ba9f Mon Sep 17 00:00:00 2001 From: jpic Date: Mon, 5 Oct 2020 18:38:03 +0200 Subject: [PATCH 66/90] test From c46b42b23ae69a962ca4f484fa66bc6d4068b204 Mon Sep 17 00:00:00 2001 From: jpic Date: Mon, 5 Oct 2020 18:41:16 +0200 Subject: [PATCH 67/90] Commenting docker job for now --- .gitlab-ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb4f630..90ddbee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,15 +7,15 @@ build: - pip3 install -U --user -e .[cli] - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build stage: build - -build-itself: - cache: - key: cache - paths: [.cache] - image: shlax:$CI_COMMIT_SHORT_SHA - script: python3 ./shlaxfile.py build - stage: test - +# commenting until we have docker again +# build-itself: +# cache: +# key: cache +# paths: [.cache] +# image: shlax:$CI_COMMIT_SHORT_SHA +# script: python3 ./shlaxfile.py build +# stage: test +# test: image: yourlabs/python stage: build From f5765c08a9c2462744cdf06570f9c86a4e7c8750 Mon Sep 17 00:00:00 2001 From: jpic Date: Wed, 7 Oct 2020 23:08:48 +0200 Subject: [PATCH 68/90] Target level cleaning had been forgotten ... when action level cleaning was added Test both while we're at it, as this is pretty critical for actions to clean up properly --- shlax/targets/base.py | 8 ++++++++ tests/test_target.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 28dbb45..f1ab1ab 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -47,10 +47,18 @@ class Target: # the calling target self.parent = target + result = Result(self, self) + result.status = 'success' + for action in actions or self.actions: if await self.action(action, reraise=bool(actions)): + result.status = 'failure' break + if getattr(self, 'clean', None): + self.output.clean(self) + await self.clean(self, result) + async def action(self, action, reraise=False): result = Result(self, action) self.output.start(action) diff --git a/tests/test_target.py b/tests/test_target.py index 7827167..f294503 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -72,6 +72,37 @@ async def test_function(): await Stub()(hello) +@pytest.mark.asyncio +async def test_action_clean(): + class Example: + def __init__(self): + self.was_called = False + async def clean(self, target, result): + self.was_called = True + async def __call__(self, target): + raise Exception('lol') + + action = Example() + target = Stub() + with pytest.raises(Exception): + await target(action) + assert action.was_called + + +@pytest.mark.asyncio +async def test_target_clean(): + class Example(Stub): + def __init__(self, action): + self.was_called = False + super().__init__(action) + async def clean(self, target, result): + self.was_called = True + + target = Example(Error()) + await target() + assert target.was_called + + @pytest.mark.asyncio async def test_method(): class Example: From 0edd573f4a1909cde5a8154259e4724cb4f167a1 Mon Sep 17 00:00:00 2001 From: jpic Date: Wed, 7 Oct 2020 23:55:10 +0200 Subject: [PATCH 69/90] Fix up cmd --- shlax/container.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shlax/container.py b/shlax/container.py index df75e14..f023885 100644 --- a/shlax/container.py +++ b/shlax/container.py @@ -51,7 +51,11 @@ class Container: return elif result['State'] in ('exited', 'configured'): target.output.info(f'{self.full_name} starting') - await target.exec('podman', 'start', self.full_name) + startargs = ['podman', 'start'] + if '-d' not in args: + startargs.append('--attach') + startargs.append(self.full_name) + await target.exec(*startargs) return cmd = [ From cada94ecc5ed89591383b137643c4db8118aedbf Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 8 Oct 2020 00:50:30 +0200 Subject: [PATCH 70/90] Fixing push --- .gitlab-ci.yml | 20 ++++++++++---------- shlax/image.py | 20 +++++++++++++------- shlax/targets/buildah.py | 6 ++++-- shlaxfile.py | 2 +- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90ddbee..fed323e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,17 +5,17 @@ build: image: yourlabs/buildah script: - pip3 install -U --user -e .[cli] - - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build + - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push stage: build -# commenting until we have docker again -# build-itself: -# cache: -# key: cache -# paths: [.cache] -# image: shlax:$CI_COMMIT_SHORT_SHA -# script: python3 ./shlaxfile.py build -# stage: test -# + +build-itself: + cache: + key: cache + paths: [.cache] + image: quay.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA + script: python3 ./shlaxfile.py build + stage: test + test: image: yourlabs/python stage: build diff --git a/shlax/image.py b/shlax/image.py index 9aade3c..45ed9ec 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -74,13 +74,19 @@ class Image: def __str__(self): return f'{self.repository}:{self.tags[-1]}' - async def push(self, *args, **kwargs): - user = os.getenv('DOCKER_USER') - passwd = os.getenv('DOCKER_PASS') - action = kwargs.get('action', self) + async def push(self, target): + user = os.getenv('IMAGES_USER') + passwd = os.getenv('IMAGES_PASS') if user and passwd: - action.output.cmd('buildah login -u ... -p ...' + self.registry) - await action.exec('buildah', 'login', '-u', user, '-p', passwd, self.registry or 'docker.io', debug=False) + target.output.cmd('buildah login -u ... -p ...' + self.registry) + await target.parent.exec( + 'buildah', 'login', '-u', user, '-p', passwd, + self.registry or 'docker.io', debug=False) for tag in self.tags: - await action.exec('buildah', 'push', f'{self.repository}:{tag}') + await target.parent.exec( + 'buildah', + 'push', + self.repository + ':final', + f'{self.registry}/{self.repository}:{tag}' + ) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 5314a61..f5951e2 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -37,10 +37,12 @@ class Buildah(Target): return 'Replacing with: buildah unshare ' + ' '.join(sys.argv) return f'Buildah({self.image})' - async def __call__(self, *actions, target=None): + async def __call__(self, *actions, target=None, push: bool=False): if target: self.parent = target + self.push = push + if not self.is_runnable(): os.execvp('buildah', ['buildah', 'unshare'] + sys.argv) return # process has been replaced @@ -124,7 +126,7 @@ class Buildah(Target): if result.status == 'success' and self.ctr: await self.commit() - if os.getenv('BUILDAH_PUSH'): + if self.push: await self.image.push(target) if self.ctr is not None: diff --git a/shlaxfile.py b/shlaxfile.py index 7a87b23..3e019fb 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -11,7 +11,7 @@ shlax = Container( Copy('setup.py', 'shlax', '/app'), Pip('/app'), base='quay.io/podman/stable', - commit='shlax', + commit='quay.io/yourlabs/shlax', ), ) From 77439b6dc34802f085b17e988a17641f42c4e100 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 6 Dec 2020 19:09:41 +0100 Subject: [PATCH 71/90] Do not skip build/commit even if up to date --- shlax/targets/buildah.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index f5951e2..6d6c45e 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -57,12 +57,8 @@ class Buildah(Target): if actions: actions = actions[len(keep):] - if not actions: - return self.output.success('Image up to date') else: self.actions = self.actions[len(keep):] - if not self.actions: - return self.output.success('Image up to date') self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) From a80e3fb48c0fa3fb69160aa923be24b4714b5be8 Mon Sep 17 00:00:00 2001 From: jpic Date: Mon, 7 Dec 2020 11:16:31 +0100 Subject: [PATCH 72/90] Add docker daemon support --- shlax/image.py | 6 ++++-- shlax/targets/buildah.py | 13 ++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/shlax/image.py b/shlax/image.py index 45ed9ec..dc58635 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -60,9 +60,11 @@ class Image: setattr(self, k, v) # docker.io currently has issues with oci format - self.format = format or 'oci' if self.registry == 'docker.io': - self.format = 'docker' + self.backend = 'docker' + + if not self.format: + self.format = 'docker' if self.backend == 'docker' else 'oci' # filter out tags which resolved to None self.tags = [t for t in self.tags if t] diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 6d6c45e..60617ab 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -145,8 +145,19 @@ class Buildah(Target): async def commit(self): await self.parent.exec( - f'buildah commit {self.ctr} {self.image.repository}:final' + 'buildah', + 'commit', + f'--format={self.image.format}', + self.ctr, + f'{self.image.repository}:final', ) + if self.image.backend == 'docker': + await self.parent.exec( + 'buildah', + 'push', + f'{self.image.repository}:final', + f'docker-daemon:{self.image.repository}:latest' + ) ENV_TAGS = ( # gitlab From ae2aa62d0ec173a26162eb69140bbd8ff2bff871 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 10:41:02 +0200 Subject: [PATCH 73/90] Glob support in Copy --- shlax/actions/copy.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/shlax/actions/copy.py b/shlax/actions/copy.py index 2fee0a1..848917d 100644 --- a/shlax/actions/copy.py +++ b/shlax/actions/copy.py @@ -1,12 +1,19 @@ import asyncio import binascii +import glob import os class Copy: def __init__(self, *args): - self.src = args[:-1] self.dst = args[-1] + self.src = [] + + for src in args[:-1]: + if '*' in src: + self.src += glob.glob(src) + else: + self.src.append(src) def listfiles(self): if getattr(self, '_listfiles', None): From 78252a439c5abd6bf2621cd9462cb0b741b48878 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 10:41:49 +0200 Subject: [PATCH 74/90] Use buildah copy to support workingdir --- shlax/targets/buildah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 60617ab..da68553 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -188,7 +188,7 @@ class Buildah(Target): return await self.parent.mkdir(*[self.path(path) for path in paths]) async def copy(self, *args): - return await self.parent.copy(*args[:-1], self.path(args[-1])) + return await self.parent.exec('buildah', 'copy', self.ctr, *args) async def write(self, path, content): return await self.write(path, content) From 84b1230146c5c62d994f89f6de81a0dcabddd67d Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 10:46:45 +0200 Subject: [PATCH 75/90] Build for docker --- shlaxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlaxfile.py b/shlaxfile.py index 3e019fb..4162b6b 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -11,7 +11,7 @@ shlax = Container( Copy('setup.py', 'shlax', '/app'), Pip('/app'), base='quay.io/podman/stable', - commit='quay.io/yourlabs/shlax', + commit='docker://yourlabs/shlax', ), ) From 1221ac5016e00cd4b3b64d76dad5af68291a0f54 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 10:55:01 +0200 Subject: [PATCH 76/90] Do not specify an image format for layers --- shlax/targets/buildah.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index da68553..71e3ebc 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -107,7 +107,6 @@ class Buildah(Target): await self.parent.exec( 'buildah', 'commit', - '--format=' + action_image.format, self.ctr, action_image, ) From 94890226e69d6860a7fe5f367b1437e649fcd9cd Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 11:37:54 +0200 Subject: [PATCH 77/90] Fixing docker support in CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fed323e..9030492 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,7 @@ build-itself: cache: key: cache paths: [.cache] - image: quay.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA + image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA script: python3 ./shlaxfile.py build stage: test From 92246fc7bee29ccddea6896ab2a7f881471326c0 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 11:58:36 +0200 Subject: [PATCH 78/90] Dont try to color non-utf8 --- shlax/output.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shlax/output.py b/shlax/output.py index 910061c..955cb78 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -101,7 +101,11 @@ class Output: ) def highlight(self, line, highlight=True): - line = line.decode('utf8') if isinstance(line, bytes) else line + try: + line = line.decode('utf8') if isinstance(line, bytes) else line + except UnicodeDecodeError: + highlight = False + if not highlight or ( '\x1b[' in line or '\033[' in line From b206a1e107ca6412052b6def2e3bca5a9e886c55 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 11:58:45 +0200 Subject: [PATCH 79/90] Specify repository --- shlaxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlaxfile.py b/shlaxfile.py index 4162b6b..e37f165 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -11,7 +11,7 @@ shlax = Container( Copy('setup.py', 'shlax', '/app'), Pip('/app'), base='quay.io/podman/stable', - commit='docker://yourlabs/shlax', + commit='docker://docker.io/yourlabs/shlax', ), ) From 5c941d38d6664e50f95ce1ce2c57d4321bef4152 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 12:19:54 +0200 Subject: [PATCH 80/90] Push argument --- .gitlab-ci.yml | 2 +- shlax/image.py | 4 ++-- shlax/targets/buildah.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9030492..467983b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ build: image: yourlabs/buildah script: - pip3 install -U --user -e .[cli] - - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push + - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA stage: build build-itself: diff --git a/shlax/image.py b/shlax/image.py index dc58635..68d66eb 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -76,7 +76,7 @@ class Image: def __str__(self): return f'{self.repository}:{self.tags[-1]}' - async def push(self, target): + async def push(self, target, name=None): user = os.getenv('IMAGES_USER') passwd = os.getenv('IMAGES_PASS') if user and passwd: @@ -90,5 +90,5 @@ class Image: 'buildah', 'push', self.repository + ':final', - f'{self.registry}/{self.repository}:{tag}' + name if isinstance(name, str) else f'{self.registry}/{self.repository}:{tag}' ) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 71e3ebc..7de8bd5 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -37,7 +37,7 @@ class Buildah(Target): return 'Replacing with: buildah unshare ' + ' '.join(sys.argv) return f'Buildah({self.image})' - async def __call__(self, *actions, target=None, push: bool=False): + async def __call__(self, *actions, target=None, push: str=False): if target: self.parent = target @@ -122,7 +122,7 @@ class Buildah(Target): if result.status == 'success' and self.ctr: await self.commit() if self.push: - await self.image.push(target) + await self.image.push(target, self.push) if self.ctr is not None: await self.parent.exec('buildah', 'rm', self.ctr) From 9e586de3126eb8c645c8ed96d2e3c8d859b1985d Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 12:33:13 +0200 Subject: [PATCH 81/90] Cache layers in CI? --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 467983b..d880ae1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ build: cache: key: cache - paths: [.cache] + paths: [.cache, .local/share/containers/] image: yourlabs/buildah script: - pip3 install -U --user -e .[cli] @@ -11,7 +11,7 @@ build: build-itself: cache: key: cache - paths: [.cache] + paths: [.cache, .local/share/containers/] image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA script: python3 ./shlaxfile.py build stage: test From d23b3945f3e83189618f44df7f66f80189937c76 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 12:34:00 +0200 Subject: [PATCH 82/90] Support DOCKER_USER and DOCKER_PASS --- shlax/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shlax/image.py b/shlax/image.py index 68d66eb..638e4f9 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -77,8 +77,8 @@ class Image: return f'{self.repository}:{self.tags[-1]}' async def push(self, target, name=None): - user = os.getenv('IMAGES_USER') - passwd = os.getenv('IMAGES_PASS') + user = os.getenv('IMAGES_USER', os.getenv('DOCKER_USER')) + passwd = os.getenv('IMAGES_PASS', os.getenv('DOCKER_PASS')) if user and passwd: target.output.cmd('buildah login -u ... -p ...' + self.registry) await target.parent.exec( From 3cd3dae6194cd19f180888acbfbaee78b09d033b Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 12:40:59 +0200 Subject: [PATCH 83/90] Fix old call --- shlax/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/image.py b/shlax/image.py index 638e4f9..69f51fb 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -83,7 +83,7 @@ class Image: target.output.cmd('buildah login -u ... -p ...' + self.registry) await target.parent.exec( 'buildah', 'login', '-u', user, '-p', passwd, - self.registry or 'docker.io', debug=False) + self.registry or 'docker.io', quiet=True) for tag in self.tags: await target.parent.exec( From 7467f79bd9b623b627dc937b74188433c5a384a6 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 12:45:32 +0200 Subject: [PATCH 84/90] Fix layer caching in CI? --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d880ae1..708e9ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ build: cache: key: cache - paths: [.cache, .local/share/containers/] + paths: [.cache, /var/lib/containers/] image: yourlabs/buildah script: - pip3 install -U --user -e .[cli] @@ -11,7 +11,7 @@ build: build-itself: cache: key: cache - paths: [.cache, .local/share/containers/] + paths: [.cache, /var/lib/containers/] image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA script: python3 ./shlaxfile.py build stage: test From 8544a76bf2bb562f8a36e3ac280ddb312649fd7e Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 12:48:17 +0200 Subject: [PATCH 85/90] Install CLI dependencies --- shlaxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlaxfile.py b/shlaxfile.py index e37f165..9eedc74 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -9,7 +9,7 @@ shlax = Container( build=Buildah( Packages('python38', 'buildah', 'unzip', 'findutils', upgrade=False), Copy('setup.py', 'shlax', '/app'), - Pip('/app'), + Pip('/app[cli]'), base='quay.io/podman/stable', commit='docker://docker.io/yourlabs/shlax', ), From a7cd98ffbbfb6dba43e1b5c61d7b40d5997d602a Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 13:10:11 +0200 Subject: [PATCH 86/90] Fix base image --- shlaxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlaxfile.py b/shlaxfile.py index 9eedc74..d9ad328 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -10,7 +10,7 @@ shlax = Container( Packages('python38', 'buildah', 'unzip', 'findutils', upgrade=False), Copy('setup.py', 'shlax', '/app'), Pip('/app[cli]'), - base='quay.io/podman/stable', + base='quay.io/buildah/stable', commit='docker://docker.io/yourlabs/shlax', ), ) From 9846425b2980a4e3eb2ee4b1ea033483ba73f990 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 13:24:34 +0200 Subject: [PATCH 87/90] Push self built to COMMIT_REF --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 708e9ee..edef314 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,7 +13,7 @@ build-itself: key: cache paths: [.cache, /var/lib/containers/] image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA - script: python3 ./shlaxfile.py build + script: python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_REF stage: test test: From 44dd1bc6a6e42980554ae773d286f27dda3df1d8 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 13:35:29 +0200 Subject: [PATCH 88/90] Fix output in case of failed command --- shlax/cli.py | 8 ++++++-- shlax/targets/base.py | 22 ++++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/shlax/cli.py b/shlax/cli.py index 6e99c7e..9d07cf5 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -60,9 +60,13 @@ class Command(cli2.Command): except ProcFailure: # just output the failure without TB, as command was already # printed anyway - pass + self.exit_code = 1 self['target'].value.output.results(self['target'].value) - return result + + if result and result.status == 'success': + self.exit_code = 0 + else: + self.exit_code = 1 class ActionCommand(cli2.Command): diff --git a/shlax/targets/base.py b/shlax/targets/base.py index f1ab1ab..d4907c3 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -6,7 +6,7 @@ import re import sys from ..output import Output -from ..proc import Proc +from ..proc import Proc, ProcFailure from ..result import Result, Results @@ -68,13 +68,19 @@ class Target: self.output.fail(action, e) result.status = 'failure' result.exception = e - if reraise: - # nested call, re-raise - raise - else: - import traceback - traceback.print_exception(type(e), e, sys.exc_info()[2]) - return True + + if not isinstance(e, ProcFailure): + # no need to reraise in case of command error + # because the command has been printed + + if reraise: + # nested call, re-raise + raise + else: + import traceback + traceback.print_exception(type(e), e, sys.exc_info()[2]) + + return True # because it failed else: if getattr(action, 'skipped', False): self.output.skip(action) From bcbe66a37fccd354f3f7616c29805426058be99f Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 14:00:27 +0200 Subject: [PATCH 89/90] Fix exit codes --- .gitlab-ci.yml | 6 ++++++ shlax/cli.py | 12 ++++++------ tests/shlaxfail.py | 18 ++++++++++++++++++ tests/shlaxsuccess.py | 17 +++++++++++++++++ 4 files changed, 47 insertions(+), 6 deletions(-) create mode 100755 tests/shlaxfail.py create mode 100755 tests/shlaxsuccess.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index edef314..080520e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,12 @@ build-itself: script: python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_REF stage: test +test-exitcode: + image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA + script: + - tests/shlaxfail.py build || [ $? -eq 1 ] + - tests/shlaxsuccess.py build + test: image: yourlabs/python stage: build diff --git a/shlax/cli.py b/shlax/cli.py index 9d07cf5..194d576 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -60,13 +60,13 @@ class Command(cli2.Command): except ProcFailure: # just output the failure without TB, as command was already # printed anyway - self.exit_code = 1 - self['target'].value.output.results(self['target'].value) + pass - if result and result.status == 'success': - self.exit_code = 0 - else: - self.exit_code = 1 + if self['target'].value.results: + if self['target'].value.results[-1].status == 'failure': + self.exit_code = 1 + self['target'].value.output.results(self['target'].value) + return result class ActionCommand(cli2.Command): diff --git a/tests/shlaxfail.py b/tests/shlaxfail.py new file mode 100755 index 0000000..88adda5 --- /dev/null +++ b/tests/shlaxfail.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +""" +Shlaxfile for shlax itself. +""" + +from shlax.shortcuts import * + +shlax = Container( + build=Buildah( + Packages('prout', upgrade=False), + base='alpine', + commit='shlaxfail', + ), +) + + +if __name__ == '__main__': + print(Group(doc=__doc__).load(shlax).entry_point()) diff --git a/tests/shlaxsuccess.py b/tests/shlaxsuccess.py new file mode 100755 index 0000000..cd5b23b --- /dev/null +++ b/tests/shlaxsuccess.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +""" +Shlaxfile for shlax itself. +""" + +from shlax.shortcuts import * + +shlax = Container( + build=Buildah( + base='alpine', + commit='shlaxsuccess', + ), +) + + +if __name__ == '__main__': + print(Group(doc=__doc__).load(shlax).entry_point()) From 28bfef29b01d816966ab13b526451e27506db27a Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 24 Apr 2021 14:28:32 +0200 Subject: [PATCH 90/90] Fix dist --- .gitlab-ci.yml | 2 +- shlax/__init__.py | 0 shlax/actions/__init__.py | 0 shlax/contrib/__init__.py | 0 shlax/repo/__init__.py | 0 shlax/targets/__init__.py | 0 6 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 shlax/__init__.py create mode 100644 shlax/actions/__init__.py create mode 100644 shlax/contrib/__init__.py create mode 100644 shlax/repo/__init__.py create mode 100644 shlax/targets/__init__.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 080520e..d796363 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ build: paths: [.cache, /var/lib/containers/] image: yourlabs/buildah script: - - pip3 install -U --user -e .[cli] + - pip3 install -U --user .[cli] - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA stage: build diff --git a/shlax/__init__.py b/shlax/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shlax/actions/__init__.py b/shlax/actions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shlax/contrib/__init__.py b/shlax/contrib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shlax/repo/__init__.py b/shlax/repo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shlax/targets/__init__.py b/shlax/targets/__init__.py new file mode 100644 index 0000000..e69de29