From 6924d39590f268a1f15957d12628a40ee356a6bb Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 13:47:33 +0100 Subject: [PATCH] Add Docker target, recursive calls --- .gitlab-ci.yml | 10 ++--- shlax/actions/base.py | 34 ++++++++++++++++- shlax/actions/packages.py | 2 +- shlax/actions/pip.py | 40 +++++++++---------- shlax/actions/run.py | 14 ++++++- shlax/cli.py | 5 ++- shlax/contrib/gitlab.py | 11 +++++- shlax/image.py | 7 ++++ shlax/output.py | 58 +++++++++++++++------------- shlax/targets/__init__.py | 1 + shlax/targets/buildah.py | 37 +++++++++--------- shlax/targets/docker.py | 78 ++++++++++++++++++++++++++++++++++++++ shlax/targets/localhost.py | 19 +++++++--- shlaxfile.py | 54 +++++++++++++++----------- 14 files changed, 268 insertions(+), 102 deletions(-) create mode 100644 shlax/targets/docker.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cca2778..d71d9db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,14 @@ build: image: yourlabs/shlax - script: pip install -U .[test] && py.test -svv tests - stage: test + script: ./shlaxfile.py build + stage: build pypi: image: yourlabs/python only: - tags - script: pypi-release + script: ./shlaxfile.py pypi stage: deploy test: - image: yourlabs/python - script: pip install -U .[test] && py.test -svv tests + image: yourlabs/shlax + script: ./shlaxfile.py test stage: test diff --git a/shlax/actions/base.py b/shlax/actions/base.py index 1b337aa..d2e0e76 100644 --- a/shlax/actions/base.py +++ b/shlax/actions/base.py @@ -1,4 +1,6 @@ +import functools import inspect +import importlib import sys from ..output import Output @@ -164,16 +166,46 @@ class Action: def callable(self): from ..targets import Localhost async def cb(*a, **k): - return await Localhost(self, quiet=True)(*a, **k) + from shlax.cli import cli + script = Localhost(self, quiet=True) + result = await script(*a, **k) + + success = functools.reduce( + lambda a, b: a + b, + [1 for c in script.children() if c.status == 'success'] or [0]) + if success: + script.output.success(f'{success} PASS') + + failures = functools.reduce( + lambda a, b: a + b, + [1 for c in script.children() if c.status == 'fail'] or [0]) + if failures: + script.output.fail(f'{failures} FAIL') + cli.exit_code = failures + return result return cb def kwargs_output(self): return self.kwargs def action(self, action, *args, **kwargs): + if isinstance(action, str): + import cli2 + a = cli2.Callable.factory(action).target + if not a: + a = cli2.Callable.factory( + '.'.join(['shlax', action]) + ).target + if a: + action = a + p = action(*args, **kwargs) for parent in self.parents(): if hasattr(parent, 'actions'): break p.parent = parent + if 'actions' not in self.__dict__: + # "mutate" to Strategy + from ..strategies.script import Actions + self.actions = Actions(self, [p]) return p diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index c509a5a..37f6d4b 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -113,7 +113,7 @@ class Packages(Action): else: mgr = await self.which(*self.mgrs.keys()) if mgr: - self.mgr = mgr.split('/')[-1] + self.mgr = mgr[0].split('/')[-1] if not self.mgr: raise Exception('Packages does not yet support this distro') diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py index 8fa1342..fcbae98 100644 --- a/shlax/actions/pip.py +++ b/shlax/actions/pip.py @@ -5,25 +5,27 @@ from .base import Action class Pip(Action): - packages = dict( - apt=['python3-pip'], - ) - def __init__(self, *pip_packages, pip=None, requirements=None): - self.pip_packages = pip_packages self.requirements = requirements super().__init__(*pip_packages, pip=pip, requirements=requirements) async def call(self, *args, **kwargs): - self.pip = await self.which('pip3', 'pip', 'pip2') - if not self.pip: + pip = await self.which('pip3', 'pip', 'pip2') + if pip: + pip = pip[0] + else: from .packages import Packages - action = self.action(Packages, 'python3,apk', 'python3-pip,apt', args=args, kwargs=kwargs) + action = self.action( + Packages, + 'python3,apk', 'python3-pip,apt', + args=args, kwargs=kwargs + ) await action(*args, **kwargs) - - self.pip = await self.which('pip3', 'pip', 'pip2') - if not self.pip: - raise Exception('Could not install a pip command') + pip = await self.which('pip3', 'pip', 'pip2') + if not pip: + raise Exception('Could not install a pip command') + else: + pip = pip[0] if 'CACHE_DIR' in os.environ: cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') @@ -33,20 +35,20 @@ class Pip(Action): if getattr(self, 'mount', None): # we are in a target which shares a mount command await self.mount(cache, '/root/.cache/pip') - await self.exec(f'{self.pip} install --upgrade pip') + await self.exec(f'{pip} install --upgrade pip') # https://github.com/pypa/pip/issues/5599 - self.pip = 'python3 -m pip' + pip = 'python3 -m pip' - source = [p for p in self.pip_packages if p.startswith('/')] + source = [p for p in self.args if p.startswith('/') or p.startswith('.')] if source: await self.exec( - f'{self.pip} install --upgrade --editable {" ".join(source)}' + f'{pip} install --upgrade --editable {" ".join(source)}' ) - nonsource = [p for p in self.pip_packages if not p.startswith('/')] + nonsource = [p for p in self.args if not p.startswith('/')] if nonsource: - await self.exec(f'{self.pip} install --upgrade {" ".join(nonsource)}') + await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}') if self.requirements: - await self.exec(f'{self.pip} install --upgrade -r {self.requirements}') + await self.exec(f'{pip} install --upgrade -r {self.requirements}') diff --git a/shlax/actions/run.py b/shlax/actions/run.py index 83ea901..e3f8730 100644 --- a/shlax/actions/run.py +++ b/shlax/actions/run.py @@ -1,6 +1,18 @@ +from ..targets.buildah import Buildah +from ..targets.docker import Docker + from .base import Action class Run(Action): async def call(self, *args, **kwargs): - return await self.exec(*self.args, **self.kwargs) + image = self.kwargs.get('image', None) + if not image: + return await self.exec(*self.args, **self.kwargs) + if isinstance(image, Buildah): + breakpoint() + result = await self.action(image, *args, **kwargs) + + return await Docker( + image=image, + ).exec(*args, **kwargs) diff --git a/shlax/cli.py b/shlax/cli.py index fc43129..fa1767f 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -8,6 +8,7 @@ shlax is a micro-framework to orchestrate commands. import asyncio import cli2 +import copy import inspect import os import sys @@ -130,8 +131,10 @@ class ConsoleScript(cli2.ConsoleScript): return super().__call__(*args, **kwargs) def call(self, command): + kwargs = copy.copy(self.parser.funckwargs) + kwargs.update(self.parser.options) try: - return super().call(command) + return command(*self.parser.funcargs, **kwargs) except WrongResult as e: print(e) self.exit_code = e.proc.rc diff --git a/shlax/contrib/gitlab.py b/shlax/contrib/gitlab.py index e3b8891..f292d47 100644 --- a/shlax/contrib/gitlab.py +++ b/shlax/contrib/gitlab.py @@ -3,9 +3,16 @@ import yaml from shlax import * -class GitLabCIConfig(Script): +class GitLabCI(Script): async def call(self, *args, write=True, **kwargs): - output = yaml.dump(self.kwargs) + output = dict() + for key, value in self.kwargs.items(): + if isinstance(value, dict): + output[key] = value + output[key]['script'] = './shlaxfile.py ' + key + else: + output[key] = value + output = yaml.dump(output) if kwargs['debug'] is True: self.output(output) if write: diff --git a/shlax/image.py b/shlax/image.py index 9ba27c8..173b8fa 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -55,3 +55,10 @@ class Image: # default tag by default ... if not self.tags: self.tags = ['latest'] + + async def __call__(self, action, *args, **kwargs): + args = list(args) + return await action.exec(*args, **self.kwargs) + + def __str__(self): + return self.repository diff --git a/shlax/output.py b/shlax/output.py index c9109ca..ee38572 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -33,8 +33,8 @@ class Output: self.write = write or sys.stdout.buffer.write self.flush = flush or sys.stdout.flush - def __call__(self, line, highlight=True, flush=True): - if self.prefix and self.prefix not in self.prefixes: + def prefix_line(self): + if self.prefix not in self.prefixes: self.prefixes[self.prefix] = self.prefix_colors[len(self.prefixes)] if len(self.prefix) > self.prefix_length: self.prefix_length = len(self.prefix) @@ -44,24 +44,24 @@ class Output: if prefix_padding: prefix_padding = ' ' + prefix_padding + ' ' - self.write(( - ( - prefix_color - + prefix_padding - + self.prefix - + ' ' - + self.colors['reset'] - + '| ' - if self.prefix - else '' - ) - + self.highlight(line, highlight) - + self.colors['reset'] - ).encode('utf8')) + return [ + prefix_color, + prefix_padding, + self.prefix, + ' ', + self.colors['reset'], + '| ' + ] + + def __call__(self, line, highlight=True, flush=True): + line = [self.highlight(line) if highlight else line] + if self.prefix: + line = self.prefix_line() + line + line = ''.join(line) + + self.write(line.encode('utf8')) if flush: - if not line.endswith('\n'): - self.write(b'\n') self.flush() def cmd(self, line): @@ -70,7 +70,8 @@ class Output: + '\x1b[1;38;5;15m' + ' ' + self.highlight(line, 'bash') - + self.colors['reset'], + + self.colors['reset'] + + '\n', highlight=False ) @@ -92,25 +93,28 @@ class Output: for regexp, colors in self.regexps.items(): line = re.sub(regexp, colors.format(**self.colors), line) + line = line + self.colors['reset'] return line def clean(self, action): - if self.debug is True or 'visit' in str(self.debug): + if self.debug is True: self(''.join([ self.colors['bluebold'], - '+ CLEAN ', + '+ CLEAN ', self.colors['reset'], action.colorized(), + '\n', ])) def start(self, action): - if self.debug is True or 'visit' in str(self.debug): + if self.debug is True: self(''.join([ self.colors['orangebold'], - '⚠ START ', + '⚠ START ', self.colors['reset'], action.colorized(), + '\n', ])) def success(self, action): @@ -119,14 +123,16 @@ class Output: self.colors['greenbold'], '✔ SUCCESS ', self.colors['reset'], - action.colorized(), + action.colorized() if hasattr(action, 'colorized') else str(action), + '\n', ])) def fail(self, action, exception=None): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['redbold'], - '✘ FAIL ', + '✘ FAIL ', self.colors['reset'], - action.colorized(), + action.colorized() if hasattr(action, 'colorized') else str(action), + '\n', ])) diff --git a/shlax/targets/__init__.py b/shlax/targets/__init__.py index c7fb36f..3f7353f 100644 --- a/shlax/targets/__init__.py +++ b/shlax/targets/__init__.py @@ -1,3 +1,4 @@ from .buildah import Buildah +from .docker import Docker from .localhost import Localhost from .ssh import Ssh diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index df55e92..bfde5bb 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -8,6 +8,8 @@ import subprocess import sys import textwrap +from ..actions.base import Action +from ..exceptions import Mistake from ..proc import Proc from ..image import Image from .localhost import Localhost @@ -20,7 +22,10 @@ class Buildah(Localhost): """ contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] - def __init__(self, base, *args, commit=None, push=False, **kwargs): + def __init__(self, base, *args, commit=None, push=False, cmd=None, **kwargs): + if isinstance(base, Action): + args = [base] + list(args) + base = 'alpine' # default selection in case of mistake super().__init__(*args, **kwargs) self.base = base self.mounts = dict() @@ -28,6 +33,9 @@ class Buildah(Localhost): self.mnt = None self.image = Image(commit) if commit else None self.push = push or os.getenv('CI') + self.config= dict( + cmd=cmd or 'sh', + ) def shargs(self, *args, user=None, buildah=True, **kwargs): if not buildah or args[0].startswith('buildah'): @@ -52,9 +60,6 @@ class Buildah(Localhost): """Run buildah config.""" return await self.exec(f'buildah config {line} {self.ctr}', buildah=False) - async def mkdir(self, *dirs): - return await self.exec(*['mkdir', '-p'] + list(dirs)) - async def copy(self, *args): """Run buildah copy to copy a file from host into container.""" src = args[:-1] @@ -81,19 +86,6 @@ class Buildah(Localhost): await self.exec(f'mount -o bind {src} {target}', buildah=False) self.mounts[src] = dst - async def which(self, *cmd): - """ - Return the first path to the cmd in the container. - - If cmd argument is a list then it will try all commands. - """ - paths = (await self.env('PATH')).split(':') - for path in paths: - for c in cmd: - p = os.path.join(self.mnt, path[1:], c) - if os.path.exists(p): - return p[len(str(self.mnt)):] - def is_wrapper(self): return not ( Proc.test @@ -101,7 +93,6 @@ class Buildah(Localhost): or getattr(self.parent, 'parent', None) ) - async def call(self, *args, **kwargs): if not self.is_wrapper(): self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out @@ -124,7 +115,11 @@ class Buildah(Localhost): argv += [ cli.parser.command.name, # script name ? ] - self.output(' '.join(argv), 'EXECUTION', flush=True) + + await self.exec(*argv) + ''' + if debug is True or 'cmd' in str(debug): + self.output.cmd(' '.join(argv)) proc = await asyncio.create_subprocess_shell( shlex.join(argv), @@ -134,11 +129,15 @@ class Buildah(Localhost): ) await proc.communicate() cli.exit_code = await proc.wait() + ''' async def commit(self): if not self.image: return + for key, value in self.config.items(): + await self.exec(f'buildah config --{key} "{value}" {self.ctr}') + self.sha = (await self.exec( 'buildah', 'commit', diff --git a/shlax/targets/docker.py b/shlax/targets/docker.py new file mode 100644 index 0000000..80db9c5 --- /dev/null +++ b/shlax/targets/docker.py @@ -0,0 +1,78 @@ +import asyncio +from pathlib import Path +import os + +from ..image import Image +from .localhost import Localhost + + +class Docker(Localhost): + contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] + + def __init__(self, *args, **kwargs): + self.image = Image(kwargs.get('image', 'alpine')) + super().__init__(*args, **kwargs) + self.context['ctr'] = None + + def shargs(self, *args, daemon=False, **kwargs): + if args[0] == 'docker': + return args, kwargs + + extra = [] + if 'user' in kwargs: + extra += ['--user', kwargs.pop('user')] + + args, kwargs = super().shargs(*args, **kwargs) + + if self.context['ctr']: + executor = 'exec' + extra = [self.context['ctr']] + return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + list(args), kwargs + + executor = 'run' + cwd = os.getcwd() + if daemon: + extra += ['-d'] + extra = extra + ['-v', f'{cwd}:{cwd}', '-w', f'{cwd}'] + return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + [str(self.image)] + list(args), kwargs + + async def call(self, *args, **kwargs): + name = kwargs.get('name', os.getcwd()).split('/')[-1] + self.context['ctr'] = ( + await self.exec( + 'docker', 'ps', '-aq', '--filter', + 'name=' + name, + raises=False + ) + ).out.split('\n')[0] + + if 'recreate' in args and self.context['ctr']: + await self.exec('docker', 'rm', '-f', self.context['ctr']) + self.context['ctr'] = None + + if self.context['ctr']: + self.context['ctr'] = (await self.exec('docker', 'start', name)).out + else: + self.context['ctr'] = ( + await self.exec('sleep', '120', daemon=True) + ).out + return await super().call(*args, **kwargs) + + async def copy(self, *args): + src = args[:-1] + dst = args[-1] + await self.mkdir(dst) + + procs = [] + for s in src: + ''' + if Path(s).is_dir(): + await self.mkdir(s) + args = ['docker', 'copy', self.ctr, s, Path(dst) / s] + else: + args = ['docker', 'copy', self.ctr, s, dst] + ''' + args = ['docker', 'cp', s, self.context['ctr'] + ':' + dst] + procs.append(self.exec(*args)) + + return await asyncio.gather(*procs) diff --git a/shlax/targets/localhost.py b/shlax/targets/localhost.py index 892383f..8bc0294 100644 --- a/shlax/targets/localhost.py +++ b/shlax/targets/localhost.py @@ -1,4 +1,5 @@ import os +import re from shlax.proc import Proc @@ -50,12 +51,20 @@ class Localhost(Script): If cmd argument is a list then it will try all commands. """ - for path in (await self.env('PATH')).split(':'): - for c in cmd: - p = os.path.join(self.root, path[1:], c) - if os.path.exists(p): - return p[len(str(self.root)):] + proc = await self.exec('type ' + ' '.join(cmd), raises=False) + result = [] + for res in proc.out.split('\n'): + match = re.match('([^ ]+) is ([^ ]+)$', res.strip()) + if match: + result.append(match.group(1)) + return result async def copy(self, *args): args = ['cp', '-ra'] + list(args) return await self.exec(*args) + + async def mount(self, *dirs): + pass + + async def mkdir(self, *dirs): + return await self.exec(*['mkdir', '-p'] + list(dirs)) diff --git a/shlaxfile.py b/shlaxfile.py index 4ae04e5..556a02a 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -3,32 +3,42 @@ from shlax.contrib.gitlab import * PYTEST = 'py.test -svv tests' -build = Buildah('alpine', +build = Buildah( Copy('shlax/', 'setup.py', '/app'), Pip('/app'), commit='yourlabs/shlax', -) - -gitlabci = GitLabCIConfig( - build=dict( - stage='test', - image='yourlabs/shlax', - script='pip install -U .[test] && ' + PYTEST, - ), - test=dict( - stage='test', - image='yourlabs/python', - script='pip install -U .[test] && ' + PYTEST, - ), - pypi=dict( - stage='deploy', - image='yourlabs/python', - script='pypi-release', - only=['tags'] - ), + workdir='/app', ) test = Script( - gitlabci, - Run('gitlab-runner exec docker test'), + Pip('.[test]'), + Run(PYTEST), +) + +buildtest = Docker( + *test.actions, + mount={'.': '/app'}, + workdir='/app', +) + +pypi = Run( + 'pypi-release', + stage='deploy', + image='yourlabs/python', +) + +gitlabci = GitLabCI( + build=dict( + stage='build', + image='yourlabs/shlax', + ), + test=dict( + stage='test', + image='yourlabs/shlax', + ), + pypi=dict( + stage='deploy', + only=['tags'], + image='yourlabs/python', + ), )