diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cca2778..b8fb8b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,15 @@ build: + cache: + key: cache + paths: [.cache] image: yourlabs/shlax - script: pip install -U .[test] && py.test -svv tests - stage: test + script: pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d + shlax build push + stage: build pypi: image: yourlabs/python - only: - - tags + only: [tags] script: pypi-release stage: deploy -test: - image: yourlabs/python - script: pip install -U .[test] && py.test -svv tests - stage: test +test: {image: yourlabs/python, script: 'pip install -U --user -e .[test] && py.test + -svv tests', stage: build} diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index b021449..cbfc7b9 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,5 +1,5 @@ - id: shlaxfile-gitlabci name: Regenerate .gitlab-ci.yml description: Regenerate gitlabci - entry: pip install -e . && ./shlaxfile.py gitlabci + entry: ./shlaxfile.py gitlabci language: python diff --git a/setup.py b/setup.py index 078df4b..b15babf 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,9 @@ setup( setup_requires='setupmeta', install_requires=['cli2'], extras_require=dict( + full=[ + 'pyyaml', + ], test=[ 'pytest', 'pytest-cov', diff --git a/shlax/actions/__init__.py b/shlax/actions/__init__.py index 7519565..44accfa 100644 --- a/shlax/actions/__init__.py +++ b/shlax/actions/__init__.py @@ -1,4 +1,3 @@ -from .commit import Commit from .copy import Copy from .packages import Packages # noqa from .base import Action # noqa diff --git a/shlax/actions/base.py b/shlax/actions/base.py index 1b337aa..d2e0e76 100644 --- a/shlax/actions/base.py +++ b/shlax/actions/base.py @@ -1,4 +1,6 @@ +import functools import inspect +import importlib import sys from ..output import Output @@ -164,16 +166,46 @@ class Action: def callable(self): from ..targets import Localhost async def cb(*a, **k): - return await Localhost(self, quiet=True)(*a, **k) + from shlax.cli import cli + script = Localhost(self, quiet=True) + result = await script(*a, **k) + + success = functools.reduce( + lambda a, b: a + b, + [1 for c in script.children() if c.status == 'success'] or [0]) + if success: + script.output.success(f'{success} PASS') + + failures = functools.reduce( + lambda a, b: a + b, + [1 for c in script.children() if c.status == 'fail'] or [0]) + if failures: + script.output.fail(f'{failures} FAIL') + cli.exit_code = failures + return result return cb def kwargs_output(self): return self.kwargs def action(self, action, *args, **kwargs): + if isinstance(action, str): + import cli2 + a = cli2.Callable.factory(action).target + if not a: + a = cli2.Callable.factory( + '.'.join(['shlax', action]) + ).target + if a: + action = a + p = action(*args, **kwargs) for parent in self.parents(): if hasattr(parent, 'actions'): break p.parent = parent + if 'actions' not in self.__dict__: + # "mutate" to Strategy + from ..strategies.script import Actions + self.actions = Actions(self, [p]) return p diff --git a/shlax/actions/commit.py b/shlax/actions/commit.py deleted file mode 100644 index ee6dd55..0000000 --- a/shlax/actions/commit.py +++ /dev/null @@ -1,87 +0,0 @@ -import os -import subprocess - -from .base import Action - -from ..exceptions import WrongResult - -CI_VARS = ( - # gitlab - 'CI_COMMIT_SHORT_SHA', - 'CI_COMMIT_REF_NAME', - 'CI_COMMIT_TAG', - # CircleCI - 'CIRCLE_SHA1', - 'CIRCLE_TAG', - 'CIRCLE_BRANCH', -) - - -class Commit(Action): - def __init__(self, repo, tags=None, format=None, push=None, registry=None): - self.repo = repo - self.registry = registry or 'localhost' - self.push = push or os.getenv('CI') - - # figure out registry host - if '/' in self.repo and not registry: - first = self.repo.split('/')[0] - if '.' in first or ':' in first: - self.registry = self.repo.split('/')[0] - - # docker.io currently has issues with oci format - self.format = format or 'oci' - if self.registry == 'docker.io': - self.format = 'docker' - - self.tags = tags or [] - - # figure tags from CI vars - if not self.tags: - for name in CI_VARS: - value = os.getenv(name) - if value: - self.tags.append(value) - - # filter out tags which resolved to None - self.tags = [t for t in self.tags if t is not None] - - # default tag by default ... - if not self.tags: - self.tags = ['latest'] - - async def call(self, *args, ctr=None, **kwargs): - self.sha = (await self.parent.parent.exec( - 'buildah', - 'commit', - '--format=' + self.format, - ctr, - )).out - - if 'master' in self.tags: - self.tags.append('latest') - - if self.tags: - tags = ' '.join([f'{self.repo}:{tag}' for tag in self.tags]) - await script.exec('buildah', 'tag', self.sha, self.repo, tags) - - if self.push: - user = os.getenv('DOCKER_USER') - passwd = os.getenv('DOCKER_PASS') - if user and passwd and os.getenv('CI') and self.registry: - await script.exec( - 'podman', - 'login', - '-u', - user, - '-p', - passwd, - self.registry, - ) - - for tag in self.tags: - await script.exec('podman', 'push', f'{self.repo}:{tag}') - await script.umount() - - def __repr__(self): - return f'Commit({self.registry}/{self.repo}:{self.tags})' diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index c509a5a..fc11109 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -113,7 +113,7 @@ class Packages(Action): else: mgr = await self.which(*self.mgrs.keys()) if mgr: - self.mgr = mgr.split('/')[-1] + self.mgr = mgr[0].split('/')[-1] if not self.mgr: raise Exception('Packages does not yet support this distro') @@ -121,7 +121,8 @@ class Packages(Action): self.cmds = self.mgrs[self.mgr] if not getattr(self, '_packages_upgraded', None): await self.update() - await self.rexec(self.cmds['upgrade']) + if self.kwargs.get('upgrade', True): + await self.rexec(self.cmds['upgrade']) self._packages_upgraded = True packages = [] @@ -141,13 +142,13 @@ class Packages(Action): cachedir = os.path.join(self.cache_root, self.mgr) await self.mount(cachedir, '/var/cache/apk') # special step to enable apk cache - await self.rexec('ln -s /var/cache/apk /etc/apk/cache') + await self.rexec('ln -sf /var/cache/apk /etc/apk/cache') return cachedir async def dnf_setup(self): cachedir = os.path.join(self.cache_root, self.mgr) await self.mount(cachedir, f'/var/cache/{self.mgr}') - await self.run('echo keepcache=True >> /etc/dnf/dnf.conf') + await self.rexec('echo keepcache=True >> /etc/dnf/dnf.conf') return cachedir async def apt_setup(self): diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py index 8fa1342..c74815c 100644 --- a/shlax/actions/pip.py +++ b/shlax/actions/pip.py @@ -5,25 +5,29 @@ from .base import Action class Pip(Action): - packages = dict( - apt=['python3-pip'], - ) - def __init__(self, *pip_packages, pip=None, requirements=None): - self.pip_packages = pip_packages self.requirements = requirements super().__init__(*pip_packages, pip=pip, requirements=requirements) async def call(self, *args, **kwargs): - self.pip = await self.which('pip3', 'pip', 'pip2') - if not self.pip: - from .packages import Packages - action = self.action(Packages, 'python3,apk', 'python3-pip,apt', args=args, kwargs=kwargs) - await action(*args, **kwargs) - - self.pip = await self.which('pip3', 'pip', 'pip2') - if not self.pip: - raise Exception('Could not install a pip command') + pip = self.kwargs.get('pip', None) + if not pip: + pip = await self.which('pip3', 'pip', 'pip2') + if pip: + pip = pip[0] + else: + from .packages import Packages + action = self.action( + Packages, + 'python3,apk', 'python3-pip,apt', + args=args, kwargs=kwargs + ) + await action(*args, **kwargs) + pip = await self.which('pip3', 'pip', 'pip2') + if not pip: + raise Exception('Could not install a pip command') + else: + pip = pip[0] if 'CACHE_DIR' in os.environ: cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') @@ -33,20 +37,21 @@ class Pip(Action): if getattr(self, 'mount', None): # we are in a target which shares a mount command await self.mount(cache, '/root/.cache/pip') - await self.exec(f'{self.pip} install --upgrade pip') + await self.exec(f'{pip} install --upgrade pip') # https://github.com/pypa/pip/issues/5599 - self.pip = 'python3 -m pip' + if 'pip' not in self.kwargs: + pip = 'python3 -m pip' - source = [p for p in self.pip_packages if p.startswith('/')] + source = [p for p in self.args if p.startswith('/') or p.startswith('.')] if source: await self.exec( - f'{self.pip} install --upgrade --editable {" ".join(source)}' + f'{pip} install --upgrade --editable {" ".join(source)}' ) - nonsource = [p for p in self.pip_packages if not p.startswith('/')] + nonsource = [p for p in self.args if not p.startswith('/')] if nonsource: - await self.exec(f'{self.pip} install --upgrade {" ".join(nonsource)}') + await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}') if self.requirements: - await self.exec(f'{self.pip} install --upgrade -r {self.requirements}') + await self.exec(f'{pip} install --upgrade -r {self.requirements}') diff --git a/shlax/actions/run.py b/shlax/actions/run.py index 83ea901..e3f8730 100644 --- a/shlax/actions/run.py +++ b/shlax/actions/run.py @@ -1,6 +1,18 @@ +from ..targets.buildah import Buildah +from ..targets.docker import Docker + from .base import Action class Run(Action): async def call(self, *args, **kwargs): - return await self.exec(*self.args, **self.kwargs) + image = self.kwargs.get('image', None) + if not image: + return await self.exec(*self.args, **self.kwargs) + if isinstance(image, Buildah): + breakpoint() + result = await self.action(image, *args, **kwargs) + + return await Docker( + image=image, + ).exec(*args, **kwargs) diff --git a/shlax/cli.py b/shlax/cli.py index fc43129..fa1767f 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -8,6 +8,7 @@ shlax is a micro-framework to orchestrate commands. import asyncio import cli2 +import copy import inspect import os import sys @@ -130,8 +131,10 @@ class ConsoleScript(cli2.ConsoleScript): return super().__call__(*args, **kwargs) def call(self, command): + kwargs = copy.copy(self.parser.funckwargs) + kwargs.update(self.parser.options) try: - return super().call(command) + return command(*self.parser.funcargs, **kwargs) except WrongResult as e: print(e) self.exit_code = e.proc.rc diff --git a/shlax/contrib/gitlab.py b/shlax/contrib/gitlab.py index e3b8891..000efc0 100644 --- a/shlax/contrib/gitlab.py +++ b/shlax/contrib/gitlab.py @@ -1,16 +1,37 @@ +from copy import deepcopy import yaml from shlax import * -class GitLabCIConfig(Script): +class GitLabCI(Script): async def call(self, *args, write=True, **kwargs): - output = yaml.dump(self.kwargs) + output = dict() + for key, value in self.kwargs.items(): + if isinstance(value, dict): + output[key] = deepcopy(value) + image = output[key].get('image', 'alpine') + if hasattr(image, 'image'): + output[key]['image'] = image.image.repository + ':$CI_COMMIT_SHORT_SHA' + else: + output[key] = value + + output = yaml.dump(output) if kwargs['debug'] is True: self.output(output) if write: with open('.gitlab-ci.yml', 'w+') as f: f.write(output) + from shlax.cli import cli + for arg in args: + job = self.kwargs[arg] + _args = [] + if not isinstance(job['image'], str): + image = str(job['image'].image) + else: + image = job['image'] + await self.action('Docker', Run(job['script']), image=image)(*_args, **kwargs) + def colorized(self): return type(self).__name__ diff --git a/shlax/image.py b/shlax/image.py index 9ba27c8..80b4562 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -55,3 +55,21 @@ class Image: # default tag by default ... if not self.tags: self.tags = ['latest'] + + async def __call__(self, action, *args, **kwargs): + args = list(args) + return await action.exec(*args, **self.kwargs) + + def __str__(self): + return f'{self.repository}:{self.tags[-1]}' + + async def push(self, *args, **kwargs): + user = os.getenv('DOCKER_USER') + passwd = os.getenv('DOCKER_PASS') + action = kwargs.get('action', self) + if user and passwd: + action.output.cmd('buildah login -u ... -p ...' + self.registry) + await action.exec('buildah', 'login', '-u', user, '-p', passwd, self.registry or 'docker.io', debug=False) + + for tag in self.tags: + await action.exec('buildah', 'push', f'{self.repository}:{tag}') diff --git a/shlax/output.py b/shlax/output.py index c9109ca..9a8955f 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -25,16 +25,17 @@ class Output: def colorize(self, code, content): return self.color(code) + content + self.color() - def __init__(self, prefix=None, regexps=None, debug=False, write=None, flush=None): + def __init__(self, prefix=None, regexps=None, debug=False, write=None, flush=None, **kwargs): self.prefix = prefix self.debug = debug self.prefix_length = 0 self.regexps = regexps or dict() self.write = write or sys.stdout.buffer.write self.flush = flush or sys.stdout.flush + self.kwargs = kwargs - def __call__(self, line, highlight=True, flush=True): - if self.prefix and self.prefix not in self.prefixes: + def prefix_line(self): + if self.prefix not in self.prefixes: self.prefixes[self.prefix] = self.prefix_colors[len(self.prefixes)] if len(self.prefix) > self.prefix_length: self.prefix_length = len(self.prefix) @@ -44,24 +45,24 @@ class Output: if prefix_padding: prefix_padding = ' ' + prefix_padding + ' ' - self.write(( - ( - prefix_color - + prefix_padding - + self.prefix - + ' ' - + self.colors['reset'] - + '| ' - if self.prefix - else '' - ) - + self.highlight(line, highlight) - + self.colors['reset'] - ).encode('utf8')) + return [ + prefix_color, + prefix_padding, + self.prefix, + ' ', + self.colors['reset'], + '| ' + ] + + def __call__(self, line, highlight=True, flush=True): + line = [self.highlight(line) if highlight else line] + if self.prefix: + line = self.prefix_line() + line + line = ''.join(line) + + self.write(line.encode('utf8')) if flush: - if not line.endswith('\n'): - self.write(b'\n') self.flush() def cmd(self, line): @@ -70,7 +71,8 @@ class Output: + '\x1b[1;38;5;15m' + ' ' + self.highlight(line, 'bash') - + self.colors['reset'], + + self.colors['reset'] + + '\n', highlight=False ) @@ -92,25 +94,38 @@ class Output: for regexp, colors in self.regexps.items(): line = re.sub(regexp, colors.format(**self.colors), line) + line = line + self.colors['reset'] return line - def clean(self, action): - if self.debug is True or 'visit' in str(self.debug): + def test(self, action): + if self.debug is True: self(''.join([ - self.colors['bluebold'], - '+ CLEAN ', + self.colors['purplebold'], + '! TEST ', self.colors['reset'], action.colorized(), + '\n', + ])) + + def clean(self, action): + if self.debug is True: + self(''.join([ + self.colors['bluebold'], + '+ CLEAN ', + self.colors['reset'], + action.colorized(), + '\n', ])) def start(self, action): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['orangebold'], - '⚠ START ', + '⚠ START ', self.colors['reset'], action.colorized(), + '\n', ])) def success(self, action): @@ -119,14 +134,16 @@ class Output: self.colors['greenbold'], '✔ SUCCESS ', self.colors['reset'], - action.colorized(), + action.colorized() if hasattr(action, 'colorized') else str(action), + '\n', ])) def fail(self, action, exception=None): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['redbold'], - '✘ FAIL ', + '✘ FAIL ', self.colors['reset'], - action.colorized(), + action.colorized() if hasattr(action, 'colorized') else str(action), + '\n', ])) diff --git a/shlax/strategies/__init__.py b/shlax/strategies/__init__.py index 3ee13c2..1712bfe 100644 --- a/shlax/strategies/__init__.py +++ b/shlax/strategies/__init__.py @@ -1,2 +1,3 @@ from .asyn import Async from .script import Script +from .pod import Pod, Container diff --git a/shlax/strategies/asyn.py b/shlax/strategies/asyn.py index ee45d87..06213e0 100644 --- a/shlax/strategies/asyn.py +++ b/shlax/strategies/asyn.py @@ -5,7 +5,7 @@ from .script import Script class Async(Script): async def call(self, *args, **kwargs): - return asyncio.gather(*[ - procs.append(action(*args, **kwargs)) + return await asyncio.gather(*[ + action(*args, **kwargs) for action in self.actions ]) diff --git a/shlax/strategies/pod.py b/shlax/strategies/pod.py new file mode 100644 index 0000000..97da07c --- /dev/null +++ b/shlax/strategies/pod.py @@ -0,0 +1,27 @@ +import os +from .script import Script + + +class Container(Script): + async def call(self, *args, **kwargs): + if not args or 'build' in args: + await self.kwargs['build'](**kwargs) + self.image = self.kwargs['build'].image + + if not args or 'test' in args: + self.output.test(self) + await self.action('Docker', + *self.kwargs['test'].actions, + image=self.image, + mount={'.': '/app'}, + workdir='/app', + )(**kwargs) + + if not args or 'push' in args: + await self.image.push(action=self) + + #name = kwargs.get('name', os.getcwd()).split('/')[-1] + + +class Pod(Script): + pass diff --git a/shlax/strategies/script.py b/shlax/strategies/script.py index 24ac09b..bd8b0d7 100644 --- a/shlax/strategies/script.py +++ b/shlax/strategies/script.py @@ -29,4 +29,6 @@ class Script(Action): async def call(self, *args, **kwargs): for action in self.actions: - await action(*args, **kwargs) + result = await action(*args, **kwargs) + if action.status != 'success': + break diff --git a/shlax/targets/__init__.py b/shlax/targets/__init__.py index c7fb36f..3f7353f 100644 --- a/shlax/targets/__init__.py +++ b/shlax/targets/__init__.py @@ -1,3 +1,4 @@ from .buildah import Buildah +from .docker import Docker from .localhost import Localhost from .ssh import Ssh diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index df55e92..bf5e28c 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -8,6 +8,8 @@ import subprocess import sys import textwrap +from ..actions.base import Action +from ..exceptions import Mistake from ..proc import Proc from ..image import Image from .localhost import Localhost @@ -18,16 +20,21 @@ class Buildah(Localhost): The build script iterates over visitors and runs the build functions, it also provides wrappers around the buildah command. """ - contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] + contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount', 'image'] - def __init__(self, base, *args, commit=None, push=False, **kwargs): + def __init__(self, base, *args, commit=None, push=False, cmd=None, **kwargs): + if isinstance(base, Action): + args = [base] + list(args) + base = 'alpine' # default selection in case of mistake super().__init__(*args, **kwargs) self.base = base self.mounts = dict() self.ctr = None self.mnt = None self.image = Image(commit) if commit else None - self.push = push or os.getenv('CI') + self.config= dict( + cmd=cmd or 'sh', + ) def shargs(self, *args, user=None, buildah=True, **kwargs): if not buildah or args[0].startswith('buildah'): @@ -52,9 +59,6 @@ class Buildah(Localhost): """Run buildah config.""" return await self.exec(f'buildah config {line} {self.ctr}', buildah=False) - async def mkdir(self, *dirs): - return await self.exec(*['mkdir', '-p'] + list(dirs)) - async def copy(self, *args): """Run buildah copy to copy a file from host into container.""" src = args[:-1] @@ -81,29 +85,14 @@ class Buildah(Localhost): await self.exec(f'mount -o bind {src} {target}', buildah=False) self.mounts[src] = dst - async def which(self, *cmd): - """ - Return the first path to the cmd in the container. - - If cmd argument is a list then it will try all commands. - """ - paths = (await self.env('PATH')).split(':') - for path in paths: - for c in cmd: - p = os.path.join(self.mnt, path[1:], c) - if os.path.exists(p): - return p[len(str(self.mnt)):] - - def is_wrapper(self): - return not ( + def is_runnable(self): + return ( Proc.test or os.getuid() == 0 - or getattr(self.parent, 'parent', None) ) - async def call(self, *args, **kwargs): - if not self.is_wrapper(): + if self.is_runnable(): self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out self.mnt = Path((await self.exec('buildah', 'mount', self.ctr, buildah=False)).out) result = await super().call(*args, **kwargs) @@ -124,21 +113,16 @@ class Buildah(Localhost): argv += [ cli.parser.command.name, # script name ? ] - self.output(' '.join(argv), 'EXECUTION', flush=True) - proc = await asyncio.create_subprocess_shell( - shlex.join(argv), - stderr=sys.stderr, - stdin=sys.stdin, - stdout=sys.stdout, - ) - await proc.communicate() - cli.exit_code = await proc.wait() + await self.exec(*argv) async def commit(self): if not self.image: return + for key, value in self.config.items(): + await self.exec(f'buildah config --{key} "{value}" {self.ctr}') + self.sha = (await self.exec( 'buildah', 'commit', @@ -148,39 +132,25 @@ class Buildah(Localhost): )).out if self.image.tags: - tags = ' '.join([f'{self.image.repository}:{tag}' for tag in self.image.tags]) - await self.exec('buildah', 'tag', self.sha, self.image.repository, tags, buildah=False) + tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags] + else: + tags = [self.image.repository] - if self.push: - user = os.getenv('DOCKER_USER') - passwd = os.getenv('DOCKER_PASS') - if user and passwd and os.getenv('CI') and self.registry: - await self.exec( - 'podman', - 'login', - '-u', - user, - '-p', - passwd, - self.registry, - buildah=False, - ) - - for tag in self.image.tags: - await self.exec('podman', 'push', f'{self.image.repository}:{tag}', buildah=False) + for tag in tags: + await self.exec('buildah', 'tag', self.sha, tag, buildah=False) async def clean(self, *args, **kwargs): - if self.is_wrapper(): - return + if self.is_runnable(): + for src, dst in self.mounts.items(): + await self.exec('umount', self.mnt / str(dst)[1:], buildah=False) - for src, dst in self.mounts.items(): - await self.exec('umount', self.mnt / str(dst)[1:], buildah=False) + if self.status == 'success': + await self.commit() + if 'push' in args: + await self.image.push(action=self) - if self.status == 'success': - await self.commit() + if self.mnt is not None: + await self.exec('buildah', 'umount', self.ctr, buildah=False) - if self.mnt is not None: - await self.exec('buildah', 'umount', self.ctr, buildah=False) - - if self.ctr is not None: - await self.exec('buildah', 'rm', self.ctr, buildah=False) + if self.ctr is not None: + await self.exec('buildah', 'rm', self.ctr, buildah=False) diff --git a/shlax/targets/docker.py b/shlax/targets/docker.py new file mode 100644 index 0000000..331c8b9 --- /dev/null +++ b/shlax/targets/docker.py @@ -0,0 +1,76 @@ +import asyncio +from pathlib import Path +import os + +from ..image import Image +from .localhost import Localhost + + +class Docker(Localhost): + contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] + + def __init__(self, *args, **kwargs): + self.image = kwargs.get('image', 'alpine') + if not isinstance(self.image, Image): + self.image = Image(self.image) + super().__init__(*args, **kwargs) + self.context['ctr'] = None + + def shargs(self, *args, daemon=False, **kwargs): + if args[0] == 'docker': + return args, kwargs + + extra = [] + if 'user' in kwargs: + extra += ['--user', kwargs.pop('user')] + + args, kwargs = super().shargs(*args, **kwargs) + + if self.context['ctr']: + executor = 'exec' + extra = [self.context['ctr']] + return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + list(args), kwargs + + executor = 'run' + cwd = os.getcwd() + if daemon: + extra += ['-d'] + extra = extra + ['-v', f'{cwd}:{cwd}', '-w', f'{cwd}'] + return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + [str(self.image)] + list(args), kwargs + + async def call(self, *args, **kwargs): + name = kwargs.get('name', os.getcwd()).split('/')[-1] + self.context['ctr'] = ( + await self.exec( + 'docker', 'ps', '-aq', '--filter', + 'name=' + name, + raises=False + ) + ).out.split('\n')[0] + + if 'recreate' in args and self.context['ctr']: + await self.exec('docker', 'rm', '-f', self.context['ctr']) + self.context['ctr'] = None + + if self.context['ctr']: + self.context['ctr'] = (await self.exec('docker', 'start', name)).out + return await super().call(*args, **kwargs) + + async def copy(self, *args): + src = args[:-1] + dst = args[-1] + await self.mkdir(dst) + + procs = [] + for s in src: + ''' + if Path(s).is_dir(): + await self.mkdir(s) + args = ['docker', 'copy', self.ctr, s, Path(dst) / s] + else: + args = ['docker', 'copy', self.ctr, s, dst] + ''' + args = ['docker', 'cp', s, self.context['ctr'] + ':' + dst] + procs.append(self.exec(*args)) + + return await asyncio.gather(*procs) diff --git a/shlax/targets/localhost.py b/shlax/targets/localhost.py index 892383f..d424179 100644 --- a/shlax/targets/localhost.py +++ b/shlax/targets/localhost.py @@ -1,4 +1,5 @@ import os +import re from shlax.proc import Proc @@ -29,7 +30,8 @@ class Localhost(Script): return args, kwargs async def exec(self, *args, **kwargs): - kwargs.setdefault('debug', self.call_kwargs.get('debug', False)) + if 'debug' not in kwargs: + kwargs['debug'] = getattr(self, 'call_kwargs', {}).get('debug', False) kwargs.setdefault('output', self.output) args, kwargs = self.shargs(*args, **kwargs) proc = await Proc(*args, **kwargs)() @@ -50,12 +52,20 @@ class Localhost(Script): If cmd argument is a list then it will try all commands. """ - for path in (await self.env('PATH')).split(':'): - for c in cmd: - p = os.path.join(self.root, path[1:], c) - if os.path.exists(p): - return p[len(str(self.root)):] + proc = await self.exec('type ' + ' '.join(cmd), raises=False) + result = [] + for res in proc.out.split('\n'): + match = re.match('([^ ]+) is ([^ ]+)$', res.strip()) + if match: + result.append(match.group(1)) + return result async def copy(self, *args): args = ['cp', '-ra'] + list(args) return await self.exec(*args) + + async def mount(self, *dirs): + pass + + async def mkdir(self, *dirs): + return await self.exec(*['mkdir', '-p'] + list(dirs)) diff --git a/shlaxfile.py b/shlaxfile.py index 4ae04e5..3921ed5 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -3,32 +3,53 @@ from shlax.contrib.gitlab import * PYTEST = 'py.test -svv tests' -build = Buildah('alpine', - Copy('shlax/', 'setup.py', '/app'), - Pip('/app'), - commit='yourlabs/shlax', +test = Script( + Pip('.[test]'), + Run(PYTEST), ) -gitlabci = GitLabCIConfig( - build=dict( - stage='test', - image='yourlabs/shlax', - script='pip install -U .[test] && ' + PYTEST, +build = Buildah( + 'quay.io/podman/stable', + Packages('python38', 'buildah', 'unzip', 'findutils', 'python3-yaml', upgrade=False), + Async( + # python3.8 on centos with pip dance ... + Run(''' + curl -o setuptools.zip https://files.pythonhosted.org/packages/42/3e/2464120172859e5d103e5500315fb5555b1e908c0dacc73d80d35a9480ca/setuptools-45.1.0.zip + unzip setuptools.zip + mkdir -p /usr/local/lib/python3.8/site-packages/ + sh -c "cd setuptools-* && python3.8 setup.py install" + easy_install-3.8 pip + echo python3.8 -m pip > /usr/bin/pip + chmod +x /usr/bin/pip + '''), + Copy('shlax/', 'setup.py', '/app'), ), + Pip('/app[full]'), + commit='docker.io/yourlabs/shlax', + workdir='/app', +) + +shlax = Container( + build=build, + test=Script(Run('./shlaxfile.py -d test')), +) + +gitlabci = GitLabCI( test=dict( - stage='test', + stage='build', + script='pip install -U --user -e .[test] && ' + PYTEST, image='yourlabs/python', - script='pip install -U .[test] && ' + PYTEST, + ), + build=dict( + stage='build', + image='yourlabs/shlax', + script='pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push', + cache=dict(paths=['.cache'], key='cache'), ), pypi=dict( stage='deploy', + only=['tags'], image='yourlabs/python', script='pypi-release', - only=['tags'] ), ) - -test = Script( - gitlabci, - Run('gitlab-runner exec docker test'), -)