Merge branch 'jpic' into 'master'

yourlabs/shlax auto build

See merge request oss/shlax!1

.gitlab-ci.yml

@@ -1,14 +1,15 @@
 build:
+  cache:
+    key: cache
+    paths: [.cache]
   image: yourlabs/shlax
-  script: pip install -U .[test] && py.test -svv tests
-  stage: test
+  script: pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d
+    shlax build push
+  stage: build
 pypi:
   image: yourlabs/python
-  only:
-  - tags
+  only: [tags]
   script: pypi-release
   stage: deploy
-test:
-  image: yourlabs/python
-  script: pip install -U .[test] && py.test -svv tests
-  stage: test
+test: {image: yourlabs/python, script: 'pip install -U --user -e .[test] && py.test
+    -svv tests', stage: build}

.pre-commit-hooks.yaml

@@ -1,5 +1,5 @@
 - id: shlaxfile-gitlabci
   name: Regenerate .gitlab-ci.yml
   description: Regenerate gitlabci
-  entry: pip install -e . && ./shlaxfile.py gitlabci
+  entry: ./shlaxfile.py gitlabci
   language: python

setup.py

@@ -7,6 +7,9 @@ setup(
     setup_requires='setupmeta',
     install_requires=['cli2'],
     extras_require=dict(
+        full=[
+            'pyyaml',
+        ],
         test=[
             'pytest',
             'pytest-cov',

shlax/actions/__init__.py

@@ -1,4 +1,3 @@
-from .commit import Commit
 from .copy import Copy
 from .packages import Packages  # noqa
 from .base import Action  # noqa

shlax/actions/base.py

@@ -1,4 +1,6 @@
+import functools
 import inspect
+import importlib
 import sys
 
 from ..output import Output
@@ -164,16 +166,46 @@ class Action:
     def callable(self):
         from ..targets import Localhost
         async def cb(*a, **k):
-            return await Localhost(self, quiet=True)(*a, **k)
+            from shlax.cli import cli
+            script = Localhost(self, quiet=True)
+            result = await script(*a, **k)
+
+            success = functools.reduce(
+                lambda a, b: a + b,
+                [1 for c in script.children() if c.status == 'success'] or [0])
+            if success:
+                script.output.success(f'{success} PASS')
+
+            failures = functools.reduce(
+                lambda a, b: a + b,
+                [1 for c in script.children() if c.status == 'fail'] or [0])
+            if failures:
+                script.output.fail(f'{failures} FAIL')
+            cli.exit_code = failures
+            return result
         return cb
 
     def kwargs_output(self):
         return self.kwargs
 
     def action(self, action, *args, **kwargs):
+        if isinstance(action, str):
+            import cli2
+            a = cli2.Callable.factory(action).target
+            if not a:
+                a = cli2.Callable.factory(
+                    '.'.join(['shlax', action])
+                ).target
+            if a:
+                action = a
+
         p = action(*args, **kwargs)
         for parent in self.parents():
             if hasattr(parent, 'actions'):
                 break
         p.parent = parent
+        if 'actions' not in self.__dict__:
+            # "mutate" to Strategy
+            from ..strategies.script import Actions
+            self.actions = Actions(self, [p])
         return p

shlax/actions/commit.py

@@ -1,87 +0,0 @@
-import os
-import subprocess
-
-from .base import Action
-
-from ..exceptions import WrongResult
-
-CI_VARS = (
-    # gitlab
-    'CI_COMMIT_SHORT_SHA',
-    'CI_COMMIT_REF_NAME',
-    'CI_COMMIT_TAG',
-    # CircleCI
-    'CIRCLE_SHA1',
-    'CIRCLE_TAG',
-    'CIRCLE_BRANCH',
-)
-
-
-class Commit(Action):
-    def __init__(self, repo, tags=None, format=None, push=None, registry=None):
-        self.repo = repo
-        self.registry = registry or 'localhost'
-        self.push = push or os.getenv('CI')
-
-        # figure out registry host
-        if '/' in self.repo and not registry:
-            first = self.repo.split('/')[0]
-            if '.' in first or ':' in first:
-                self.registry = self.repo.split('/')[0]
-
-        # docker.io currently has issues with oci format
-        self.format = format or 'oci'
-        if self.registry == 'docker.io':
-            self.format = 'docker'
-
-        self.tags = tags or []
-
-        # figure tags from CI vars
-        if not self.tags:
-            for name in CI_VARS:
-                value = os.getenv(name)
-                if value:
-                    self.tags.append(value)
-
-        # filter out tags which resolved to None
-        self.tags = [t for t in self.tags if t is not None]
-
-        # default tag by default ...
-        if not self.tags:
-            self.tags = ['latest']
-
-    async def call(self, *args, ctr=None, **kwargs):
-        self.sha = (await self.parent.parent.exec(
-            'buildah',
-            'commit',
-            '--format=' + self.format,
-            ctr,
-        )).out
-
-        if 'master' in self.tags:
-            self.tags.append('latest')
-
-        if self.tags:
-            tags = ' '.join([f'{self.repo}:{tag}' for tag in self.tags])
-            await script.exec('buildah', 'tag', self.sha, self.repo, tags)
-
-            if self.push:
-                user = os.getenv('DOCKER_USER')
-                passwd = os.getenv('DOCKER_PASS')
-                if user and passwd and os.getenv('CI') and self.registry:
-                    await script.exec(
-                        'podman',
-                        'login',
-                        '-u',
-                        user,
-                        '-p',
-                        passwd,
-                        self.registry,
-                    )
-
-                for tag in self.tags:
-                    await script.exec('podman', 'push', f'{self.repo}:{tag}')
-        await script.umount()
-
-    def __repr__(self):
-        return f'Commit({self.registry}/{self.repo}:{self.tags})'

shlax/actions/packages.py

@@ -113,7 +113,7 @@ class Packages(Action):
         else:
             mgr = await self.which(*self.mgrs.keys())
             if mgr:
-                self.mgr = mgr.split('/')[-1]
+                self.mgr = mgr[0].split('/')[-1]
 
         if not self.mgr:
             raise Exception('Packages does not yet support this distro')
@@ -121,6 +121,7 @@ class Packages(Action):
         self.cmds = self.mgrs[self.mgr]
         if not getattr(self, '_packages_upgraded', None):
             await self.update()
+            if self.kwargs.get('upgrade', True):
                 await self.rexec(self.cmds['upgrade'])
             self._packages_upgraded = True
 
@@ -141,13 +142,13 @@ class Packages(Action):
         cachedir = os.path.join(self.cache_root, self.mgr)
         await self.mount(cachedir, '/var/cache/apk')
         # special step to enable apk cache
-        await self.rexec('ln -s /var/cache/apk /etc/apk/cache')
+        await self.rexec('ln -sf /var/cache/apk /etc/apk/cache')
         return cachedir
 
     async def dnf_setup(self):
         cachedir = os.path.join(self.cache_root, self.mgr)
         await self.mount(cachedir, f'/var/cache/{self.mgr}')
-        await self.run('echo keepcache=True >> /etc/dnf/dnf.conf')
+        await self.rexec('echo keepcache=True >> /etc/dnf/dnf.conf')
         return cachedir
 
     async def apt_setup(self):

shlax/actions/pip.py

@@ -5,25 +5,29 @@ from .base import Action
 
 
 class Pip(Action):
-    packages = dict(
-        apt=['python3-pip'],
-    )
-
     def __init__(self, *pip_packages, pip=None, requirements=None):
-        self.pip_packages = pip_packages
         self.requirements = requirements
         super().__init__(*pip_packages, pip=pip, requirements=requirements)
 
     async def call(self, *args, **kwargs):
-        self.pip = await self.which('pip3', 'pip', 'pip2')
-        if not self.pip:
+        pip = self.kwargs.get('pip', None)
+        if not pip:
+            pip = await self.which('pip3', 'pip', 'pip2')
+            if pip:
+                pip = pip[0]
+            else:
                 from .packages import Packages
-            action = self.action(Packages, 'python3,apk', 'python3-pip,apt', args=args, kwargs=kwargs)
+                action = self.action(
+                    Packages,
+                    'python3,apk', 'python3-pip,apt',
+                    args=args, kwargs=kwargs
+                )
                 await action(*args, **kwargs)
-
-        self.pip = await self.which('pip3', 'pip', 'pip2')
-        if not self.pip:
+                pip = await self.which('pip3', 'pip', 'pip2')
+                if not pip:
                     raise Exception('Could not install a pip command')
+                else:
+                    pip = pip[0]
 
         if 'CACHE_DIR' in os.environ:
             cache = os.path.join(os.getenv('CACHE_DIR'), 'pip')
@@ -33,20 +37,21 @@ class Pip(Action):
         if getattr(self, 'mount', None):
             # we are in a target which shares a mount command
             await self.mount(cache, '/root/.cache/pip')
-        await self.exec(f'{self.pip} install --upgrade pip')
+        await self.exec(f'{pip} install --upgrade pip')
 
         # https://github.com/pypa/pip/issues/5599
-        self.pip = 'python3 -m pip'
+        if 'pip' not in self.kwargs:
+            pip = 'python3 -m pip'
 
-        source = [p for p in self.pip_packages if p.startswith('/')]
+        source = [p for p in self.args if p.startswith('/') or p.startswith('.')]
         if source:
             await self.exec(
-                f'{self.pip} install --upgrade --editable {" ".join(source)}'
+                f'{pip} install --upgrade --editable {" ".join(source)}'
             )
 
-        nonsource = [p for p in self.pip_packages if not p.startswith('/')]
+        nonsource = [p for p in self.args if not p.startswith('/')]
         if nonsource:
-            await self.exec(f'{self.pip} install --upgrade {" ".join(nonsource)}')
+            await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}')
 
         if self.requirements:
-            await self.exec(f'{self.pip} install --upgrade -r {self.requirements}')
+            await self.exec(f'{pip} install --upgrade -r {self.requirements}')

shlax/actions/run.py

@@ -1,6 +1,18 @@
+from ..targets.buildah import Buildah
+from ..targets.docker import Docker
+
 from .base import Action
 
 
 class Run(Action):
     async def call(self, *args, **kwargs):
+        image = self.kwargs.get('image', None)
+        if not image:
             return await self.exec(*self.args, **self.kwargs)
+        if isinstance(image, Buildah):
+            breakpoint()
+            result = await self.action(image, *args, **kwargs)
+
+        return await Docker(
+            image=image,
+        ).exec(*args, **kwargs)

shlax/cli.py

@@ -8,6 +8,7 @@ shlax is a micro-framework to orchestrate commands.
 
 import asyncio
 import cli2
+import copy
 import inspect
 import os
 import sys
@@ -130,8 +131,10 @@ class ConsoleScript(cli2.ConsoleScript):
         return super().__call__(*args, **kwargs)
 
     def call(self, command):
+        kwargs = copy.copy(self.parser.funckwargs)
+        kwargs.update(self.parser.options)
         try:
-            return super().call(command)
+            return command(*self.parser.funcargs, **kwargs)
         except WrongResult as e:
             print(e)
             self.exit_code = e.proc.rc

shlax/contrib/gitlab.py

@@ -1,16 +1,37 @@
+from copy import deepcopy
 import yaml
 
 from shlax import *
 
 
-class GitLabCIConfig(Script):
+class GitLabCI(Script):
     async def call(self, *args, write=True, **kwargs):
-        output = yaml.dump(self.kwargs)
+        output = dict()
+        for key, value in self.kwargs.items():
+            if isinstance(value, dict):
+                output[key] = deepcopy(value)
+                image = output[key].get('image', 'alpine')
+                if hasattr(image, 'image'):
+                    output[key]['image'] = image.image.repository + ':$CI_COMMIT_SHORT_SHA'
+            else:
+                output[key] = value
+
+        output = yaml.dump(output)
         if kwargs['debug'] is True:
             self.output(output)
         if write:
             with open('.gitlab-ci.yml', 'w+') as f:
                 f.write(output)
 
+        from shlax.cli import cli
+        for arg in args:
+            job = self.kwargs[arg]
+            _args = []
+            if not isinstance(job['image'], str):
+                image = str(job['image'].image)
+            else:
+                image = job['image']
+            await self.action('Docker', Run(job['script']), image=image)(*_args, **kwargs)
+
     def colorized(self):
         return type(self).__name__

shlax/image.py

@@ -55,3 +55,21 @@ class Image:
         # default tag by default ...
         if not self.tags:
             self.tags = ['latest']
+
+    async def __call__(self, action, *args, **kwargs):
+        args = list(args)
+        return await action.exec(*args, **self.kwargs)
+
+    def __str__(self):
+        return f'{self.repository}:{self.tags[-1]}'
+
+    async def push(self, *args, **kwargs):
+        user = os.getenv('DOCKER_USER')
+        passwd = os.getenv('DOCKER_PASS')
+        action = kwargs.get('action', self)
+        if user and passwd:
+            action.output.cmd('buildah login -u ... -p ...' + self.registry)
+            await action.exec('buildah', 'login', '-u', user, '-p', passwd, self.registry or 'docker.io', debug=False)
+
+        for tag in self.tags:
+            await action.exec('buildah', 'push', f'{self.repository}:{tag}')

shlax/output.py

@@ -25,16 +25,17 @@ class Output:
     def colorize(self, code, content):
         return self.color(code) + content + self.color()
 
-    def __init__(self, prefix=None, regexps=None, debug=False, write=None, flush=None):
+    def __init__(self, prefix=None, regexps=None, debug=False, write=None, flush=None, **kwargs):
         self.prefix = prefix
         self.debug = debug
         self.prefix_length = 0
         self.regexps = regexps or dict()
         self.write = write or sys.stdout.buffer.write
         self.flush = flush or sys.stdout.flush
+        self.kwargs = kwargs
 
-    def __call__(self, line, highlight=True, flush=True):
-        if self.prefix and self.prefix not in self.prefixes:
+    def prefix_line(self):
+        if self.prefix not in self.prefixes:
             self.prefixes[self.prefix] = self.prefix_colors[len(self.prefixes)]
             if len(self.prefix) > self.prefix_length:
                 self.prefix_length = len(self.prefix)
@@ -44,24 +45,24 @@ class Output:
         if prefix_padding:
             prefix_padding = ' ' + prefix_padding + ' '
 
-        self.write((
-            (
-                prefix_color
-                + prefix_padding
-                + self.prefix
-                + ' '
-                + self.colors['reset']
-                + '| '
-                if self.prefix
-                else ''
-            )
-            + self.highlight(line, highlight)
-            + self.colors['reset']
-        ).encode('utf8'))
+        return [
+            prefix_color,
+            prefix_padding,
+            self.prefix,
+            ' ',
+            self.colors['reset'],
+            '| '
+        ]
+
+    def __call__(self, line, highlight=True, flush=True):
+        line = [self.highlight(line) if highlight else line]
+        if self.prefix:
+            line = self.prefix_line() + line
+        line = ''.join(line)
+
+        self.write(line.encode('utf8'))
 
         if flush:
-            if not line.endswith('\n'):
-                self.write(b'\n')
             self.flush()
 
     def cmd(self, line):
@@ -70,7 +71,8 @@ class Output:
             + '\x1b[1;38;5;15m'
             + ' '
             + self.highlight(line, 'bash')
-            + self.colors['reset'],
+            + self.colors['reset']
+            + '\n',
             highlight=False
         )
 
@@ -92,16 +94,28 @@ class Output:
 
         for regexp, colors in self.regexps.items():
             line = re.sub(regexp, colors.format(**self.colors), line)
+        line = line + self.colors['reset']
 
         return line
 
+    def test(self, action):
+        if self.debug is True:
+            self(''.join([
+                self.colors['purplebold'],
+                '!  TEST   ',
+                self.colors['reset'],
+                action.colorized(),
+                '\n',
+            ]))
+
     def clean(self, action):
-        if self.debug is True or 'visit' in str(self.debug):
+        if self.debug is True:
             self(''.join([
                 self.colors['bluebold'],
                 '+  CLEAN  ',
                 self.colors['reset'],
                 action.colorized(),
+                '\n',
             ]))
 
     def start(self, action):
@@ -111,6 +125,7 @@ class Output:
                 '⚠  START  ',
                 self.colors['reset'],
                 action.colorized(),
+                '\n',
             ]))
 
     def success(self, action):
@@ -119,7 +134,8 @@ class Output:
                 self.colors['greenbold'],
                 '✔ SUCCESS ',
                 self.colors['reset'],
-                action.colorized(),
+                action.colorized() if hasattr(action, 'colorized') else str(action),
+                '\n',
             ]))
 
     def fail(self, action, exception=None):
@@ -128,5 +144,6 @@ class Output:
                 self.colors['redbold'],
                 '✘  FAIL   ',
                 self.colors['reset'],
-                action.colorized(),
+                action.colorized() if hasattr(action, 'colorized') else str(action),
+                '\n',
             ]))

shlax/strategies/__init__.py

@@ -1,2 +1,3 @@
 from .asyn import Async
 from .script import Script
+from .pod import Pod, Container

shlax/strategies/asyn.py

@@ -5,7 +5,7 @@ from .script import Script
 
 class Async(Script):
     async def call(self, *args, **kwargs):
-        return asyncio.gather(*[
-            procs.append(action(*args, **kwargs))
+        return await asyncio.gather(*[
+            action(*args, **kwargs)
             for action in self.actions
         ])

shlax/strategies/pod.py

@@ -0,0 +1,27 @@
+import os
+from .script import Script
+
+
+class Container(Script):
+    async def call(self, *args, **kwargs):
+        if not args or 'build' in args:
+            await self.kwargs['build'](**kwargs)
+        self.image = self.kwargs['build'].image
+
+        if not args or 'test' in args:
+            self.output.test(self)
+            await self.action('Docker',
+                *self.kwargs['test'].actions,
+                image=self.image,
+                mount={'.': '/app'},
+                workdir='/app',
+            )(**kwargs)
+
+        if not args or 'push' in args:
+            await self.image.push(action=self)
+
+        #name = kwargs.get('name', os.getcwd()).split('/')[-1]
+
+
+class Pod(Script):
+    pass

shlax/strategies/script.py

@@ -29,4 +29,6 @@ class Script(Action):
 
     async def call(self, *args, **kwargs):
         for action in self.actions:
-            await action(*args, **kwargs)
+            result = await action(*args, **kwargs)
+            if action.status != 'success':
+                break

shlax/targets/__init__.py

@@ -1,3 +1,4 @@
 from .buildah import Buildah
+from .docker import Docker
 from .localhost import Localhost
 from .ssh import Ssh

shlax/targets/buildah.py

@@ -8,6 +8,8 @@ import subprocess
 import sys
 import textwrap
 
+from ..actions.base import Action
+from ..exceptions import Mistake
 from ..proc import Proc
 from ..image import Image
 from .localhost import Localhost
@@ -18,16 +20,21 @@ class Buildah(Localhost):
     The build script iterates over visitors and runs the build functions, it
     also provides wrappers around the buildah command.
     """
-    contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount']
+    contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount', 'image']
 
-    def __init__(self, base, *args, commit=None, push=False, **kwargs):
+    def __init__(self, base, *args, commit=None, push=False, cmd=None, **kwargs):
+        if isinstance(base, Action):
+            args = [base] + list(args)
+            base = 'alpine'  # default selection in case of mistake
         super().__init__(*args, **kwargs)
         self.base = base
         self.mounts = dict()
         self.ctr = None
         self.mnt = None
         self.image = Image(commit) if commit else None
-        self.push = push or os.getenv('CI')
+        self.config= dict(
+            cmd=cmd or 'sh',
+        )
 
     def shargs(self, *args, user=None, buildah=True, **kwargs):
         if not buildah or args[0].startswith('buildah'):
@@ -52,9 +59,6 @@ class Buildah(Localhost):
         """Run buildah config."""
         return await self.exec(f'buildah config {line} {self.ctr}', buildah=False)
 
-    async def mkdir(self, *dirs):
-        return await self.exec(*['mkdir', '-p'] + list(dirs))
-
     async def copy(self, *args):
         """Run buildah copy to copy a file from host into container."""
         src = args[:-1]
@@ -81,29 +85,14 @@ class Buildah(Localhost):
         await self.exec(f'mount -o bind {src} {target}', buildah=False)
         self.mounts[src] = dst
 
-    async def which(self, *cmd):
-        """
-        Return the first path to the cmd in the container.
-
-        If cmd argument is a list then it will try all commands.
-        """
-        paths = (await self.env('PATH')).split(':')
-        for path in paths:
-            for c in cmd:
-                p = os.path.join(self.mnt, path[1:], c)
-                if os.path.exists(p):
-                    return p[len(str(self.mnt)):]
-
-    def is_wrapper(self):
-        return not (
+    def is_runnable(self):
+        return (
             Proc.test
             or os.getuid() == 0
-            or getattr(self.parent, 'parent', None)
         )
 
-
     async def call(self, *args, **kwargs):
-        if not self.is_wrapper():
+        if self.is_runnable():
             self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out
             self.mnt = Path((await self.exec('buildah', 'mount', self.ctr, buildah=False)).out)
             result = await super().call(*args, **kwargs)
@@ -124,21 +113,16 @@ class Buildah(Localhost):
         argv += [
             cli.parser.command.name,  # script name ?
         ]
-        self.output(' '.join(argv), 'EXECUTION', flush=True)
 
-        proc = await asyncio.create_subprocess_shell(
-            shlex.join(argv),
-            stderr=sys.stderr,
-            stdin=sys.stdin,
-            stdout=sys.stdout,
-        )
-        await proc.communicate()
-        cli.exit_code = await proc.wait()
+        await self.exec(*argv)
 
     async def commit(self):
         if not self.image:
             return
 
+        for key, value in self.config.items():
+            await self.exec(f'buildah config --{key} "{value}" {self.ctr}')
+
         self.sha = (await self.exec(
             'buildah',
             'commit',
@@ -148,36 +132,22 @@ class Buildah(Localhost):
         )).out
 
         if self.image.tags:
-            tags = ' '.join([f'{self.image.repository}:{tag}' for tag in self.image.tags])
-            await self.exec('buildah', 'tag', self.sha, self.image.repository, tags, buildah=False)
+            tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags]
+        else:
+            tags = [self.image.repository]
 
-            if self.push:
-                user = os.getenv('DOCKER_USER')
-                passwd = os.getenv('DOCKER_PASS')
-                if user and passwd and os.getenv('CI') and self.registry:
-                    await self.exec(
-                        'podman',
-                        'login',
-                        '-u',
-                        user,
-                        '-p',
-                        passwd,
-                        self.registry,
-                        buildah=False,
-                    )
-
-                for tag in self.image.tags:
-                    await self.exec('podman', 'push', f'{self.image.repository}:{tag}', buildah=False)
+        for tag in tags:
+            await self.exec('buildah', 'tag', self.sha, tag, buildah=False)
 
     async def clean(self, *args, **kwargs):
-        if self.is_wrapper():
-            return
-
+        if self.is_runnable():
             for src, dst in self.mounts.items():
                 await self.exec('umount', self.mnt / str(dst)[1:], buildah=False)
 
             if self.status == 'success':
                 await self.commit()
+                if 'push' in args:
+                    await self.image.push(action=self)
 
             if self.mnt is not None:
                 await self.exec('buildah', 'umount', self.ctr, buildah=False)

shlax/targets/docker.py

@@ -0,0 +1,76 @@
+import asyncio
+from pathlib import Path
+import os
+
+from ..image import Image
+from .localhost import Localhost
+
+
+class Docker(Localhost):
+    contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount']
+
+    def __init__(self, *args, **kwargs):
+        self.image = kwargs.get('image', 'alpine')
+        if not isinstance(self.image, Image):
+            self.image = Image(self.image)
+        super().__init__(*args, **kwargs)
+        self.context['ctr'] = None
+
+    def shargs(self, *args, daemon=False, **kwargs):
+        if args[0] == 'docker':
+            return args, kwargs
+
+        extra = []
+        if 'user' in kwargs:
+            extra += ['--user', kwargs.pop('user')]
+
+        args, kwargs = super().shargs(*args, **kwargs)
+
+        if self.context['ctr']:
+            executor = 'exec'
+            extra = [self.context['ctr']]
+            return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + list(args), kwargs
+
+        executor = 'run'
+        cwd = os.getcwd()
+        if daemon:
+            extra += ['-d']
+        extra = extra + ['-v', f'{cwd}:{cwd}', '-w', f'{cwd}']
+        return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + [str(self.image)] + list(args), kwargs
+
+    async def call(self, *args, **kwargs):
+        name = kwargs.get('name', os.getcwd()).split('/')[-1]
+        self.context['ctr'] = (
+            await self.exec(
+                'docker', 'ps', '-aq', '--filter',
+                'name=' + name,
+                raises=False
+            )
+        ).out.split('\n')[0]
+
+        if 'recreate' in args and self.context['ctr']:
+            await self.exec('docker', 'rm', '-f', self.context['ctr'])
+            self.context['ctr'] = None
+
+        if self.context['ctr']:
+            self.context['ctr'] = (await self.exec('docker', 'start', name)).out
+        return await super().call(*args, **kwargs)
+
+    async def copy(self, *args):
+        src = args[:-1]
+        dst = args[-1]
+        await self.mkdir(dst)
+
+        procs = []
+        for s in src:
+            '''
+            if Path(s).is_dir():
+                await self.mkdir(s)
+                args = ['docker', 'copy', self.ctr, s, Path(dst) / s]
+            else:
+                args = ['docker', 'copy', self.ctr, s, dst]
+            '''
+            args = ['docker', 'cp', s, self.context['ctr'] + ':' + dst]
+            procs.append(self.exec(*args))
+
+        return await asyncio.gather(*procs)

shlax/targets/localhost.py

@@ -1,4 +1,5 @@
 import os
+import re
 
 from shlax.proc import Proc
 
@@ -29,7 +30,8 @@ class Localhost(Script):
             return args, kwargs
 
     async def exec(self, *args, **kwargs):
-        kwargs.setdefault('debug', self.call_kwargs.get('debug', False))
+        if 'debug' not in kwargs:
+            kwargs['debug'] = getattr(self, 'call_kwargs', {}).get('debug', False)
         kwargs.setdefault('output', self.output)
         args, kwargs = self.shargs(*args, **kwargs)
         proc = await Proc(*args, **kwargs)()
@@ -50,12 +52,20 @@ class Localhost(Script):
 
         If cmd argument is a list then it will try all commands.
         """
-        for path in (await self.env('PATH')).split(':'):
-            for c in cmd:
-                p = os.path.join(self.root, path[1:], c)
-                if os.path.exists(p):
-                    return p[len(str(self.root)):]
+        proc = await self.exec('type ' + ' '.join(cmd), raises=False)
+        result = []
+        for res in proc.out.split('\n'):
+            match = re.match('([^ ]+) is ([^ ]+)$', res.strip())
+            if match:
+                result.append(match.group(1))
+        return result
 
     async def copy(self, *args):
         args = ['cp', '-ra'] + list(args)
         return await self.exec(*args)
+
+    async def mount(self, *dirs):
+        pass
+
+    async def mkdir(self, *dirs):
+        return await self.exec(*['mkdir', '-p'] + list(dirs))

shlaxfile.py

@@ -3,32 +3,53 @@ from shlax.contrib.gitlab import *
 
 PYTEST = 'py.test -svv tests'
 
-build = Buildah('alpine',
-    Copy('shlax/', 'setup.py', '/app'),
-    Pip('/app'),
-    commit='yourlabs/shlax',
+test = Script(
+    Pip('.[test]'),
+    Run(PYTEST),
 )
 
-gitlabci = GitLabCIConfig(
-    build=dict(
-        stage='test',
-        image='yourlabs/shlax',
-        script='pip install -U .[test] && ' + PYTEST,
+build = Buildah(
+    'quay.io/podman/stable',
+    Packages('python38', 'buildah', 'unzip', 'findutils', 'python3-yaml', upgrade=False),
+    Async(
+        # python3.8 on centos with pip dance ...
+        Run('''
+        curl -o setuptools.zip https://files.pythonhosted.org/packages/42/3e/2464120172859e5d103e5500315fb5555b1e908c0dacc73d80d35a9480ca/setuptools-45.1.0.zip
+        unzip setuptools.zip
+        mkdir -p /usr/local/lib/python3.8/site-packages/
+        sh -c "cd setuptools-* && python3.8 setup.py install"
+        easy_install-3.8 pip
+        echo python3.8 -m pip > /usr/bin/pip
+        chmod +x /usr/bin/pip
+        '''),
+        Copy('shlax/', 'setup.py', '/app'),
     ),
+    Pip('/app[full]'),
+    commit='docker.io/yourlabs/shlax',
+    workdir='/app',
+)
+
+shlax = Container(
+    build=build,
+    test=Script(Run('./shlaxfile.py -d test')),
+)
+
+gitlabci = GitLabCI(
     test=dict(
-        stage='test',
+        stage='build',
+        script='pip install -U --user -e .[test] && ' + PYTEST,
         image='yourlabs/python',
-        script='pip install -U .[test] && ' + PYTEST,
+    ),
+    build=dict(
+        stage='build',
+        image='yourlabs/shlax',
+        script='pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push',
+        cache=dict(paths=['.cache'], key='cache'),
     ),
     pypi=dict(
         stage='deploy',
+        only=['tags'],
         image='yourlabs/python',
         script='pypi-release',
-        only=['tags']
     ),
 )
-
-test = Script(
-    gitlabci,
-    Run('gitlab-runner exec docker test'),
-)