Add Docker target, recursive calls

.gitlab-ci.yml

@@ -1,14 +1,14 @@
 build:
   image: yourlabs/shlax
-  script: pip install -U .[test] && py.test -svv tests
-  stage: test
+  script: ./shlaxfile.py build
+  stage: build
 pypi:
   image: yourlabs/python
   only:
   - tags
-  script: pypi-release
+  script: ./shlaxfile.py pypi
   stage: deploy
 test:
-  image: yourlabs/python
-  script: pip install -U .[test] && py.test -svv tests
+  image: yourlabs/shlax
+  script: ./shlaxfile.py test
   stage: test

shlax/actions/base.py

@@ -1,4 +1,6 @@
+import functools
 import inspect
+import importlib
 import sys
 
 from ..output import Output
@@ -164,16 +166,46 @@ class Action:
     def callable(self):
         from ..targets import Localhost
         async def cb(*a, **k):
-            return await Localhost(self, quiet=True)(*a, **k)
+            from shlax.cli import cli
+            script = Localhost(self, quiet=True)
+            result = await script(*a, **k)
+
+            success = functools.reduce(
+                lambda a, b: a + b,
+                [1 for c in script.children() if c.status == 'success'] or [0])
+            if success:
+                script.output.success(f'{success} PASS')
+
+            failures = functools.reduce(
+                lambda a, b: a + b,
+                [1 for c in script.children() if c.status == 'fail'] or [0])
+            if failures:
+                script.output.fail(f'{failures} FAIL')
+            cli.exit_code = failures
+            return result
         return cb
 
     def kwargs_output(self):
         return self.kwargs
 
     def action(self, action, *args, **kwargs):
+        if isinstance(action, str):
+            import cli2
+            a = cli2.Callable.factory(action).target
+            if not a:
+                a = cli2.Callable.factory(
+                    '.'.join(['shlax', action])
+                ).target
+            if a:
+                action = a
+
         p = action(*args, **kwargs)
         for parent in self.parents():
             if hasattr(parent, 'actions'):
                 break
         p.parent = parent
+        if 'actions' not in self.__dict__:
+            # "mutate" to Strategy
+            from ..strategies.script import Actions
+            self.actions = Actions(self, [p])
         return p

shlax/actions/packages.py

@@ -113,7 +113,7 @@ class Packages(Action):
         else:
             mgr = await self.which(*self.mgrs.keys())
             if mgr:
-                self.mgr = mgr.split('/')[-1]
+                self.mgr = mgr[0].split('/')[-1]
 
         if not self.mgr:
             raise Exception('Packages does not yet support this distro')

shlax/actions/pip.py

@@ -5,25 +5,27 @@ from .base import Action
 
 
 class Pip(Action):
-    packages = dict(
-        apt=['python3-pip'],
-    )
-
     def __init__(self, *pip_packages, pip=None, requirements=None):
-        self.pip_packages = pip_packages
         self.requirements = requirements
         super().__init__(*pip_packages, pip=pip, requirements=requirements)
 
     async def call(self, *args, **kwargs):
-        self.pip = await self.which('pip3', 'pip', 'pip2')
-        if not self.pip:
+        pip = await self.which('pip3', 'pip', 'pip2')
+        if pip:
+            pip = pip[0]
+        else:
             from .packages import Packages
-            action = self.action(Packages, 'python3,apk', 'python3-pip,apt', args=args, kwargs=kwargs)
+            action = self.action(
+                Packages,
+                'python3,apk', 'python3-pip,apt',
+                args=args, kwargs=kwargs
+            )
             await action(*args, **kwargs)
-
-        self.pip = await self.which('pip3', 'pip', 'pip2')
-        if not self.pip:
-            raise Exception('Could not install a pip command')
+            pip = await self.which('pip3', 'pip', 'pip2')
+            if not pip:
+                raise Exception('Could not install a pip command')
+            else:
+                pip = pip[0]
 
         if 'CACHE_DIR' in os.environ:
             cache = os.path.join(os.getenv('CACHE_DIR'), 'pip')
@@ -33,20 +35,20 @@ class Pip(Action):
         if getattr(self, 'mount', None):
             # we are in a target which shares a mount command
             await self.mount(cache, '/root/.cache/pip')
-        await self.exec(f'{self.pip} install --upgrade pip')
+        await self.exec(f'{pip} install --upgrade pip')
 
         # https://github.com/pypa/pip/issues/5599
-        self.pip = 'python3 -m pip'
+        pip = 'python3 -m pip'
 
-        source = [p for p in self.pip_packages if p.startswith('/')]
+        source = [p for p in self.args if p.startswith('/') or p.startswith('.')]
         if source:
             await self.exec(
-                f'{self.pip} install --upgrade --editable {" ".join(source)}'
+                f'{pip} install --upgrade --editable {" ".join(source)}'
             )
 
-        nonsource = [p for p in self.pip_packages if not p.startswith('/')]
+        nonsource = [p for p in self.args if not p.startswith('/')]
         if nonsource:
-            await self.exec(f'{self.pip} install --upgrade {" ".join(nonsource)}')
+            await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}')
 
         if self.requirements:
-            await self.exec(f'{self.pip} install --upgrade -r {self.requirements}')
+            await self.exec(f'{pip} install --upgrade -r {self.requirements}')

shlax/actions/run.py

@@ -1,6 +1,18 @@
+from ..targets.buildah import Buildah
+from ..targets.docker import Docker
+
 from .base import Action
 
 
 class Run(Action):
     async def call(self, *args, **kwargs):
-        return await self.exec(*self.args, **self.kwargs)
+        image = self.kwargs.get('image', None)
+        if not image:
+            return await self.exec(*self.args, **self.kwargs)
+        if isinstance(image, Buildah):
+            breakpoint()
+            result = await self.action(image, *args, **kwargs)
+
+        return await Docker(
+            image=image,
+        ).exec(*args, **kwargs)

shlax/cli.py

@@ -8,6 +8,7 @@ shlax is a micro-framework to orchestrate commands.
 
 import asyncio
 import cli2
+import copy
 import inspect
 import os
 import sys
@@ -130,8 +131,10 @@ class ConsoleScript(cli2.ConsoleScript):
         return super().__call__(*args, **kwargs)
 
     def call(self, command):
+        kwargs = copy.copy(self.parser.funckwargs)
+        kwargs.update(self.parser.options)
         try:
-            return super().call(command)
+            return command(*self.parser.funcargs, **kwargs)
         except WrongResult as e:
             print(e)
             self.exit_code = e.proc.rc

shlax/contrib/gitlab.py

@@ -3,9 +3,16 @@ import yaml
 from shlax import *
 
 
-class GitLabCIConfig(Script):
+class GitLabCI(Script):
     async def call(self, *args, write=True, **kwargs):
-        output = yaml.dump(self.kwargs)
+        output = dict()
+        for key, value in self.kwargs.items():
+            if isinstance(value, dict):
+                output[key] = value
+                output[key]['script'] = './shlaxfile.py ' + key
+            else:
+                output[key] = value
+        output = yaml.dump(output)
         if kwargs['debug'] is True:
             self.output(output)
         if write:

shlax/image.py

@@ -55,3 +55,10 @@ class Image:
         # default tag by default ...
         if not self.tags:
             self.tags = ['latest']
+
+    async def __call__(self, action, *args, **kwargs):
+        args = list(args)
+        return await action.exec(*args, **self.kwargs)
+
+    def __str__(self):
+        return self.repository

shlax/output.py

@@ -33,8 +33,8 @@ class Output:
         self.write = write or sys.stdout.buffer.write
         self.flush = flush or sys.stdout.flush
 
-    def __call__(self, line, highlight=True, flush=True):
-        if self.prefix and self.prefix not in self.prefixes:
+    def prefix_line(self):
+        if self.prefix not in self.prefixes:
             self.prefixes[self.prefix] = self.prefix_colors[len(self.prefixes)]
             if len(self.prefix) > self.prefix_length:
                 self.prefix_length = len(self.prefix)
@@ -44,24 +44,24 @@ class Output:
         if prefix_padding:
             prefix_padding = ' ' + prefix_padding + ' '
 
-        self.write((
-            (
-                prefix_color
-                + prefix_padding
-                + self.prefix
-                + ' '
-                + self.colors['reset']
-                + '| '
-                if self.prefix
-                else ''
-            )
-            + self.highlight(line, highlight)
-            + self.colors['reset']
-        ).encode('utf8'))
+        return [
+            prefix_color,
+            prefix_padding,
+            self.prefix,
+            ' ',
+            self.colors['reset'],
+            '| '
+        ]
+
+    def __call__(self, line, highlight=True, flush=True):
+        line = [self.highlight(line) if highlight else line]
+        if self.prefix:
+            line = self.prefix_line() + line
+        line = ''.join(line)
+
+        self.write(line.encode('utf8'))
 
         if flush:
-            if not line.endswith('\n'):
-                self.write(b'\n')
             self.flush()
 
     def cmd(self, line):
@@ -70,7 +70,8 @@ class Output:
             + '\x1b[1;38;5;15m'
             + ' '
             + self.highlight(line, 'bash')
-            + self.colors['reset'],
+            + self.colors['reset']
+            + '\n',
             highlight=False
         )
 
@@ -92,25 +93,28 @@ class Output:
 
         for regexp, colors in self.regexps.items():
             line = re.sub(regexp, colors.format(**self.colors), line)
+        line = line + self.colors['reset']
 
         return line
 
     def clean(self, action):
-        if self.debug is True or 'visit' in str(self.debug):
+        if self.debug is True:
             self(''.join([
                 self.colors['bluebold'],
-                '+ CLEAN ',
+                '+  CLEAN  ',
                 self.colors['reset'],
                 action.colorized(),
+                '\n',
             ]))
 
     def start(self, action):
-        if self.debug is True or 'visit' in str(self.debug):
+        if self.debug is True:
             self(''.join([
                 self.colors['orangebold'],
-                '⚠ START ',
+                '⚠  START  ',
                 self.colors['reset'],
                 action.colorized(),
+                '\n',
             ]))
 
     def success(self, action):
@@ -119,14 +123,16 @@ class Output:
                 self.colors['greenbold'],
                 '✔ SUCCESS ',
                 self.colors['reset'],
-                action.colorized(),
+                action.colorized() if hasattr(action, 'colorized') else str(action),
+                '\n',
             ]))
 
     def fail(self, action, exception=None):
         if self.debug is True or 'visit' in str(self.debug):
             self(''.join([
                 self.colors['redbold'],
-                '✘ FAIL ',
+                '✘  FAIL   ',
                 self.colors['reset'],
-                action.colorized(),
+                action.colorized() if hasattr(action, 'colorized') else str(action),
+                '\n',
             ]))

shlax/targets/__init__.py

@@ -1,3 +1,4 @@
 from .buildah import Buildah
+from .docker import Docker
 from .localhost import Localhost
 from .ssh import Ssh

shlax/targets/buildah.py

@@ -8,6 +8,8 @@ import subprocess
 import sys
 import textwrap
 
+from ..actions.base import Action
+from ..exceptions import Mistake
 from ..proc import Proc
 from ..image import Image
 from .localhost import Localhost
@@ -20,7 +22,10 @@ class Buildah(Localhost):
     """
     contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount']
 
-    def __init__(self, base, *args, commit=None, push=False, **kwargs):
+    def __init__(self, base, *args, commit=None, push=False, cmd=None, **kwargs):
+        if isinstance(base, Action):
+            args = [base] + list(args)
+            base = 'alpine'  # default selection in case of mistake
         super().__init__(*args, **kwargs)
         self.base = base
         self.mounts = dict()
@@ -28,6 +33,9 @@ class Buildah(Localhost):
         self.mnt = None
         self.image = Image(commit) if commit else None
         self.push = push or os.getenv('CI')
+        self.config= dict(
+            cmd=cmd or 'sh',
+        )
 
     def shargs(self, *args, user=None, buildah=True, **kwargs):
         if not buildah or args[0].startswith('buildah'):
@@ -52,9 +60,6 @@ class Buildah(Localhost):
         """Run buildah config."""
         return await self.exec(f'buildah config {line} {self.ctr}', buildah=False)
 
-    async def mkdir(self, *dirs):
-        return await self.exec(*['mkdir', '-p'] + list(dirs))
-
     async def copy(self, *args):
         """Run buildah copy to copy a file from host into container."""
         src = args[:-1]
@@ -81,19 +86,6 @@ class Buildah(Localhost):
         await self.exec(f'mount -o bind {src} {target}', buildah=False)
         self.mounts[src] = dst
 
-    async def which(self, *cmd):
-        """
-        Return the first path to the cmd in the container.
-
-        If cmd argument is a list then it will try all commands.
-        """
-        paths = (await self.env('PATH')).split(':')
-        for path in paths:
-            for c in cmd:
-                p = os.path.join(self.mnt, path[1:], c)
-                if os.path.exists(p):
-                    return p[len(str(self.mnt)):]
-
     def is_wrapper(self):
         return not (
             Proc.test
@@ -101,7 +93,6 @@ class Buildah(Localhost):
             or getattr(self.parent, 'parent', None)
         )
 
-
     async def call(self, *args, **kwargs):
         if not self.is_wrapper():
             self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out
@@ -124,7 +115,11 @@ class Buildah(Localhost):
         argv += [
             cli.parser.command.name,  # script name ?
         ]
-        self.output(' '.join(argv), 'EXECUTION', flush=True)
+
+        await self.exec(*argv)
+        '''
+        if debug is True or 'cmd' in str(debug):
+            self.output.cmd(' '.join(argv))
 
         proc = await asyncio.create_subprocess_shell(
             shlex.join(argv),
@@ -134,11 +129,15 @@ class Buildah(Localhost):
         )
         await proc.communicate()
         cli.exit_code = await proc.wait()
+        '''
 
     async def commit(self):
         if not self.image:
             return
 
+        for key, value in self.config.items():
+            await self.exec(f'buildah config --{key} "{value}" {self.ctr}')
+
         self.sha = (await self.exec(
             'buildah',
             'commit',

shlax/targets/docker.py

@@ -0,0 +1,78 @@
+import asyncio
+from pathlib import Path
+import os
+
+from ..image import Image
+from .localhost import Localhost
+
+
+class Docker(Localhost):
+    contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount']
+
+    def __init__(self, *args, **kwargs):
+        self.image = Image(kwargs.get('image', 'alpine'))
+        super().__init__(*args, **kwargs)
+        self.context['ctr'] = None
+
+    def shargs(self, *args, daemon=False, **kwargs):
+        if args[0] == 'docker':
+            return args, kwargs
+
+        extra = []
+        if 'user' in kwargs:
+            extra += ['--user', kwargs.pop('user')]
+
+        args, kwargs = super().shargs(*args, **kwargs)
+
+        if self.context['ctr']:
+            executor = 'exec'
+            extra = [self.context['ctr']]
+            return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + list(args), kwargs
+
+        executor = 'run'
+        cwd = os.getcwd()
+        if daemon:
+            extra += ['-d']
+        extra = extra + ['-v', f'{cwd}:{cwd}', '-w', f'{cwd}']
+        return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + [str(self.image)] + list(args), kwargs
+
+    async def call(self, *args, **kwargs):
+        name = kwargs.get('name', os.getcwd()).split('/')[-1]
+        self.context['ctr'] = (
+            await self.exec(
+                'docker', 'ps', '-aq', '--filter',
+                'name=' + name,
+                raises=False
+            )
+        ).out.split('\n')[0]
+
+        if 'recreate' in args and self.context['ctr']:
+            await self.exec('docker', 'rm', '-f', self.context['ctr'])
+            self.context['ctr'] = None
+
+        if self.context['ctr']:
+            self.context['ctr'] = (await self.exec('docker', 'start', name)).out
+        else:
+            self.context['ctr'] = (
+                await self.exec('sleep', '120', daemon=True)
+            ).out
+        return await super().call(*args, **kwargs)
+
+    async def copy(self, *args):
+        src = args[:-1]
+        dst = args[-1]
+        await self.mkdir(dst)
+
+        procs = []
+        for s in src:
+            '''
+            if Path(s).is_dir():
+                await self.mkdir(s)
+                args = ['docker', 'copy', self.ctr, s, Path(dst) / s]
+            else:
+                args = ['docker', 'copy', self.ctr, s, dst]
+            '''
+            args = ['docker', 'cp', s, self.context['ctr'] + ':' + dst]
+            procs.append(self.exec(*args))
+
+        return await asyncio.gather(*procs)

shlax/targets/localhost.py

@@ -1,4 +1,5 @@
 import os
+import re
 
 from shlax.proc import Proc
 
@@ -50,12 +51,20 @@ class Localhost(Script):
 
         If cmd argument is a list then it will try all commands.
         """
-        for path in (await self.env('PATH')).split(':'):
-            for c in cmd:
-                p = os.path.join(self.root, path[1:], c)
-                if os.path.exists(p):
-                    return p[len(str(self.root)):]
+        proc = await self.exec('type ' + ' '.join(cmd), raises=False)
+        result = []
+        for res in proc.out.split('\n'):
+            match = re.match('([^ ]+) is ([^ ]+)$', res.strip())
+            if match:
+                result.append(match.group(1))
+        return result
 
     async def copy(self, *args):
         args = ['cp', '-ra'] + list(args)
         return await self.exec(*args)
+
+    async def mount(self, *dirs):
+        pass
+
+    async def mkdir(self, *dirs):
+        return await self.exec(*['mkdir', '-p'] + list(dirs))

shlaxfile.py

@@ -3,32 +3,42 @@ from shlax.contrib.gitlab import *
 
 PYTEST = 'py.test -svv tests'
 
-build = Buildah('alpine',
+build = Buildah(
     Copy('shlax/', 'setup.py', '/app'),
     Pip('/app'),
     commit='yourlabs/shlax',
-)
-
-gitlabci = GitLabCIConfig(
-    build=dict(
-        stage='test',
-        image='yourlabs/shlax',
-        script='pip install -U .[test] && ' + PYTEST,
-    ),
-    test=dict(
-        stage='test',
-        image='yourlabs/python',
-        script='pip install -U .[test] && ' + PYTEST,
-    ),
-    pypi=dict(
-        stage='deploy',
-        image='yourlabs/python',
-        script='pypi-release',
-        only=['tags']
-    ),
+    workdir='/app',
 )
 
 test = Script(
-    gitlabci,
-    Run('gitlab-runner exec docker test'),
+    Pip('.[test]'),
+    Run(PYTEST),
+)
+
+buildtest = Docker(
+    *test.actions,
+    mount={'.': '/app'},
+    workdir='/app',
+)
+
+pypi = Run(
+    'pypi-release',
+    stage='deploy',
+    image='yourlabs/python',
+)
+
+gitlabci = GitLabCI(
+    build=dict(
+        stage='build',
+        image='yourlabs/shlax',
+    ),
+    test=dict(
+        stage='test',
+        image='yourlabs/shlax',
+    ),
+    pypi=dict(
+        stage='deploy',
+        only=['tags'],
+        image='yourlabs/python',
+    ),
 )