Fix cli that would play all scripts

shlax/__init__.py

@@ -1,6 +1,7 @@
 from .actions import *
 from .image import Image
 from .strategies import *
+from .output import Output
 from .proc import Proc
 from .targets import *
 from .shlaxfile import Shlaxfile

shlax/actions/__init__.py

@@ -1,5 +1,7 @@
 from .commit import Commit
+from .copy import Copy
 from .packages import Packages  # noqa
 from .base import Action  # noqa
 from .run import Run  # noqa
+from .pip import Pip
 from .service import Service

shlax/actions/base.py

@@ -8,6 +8,9 @@ from ..exceptions import WrongResult
 class Action:
     parent = None
     contextualize = []
+    colorize = {
+        '[^ ]*([^:]*):': {1: 0},
+    }
 
     def __init__(self, *args, **kwargs):
         self.args = args
@@ -84,7 +87,8 @@ class Action:
         sys.exit(1)
 
     def output_factory(self, *args, **kwargs):
-        return Output(*args, kwargs)
+        kwargs.setdefault('regexps', self.colorize)
+        return Output(*args, **kwargs)
 
     async def __call__(self, *args, **kwargs):
         self.status = 'running'
@@ -98,3 +102,8 @@ class Action:
             if self.status == 'running':
                 self.status = 'success'
         return result
+
+    def callable(self):
+        async def cb(*a, **k):
+            return await self(*a, **k)
+        return cb

shlax/actions/copy.py

@@ -0,0 +1,10 @@
+from .base import Action
+
+
+class Copy(Action):
+    def __init__(self, *args):
+        self.src = args[:-1]
+        self.dst = args[-1]
+
+    def call(self, *args, **kwargs):
+        self.copy(self.src, self.dst)

shlax/actions/pip.py

@@ -0,0 +1,51 @@
+from glob import glob
+import os
+
+from .base import Action
+
+
+class Pip(Action):
+    packages = dict(
+        apt=['python3-pip'],
+    )
+
+    def __init__(self, *pip_packages, pip=None, requirements=None):
+        self.pip_packages = pip_packages
+        self.requirements = requirements
+
+    async def call(self, *args, **kwargs):
+        breakpoint()
+        self.pip = await self.which(('pip3', 'pip', 'pip2'))
+        if not self.pip:
+            raise Exception('Could not find pip command')
+
+        if 'CACHE_DIR' in os.environ:
+            cache = os.path.join(os.getenv('CACHE_DIR'), 'pip')
+        else:
+            cache = os.path.join(os.getenv('HOME'), '.cache', 'pip')
+
+        await script.mount(cache, '/root/.cache/pip')
+        await script.crexec(f'{self.pip} install --upgrade pip')
+
+        # https://github.com/pypa/pip/issues/5599
+        self.pip = 'python3 -m pip'
+
+        pip_packages = []
+        for visitor in script.container.visitors:
+            pp = getattr(visitor, 'pip_packages', None)
+            if not pp:
+                continue
+            pip_packages += pip_packages
+
+        source = [p for p in pip_packages if p.startswith('/')]
+        if source:
+            await script.crexec(
+                f'{self.pip} install --upgrade --editable {" ".join(source)}'
+            )
+
+        nonsource = [p for p in pip_packages if not p.startswith('/')]
+        if nonsource:
+            await script.crexec(f'{self.pip} install --upgrade {" ".join(nonsource)}')
+
+        if self.requirements:
+            await script.crexec(f'{self.pip} install --upgrade -r {self.requirements}')

shlax/cli.py

@@ -24,7 +24,6 @@ async def runall(*args, **kwargs):
 
 @cli2.option('debug', alias='d', help='Display debug output.')
 async def test(*args, **kwargs):
-    breakpoint()
     """Run podctl test over a bunch of paths."""
     report = []
 
@@ -112,28 +111,20 @@ async def test(*args, **kwargs):
 
 
 class ConsoleScript(cli2.ConsoleScript):
-    def call(self, *args, **kwargs):
+    def __call__(self, *args, **kwargs):
         self.shlaxfile = None
         shlaxfile = sys.argv.pop(1) if len(sys.argv) > 1 else ''
         if os.path.exists(shlaxfile.split('::')[0]):
             self.shlaxfile = Shlaxfile()
             self.shlaxfile.parse(shlaxfile)
             for name, action in self.shlaxfile.actions.items():
-                async def cb(*args, **kwargs):
-                    return await Localhost(action)(*args, **kwargs)
                 self[name] = cli2.Callable(
                     name,
-                    cb,
+                    action.callable(),
                     color=getattr(action, 'color', cli2.YELLOW),
                 )
         return super().__call__(*args, **kwargs)
 
-    def __call__(self, command):
-        args = self.parser.funcargs
-        kwargs = self.parser.funckwargs
-        breakpoint()
-        return command(*args, **kwargs)
-
     def call(self, command):
         try:
             return super().call(command)

shlax/contrib/gitlab.py

@@ -5,12 +5,12 @@ from shlax import *
 
 class GitLabCIConfig(Script):
     async def call(self, *args, write=True, **kwargs):
-        await super().__call__(*args, **kwargs)
+        await super().call(*args, **kwargs)
         self.kwargs = kwargs
         for name, definition in self.context.items():
             self.kwargs[name] = definition
         output = yaml.dump(self.kwargs)
-        print(output)
+        self.output(output)
         if write:
             with open('.gitlab-ci.yml', 'w+') as f:
                 f.write(output)

shlax/output.py

@@ -1,6 +1,9 @@
+import re
+import sys
 
 
 class Output:
+    prefixes = dict()
     colors = (
         '\x1b[1;36;45m',
         '\x1b[1;36;41m',
@@ -8,53 +11,59 @@ class Output:
         '\x1b[1;37;45m',
         '\x1b[1;32m',
         '\x1b[1;37;44m',
+        '\u001b[30;1m',
     )
 
-    def __init__(self, prefix=None):
+    def color(self, code=None):
-        self.prefix = prefix
+        if not code:
-        self.prefixes = dict()
+            return '\u001b[0m'
-        self.prefix_length = 0
+        code = str(code)
+        return u"\u001b[38;5;" + code + "m"
 
-    def call(self, line, prefix, highlight=True, flush=True):
+    def colorize(self, code, content):
-        if prefix and prefix not in self.prefixes:
+        return self.color(code) + content + self.color()
-            self.prefixes[prefix] = (
+
+    def __init__(self, prefix=None, regexps=None, debug=True, write=None, flush=None):
+        self.prefix = prefix
+        self.debug = debug
+        self.prefix_length = 0
+        self.regexps = regexps or dict()
+        self.write = write or sys.stdout.buffer.write
+        self.flush = flush or sys.stdout.flush
+
+    def __call__(self, line, highlight=True, flush=True):
+        if self.prefix and self.prefix not in self.prefixes:
+            self.prefixes[self.prefix] = (
                 self.colors[len([*self.prefixes.keys()]) - 1]
             )
-            if len(prefix) > self.prefix_length:
+            if len(self.prefix) > self.prefix_length:
-                self.prefix_length = len(prefix)
+                self.prefix_length = len(self.prefix)
 
-        prefix_color = self.prefixes[prefix] if prefix else ''
+        prefix_color = self.prefixes[self.prefix] if self.prefix else ''
-        prefix_padding = '.' * (self.prefix_length - len(prefix) - 2) if prefix else ''
+        prefix_padding = '.' * (self.prefix_length - len(self.prefix) - 2) if self.prefix else ''
         if prefix_padding:
             prefix_padding = ' ' + prefix_padding + ' '
 
-        sys.stdout.buffer.write((
+        self.write((
             (
                 prefix_color
                 + prefix_padding
-                + prefix
+                + self.prefix
                 + ' '
-                + Back.RESET
-                + Style.RESET_ALL
-                + Fore.LIGHTBLACK_EX
                 + '| '
-                + Style.RESET_ALL
+                if self.prefix
-                if prefix
                 else ''
             )
             + self.highlight(line, highlight)
         ).encode('utf8'))
 
         if flush:
-            sys.stdout.flush()
+            self.flush()
 
-    def cmd(self, line, prefix):
+    def cmd(self, line):
         self(
-            Fore.LIGHTBLACK_EX
+            self.colorize(251, '+ ')
-            + '+ '
-            + Style.RESET_ALL
             + self.highlight(line, 'bash'),
-            prefix,
             highlight=False
         )
 
@@ -73,4 +82,16 @@ class Output:
             or '\\e[' in line
         ):
             return line
+
+        for regexp, colors in self.regexps.items():
+            match = re.match(regexp, line)
+            if not match:
+                continue
+
+            for group, color in colors.items():
+                res = match.group(group)
+                if not res:
+                    continue
+                line = line.replace(res, self.colorize(color, res))
+
         return line

shlax/proc.py

@@ -22,16 +22,16 @@ class PrefixStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol):
         super().__init__(*args, **kwargs)
 
     def pipe_data_received(self, fd, data):
-        if (self.debug is True or 'out' in str(self.debug)) and fd in (1, 2):
+        if (self.output.debug is True or 'out' in str(self.output.debug)) and fd in (1, 2):
             self.output(data, flush=False)
             sys.stdout.flush()
         super().pipe_data_received(fd, data)
 
 
-def protocol_factory(prefix):
+def protocol_factory(output):
     def _p():
         return PrefixStreamProtocol(
-            prefix,
+            output,
             limit=asyncio.streams._DEFAULT_LIMIT,
             loop=asyncio.events.get_event_loop()
         )
@@ -79,12 +79,16 @@ class Proc:
                 args = ['sh', '-euc', ' '.join(args)]
         return args
 
+    def output_factory(self, *args, **kwargs):
+        args = tuple(self.prefix) + args
+        return Output(*args, kwargs)
+
     async def __call__(self, wait=True):
         if self.called:
             raise Exception('Already called: ' + self.cmd)
 
         if self.debug is True or 'cmd' in str(self.debug):
-            output.cmd(self.cmd, self.prefix)
+            self.output.cmd(self.cmd)
 
         if self.test:
             if self.test is True:
@@ -94,7 +98,7 @@ class Proc:
 
         loop = asyncio.events.get_event_loop()
         transport, protocol = await loop.subprocess_exec(
-            protocol_factory(self.prefix), *self.args)
+            protocol_factory(self.output), *self.args)
         self.proc = asyncio.subprocess.Process(transport, protocol, loop)
         self.called = True
 

shlax/targets/buildah.py

@@ -53,7 +53,7 @@ class Buildah(Localhost):
 
     async def copy(self, src, dst):
         """Run buildah copy to copy a file from host into container."""
-        return await self.exec(f'buildah copy {self.ctr} {src} {dst}')
+        return await self.exec(f'buildah copy {self.ctr} {src} {self.mnt}{dst}')
 
     async def mount(self, src, dst):
         """Mount a host directory into the container."""

shlaxfile.py

@@ -3,6 +3,12 @@ from shlax.contrib.gitlab import *
 
 PYTEST = 'py.test -svv tests'
 
+build = Buildah('alpine',
+    Copy('shlax', 'setup.py', '/app'),
+    Pip('/app'),
+    commit='yourlabs/shlax',
+)
+
 gitlabci = GitLabCIConfig(
     Job('test',
         stage='test',

tests/test_output.py

@@ -0,0 +1,24 @@
+import pytest
+from shlax import Output
+
+
+class Write:
+    def __init__(self):
+        self.output = ''
+    def __call__(self, out):
+        self.output += out.decode('utf8')
+
+
+@pytest.fixture
+def write():
+    return Write()
+
+
+def test_output_regexps(write):
+    output = Output(
+        regexps={'.*': {0: 0}},
+        write=write,
+        flush=lambda: None,
+    )
+    output('foo')
+    assert write.output == output.colorize(0, 'foo')