.gitignore

@@ -1,10 +0,0 @@
-*.pyc
-__pycache__
-.cache/
-.coverage
-.eggs/
-.podctl_build_django.sh
-.podctl_build_podctl.sh
-.setupmeta.version
-.testmondata
-*.egg-info

.gitlab-ci.yml

@@ -1,15 +1,20 @@
-build:
-  cache:
-    key: cache
-    paths: [.cache]
-  image: yourlabs/shlax
-  script: pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d
-    shlax build push
-  stage: build
+image: yourlabs/python-arch
+
+qa:
+  stage: test
+  script: flake8
+
+pytest:
+  stage: test
+  script:
+  - pip install --user -e .
+  - pytest -vv --cov shlax --cov-report=xml:coverage.xml --junitxml=report.xml --cov-report=term-missing --strict tests
+  - CI_PROJECT_PATH=yourlabs/shlax CI_BUILD_REPO=https://github.com/yourlabs/cli2 codecov-bash -f coverage.xml
+  artifacts:
+    reports:
+      junit: report.xml
+
 pypi:
-  image: yourlabs/python
-  only: [tags]
-  script: pypi-release
   stage: deploy
-test: {image: yourlabs/python, script: 'pip install -U --user -e .[test] && py.test
-    -svv tests', stage: build}
+  script: pypi-release
+  only: [tags]

.pre-commit-config.yaml

@@ -1,14 +0,0 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
-repos:
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v2.4.0
-  hooks:
-  - id: trailing-whitespace
-  - id: end-of-file-fixer
-  - id: check-yaml
-  - id: check-added-large-files
-- repo: https://yourlabs.io/oss/shlax
-  rev: master
-  hooks:
-  - id: shlaxfile-gitlabci

.pre-commit-hooks.yaml

@@ -1,5 +0,0 @@
-- id: shlaxfile-gitlabci
-  name: Regenerate .gitlab-ci.yml
-  description: Regenerate gitlabci
-  entry: ./shlaxfile.py gitlabci
-  language: python

README.rst

@@ -0,0 +1,110 @@
+Shlax: Beautiful Async Subprocess executor
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Why?
+====
+
+In Python we now have async subprocesses which allows to execute several
+subprocesses at the same time. The purpose of this library is to:
+
+- stream stderr and stdout in real time while capturing it,
+- real time output must be prefixed for when you execute several commands at
+  the time so that you know which line is for which process, like with
+  docker-compose logs,
+- output coloration in real time with regexps to make even more readable.
+
+This code was copy/pasted between projects and finally extracted on its own.
+
+Demo
+====
+
+.. image:: https://yourlabs.io/oss/shlax/-/raw/master/demo.png
+
+You will find the demo script in demo.py in this repository.
+
+Usage
+=====
+
+Basics
+------
+
+Basic example, this will both stream output and capture it:
+
+.. code-block:: python
+
+    from shlax import Subprocess
+    proc = await Subprocess('echo hi').wait()
+    print(proc.rc, proc.out, proc.err, proc.out_raw, proc.err_raw)
+
+Longer
+------
+
+If you want to start the command and wait for completion elsewhere then call
+any of ``start()`` and ``wait()``, or both, explicitely:
+
+.. code-block:: python
+
+    proc = Subprocess('echo hi')
+    await proc.start()  # start the process
+    await proc.wait()   # wait for completion
+
+Proc alias
+----------
+
+Note that shlax defines an alias ``Proc`` to ``Subprocess`` so this also works:
+
+.. code-block:: python
+
+    from shlax import Proc
+    proc = await Proc('echo hi').wait()
+
+Quiet
+-----
+
+To disable real time output streaming use the ``quiet`` argument:
+
+.. code-block:: python
+
+    proc = await Subprocess('echo hi', quiet=True).wait()
+
+Prefix
+------
+
+Using prefixes, you can have real time outputs of parallel commands and at the
+same time know which output belongs to which process:
+
+.. code-block:: python
+
+    proc0 = Subprocess('find /', prefix='first')
+    proc1 = Subprocess('find /', prefix='second')
+    await asyncio.gather(proc0.wait(), proc1.wait())
+
+Coloration and output patching
+------------------------------
+
+You can add coloration or patch real time output with regexps, note that it
+will be applied line by line:
+
+.. code-block:: python
+
+    import sys
+    regexps = {
+        '^(.*).py$': '{cyan}\\1',
+    }
+    await asyncio.gather(*[
+        Subprocess(
+            f'find {path}',
+            regexps=regexps,
+        ).wait()
+        for path in sys.path
+    ])
+
+Where is the rest?
+==================
+
+Shlax used to be the name of a much more ambitious poc-project, that you can
+still find in the ``OLD`` branch of this repository. It has been extracted in
+two projects with clear boundaries, namely `sysplan
+<https://yourlabs.io/oss/sysplan>`_ and `podplan
+<https://yourlabs.io/oss/podplan>`_ which are still in alpha state, although
+Shlax as it now, is feature complete and stable.

demo.py

@@ -0,0 +1,24 @@
+import asyncio
+
+from shlax import Subprocess
+
+
+async def main():
+    colors = {
+        '^(.*).txt$': '{green}\\1.txt',
+        '^(.*).py$': '{bred}\\1.py',
+    }
+    await asyncio.gather(
+        Subprocess(
+            'for i in $(find .. | head); do echo $i; sleep .2; done',
+            regexps=colors,
+            prefix='parent',
+        ).wait(),
+        Subprocess(
+            'for i in $(find . | head); do echo $i; sleep .3; done',
+            regexps=colors,
+            prefix='cwd',
+        ).wait(),
+    )
+
+asyncio.run(main())

setup.py

@@ -5,11 +5,7 @@ setup(
     name='shlax',
     versioning='dev',
     setup_requires='setupmeta',
-    install_requires=['cli2'],
     extras_require=dict(
-        full=[
-            'pyyaml',
-        ],
         test=[
             'pytest',
             'pytest-cov',
@@ -21,11 +17,6 @@ setup(
     url='https://yourlabs.io/oss/shlax',
     include_package_data=True,
     license='MIT',
-    keywords='cli automation ansible',
+    keywords='async subprocess',
     python_requires='>=3',
-    entry_points={
-        'console_scripts': [
-            'shlax = shlax.cli:cli',
-        ],
-    },
 )

shlax/__init__.py

@@ -0,0 +1,3 @@
+from .subprocess import Subprocess  # noqa
+Proc = Subprocess                   # noqa
+from .colors import colors, c       # noqa

shlax/actions/base.py

@@ -1,235 +0,0 @@
-from copy import deepcopy
-import functools
-import inspect
-import importlib
-import sys
-
-from ..output import Output
-from ..exceptions import WrongResult
-from ..result import Result
-
-
-
-
-class class_or_instance_method:
-    def __init__(self, f):
-        self.f = f
-    def __get__(self, instance, owner):
-        def newfunc(*args, **kwargs):
-            return self.f(
-                instance if instance is not None else owner,
-                *args,
-                **kwargs
-            )
-        return newfunc
-
-
-class Action:
-    display_variables = []
-    hide_variables = ['output']
-    default_steps = ['apply']
-    parent = None
-    contextualize = []
-    regexps = {
-        r'([\w]+):': '{cyan}\\1{gray}:{reset}',
-        r'(^|\n)( *)\- ': '\\1\\2{red}-{reset} ',
-    }
-    options = dict(
-        debug=dict(
-            alias='d',
-            default='visit',
-            help='''
-            Display debug output. Supports values (combinable): cmd,out,visit
-            '''.strip(),
-            immediate=True,
-        ),
-    )
-
-    def __init__(self, *args, **kwargs):
-        self.args = args
-        for key, value in kwargs.items():
-            setattr(self, key, value)
-            if isinstance(value, Action):
-                getattr(self, key).shlaxstep = True
-
-    def actions_filter(self, results, f=None, **filters):
-        if f:
-            def ff(a):
-                try:
-                    return f(a)
-                except:
-                    return False
-            results = [*filter(ff, results)]
-
-        for k, v in filters.items():
-            if k == 'type':
-                results = [*filter(
-                    lambda s: type(s).__name__.lower() == str(v).lower(),
-                    results
-                )]
-            else:
-                results = [*filter(
-                    lambda s: getattr(s, k, None) == v,
-                    results
-                )]
-
-        return results
-
-
-    def sibblings(self, f=None, **filters):
-        if not self.parent:
-            return []
-        return self.actions_filter(
-            [a for a in self.parent.actions if a is not self],
-            f, **filters
-        )
-
-    def parents(self, f=None, **filters):
-        if self.parent:
-            return self.actions_filter(
-                [self.parent] + self.parent.parents(),
-                f, **filters
-            )
-        return []
-
-    def children(self, f=None, **filters):
-        children = []
-        def add(parent):
-            if parent != self:
-                children.append(parent)
-            if 'actions' not in parent.__dict__:
-                return
-
-            for action in parent.actions:
-                add(action)
-        add(self)
-        return self.actions_filter(children, f, **filters)
-
-    async def __call__(self, *targets, **options):
-        if not targets:
-            from ..targets.localhost import Localhost
-            targets = [Localhost()]
-
-        output = Output(regexp=self.regexps, debug=True)
-        results = []
-        for target in targets:
-            target.output = output
-            if len(targets) > 1:
-                output.prefix = target
-            from copy import deepcopy
-            action = deepcopy(self)
-            action.target = target
-            result = Result(action, target)
-            results.append(result)
-            action.result = result
-            action.output = output
-            for step in options.get('steps', None) or self.default_steps:
-                if step not in action.steps():
-                    print(f'Failed to find {type(action).__name__}.{step}')
-                    continue
-                action.step = step
-                output.start(action)
-                try:
-                    await getattr(action, step)()
-                except Exception as e:
-                    output.fail(action, e)
-                    action.result.status = 'fail'
-                    proc = getattr(e, 'proc', None)
-                    if proc:
-                        result = proc.rc
-                    else:
-                        raise
-                else:
-                    output.success(action)
-                    result.status = 'success'
-                finally:
-                    clean = getattr(action, 'clean', None)
-                    if clean:
-                        output.clean(action)
-                        await clean(target)
-
-        return results
-
-    def __repr__(self):
-        return ' '.join([type(self).__name__] + [
-            f'{k}={v}'
-            for k, v in self.__dict__.items()
-            if (k in self.display_variables or not self.display_variables)
-            and (k not in self.hide_variables)
-        ])
-
-    def colorized(self, colors):
-        return ' '.join([
-            colors['pink1']
-            + type(self).__name__
-            + '.'
-            + self.step
-            + colors['yellow']
-        ] + [
-            f'{colors["blue"]}{k}{colors["gray"]}={colors["green2"]}{v}'
-            for k, v in self.__dict__.items()
-            if (k in self.display_variables or not self.display_variables)
-            and (k not in self.hide_variables)
-        ] + [colors['reset']])
-
-    def callable(self):
-        from ..targets import Localhost
-        async def cb(*a, **k):
-            from shlax.cli import cli
-            script = Localhost(self, quiet=True)
-            result = await script(*a, **k)
-
-            success = functools.reduce(
-                lambda a, b: a + b,
-                [1 for c in script.children() if c.status == 'success'] or [0])
-            if success:
-                script.output.success(f'{success} PASS')
-
-            failures = functools.reduce(
-                lambda a, b: a + b,
-                [1 for c in script.children() if c.status == 'fail'] or [0])
-            if failures:
-                script.output.fail(f'{failures} FAIL')
-            cli.exit_code = failures
-            return result
-        return cb
-
-    def kwargs_output(self):
-        return self.kwargs
-
-    def action(self, action, *args, **kwargs):
-        if isinstance(action, str):
-            # import dotted module path string to action
-            import cli2
-            a = cli2.Callable.factory(action).target
-            if not a:
-                a = cli2.Callable.factory(
-                    '.'.join(['shlax', action])
-                ).target
-            if a:
-                action = a
-
-        p = action(*args, **kwargs)
-        p.parent = self
-        for parent in self.parents():
-            if hasattr(parent, 'actions'):
-                p.parent = parent
-                break
-
-        if 'actions' not in self.__dict__:
-            # "mutate" to Strategy
-            from ..strategies.script import Actions
-            self.actions = Actions(self, [p])
-        return p
-
-    @class_or_instance_method
-    def steps(self):
-        return {
-            key: getattr(self, key)
-            for key in dir(self)
-            if key != 'steps'  # avoid recursion
-            and (
-                key in self.default_steps
-                or getattr(getattr(self, key), 'shlaxstep', False)
-            )
-        }

shlax/actions/copy.py

@@ -1,7 +0,0 @@
-from .base import Action
-
-
-class Copy(Action):
-    """Copy files or directories to target."""
-    async def call(self, *args, **kwargs):
-        await self.copy(*self.args)

shlax/actions/htpasswd.py

@@ -1,37 +0,0 @@
-import hashlib
-import secrets
-import string
-
-from .base import Action
-
-
-class Htpasswd(Action):
-    """Ensure a user is present in an htpasswd file."""
-    display_variables = ('user', 'path')
-    regexps = {
-        r'(.*)': '{red}\\1{gray}:${blue}\\2${blue}',
-        r'([^:]*):\\$([^$]*)\\$(.*)$': '{red}\\1{gray}:${blue}\\2${blue}\\3',
-    }
-
-    def __init__(self, user, path, **kwargs):
-        self.user = user
-        self.path = path
-        super().__init__(**kwargs)
-
-    async def apply(self):
-        found = False
-        htpasswd = await self.target.exec(
-            'cat', self.path, raises=False)
-        if htpasswd.rc == 0:
-            for line in htpasswd.out.split('\n'):
-                if line.startswith(self.user + ':'):
-                    found = True
-                    break
-
-        if not found:
-            self.password = ''.join(secrets.choice(
-                string.ascii_letters + string.digits
-            ) for i in range(20))
-            hashed = hashlib.sha1(self.password.encode('utf8'))
-            line = f'{self.user}:\\$sha1\\${hashed.hexdigest()}'
-            await self.target.exec(f'echo {line} >> {self.path}')

shlax/actions/packages.py

@@ -1,169 +0,0 @@
-import asyncio
-import copy
-
-from datetime import datetime
-from glob import glob
-import os
-import subprocess
-from textwrap import dedent
-
-from .base import Action
-
-
-class Packages(Action):
-    """
-    Package manager abstract layer with caching.
-
-    It's a central piece of the build process, and does iterate over other
-    container visitors in order to pick up packages. For example, the Pip
-    visitor will declare ``self.packages = dict(apt=['python3-pip'])``, and the
-    Packages visitor will pick it up.
-    """
-    regexps = {
-        #r'Installing ([\w\d-]+)': '{cyan}\\1',
-        r'Installing': '{cyan}lol',
-    }
-
-    mgrs = dict(
-        apk=dict(
-            update='apk update',
-            upgrade='apk upgrade',
-            install='apk add',
-        ),
-        apt=dict(
-            update='apt-get -y update',
-            upgrade='apt-get -y upgrade',
-            install='apt-get -y --no-install-recommends install',
-        ),
-        pacman=dict(
-            update='pacman -Sy',
-            upgrade='pacman -Su --noconfirm',
-            install='pacman -S --noconfirm',
-        ),
-        dnf=dict(
-            update='dnf makecache --assumeyes',
-            upgrade='dnf upgrade --best --assumeyes --skip-broken',  # noqa
-            install='dnf install --setopt=install_weak_deps=False --best --assumeyes',  # noqa
-        ),
-        yum=dict(
-            update='yum update',
-            upgrade='yum upgrade',
-            install='yum install',
-        ),
-    )
-
-    installed = []
-
-    def __init__(self, *packages, **kwargs):
-        self.packages = []
-        for package in packages:
-            line = dedent(package).strip().replace('\n', ' ')
-            self.packages += line.split(' ')
-        super().__init__(*packages, **kwargs)
-
-    @property
-    def cache_root(self):
-        if 'CACHE_DIR' in os.environ:
-            return os.path.join(os.getenv('CACHE_DIR'))
-        else:
-            return os.path.join(os.getenv('HOME'), '.cache')
-
-    async def update(self):
-        # run pkgmgr_setup functions ie. apk_setup
-        cachedir = await getattr(self, self.mgr + '_setup')()
-
-        lastupdate = None
-        if os.path.exists(cachedir + '/lastupdate'):
-            with open(cachedir + '/lastupdate', 'r') as f:
-                try:
-                    lastupdate = int(f.read().strip())
-                except:
-                    pass
-
-        if not os.path.exists(cachedir):
-            os.makedirs(cachedir)
-
-        now = int(datetime.now().strftime('%s'))
-        # cache for a week
-        if not lastupdate or now - lastupdate > 604800:
-            # crude lockfile implementation, should work against *most*
-            # race-conditions ...
-            lockfile = cachedir + '/update.lock'
-            if not os.path.exists(lockfile):
-                with open(lockfile, 'w+') as f:
-                    f.write(str(os.getpid()))
-
-                try:
-                    await self.rexec(self.cmds['update'])
-                finally:
-                    os.unlink(lockfile)
-
-                with open(cachedir + '/lastupdate', 'w+') as f:
-                    f.write(str(now))
-            else:
-                while os.path.exists(lockfile):
-                    print(f'{self.container.name} | Waiting for update ...')
-                    await asyncio.sleep(1)
-
-    async def apply(self):
-        cached = getattr(self.target, 'pkgmgr', None)
-        if cached:
-            self.mgr = cached
-        else:
-            mgr = await self.target.which(*self.mgrs.keys())
-            if mgr:
-                self.mgr = mgr[0].split('/')[-1]
-
-        if not self.mgr:
-            raise Exception('Packages does not yet support this distro')
-
-        self.cmds = self.mgrs[self.mgr]
-        if not getattr(self, '_packages_upgraded', None):
-            await self.update()
-            if self.kwargs.get('upgrade', True):
-                await self.target.exec(self.cmds['upgrade'], user='root')
-            self._packages_upgraded = True
-
-        packages = []
-        for package in self.packages:
-            if ',' in package:
-                parts = package.split(',')
-                package = parts[0]
-                if self.mgr in parts[1:]:
-                    # include apt on apt
-                    packages.append(package)
-            else:
-                packages.append(package)
-
-        await self.target.exec(*self.cmds['install'].split(' ') + packages, user='root')
-
-    async def apk_setup(self):
-        cachedir = os.path.join(self.cache_root, self.mgr)
-        await self.mount(cachedir, '/var/cache/apk')
-        # special step to enable apk cache
-        await self.rexec('ln -sf /var/cache/apk /etc/apk/cache')
-        return cachedir
-
-    async def dnf_setup(self):
-        cachedir = os.path.join(self.cache_root, self.mgr)
-        await self.mount(cachedir, f'/var/cache/{self.mgr}')
-        await self.rexec('echo keepcache=True >> /etc/dnf/dnf.conf')
-        return cachedir
-
-    async def apt_setup(self):
-        codename = (await self.rexec(
-            f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME'
-        )).out
-        cachedir = os.path.join(self.cache_root, self.mgr, codename)
-        await self.rexec('rm /etc/apt/apt.conf.d/docker-clean')
-        cache_archives = os.path.join(cachedir, 'archives')
-        await self.mount(cache_archives, f'/var/cache/apt/archives')
-        cache_lists = os.path.join(cachedir, 'lists')
-        await self.mount(cache_lists, f'/var/lib/apt/lists')
-        return cachedir
-
-    async def pacman_setup(self):
-        return self.cache_root + '/pacman'
-
-    def __repr__(self):
-        return f'Packages({self.packages})'

shlax/actions/pip.py

@@ -1,59 +0,0 @@
-from glob import glob
-import os
-
-from .base import Action
-
-
-class Pip(Action):
-    """Pip abstraction layer."""
-
-    def __init__(self, *pip_packages, pip=None, requirements=None):
-        self.requirements = requirements
-        super().__init__(*pip_packages, pip=pip, requirements=requirements)
-
-    async def call(self, *args, **kwargs):
-        pip = self.kwargs.get('pip', None)
-        if not pip:
-            pip = await self.which('pip3', 'pip', 'pip2')
-            if pip:
-                pip = pip[0]
-            else:
-                from .packages import Packages
-                action = self.action(
-                    Packages,
-                    'python3,apk', 'python3-pip,apt',
-                    args=args, kwargs=kwargs
-                )
-                await action(*args, **kwargs)
-                pip = await self.which('pip3', 'pip', 'pip2')
-                if not pip:
-                    raise Exception('Could not install a pip command')
-                else:
-                    pip = pip[0]
-
-        if 'CACHE_DIR' in os.environ:
-            cache = os.path.join(os.getenv('CACHE_DIR'), 'pip')
-        else:
-            cache = os.path.join(os.getenv('HOME'), '.cache', 'pip')
-
-        if getattr(self, 'mount', None):
-            # we are in a target which shares a mount command
-            await self.mount(cache, '/root/.cache/pip')
-        await self.exec(f'{pip} install --upgrade pip')
-
-        # https://github.com/pypa/pip/issues/5599
-        if 'pip' not in self.kwargs:
-            pip = 'python3 -m pip'
-
-        source = [p for p in self.args if p.startswith('/') or p.startswith('.')]
-        if source:
-            await self.exec(
-                f'{pip} install --upgrade --editable {" ".join(source)}'
-            )
-
-        nonsource = [p for p in self.args if not p.startswith('/')]
-        if nonsource:
-            await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}')
-
-        if self.requirements:
-            await self.exec(f'{pip} install --upgrade -r {self.requirements}')

shlax/actions/run.py

@@ -1,18 +0,0 @@
-from .base import Action
-
-
-class Run(Action):
-    """Run a script or command on a target."""
-    async def call(self, *args, **kwargs):
-        image = self.kwargs.get('image', None)
-        if not image:
-            return await self.exec(*self.args, **self.kwargs)
-        from ..targets.buildah import Buildah
-        from ..targets.docker import Docker
-        if isinstance(image, Buildah):
-            breakpoint()
-            result = await self.action(image, *args, **kwargs)
-
-        return await Docker(
-            image=image,
-        ).exec(*args, **kwargs)

shlax/actions/service.py

@@ -1,19 +0,0 @@
-import asyncio
-
-from .base import Action
-
-
-class Service(Action):
-    """
-    Manage a systemd service.
-    """
-    def __init__(self, *names, state=None):
-        self.state = state or 'started'
-        self.names = names
-        super().__init__()
-
-    async def call(self, *args, **kwargs):
-        return asyncio.gather(*[
-            self.exec('systemctl', 'start', name, user='root')
-            for name in self.names
-        ])

shlax/cli.py

@@ -1,170 +0,0 @@
-"""
-Shlax automation tool manual
-
-Shlax is built mostly around 3 moving pieces:
-- Target: a target host and protocol
-- Action: execute a shlax action
-- Strategy: defines how to apply actions on targets (scripted only)
-
-Shlax executes mostly in 3 ways:
-- Execute actions on targets with the command line
-- With your shlaxfile as first argument: offer defined Actions
-- With the name of a module in shlax.repo: a community maintained shlaxfile
-"""
-
-import copy
-import cli2
-import inspect
-import importlib
-import glob
-import os
-import sys
-
-from .actions.base import Action
-from .exceptions import ShlaxException, WrongResult
-from .strategies import Script
-
-
-class ConsoleScript(cli2.ConsoleScript):
-    class Parser(cli2.Parser):
-        def __init__(self, *args, **kwargs):
-            self.targets = dict()
-            super().__init__(*args, **kwargs)
-
-        def append(self, arg):
-            if '=' not in arg and '@' in arg:
-                if '://' in arg:
-                    kind, spec = arg.split('://')
-                else:
-                    kind = 'ssh'
-                    spec = arg
-
-                mod = importlib.import_module('shlax.targets.' + kind)
-                target = getattr(mod, kind.capitalize())(spec)
-                self.targets[str(target)] = target
-            else:
-                super().append(arg)
-
-    def __call__(self):
-        if len(sys.argv) > 1 and os.path.exists(sys.argv[1]):
-            pass
-        else:
-            scripts = glob.glob(os.path.join(
-                os.path.dirname(__file__), 'actions', '*.py'))
-            for script in scripts:
-                modname = script.split('/')[-1].replace('.py', '')
-                mod = importlib.import_module('shlax.actions.' + modname)
-                for key, value in mod.__dict__.items():
-                    if key.lower() != modname:
-                        continue
-                    break
-                self[modname] = cli2.Callable(
-                    modname, self.action_class(value))
-
-            scripts = glob.glob(os.path.join(
-                os.path.dirname(__file__), 'repo', '*.py'))
-            for script in scripts:
-                modname = script.split('/')[-1].replace('.py', '')
-                mod = importlib.import_module('shlax.repo.' + modname)
-                self[modname] = cli2.Group(key, doc=inspect.getdoc(mod))
-                for key, value in mod.__dict__.items():
-                    if not isinstance(value, Action):
-                        continue
-                    doc = (inspect.getdoc(mod) or '').split("\n")[0]
-                    if key == 'main':
-                        if len(value.steps()) == 1:
-                            self[modname] = cli2.Callable(
-                                modname, self.action(value), doc=doc)
-                        else:
-                            for name, method in value.steps().items():
-                                self[modname][name] = cli2.Callable(
-                                    modname, self.action(value),
-                                    doc=inspect.getdoc(method)
-                                )
-                    else:
-                        if len(value.steps()) == 1:
-                            self[modname][key] = cli2.Callable(
-                                modname, self.action(value), doc=doc)
-                        else:
-                            self[modname][key] = cli2.Group('steps')
-                            for step in value.steps():
-                                self[modname][key][step] = cli2.Callable(
-                                    modname, self.action(value), doc='lol')
-
-        return super().__call__()
-
-    def action(self, action):
-        async def cb(*args, **kwargs):
-            options = dict(steps=args)
-            options.update(self.parser.options)
-            # UnboundLocalError: local variable 'action' referenced before assignment
-            # ??? gotta be missing something, commenting meanwhile
-            # action = copy.deepcopy(action)
-            return await action(*self.parser.targets, **options)
-        return cb
-
-    def action_class(self, action_class):
-        async def cb(*args, **kwargs):
-            argspec = inspect.getfullargspec(action_class)
-            required = argspec.args[1:]
-            missing = []
-            for i, name in enumerate(required):
-                if len(args) - 1 <= i:
-                    continue
-                if name in kwargs:
-                    continue
-                missing.append(name)
-            if missing:
-                if not args:
-                    print('No args provided after action name ' + action_class.__name__.lower())
-                print('Required arguments: ' + ', '.join(argspec.args[1:]))
-                if args:
-                    print('Provided: ' + ', '.join(args))
-                print('Missing arguments: ' + ', '.join(missing))
-                print('Try to just add args on the command line separated with a space')
-                print(inspect.getdoc(action_class))
-                example = 'Example: shlax action '
-                example += action_class.__name__.lower()
-                if args:
-                    example += ' ' + ' '.join(args)
-                example += ' ' + ' '.join(missing)
-                print(example)
-                return
-
-            _args = []
-            steps = []
-            for arg in args:
-                if arg in action_class.steps():
-                    steps.append(arg)
-                else:
-                    _args.append(arg)
-
-            options = dict(steps=steps)
-
-            '''
-            varargs = argspec.varargs
-            if varargs:
-                extra = args[len(argspec.args) - 1:]
-                args = args[:len(argspec.args) - 1]
-                options = dict(steps=extra)
-            else:
-                extra = args[len(argspec.args) - 1:]
-                args = args[:len(argspec.args) - 1]
-                options = dict(steps=extra)
-            '''
-            options.update(self.parser.options)
-            return await action_class(*_args, **kwargs)(*self.parser.targets, **options)
-        cb.__doc__ = (inspect.getdoc(action_class) or '').split("\n")[0]
-        return cb
-
-    def call(self, command):
-        try:
-            return super().call(command)
-        except WrongResult as e:
-            print(e)
-            self.exit_code = e.proc.rc
-        except ShlaxException as e:
-            print(e)
-            self.exit_code = 1
-
-cli = ConsoleScript(__doc__).add_module('shlax.cli')

shlax/colors.py

@@ -1,68 +1,73 @@
-colors = dict(
-    cyan='\u001b[38;5;51m',
-    cyan1='\u001b[38;5;87m',
-    cyan2='\u001b[38;5;123m',
-    cyan3='\u001b[38;5;159m',
-    blue='\u001b[38;5;33m',
-    blue1='\u001b[38;5;69m',
-    blue2='\u001b[38;5;75m',
-    blue3='\u001b[38;5;81m',
-    blue4='\u001b[38;5;111m',
-    blue5='\u001b[38;5;27m',
-    green='\u001b[38;5;10m',
-    green1='\u001b[38;5;2m',
-    green2='\u001b[38;5;46m',
-    green3='\u001b[38;5;47m',
-    green4='\u001b[38;5;48m',
-    green5='\u001b[38;5;118m',
-    green6='\u001b[38;5;119m',
-    green7='\u001b[38;5;120m',
-    purple='\u001b[38;5;5m',
-    purple1='\u001b[38;5;6m',
-    purple2='\u001b[38;5;13m',
-    purple3='\u001b[38;5;164m',
-    purple4='\u001b[38;5;165m',
-    purple5='\u001b[38;5;176m',
-    purple6='\u001b[38;5;145m',
-    purple7='\u001b[38;5;213m',
-    purple8='\u001b[38;5;201m',
-    red='\u001b[38;5;1m',
-    red1='\u001b[38;5;9m',
-    red2='\u001b[38;5;196m',
-    red3='\u001b[38;5;160m',
-    red4='\u001b[38;5;197m',
-    red5='\u001b[38;5;198m',
-    red6='\u001b[38;5;199m',
-    yellow='\u001b[38;5;226m',
-    yellow1='\u001b[38;5;227m',
-    yellow2='\u001b[38;5;226m',
-    yellow3='\u001b[38;5;229m',
-    yellow4='\u001b[38;5;220m',
-    yellow5='\u001b[38;5;230m',
-    gray='\u001b[38;5;250m',
-    gray1='\u001b[38;5;251m',
-    gray2='\u001b[38;5;252m',
-    gray3='\u001b[38;5;253m',
-    gray4='\u001b[38;5;254m',
-    gray5='\u001b[38;5;255m',
-    gray6='\u001b[38;5;249m',
-    pink='\u001b[38;5;197m',
-    pink1='\u001b[38;5;198m',
-    pink2='\u001b[38;5;199m',
-    pink3='\u001b[38;5;200m',
-    pink4='\u001b[38;5;201m',
-    pink5='\u001b[38;5;207m',
-    pink6='\u001b[38;5;213m',
-    orange='\u001b[38;5;202m',
-    orange1='\u001b[38;5;208m',
-    orange2='\u001b[38;5;214m',
-    orange3='\u001b[38;5;220m',
-    orange4='\u001b[38;5;172m',
-    orange5='\u001b[38;5;166m',
-    reset='\u001b[0m',
+theme = dict(
+    cyan='\033[38;5;51m',
+    cyan1='\033[38;5;87m',
+    cyan2='\033[38;5;123m',
+    cyan3='\033[38;5;159m',
+    blue='\033[38;5;33m',
+    blue1='\033[38;5;69m',
+    blue2='\033[38;5;75m',
+    blue3='\033[38;5;81m',
+    blue4='\033[38;5;111m',
+    blue5='\033[38;5;27m',
+    green='\033[38;5;10m',
+    green1='\033[38;5;2m',
+    green2='\033[38;5;46m',
+    green3='\033[38;5;47m',
+    green4='\033[38;5;48m',
+    green5='\033[38;5;118m',
+    green6='\033[38;5;119m',
+    green7='\033[38;5;120m',
+    purple='\033[38;5;5m',
+    purple1='\033[38;5;6m',
+    purple2='\033[38;5;13m',
+    purple3='\033[38;5;164m',
+    purple4='\033[38;5;165m',
+    purple5='\033[38;5;176m',
+    purple6='\033[38;5;145m',
+    purple7='\033[38;5;213m',
+    purple8='\033[38;5;201m',
+    red='\033[38;5;1m',
+    red1='\033[38;5;9m',
+    red2='\033[38;5;196m',
+    red3='\033[38;5;160m',
+    red4='\033[38;5;197m',
+    red5='\033[38;5;198m',
+    red6='\033[38;5;199m',
+    yellow='\033[38;5;226m',
+    yellow1='\033[38;5;227m',
+    yellow2='\033[38;5;226m',
+    yellow3='\033[38;5;229m',
+    yellow4='\033[38;5;220m',
+    yellow5='\033[38;5;230m',
+    gray='\033[38;5;250m',
+    gray1='\033[38;5;251m',
+    gray2='\033[38;5;252m',
+    gray3='\033[38;5;253m',
+    gray4='\033[38;5;254m',
+    gray5='\033[38;5;255m',
+    gray6='\033[38;5;249m',
+    pink='\033[38;5;197m',
+    pink1='\033[38;5;198m',
+    pink2='\033[38;5;199m',
+    pink3='\033[38;5;200m',
+    pink4='\033[38;5;201m',
+    pink5='\033[38;5;207m',
+    pink6='\033[38;5;213m',
+    orange='\033[38;5;202m',
+    orange1='\033[38;5;208m',
+    orange2='\033[38;5;214m',
+    orange3='\033[38;5;220m',
+    orange4='\033[38;5;172m',
+    orange5='\033[38;5;166m',
+    reset='\033[0m',
 )
 
-colors.update({
-    k + 'bold': v.replace('[', '[1;')
-    for k, v in colors.items()
-})
+
+class Colors:
+    def __init__(self, **theme):
+        for name, value in theme.items():
+            setattr(self, name, value)
+            setattr(self, f'b{name}', value.replace('[', '[1;'))
+
+
+c = colors = Colors(**theme)

shlax/exceptions.py

@@ -1,22 +0,0 @@
-class ShlaxException(Exception):
-    pass
-
-
-class Mistake(ShlaxException):
-    pass
-
-
-class WrongResult(ShlaxException):
-    def __init__(self, proc):
-        self.proc = proc
-
-        msg = f'FAIL exit with {proc.rc} ' + proc.args[0]
-
-        if not proc.debug or 'cmd' not in str(proc.debug):
-            msg += '\n' + proc.cmd
-
-        if not proc.debug or 'out' not in str(proc.debug):
-            msg += '\n' + proc.out
-            msg += '\n' + proc.err
-
-        super().__init__(msg)

shlax/image.py

@@ -1,75 +0,0 @@
-import os
-import re
-
-class Image:
-    ENV_TAGS = (
-        # gitlab
-        'CI_COMMIT_SHORT_SHA',
-        'CI_COMMIT_REF_NAME',
-        'CI_COMMIT_TAG',
-        # CircleCI
-        'CIRCLE_SHA1',
-        'CIRCLE_TAG',
-        'CIRCLE_BRANCH',
-        # contributions welcome here
-    )
-
-    PATTERN = re.compile(
-        '^((?P<backend>[a-z]*)://)?((?P<registry>[^/]*[.][^/]*)/)?((?P<repository>[^:]+))?(:(?P<tags>.*))?$'  # noqa
-        , re.I
-    )
-
-    def __init__(self, arg=None, format=None, backend=None, registry=None, repository=None, tags=None):
-        self.arg = arg
-        self.format = format
-        self.backend = backend
-        self.registry = registry
-        self.repository = repository
-        self.tags = tags or []
-
-        match = re.match(self.PATTERN, arg)
-        if match:
-            for k, v in match.groupdict().items():
-                if getattr(self, k):
-                    continue
-                if not v:
-                    continue
-                if k == 'tags':
-                    v = v.split(',')
-                setattr(self, k, v)
-
-        # docker.io currently has issues with oci format
-        self.format = format or 'oci'
-        if self.registry == 'docker.io':
-            self.format = 'docker'
-
-        # figure tags from CI vars
-        for name in self.ENV_TAGS:
-            value = os.getenv(name)
-            if value:
-                self.tags.append(value)
-
-        # filter out tags which resolved to None
-        self.tags = [t for t in self.tags if t]
-
-        # default tag by default ...
-        if not self.tags:
-            self.tags = ['latest']
-
-    async def __call__(self, action, *args, **kwargs):
-        args = list(args)
-        return await action.exec(*args, **self.kwargs)
-
-    def __str__(self):
-        return f'{self.repository}:{self.tags[-1]}'
-
-    async def push(self, *args, **kwargs):
-        user = os.getenv('DOCKER_USER')
-        passwd = os.getenv('DOCKER_PASS')
-        action = kwargs.get('action', self)
-        if user and passwd:
-            action.output.cmd('buildah login -u ... -p ...' + self.registry)
-            await action.exec('buildah', 'login', '-u', user, '-p', passwd, self.registry or 'docker.io', debug=False)
-
-        for tag in self.tags:
-            await action.exec('buildah', 'push', f'{self.repository}:{tag}')

shlax/output.py

@@ -1,149 +0,0 @@
-import re
-import sys
-
-from .colors import colors
-
-
-class Output:
-    prefixes = dict()
-    colors = colors
-    prefix_colors = (
-        '\x1b[1;36;45m',
-        '\x1b[1;36;41m',
-        '\x1b[1;36;40m',
-        '\x1b[1;37;45m',
-        '\x1b[1;32m',
-        '\x1b[1;37;44m',
-    )
-
-    def color(self, code=None):
-        if not code:
-            return '\u001b[0m'
-        code = str(code)
-        return u"\u001b[38;5;" + code + "m"
-
-    def colorize(self, code, content):
-        return self.color(code) + content + self.color()
-
-    def __init__(self, prefix=None, regexps=None, debug=False, write=None, flush=None, **kwargs):
-        self.prefix = prefix
-        self.debug = debug
-        self.prefix_length = 0
-        self.regexps = regexps or dict()
-        self.write = write or sys.stdout.buffer.write
-        self.flush = flush or sys.stdout.flush
-        self.kwargs = kwargs
-
-    def prefix_line(self):
-        if self.prefix not in self.prefixes:
-            self.prefixes[self.prefix] = self.prefix_colors[len(self.prefixes)]
-            if len(self.prefix) > self.prefix_length:
-                self.prefix_length = len(self.prefix)
-
-        prefix_color = self.prefixes[self.prefix] if self.prefix else ''
-        prefix_padding = '.' * (self.prefix_length - len(self.prefix) - 2) if self.prefix else ''
-        if prefix_padding:
-            prefix_padding = ' ' + prefix_padding + ' '
-
-        return [
-            prefix_color,
-            prefix_padding,
-            self.prefix,
-            ' ',
-            self.colors['reset'],
-            '| '
-        ]
-
-    def __call__(self, line, highlight=True, flush=True):
-        line = [self.highlight(line) if highlight else line]
-        if self.prefix:
-            line = self.prefix_line() + line
-        line = ''.join(line)
-
-        self.write(line.encode('utf8'))
-
-        if flush:
-            self.flush()
-
-    def cmd(self, line):
-        self(
-            self.colorize(251, '+')
-            + '\x1b[1;38;5;15m'
-            + ' '
-            + self.highlight(line, 'bash')
-            + self.colors['reset']
-            + '\n',
-            highlight=False
-        )
-
-    def print(self, content):
-        self(
-            content,
-            prefix=None,
-            highlight=False
-        )
-
-    def highlight(self, line, highlight=True):
-        line = line.decode('utf8') if isinstance(line, bytes) else line
-        if not highlight or (
-            '\x1b[' in line
-            or '\033[' in line
-            or '\\e[' in line
-        ):
-            return line
-
-        for regexp, colors in self.regexps.items():
-            line = re.sub(regexp, colors.format(**self.colors), line)
-        line = line + self.colors['reset']
-
-        return line
-
-    def test(self, action):
-        if self.debug is True:
-            self(''.join([
-                self.colors['purplebold'],
-                '!  TEST   ',
-                self.colors['reset'],
-                action.colorized(self.colors),
-                '\n',
-            ]))
-
-    def clean(self, action):
-        if self.debug is True:
-            self(''.join([
-                self.colors['bluebold'],
-                '+  CLEAN  ',
-                self.colors['reset'],
-                action.colorized(self.colors),
-                '\n',
-            ]))
-
-    def start(self, action):
-        if self.debug is True or 'visit' in str(self.debug):
-            self(''.join([
-                self.colors['orangebold'],
-                '⚠  START  ',
-                self.colors['reset'],
-                action.colorized(self.colors),
-                '\n',
-            ]))
-
-    def success(self, action):
-        if self.debug is True or 'visit' in str(self.debug):
-            self(''.join([
-                self.colors['greenbold'],
-                '✔ SUCCESS ',
-                self.colors['reset'],
-                action.colorized(self.colors) if hasattr(action, 'colorized') else str(action),
-                '\n',
-            ]))
-
-    def fail(self, action, exception=None):
-        if self.debug is True or 'visit' in str(self.debug):
-            self(''.join([
-                self.colors['redbold'],
-                '✘  FAIL   ',
-                self.colors['reset'],
-                action.colorized(self.colors) if hasattr(action, 'colorized') else str(action),
-                '\n',
-            ]))

shlax/play.py

@@ -1,9 +0,0 @@
-
-from .targets import Localhost
-
-
-class Play:
-    def __init__(self, *actions, targets=None, options=None):
-        self.options = options or {}
-        self.targets = targets or dict(localhost=Localhost())
-        self.actions =

shlax/proc.py

@@ -1,142 +0,0 @@
-"""
-Asynchronous process execution wrapper.
-"""
-
-import asyncio
-import os
-import shlex
-import sys
-
-from .exceptions import WrongResult
-from .output import Output
-
-
-class PrefixStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol):
-    """
-    Internal subprocess stream protocol to add a prefix in front of output to
-    make asynchronous output readable.
-    """
-
-    def __init__(self, output, *args, **kwargs):
-        self.output = output
-        super().__init__(*args, **kwargs)
-
-    def pipe_data_received(self, fd, data):
-        if self.output.debug is True or 'out' in str(self.output.debug):
-            if fd in (1, 2):
-                self.output(data)
-        super().pipe_data_received(fd, data)
-
-
-def protocol_factory(output):
-    def _p():
-        return PrefixStreamProtocol(
-            output,
-            limit=asyncio.streams._DEFAULT_LIMIT,
-            loop=asyncio.events.get_event_loop()
-        )
-    return _p
-
-
-class Proc:
-    """
-    Subprocess wrapper.
-
-    Example usage::
-
-        proc = Proc('find', '/', prefix='containername')
-
-        await proc()     # execute
-
-        print(proc.out)  # stdout
-        print(proc.err)  # stderr
-        print(proc.rc)   # return code
-    """
-    test = False
-
-    def __init__(self, *args, prefix=None, raises=True, debug=None, output=None):
-        self.debug = debug if not self.test else False
-        self.output = output or Output()
-        self.cmd = ' '.join(args)
-        self.args = args
-        self.prefix = prefix
-        self.raises = raises
-        self.called = False
-        self.communicated = False
-        self.out_raw = b''
-        self.err_raw = b''
-        self.out = ''
-        self.err = ''
-        self.rc = None
-
-    @staticmethod
-    def split(*args):
-        args = [str(a) for a in args]
-        if len(args) == 1:
-            if isinstance(args[0], (list, tuple)):
-                args = args[0]
-            else:
-                args = ['sh', '-euc', ' '.join(args)]
-        return args
-
-    def output_factory(self, *args, **kwargs):
-        args = tuple(self.prefix) + args
-        return Output(*args, kwargs)
-
-    async def __call__(self, wait=True):
-        if self.called:
-            raise Exception('Already called: ' + self.cmd)
-
-        if self.debug is True or 'cmd' in str(self.debug):
-            self.output.cmd(self.cmd)
-
-        if self.test:
-            if self.test is True:
-                type(self).test = []
-            self.test.append(self.args)
-            return self
-
-        loop = asyncio.events.get_event_loop()
-        transport, protocol = await loop.subprocess_exec(
-            protocol_factory(self.output), *self.args)
-        self.proc = asyncio.subprocess.Process(transport, protocol, loop)
-        self.called = True
-
-        if wait:
-            await self.wait()
-
-        return self
-
-    async def communicate(self):
-        self.out_raw, self.err_raw = await self.proc.communicate()
-        self.out = self.out_raw.decode('utf8').strip()
-        self.err = self.err_raw.decode('utf8').strip()
-        self.rc = self.proc.returncode
-        self.communicated = True
-        return self
-
-    async def wait(self):
-        if self.test:
-            return self
-        if not self.called:
-            await self()
-        if not self.communicated:
-            await self.communicate()
-        if self.raises and self.proc.returncode:
-            raise WrongResult(self)
-        return self
-
-    @property
-    def json(self):
-        import json
-        return json.loads(self.out)
-
-    def mock():
-        """Context manager for testing purpose."""
-        cls = Proc
-        class Mock:
-            def __enter__(_):
-                cls.test = True
-            def __exit__(_, exc_type, exc_value, traceback):
-                cls.test = False
-        return Mock()

shlax/repo/traefik.py

@@ -1,35 +0,0 @@
-#!/usr/bin/env shlax
-"""
-Manage a traefik container maintained by Shlax community.
-"""
-
-from shlax.shortcuts import *
-
-
-main = Docker(
-    name='traefik',
-    image='traefik:v2.0.0',
-    install=Htpasswd(
-        './htpasswd', 'root', doc='Install root user in ./htpasswd'
-    ),
-    networks=['web'],
-    command=[
-        '--entrypoints.web.address=:80',
-        '--providers.docker',
-        '--api',
-    ],
-    ports=[
-        '80:80',
-        '443:443',
-    ],
-    volumes=[
-        '/var/run/docker.sock:/var/run/docker.sock:ro',
-        '/etc/traefik/acme/:/etc/traefik/acme/',
-        '/etc/traefik/htpasswd:/htpasswd:ro',
-    ],
-    labels=[
-        'traefik.http.routers.traefik.rule=Host(`{{ url.split("/")[2] }}`)',
-        'traefik.http.routers.traefik.service=api@internal',
-        'traefik.http.routers.traefik.entrypoints=web',
-    ],
-)

shlax/result.py

@@ -1,7 +0,0 @@
-
-
-class Result:
-    def __init__(self, action, target):
-        self.action = action
-        self.target = target
-        self.status = 'pending'

shlax/shlaxfile.py

@@ -1,28 +0,0 @@
-import importlib
-import os
-
-from .actions.base import Action
-
-
-class Shlaxfile:
-    def __init__(self, actions=None, tests=None):
-        self.actions = actions or {}
-        self.tests = tests or {}
-        self.paths = []
-
-    def parse(self, path):
-        spec = importlib.util.spec_from_file_location('shlaxfile', path)
-        mod = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(mod)
-        for name, value in mod.__dict__.items():
-            if isinstance(value, Action):
-                value.__name__ = name
-                self.actions[name] = value
-            elif callable(value) and getattr(value, '__name__', '').startswith('test_'):
-                self.tests[value.__name__] = value
-
-        self.paths.append(path)
-
-    @property
-    def path(self):
-        return self.paths[0]

shlax/shortcuts.py

@@ -1,14 +0,0 @@
-from .actions.copy import Copy
-from .actions.packages import Packages  # noqa
-from .actions.base import Action  # noqa
-from .actions.htpasswd import Htpasswd
-from .actions.run import Run  # noqa
-from .actions.pip import Pip
-from .actions.service import Service
-
-from .targets.buildah import Buildah
-from .targets.docker import Docker
-from .targets.localhost import Localhost
-from .targets.ssh import Ssh
-
-

shlax/strategies/__init__.py

@@ -1,3 +0,0 @@
-from .asyn import Async
-from .script import Script
-from .pod import Pod, Container

shlax/strategies/asyn.py

@@ -1,11 +0,0 @@
-import asyncio
-
-from .script import Script
-
-
-class Async(Script):
-    async def call(self, *args, **kwargs):
-        return await asyncio.gather(*[
-            action(*args, **kwargs)
-            for action in self.actions
-        ])

shlax/strategies/pod.py

@@ -1,44 +0,0 @@
-import os
-from .script import Script
-from ..image import Image
-
-
-class Container(Script):
-    """
-    Wolcome to crazy container control cli
-
-    Such wow
-    """
-    def __init__(self, *args, **kwargs):
-        kwargs.setdefault('start', dict())
-        super().__init__(*args, **kwargs)
-
-    async def call(self, *args, **kwargs):
-        if step('build'):
-            await self.kwargs['build'](**kwargs)
-            self.image = self.kwargs['build'].image
-        else:
-            self.image = kwargs.get('image', 'alpine')
-        if isinstance(self.image, str):
-            self.image = Image(self.image)
-
-        if step('install'):
-            await self.install(*args, **kwargs)
-
-        if step('test'):
-            self.output.test(self)
-            await self.action('Docker',
-                *self.kwargs['test'].actions,
-                image=self.image,
-                mount={'.': '/app'},
-                workdir='/app',
-            )(**kwargs)
-
-        if step('push'):
-            await self.image.push(action=self)
-
-        #name = kwargs.get('name', os.getcwd()).split('/')[-1]
-
-
-class Pod(Script):
-    pass

shlax/strategies/script.py

@@ -1,41 +0,0 @@
-import copy
-import os
-
-from ..exceptions import WrongResult
-from ..actions.base import Action
-from ..proc import Proc
-
-
-class Actions(list):
-    def __init__(self, owner, actions):
-        self.owner = owner
-        super().__init__()
-        for action in actions:
-            self.append(action)
-
-    def append(self, value):
-        action = copy.deepcopy(value)
-        action.parent = self.owner
-        action.status = 'pending'
-        super().append(action)
-
-
-class Script(Action):
-    contextualize = ['shargs', 'exec', 'rexec', 'env', 'which', 'copy']
-
-    def __init__(self, *actions, **kwargs):
-        self.home = kwargs.pop('home', os.getcwd())
-        super().__init__(**kwargs)
-        self.actions = Actions(self, actions)
-
-    async def call(self, *args, **kwargs):
-        for action in self.actions:
-            result = await action(*args, **kwargs)
-            if action.status != 'success':
-                break
-
-    def pollute(self, gbls):
-        for name, script in self.kwargs.items():
-            if not isinstance(script, Script):
-                continue
-            gbls[name] = script

shlax/strategies/test.py

@@ -1,7 +0,0 @@
-from .script import Script
-
-class Test(Script):
-    async def call(self, *args, backend=None, **kwargs):
-        backend = backend or 'Docker'
-        breakpoint()
-        return await self.action(backend, self.actions, **kwargs)

shlax/subprocess.py

@@ -0,0 +1,182 @@
+import asyncio
+import functools
+import re
+import shlex
+import sys
+
+from .colors import colors
+
+
+class SubprocessProtocol(asyncio.SubprocessProtocol):
+    def __init__(self, proc):
+        self.proc = proc
+        self.output = bytearray()
+
+    def pipe_data_received(self, fd, data):
+        if fd == 1:
+            self.proc.stdout(data)
+        elif fd == 2:
+            self.proc.stderr(data)
+
+    def process_exited(self):
+        self.proc.exit_future.set_result(True)
+
+
+class Subprocess:
+    colors = colors
+
+    # arbitrary list of colors
+    prefix_colors = (
+        colors.cyan,
+        colors.blue,
+        colors.green,
+        colors.purple,
+        colors.red,
+        colors.yellow,
+        colors.gray,
+        colors.pink,
+        colors.orange,
+    )
+
+    # class variables, meant to grow as new prefixes are discovered to ensure
+    # output alignment
+    prefixes = dict()
+    prefix_length = 0
+
+    def __init__(
+        self,
+        *args,
+        quiet=None,
+        prefix=None,
+        regexps=None,
+        write=None,
+        flush=None,
+    ):
+        if len(args) == 1 and ' ' in args[0]:
+            args = ['sh', '-euc', args[0]]
+
+        self.args = args
+        self.quiet = quiet if quiet is not None else False
+        self.prefix = prefix
+        self.write = write or sys.stdout.buffer.write
+        self.flush = flush or sys.stdout.flush
+        self.started = False
+        self.waited = False
+        self.out_raw = bytearray()
+        self.err_raw = bytearray()
+
+        self.regexps = dict()
+        if regexps:
+            for search, replace in regexps.items():
+                if isinstance(search, str):
+                    search = search.encode()
+                search = re.compile(search)
+                replace = replace.format(**self.colors.__dict__).encode()
+                self.regexps[search] = replace
+
+    async def start(self, wait=True):
+        if not self.quiet:
+            self.output(
+                self.colors.bgray.encode()
+                + b'+ '
+                + shlex.join([
+                    arg.replace('\n', '\\n')
+                    for arg in self.args
+                ]).encode()
+                + self.colors.reset.encode(),
+                highlight=False
+            )
+
+        # Get a reference to the event loop as we plan to use
+        # low-level APIs.
+        loop = asyncio.get_running_loop()
+
+        self.exit_future = asyncio.Future(loop=loop)
+
+        # Create the subprocess controlled by DateProtocol;
+        # redirect the standard output into a pipe.
+        self.transport, self.protocol = await loop.subprocess_exec(
+            lambda: SubprocessProtocol(self),
+            *self.args,
+            stdin=None,
+        )
+        self.started = True
+
+    async def wait(self, *args, **kwargs):
+        if not self.started:
+            await self.start()
+
+        if not self.waited:
+            # Wait for the subprocess exit using the process_exited()
+            # method of the protocol.
+            await self.exit_future
+
+            # Close the stdout pipe.
+            self.transport.close()
+
+            self.waited = True
+
+        return self
+
+    def stdout(self, data):
+        self.out_raw.extend(data)
+        if not self.quiet:
+            self.output(data)
+
+    def stderr(self, data):
+        self.err_raw.extend(data)
+        if not self.quiet:
+            self.output(data)
+
+    @functools.cached_property
+    def out(self):
+        return self.out_raw.decode().strip()
+
+    @functools.cached_property
+    def err(self):
+        return self.err_raw.decode().strip()
+
+    @functools.cached_property
+    def rc(self):
+        return self.transport.get_returncode()
+
+    def output(self, data, highlight=True, flush=True):
+        for line in data.strip().split(b'\n'):
+            line = [self.highlight(line) if highlight else line]
+            if self.prefix:
+                line = self.prefix_line() + line
+            line.append(b'\n')
+            line = b''.join(line)
+            self.write(line)
+
+        if flush:
+            self.flush()
+
+    def highlight(self, line, highlight=True):
+        if not highlight or (
+            b'\x1b[' in line
+            or b'\033[' in line
+            or b'\\e[' in line
+        ):
+            return line
+
+        for search, replace in self.regexps.items():
+            line = re.sub(search, replace, line)
+        line = line + self.colors.reset.encode()
+
+        return line
+
+    def prefix_line(self):
+        if self.prefix not in self.prefixes:
+            self.prefixes[self.prefix] = self.prefix_colors[len(self.prefixes)]
+            if len(self.prefix) > self.prefix_length:
+                type(self).prefix_length = len(self.prefix)
+
+        return [
+            self.prefixes[self.prefix].encode(),
+            b' ' * (self.prefix_length - len(self.prefix)),
+            self.prefix.encode(),
+            b' ',
+            self.colors.reset.encode(),
+            b'| '
+        ]

shlax/targets/buildah.py

@@ -1,156 +0,0 @@
-import asyncio
-import os
-import asyncio
-from pathlib import Path
-import signal
-import shlex
-import subprocess
-import sys
-import textwrap
-
-from ..actions.base import Action
-from ..exceptions import Mistake
-from ..proc import Proc
-from ..image import Image
-from .localhost import Localhost
-
-
-class Buildah(Localhost):
-    """
-    The build script iterates over visitors and runs the build functions, it
-    also provides wrappers around the buildah command.
-    """
-    contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount', 'image']
-
-    def __init__(self, base, *args, commit=None, push=False, cmd=None, **kwargs):
-        if isinstance(base, Action):
-            args = [base] + list(args)
-            base = 'alpine'  # default selection in case of mistake
-        super().__init__(*args, **kwargs)
-        self.base = base
-        self.mounts = dict()
-        self.ctr = None
-        self.mnt = None
-        self.image = Image(commit) if commit else None
-        self.config= dict(
-            cmd=cmd or 'sh',
-        )
-
-    def shargs(self, *args, user=None, buildah=True, **kwargs):
-        if not buildah or args[0].startswith('buildah'):
-            return super().shargs(*args, user=user, **kwargs)
-
-        _args = ['buildah', 'run']
-        if user:
-            _args += ['--user', user]
-        _args += [self.ctr, '--', 'sh', '-euc']
-        return super().shargs(
-            *(
-                _args
-                + [' '.join([str(a) for a in args])]
-            ),
-            **kwargs
-        )
-
-    def __repr__(self):
-        return f'Base({self.base})'
-
-    async def config(self, line):
-        """Run buildah config."""
-        return await self.exec(f'buildah config {line} {self.ctr}', buildah=False)
-
-    async def copy(self, *args):
-        """Run buildah copy to copy a file from host into container."""
-        src = args[:-1]
-        dst = args[-1]
-        await self.mkdir(dst)
-
-        procs = []
-        for s in src:
-            if Path(s).is_dir():
-                target = self.mnt / s
-                if not target.exists():
-                    await self.mkdir(target)
-                args = ['buildah', 'copy', self.ctr, s, Path(dst) / s]
-            else:
-                args = ['buildah', 'copy', self.ctr, s, dst]
-            procs.append(self.exec(*args, buildah=False))
-
-        return await asyncio.gather(*procs)
-
-    async def mount(self, src, dst):
-        """Mount a host directory into the container."""
-        target = self.mnt / str(dst)[1:]
-        await self.exec(f'mkdir -p {src} {target}', buildah=False)
-        await self.exec(f'mount -o bind {src} {target}', buildah=False)
-        self.mounts[src] = dst
-
-    def is_runnable(self):
-        return (
-            Proc.test
-            or os.getuid() == 0
-        )
-
-    async def call(self, *args, **kwargs):
-        if self.is_runnable():
-            self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out
-            self.mnt = Path((await self.exec('buildah', 'mount', self.ctr, buildah=False)).out)
-            result = await super().call(*args, **kwargs)
-            return result
-
-        from shlax.cli import cli
-        debug = kwargs.get('debug', False)
-        # restart under buildah unshare environment
-        argv = [
-            'buildah', 'unshare',
-            sys.argv[0],  # current script location
-            cli.shlaxfile.path,  # current shlaxfile location
-        ]
-        if debug is True:
-            argv.append('-d')
-        elif isinstance(debug, str) and debug:
-            argv.append('-d=' + debug)
-        argv += [
-            cli.parser.command.name,  # script name ?
-        ]
-
-        await self.exec(*argv)
-
-    async def commit(self):
-        if not self.image:
-            return
-
-        for key, value in self.config.items():
-            await self.exec(f'buildah config --{key} "{value}" {self.ctr}')
-
-        self.sha = (await self.exec(
-            'buildah',
-            'commit',
-            '--format=' + self.image.format,
-            self.ctr,
-            buildah=False,
-        )).out
-
-        if self.image.tags:
-            tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags]
-        else:
-            tags = [self.image.repository]
-
-        for tag in tags:
-            await self.exec('buildah', 'tag', self.sha, tag, buildah=False)
-
-    async def clean(self, *args, **kwargs):
-        if self.is_runnable():
-            for src, dst in self.mounts.items():
-                await self.exec('umount', self.mnt / str(dst)[1:], buildah=False)
-
-            if self.status == 'success':
-                await self.commit()
-                if 'push' in args:
-                    await self.image.push(action=self)
-
-            if self.mnt is not None:
-                await self.exec('buildah', 'umount', self.ctr, buildah=False)
-
-            if self.ctr is not None:
-                await self.exec('buildah', 'rm', self.ctr, buildah=False)

shlax/targets/docker.py

@@ -1,111 +0,0 @@
-import asyncio
-from pathlib import Path
-import os
-
-from ..image import Image
-from .localhost import Localhost
-
-
-class Docker(Localhost):
-    """Manage a docker container."""
-    default_steps = ['install', 'up']
-
-    def __init__(self, *args, **kwargs):
-        self.image = kwargs.get('image', 'alpine')
-        self.name = kwargs.get('name', os.getcwd().split('/')[-1])
-
-        if not isinstance(self.image, Image):
-            self.image = Image(self.image)
-
-        super().__init__(*args, **kwargs)
-
-    def shargs(self, *args, daemon=False, **kwargs):
-        if args[0] == 'docker':
-            return args, kwargs
-
-        extra = []
-        if 'user' in kwargs:
-            extra += ['--user', kwargs.pop('user')]
-
-        args, kwargs = super().shargs(*args, **kwargs)
-
-        if self.name:
-            executor = 'exec'
-            extra = [self.name]
-            return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + list(args), kwargs
-
-        executor = 'run'
-        cwd = os.getcwd()
-        if daemon:
-            extra += ['-d']
-        extra = extra + ['-v', f'{cwd}:{cwd}', '-w', f'{cwd}']
-        return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + [str(self.image)] + list(args), kwargs
-
-    async def call(self, *args, **kwargs):
-        def step(step):
-            return not args or step in args
-
-        # self.name = (
-        #     await self.exec(
-        #         'docker', 'ps', '-aq', '--filter',
-        #         'name=' + self.name,
-        #         raises=False
-        #     )
-        # ).out.split('\n')[0]
-        if step('install') and 'install' in self.kwargs:
-            breakpoint()
-            await self.action(self.kwargs['install'], *args, **kwargs)
-
-        if step('rm') and await self.exists():
-            await self.exec('docker', 'rm', '-f', self.name)
-
-        if step('up'):
-            if await self.exists():
-                self.name = (await self.exec('docker', 'start', self.name)).out
-            else:
-                self.id = (await self.exec(
-                    'docker', 'run', '-d', '--name', self.name, str(self.image))
-                ).out
-        return await super().call(*args, **kwargs)
-
-    async def exists(self):
-        proc = await self.exec(
-            'docker', 'ps', '-aq', '--filter',
-            'name=' + self.name,
-            raises=False
-        )
-        return bool(proc.out.strip())
-
-    async def copy(self, *args):
-        src = args[:-1]
-        dst = args[-1]
-        await self.mkdir(dst)
-
-        procs = []
-        for s in src:
-            '''
-            if Path(s).is_dir():
-                await self.mkdir(s)
-                args = ['docker', 'copy', self.ctr, s, Path(dst) / s]
-            else:
-                args = ['docker', 'copy', self.ctr, s, dst]
-            '''
-            args = ['docker', 'cp', s, self.name + ':' + dst]
-            procs.append(self.exec(*args))
-
-        return await asyncio.gather(*procs)
-
-    async def up(self):
-        """Ensure container is up and running."""
-        if await self.exists():
-            self.name = (await self.exec('docker', 'start', self.name)).out
-        else:
-            self.id = (await self.exec(
-                'docker', 'run', '-d', '--name', self.name, str(self.image))
-            ).out
-    up.shlaxstep = True
-
-    async def rm(self):
-        """Remove container."""
-        await self.exec('docker', 'rm', '-f', self.name)
-    rm.shlaxstep = True

shlax/targets/localhost.py

@@ -1,80 +0,0 @@
-import os
-import re
-
-from shlax.proc import Proc
-
-from ..strategies.script import Script
-
-
-class Localhost(Script):
-    root = '/'
-    contextualize = Script.contextualize + ['home']
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.home = kwargs.pop('home', os.getcwd())
-
-    def shargs(self, *args, **kwargs):
-        user = kwargs.pop('user', None)
-        args = [str(arg) for arg in args if args is not None]
-
-        if args and ' ' in args[0]:
-            if len(args) == 1:
-                args = ['sh', '-euc', args[0]]
-            else:
-                args = ['sh', '-euc'] + list(args)
-
-        if user == 'root':
-            args = ['sudo'] + args
-        elif user:
-            args = ['sudo', '-u', user] + args
-
-        if self.parent:
-            return self.parent.shargs(*args, **kwargs)
-        else:
-            return args, kwargs
-
-    async def exec(self, *args, **kwargs):
-        kwargs['output'] = self.output
-        args, kwargs = self.shargs(*args, **kwargs)
-        proc = await Proc(*args, **kwargs)()
-        if kwargs.get('wait', True):
-            await proc.wait()
-        return proc
-
-    async def rexec(self, *args, **kwargs):
-        kwargs['user'] = 'root'
-        return await self.exec(*args, **kwargs)
-
-    async def env(self, name):
-        return (await self.exec('echo $' + name)).out
-
-    async def exists(self, *paths):
-        proc = await self.exec('type ' + ' '.join(cmd), raises=False)
-
-    async def which(self, *cmd):
-        """
-        Return the first path to the cmd in the container.
-
-        If cmd argument is a list then it will try all commands.
-        """
-        proc = await self.exec('type ' + ' '.join(cmd), raises=False)
-        result = []
-        for res in proc.out.split('\n'):
-            match = re.match('([^ ]+) is ([^ ]+)$', res.strip())
-            if match:
-                result.append(match.group(1))
-        return result
-
-    async def copy(self, *args):
-        if args[-1].startswith('./'):
-            args = list(args)
-            args[-1] = self.home + '/' + args[-1][2:]
-        args = ['cp', '-rua'] + list(args)
-        return await self.exec(*args)
-
-    async def mount(self, *dirs):
-        pass
-
-    async def mkdir(self, *dirs):
-        return await self.exec(*['mkdir', '-p'] + list(dirs))

shlax/targets/ssh.py

@@ -1,17 +0,0 @@
-import os
-
-from shlax.proc import Proc
-
-from .localhost import Localhost
-
-
-class Ssh(Localhost):
-    root = '/'
-
-    def __init__(self, host, *args, **kwargs):
-        self.host = host
-        super().__init__(*args, **kwargs)
-
-    def shargs(self, *args, **kwargs):
-        args, kwargs = super().shargs(*args, **kwargs)
-        return (['ssh', self.host] + list(args)), kwargs

shlaxfile.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env shlax
-from shlax.contrib.gitlab import *
-
-PYTEST = 'py.test -svv tests'
-
-test = Script(
-    Pip('.[test]'),
-    Run(PYTEST),
-)
-
-build = Buildah(
-    'quay.io/podman/stable',
-    Packages('python38', 'buildah', 'unzip', 'findutils', 'python3-yaml', upgrade=False),
-    Async(
-        # dancing for pip on centos python3.8
-        Run('''
-        curl -o setuptools.zip https://files.pythonhosted.org/packages/42/3e/2464120172859e5d103e5500315fb5555b1e908c0dacc73d80d35a9480ca/setuptools-45.1.0.zip
-        unzip setuptools.zip
-        mkdir -p /usr/local/lib/python3.8/site-packages/
-        sh -c "cd setuptools-* && python3.8 setup.py install"
-        easy_install-3.8 pip
-        echo python3.8 -m pip > /usr/bin/pip
-        chmod +x /usr/bin/pip
-        '''),
-        Copy('shlax/', 'setup.py', '/app'),
-    ),
-    Pip('/app[full]'),
-    commit='docker.io/yourlabs/shlax',
-    workdir='/app',
-)
-
-shlax = Container(
-    build=build,
-    test=Script(Run('./shlaxfile.py -d test')),
-)
-
-gitlabci = GitLabCI(
-    test=dict(
-        stage='build',
-        script='pip install -U --user -e .[test] && ' + PYTEST,
-        image='yourlabs/python',
-    ),
-    build=dict(
-        stage='build',
-        image='yourlabs/shlax',
-        script='pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push',
-        cache=dict(paths=['.cache'], key='cache'),
-    ),
-    pypi=dict(
-        stage='deploy',
-        only=['tags'],
-        image='yourlabs/python',
-        script='pypi-release',
-    ),
-)

tests/actions/test_base.py

@@ -1,53 +0,0 @@
-from shlax import *
-
-inner = Run()
-other = Run('ls')
-middle = Buildah('alpine', inner, other)
-outer = Localhost(middle)
-middle = outer.actions[0]
-other = middle.actions[1]
-inner = middle.actions[0]
-
-
-def test_action_init_args():
-    assert other.args == ('ls',)
-
-
-def test_action_parent_autoset():
-    assert list(outer.actions) == [middle]
-    assert middle.parent == outer
-    assert inner.parent == middle
-    assert other.parent == middle
-
-
-def test_action_context():
-    assert outer.context is inner.context
-    assert middle.context is inner.context
-    assert middle.context is outer.context
-    assert other.context is outer.context
-
-
-def test_action_sibblings():
-    assert inner.sibblings() == [other]
-    assert inner.sibblings(lambda s: s.args[0] == 'ls') == [other]
-    assert inner.sibblings(lambda s: s.args[0] == 'foo') == []
-    assert inner.sibblings(type='run') == [other]
-    assert inner.sibblings(args=('ls',)) == [other]
-
-
-def test_actions_parents():
-    assert other.parents() == [middle, outer]
-    assert other.parents(lambda p: p.base == 'alpine') == [middle]
-    assert inner.parents(type='localhost') == [outer]
-    assert inner.parents(type='buildah') == [middle]
-
-
-def test_action_childrens():
-    assert middle.children() == [inner, other]
-    assert middle.children(lambda a: a.args[0] == 'ls') == [other]
-    assert outer.children() == [middle, inner, other]
-
-
-def test_action_getattr():
-    assert other.exec == middle.exec
-    assert other.shargs == middle.shargs

tests/test_colors.py

@@ -0,0 +1,7 @@
+import shlax
+
+
+def test_colors():
+    assert shlax.colors.cyan == '\u001b[38;5;51m'
+    assert shlax.colors.bcyan == '\u001b[1;38;5;51m'
+    assert shlax.colors.reset == '\u001b[0m'

tests/test_image.py

@@ -1,33 +0,0 @@
-import pytest
-import os
-
-from shlax import Image
-
-
-tests = {
-    'docker://a.b:1337/re/po:x,y': ('docker', 'a.b:1337', 're/po', 'x,y'),
-    'docker://a.b/re/po:x,y': ('docker', 'a.b', 're/po', 'x,y'),
-    'a.b:1337/re/po:x,y': (None, 'a.b:1337', 're/po', 'x,y'),
-    'a.b/re/po:x,y': (None, 'a.b', 're/po', 'x,y'),
-    're/po:x,y': (None, None, 're/po', 'x,y'),
-    're/po': (None, None, 're/po', 'latest'),
-    'docker://re/po': ('docker', None, 're/po', 'latest'),
-    'docker://re/po:x,y': ('docker', None, 're/po', 'x,y'),
-}
-
-@pytest.mark.parametrize(
-    'arg,expected', [(k, dict(
-        backend=v[0], registry=v[1], repository=v[2], tags=v[3].split(',')
-    )) for k, v in tests.items()]
-)
-def test_args(arg, expected):
-    Image.ENV_TAGS = []
-    im = Image(arg)
-    for k, v in expected.items():
-        assert getattr(im, k) == v
-
-def test_args_env():
-    os.environ['IMAGE_TEST_ARGS_ENV'] = 'foo'
-    Image.ENV_TAGS = ['IMAGE_TEST_ARGS_ENV']
-    im = Image('re/po:x,y')
-    assert im.tags == ['x', 'y', 'foo']

tests/test_output.py

@@ -1,24 +0,0 @@
-import pytest
-from shlax import Output
-
-
-class Write:
-    def __init__(self):
-        self.output = ''
-    def __call__(self, out):
-        self.output += out.decode('utf8')
-
-
-@pytest.fixture
-def write():
-    return Write()
-
-
-def test_output_regexps(write):
-    output = Output(
-        regexps={'^(.*)$': '{red}\\1'},
-        write=write,
-        flush=lambda: None,
-    )
-    output('foo')
-    assert write.output.strip() == output.colors['red'] + 'foo' + output.colors['reset']

tests/test_proc.py

@@ -1,65 +1,197 @@
 import pytest
+from unittest.mock import Mock, call
 
-from unittest.mock import patch
-
-from shlax import *
-from shlax import proc
+from shlax import Proc
 
 
-test_args_params = [
-    (
-        Localhost(Run('echo hi')),
-        [('sh', '-euc', 'echo hi')]
-    ),
-    (
-        Localhost(Run('echo hi', user='jimi')),
-        [('sudo', '-u', 'jimi', 'sh', '-euc', 'echo hi')]
-    ),
-    (
-        Localhost(Run('echo hi', user='root')),
-        [('sudo', 'sh', '-euc', 'echo hi')]
-    ),
-    (
-        Ssh('host', Run('echo hi', user='root')),
-        [('ssh', 'host', 'sudo', 'sh', '-euc', 'echo hi')]
-    ),
-    (
-        Buildah('alpine', Run('echo hi')),
-        [
-            ('buildah', 'from', 'alpine'),
-            ('buildah', 'mount', ''),
-            ('buildah', 'run', '', '--', 'sh', '-euc', 'echo hi'),
-            ('buildah', 'umount', ''),
-            ('buildah', 'rm', ''),
-        ]
-    ),
-    (
-        Buildah('alpine', Run('echo hi', user='root')),
-        [
-            ('buildah', 'from', 'alpine'),
-            ('buildah', 'mount', ''),
-            ('buildah', 'run', '--user', 'root', '', '--', 'sh', '-euc', 'echo hi'),
-            ('buildah', 'umount', ''),
-            ('buildah', 'rm', ''),
-        ]
-    ),
-    (
-        Ssh('host', Buildah('alpine', Run('echo hi', user='root'))),
-        [
-            ('ssh', 'host', 'buildah', 'from', 'alpine'),
-            ('ssh', 'host', 'buildah', 'mount', ''),
-            ('ssh', 'host', 'buildah', 'run', '--user', 'root', '', '--', 'sh', '-euc', 'echo hi'),
-            ('ssh', 'host', 'buildah', 'umount', ''),
-            ('ssh', 'host', 'buildah', 'rm', ''),
-        ]
-    ),
-]
-@pytest.mark.parametrize(
-    'script,commands',
-    test_args_params
-)
 @pytest.mark.asyncio
-async def test_args(script, commands):
-    with Proc.mock():
-        await script()
-        assert commands == Proc.test
+@pytest.mark.parametrize(
+    'args',
+    (
+        ['sh', '-c', 'echo hi'],
+        ['echo hi'],
+        ['sh -c "echo hi"'],
+    )
+)
+async def test_proc(args):
+    proc = Proc(*args, quiet=True)
+    assert not proc.waited
+    assert not proc.started
+    await proc.wait()
+    assert proc.waited
+    assert proc.started
+    assert proc.out == 'hi'
+    assert proc.err == ''
+    assert proc.out_raw == b'hi\n'
+    assert proc.err_raw == b''
+    assert proc.rc == 0
+
+
+@pytest.mark.asyncio
+async def test_wait_unbound():
+    proc = await Proc('echo hi', quiet=True).wait()
+    assert proc.out == 'hi'
+
+
+@pytest.mark.asyncio
+async def test_rc_1():
+    proc = await Proc(
+        'NON EXISTING COMMAND',
+        quiet=True,
+    ).wait()
+    assert proc.rc != 0
+    assert proc.err == 'sh: line 1: NON: command not found'
+
+
+@pytest.mark.asyncio
+async def test_prefix():
+    """
+    Test output prefixes for when executing multiple commands in parallel.
+    """
+    Proc.prefix_length = 0  # reset
+
+    write = Mock()
+    await Proc(
+        'echo hi',
+        write=write,
+        prefix='test_prefix',
+    ).wait()
+    await Proc(
+        'echo hi',
+        write=write,
+        prefix='test_prefix_1'
+    ).wait()
+    await Proc(
+        'echo hi',
+        write=write,
+        prefix='test_prefix',
+    ).wait()
+
+    assert write.mock_calls == [
+        call(
+            Proc.prefix_colors[0].encode()
+            + b'test_prefix '
+            + Proc.colors.reset.encode()
+            + b'| '
+            + Proc.colors.bgray.encode()
+            + b'+ sh -euc \'echo hi\''
+            + Proc.colors.reset.encode()
+            + b'\n'
+        ),
+        call(
+            Proc.prefix_colors[0].encode()
+            + b'test_prefix '
+            + Proc.colors.reset.encode()
+            + b'| hi'
+            + Proc.colors.reset.encode()
+            + b'\n'
+        ),
+        call(
+            Proc.prefix_colors[1].encode()
+            + b'test_prefix_1 '
+            + Proc.colors.reset.encode()
+            + b'| '
+            + Proc.colors.bgray.encode()
+            + b'+ sh -euc \'echo hi\''
+            + Proc.colors.reset.encode()
+            + b'\n'
+        ),
+        call(
+            Proc.prefix_colors[1].encode()
+            # padding has been added because of output1
+            + b'test_prefix_1 '
+            + Proc.colors.reset.encode()
+            + b'| hi'
+            + Proc.colors.reset.encode()
+            + b'\n'
+        ),
+        call(
+            Proc.prefix_colors[0].encode()
+            # padding has been added because of output1
+            + b'  test_prefix '
+            + Proc.colors.reset.encode()
+            + b'| '
+            + Proc.colors.bgray.encode()
+            + b'+ sh -euc \'echo hi\''
+            + Proc.colors.reset.encode()
+            + b'\n'
+        ),
+        call(
+            Proc.prefix_colors[0].encode()
+            # padding has been added because of output1
+            + b'  test_prefix '
+            + Proc.colors.reset.encode()
+            + b'| hi'
+            + Proc.colors.reset.encode()
+            + b'\n'
+        )
+    ]
+
+
+@pytest.mark.asyncio
+async def test_prefix_multiline():
+    Proc.prefix_length = 0  # reset
+    proc = await Proc(
+        'echo -e "a\nb"',
+        write=Mock(),
+        prefix='test_prefix',
+    ).wait()
+    assert proc.write.mock_calls == [
+        call(
+            Proc.prefix_colors[0].encode()
+            + b'test_prefix '
+            + Proc.colors.reset.encode()
+            + b'| '
+            + Proc.colors.bgray.encode()
+            + b'+ sh -euc \'echo -e "a\\nb"\''
+            + Proc.colors.reset.encode()
+            + b'\n'
+        ),
+        call(
+            Proc.prefix_colors[0].encode()
+            + b'test_prefix '
+            + Proc.colors.reset.encode()
+            + b'| a'
+            + Proc.colors.reset.encode()
+            + b'\n'
+        ),
+        call(
+            Proc.prefix_colors[0].encode()
+            # padding has been added because of output1
+            + b'test_prefix '
+            + Proc.colors.reset.encode()
+            + b'| b'
+            + Proc.colors.reset.encode()
+            + b'\n'
+        ),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_highlight():
+    """
+    Test that we can color output with regexps.
+    """
+    proc = await Proc(
+        'echo hi',
+        write=Mock(),
+        regexps={
+            r'h([\w\d-]+)': 'h{cyan}\\1',
+        }
+    ).wait()
+    proc.write.assert_called_with(b'h\x1b[38;5;51mi\x1b[0m\n')
+
+
+@pytest.mark.asyncio
+async def test_highlight_if_not_colored():
+    """
+    Test that coloration does not apply on output that is already colored.
+    """
+    proc = await Proc(
+        'echo -e h"\\e[31m"i',
+        write=Mock(),
+        regexps={
+            r'h([\w\d-]+)': 'h{cyan}\\1',
+        }
+    ).wait()
+    proc.write.assert_called_with(b'h\x1b[31mi\n')