dnf stuff

See merge request oss/shlax!2

.gitlab-ci.yml

@@ -4,11 +4,9 @@ build:
     paths: [.cache]
   image: quay.io/buildah/stable
   script:
-  - dnf install -y curl python38
+  - dnf install -y python3-pip
-  - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+  - pip3 install -U --user -e .[cli]
-  - python3.8 get-pip.py
+  - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build
-  - pip3.8 install -U --user -e .[cli]
-  - CACHE_DIR=$(pwd)/.cache ~/.local/bin/shlax ./shlaxfile.py build
   stage: build
 
 test:

README.md

@@ -1,9 +1,8 @@
 # Shlax: Pythonic automation tool
 
 Shlax is a Python framework for system automation, initially with the purpose
-of replacing docker, docker-compose and ansible with a single tool, with the
+of replacing docker, docker-compose and ansible with a single tool with the
-purpose of code-reuse. It may be viewed as "async fabric rewrite by a
+purpose of code-reuse made possible by target abstraction.
-megalomanic Django fanboy".
 
 The pattern resolves around two moving parts: Actions and Targets.
 
@@ -187,12 +186,13 @@ class Docker(Target):
         return await self.parent.exec(*['docker', 'exec', self.name] + args)
 ```
 
-Don't worry about `self.parent` being set, it is enforced to `Localhost` if
+This also means that you always need a parent with an exec implementation,
-unset so that we always have something that actually spawns a process in the
+there are two:
-chain ;)
 
-The result of that design is that the following use cases are open for
+- Localhost, executes on localhost
-business:
+- Stub, for testing
+
+The result of that design is that the following use cases are available:
 
 ```python
 # This action installs my favorite package on any distro
@@ -215,14 +215,44 @@ Ssh(host='yourhost')(build)
 
 # Or on a server behingh a bastion:
 # ssh yourbastion ssh yourhost build exec apt install python3
-Ssh(host='bastion')(Ssh(host='yourhost')(build))
+Localhost()(Ssh(host='bastion')(Ssh(host='yourhost')(build))
 
 # That's going to do the same
-Ssh(
+Localhost(Ssh(
     Ssh(
         build,
         host='yourhost'
     ),
     host='bastion'
-)()
+))()
 ```
+
+## CLI
+
+You can execute Shlax actions directly on the command line with the `shlax` CLI
+command.
+
+For your own Shlaxfiles, you can build your CLI with your favorite CLI
+framework. If you decide to use `cli2`, then Shlax provides a thin layer on top
+of it: Group and Command objects made for Shlax objects.
+
+For example:
+
+```python
+yourcontainer = Container(
+    build=Buildah(
+        User('app', '/app', 1000),
+        Packages('python', 'unzip', 'findutils'),
+        Copy('setup.py', 'yourdir', '/app'),
+        base='archlinux',
+        commit='yourimage',
+    ),
+)
+
+
+if __name__ == '__main__':
+    print(Group(doc=__doc__).load(yourcontainer).entry_point())
+```
+
+The above will execute a cli2 command with each method of yourcontainer as a
+sub-command.

setup.py

@@ -7,7 +7,7 @@ setup(
     setup_requires='setupmeta',
     extras_require=dict(
         cli=[
-            'cli2',
+            'cli2>=2.2.2',
         ],
         test=[
             'pytest',
@@ -24,7 +24,7 @@ setup(
     python_requires='>=3',
     entry_points={
         'console_scripts': [
-            'shlax = shlax.cli:cli',
+            'shlax = shlax.cli:cli.entry_point',
         ],
     },
 )

shlax/actions/copy.py

@@ -0,0 +1,56 @@
+import asyncio
+import binascii
+import os
+
+
+class Copy:
+    def __init__(self, *args):
+        self.src = args[:-1]
+        self.dst = args[-1]
+
+    def listfiles(self):
+        if getattr(self, '_listfiles', None):
+            return self._listfiles
+
+        result = []
+        for src in self.src:
+            if os.path.isfile(src):
+                result.append(src)
+                continue
+
+            for root, dirs, files in os.walk(src):
+                if '__pycache__' in root:
+                    continue
+                result += [
+                    os.path.join(root, f)
+                    for f in files
+                    if not f.endswith('.pyc')
+                ]
+        self._listfiles = result
+        return result
+
+    async def __call__(self, target):
+        await target.mkdir(self.dst)
+
+        for path in self.listfiles():
+            if os.path.isdir(path):
+                await target.mkdir(os.path.join(self.dst, path))
+            elif '/' in path:
+                dirname = os.path.join(
+                    self.dst,
+                    '/'.join(path.split('/')[:-1])
+                )
+                await target.mkdir(dirname)
+                await target.copy(path, dirname)
+            else:
+                await target.copy(path, self.dst)
+
+    def __str__(self):
+        return f'Copy({", ".join(self.src)}, {self.dst})'
+
+    async def cachekey(self):
+        async def chksum(path):
+            with open(path, 'rb') as f:
+                return (path, str(binascii.crc32(f.read())))
+        results = await asyncio.gather(*[chksum(f) for f in self.listfiles()])
+        return {path: chks for path, chks in results}

shlax/actions/packages.py

@@ -52,8 +52,9 @@ class Packages:
 
     installed = []
 
-    def __init__(self, *packages):
+    def __init__(self, *packages, upgrade=True):
         self.packages = []
+        self.upgrade = upgrade
         for package in packages:
             line = dedent(package).strip().replace('\n', ' ')
             self.packages += line.split(' ')
@@ -116,6 +117,7 @@ class Packages:
 
         self.cmds = self.mgrs[self.mgr]
         await self.update(target)
+        if self.upgrade:
             await target.rexec(self.cmds['upgrade'])
 
         packages = []
@@ -159,5 +161,5 @@ class Packages:
     async def pacman_setup(self, target):
         return self.cache_root + '/pacman'
 
-    def __repr__(self):
+    def __str__(self):
-        return f'Packages({self.packages})'
+        return f'Packages({self.packages}, upgrade={self.upgrade})'

shlax/actions/pip.py

@@ -0,0 +1,69 @@
+from glob import glob
+import os
+from urllib import request
+
+from .base import Action
+
+
+class Pip(Action):
+    """Pip abstraction layer."""
+    def __init__(self, *pip_packages):
+        self.pip_packages = pip_packages
+
+    async def __call__(self, target):
+        # ensure python presence
+        results = await target.which('python3', 'python')
+        if results:
+            python = results[0]
+        else:
+            raise Exception('Could not find pip nor python')
+
+        # ensure pip module presence
+        result = await target.exec(python, '-m', 'pip', raises=False)
+        if result.rc != 0:
+            if not os.path.exists('get-pip.py'):
+                req = request.urlopen(
+                    'https://bootstrap.pypa.io/get-pip.py'
+                )
+                content = req.read()
+                with open('get-pip.py', 'wb+') as f:
+                    f.write(content)
+
+            await target.copy('get-pip.py', '.')
+            await target.exec(python, 'get-pip.py')
+
+        # choose a cache directory
+        if 'CACHE_DIR' in os.environ:
+            cache = os.path.join(os.getenv('CACHE_DIR'), 'pip')
+        else:
+            cache = os.path.join(os.getenv('HOME'), '.cache', 'pip')
+
+        # and mount it
+        if getattr(target, 'mount', None):
+            # we are in a target which shares a mount command
+            await target.mount(cache, '/root/.cache/pip')
+
+        source = []
+        nonsource = []
+        for package in self.pip_packages:
+            if os.path.exists(package):
+                source.append(package)
+            else:
+                nonsource.append(package)
+
+        if nonsource:
+            await target.exec(
+                python, '-m', 'pip',
+                'install', '--upgrade',
+                *nonsource
+            )
+
+        if source:
+            await target.exec(
+                python, '-m', 'pip',
+                'install', '--upgrade', '--editable',
+                *source
+            )
+
+    def __str__(self):
+        return f'Pip({", ".join(self.pip_packages)})'

shlax/actions/user.py

@@ -0,0 +1,42 @@
+import os
+import re
+
+from .packages import Packages
+
+
+class User:
+    """
+    Create a user.
+
+    Example:
+
+        User('app', '/app', getenv('_CONTAINERS_ROOTLESS_UID', 1000)),
+
+    _CONTAINERS_ROOTLESS_UID allows to get your UID during build, which happens
+    in buildah unshare.
+    """
+    def __init__(self, username, home, uid):
+        self.username = username
+        self.home = home
+        self.uid = uid
+
+    def __str__(self):
+        return f'User({self.username}, {self.home}, {self.uid})'
+
+    async def __call__(self, target):
+        result = await target.rexec('id', self.uid)
+        if result.rc == 0:
+            old = re.match('.*\(([^)]*)\).*', result.out).group(1)
+            await target.rexec(
+                'usermod',
+                '-d', self.home,
+                '-l', self.username,
+                old
+            )
+        else:
+            await target.rexec(
+                'useradd',
+                '-d', self.home,
+                '-u', self.uid,
+                self.username
+            )

shlax/cli.py

@@ -5,6 +5,7 @@ Shlax executes mostly in 3 ways:
 - With the name of a module in shlax.repo: a community maintained shlaxfile
 """
 import ast
+import asyncio
 import cli2
 import glob
 import inspect
@@ -13,43 +14,67 @@ import os
 import sys
 
 
-class ConsoleScript(cli2.ConsoleScript):
+class Group(cli2.Group):
-    def __call__(self):
+    def __init__(self, *args, **kwargs):
-        repo = os.path.join(os.path.dirname(__file__), 'repo')
+        super().__init__(*args, **kwargs)
-
+        self.cmdclass = Command
-        if len(self.argv) > 1:
-            repofile = os.path.join(repo, sys.argv[1] + '.py')
-            if os.path.isfile(self.argv[1]):
-                self.argv = sys.argv[1:]
-                self.load_shlaxfile(sys.argv[1])
-            elif os.path.isfile(repofile):
-                self.argv = sys.argv[1:]
-                self.load_shlaxfile(repofile)
-            else:
-                raise Exception('File not found ' + sys.argv[1])
-        else:
-            available = glob.glob(os.path.join(repo, '*.py'))
-        return super().__call__()
 
 
-    def load_shlaxfile(self, path):
+class Command(cli2.Command):
-        with open(path) as f:
+    def call(self, *args, **kwargs):
-            src = f.read()
+        return self.shlax_target(self.target)
-        tree = ast.parse(src)
 
-        members = []
+    def __call__(self, *argv):
-        for node in tree.body:
+        from shlax.targets.base import Target
-            if not isinstance(node, ast.Assign):
+        self.shlax_target = Target()
+        result = super().__call__(*argv)
+        self.shlax_target.output.results(self.shlax_target)
+        return result
+
+
+class ActionCommand(Command):
+    def call(self, *args, **kwargs):
+        self.target = self.target(*args, **kwargs)
+        return super().call(*args, **kwargs)
+
+
+class ConsoleScript(Group):
+    def __call__(self, *argv):
+        self.load_actions()
+        #self.load_shlaxfiles()  # wip
+        return super().__call__(*argv)
+
+    def load_shlaxfiles(self):
+        filesdir = os.path.dirname(__file__) + '/shlaxfiles/'
+        for filename in os.listdir(filesdir):
+            filepath = filesdir + filename
+            if not os.path.isfile(filepath):
                 continue
-            if not isinstance(node.value, ast.Call):
+
+            with open(filepath, 'r') as f:
+                tree = ast.parse(f.read())
+            group = self.group(filename[:-3])
+
+            main = Group(doc=__doc__).load(shlax)
+
+    def load_actions(self):
+        actionsdir = os.path.dirname(__file__) + '/actions/'
+        for filename in os.listdir(actionsdir):
+            filepath = actionsdir + filename
+            if not os.path.isfile(filepath):
                 continue
-            members.append(node.targets[0].id)
+            with open(filepath, 'r') as f:
+                tree = ast.parse(f.read())
+            cls = [
+                node
+                for node in tree.body
+                if isinstance(node, ast.ClassDef)
+            ]
+            if not cls:
+                continue
+            mod = importlib.import_module('shlax.actions.' + filename[:-3])
+            cls = getattr(mod, cls[0].name)
+            self.add(cls, name=filename[:-3], cmdclass=ActionCommand)
 
-        spec = importlib.util.spec_from_file_location('shlaxfile', sys.argv[1])
-        mod = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(mod)
-        for member in members:
-            from shlax.targets.localhost import Localhost
-            self[member] = cli2.Callable(member, Localhost(getattr(mod, member)))
 
-cli = ConsoleScript(__doc__)
+cli = ConsoleScript(doc=__doc__)

shlax/container.py

@@ -0,0 +1,32 @@
+import os
+
+from .image import Image
+
+
+class Container:
+    def __init__(self, build=None, image=None):
+        self.build = build
+        self.image = self.build.image
+        prefix = os.getcwd().split('/')[-1]
+        repo = self.image.repository.replace('/', '-')
+        if prefix == repo:
+            self.name = repo
+        else:
+            self.name = '-'.join([prefix, repo])
+
+    async def start(self, target):
+        """Start the container"""
+        await target.rexec(
+            'podman',
+            'run',
+            '--name',
+            self.name,
+            str(self.image),
+        )
+
+    async def stop(self, target):
+        """Start the container"""
+        await target.rexec('podman', 'stop', self.name)
+
+    def __str__(self):
+        return f'Container(name={self.name}, image={self.image})'

shlax/image.py

@@ -1,20 +1,9 @@
+import copy
 import os
 import re
 
 
 class Image:
-    ENV_TAGS = (
-        # gitlab
-        'CI_COMMIT_SHORT_SHA',
-        'CI_COMMIT_REF_NAME',
-        'CI_COMMIT_TAG',
-        # CircleCI
-        'CIRCLE_SHA1',
-        'CIRCLE_TAG',
-        'CIRCLE_BRANCH',
-        # contributions welcome here
-    )
-
     PATTERN = re.compile(
         '^((?P<backend>[a-z]*)://)?((?P<registry>[^/]*[.][^/]*)/)?((?P<repository>[^:]+))?(:(?P<tags>.*))?$'  # noqa
         , re.I
@@ -44,12 +33,6 @@ class Image:
         if self.registry == 'docker.io':
             self.format = 'docker'
 
-        # figure tags from CI vars
-        for name in self.ENV_TAGS:
-            value = os.getenv(name)
-            if value:
-                self.tags.append(value)
-
         # filter out tags which resolved to None
         self.tags = [t for t in self.tags if t]
 
@@ -57,10 +40,6 @@ class Image:
         if not self.tags:
             self.tags = ['latest']
 
-    async def __call__(self, action, *args, **kwargs):
-        args = list(args)
-        return await action.exec(*args, **self.kwargs)
-
     def __str__(self):
         return f'{self.repository}:{self.tags[-1]}'
 
@@ -74,3 +53,8 @@ class Image:
 
         for tag in self.tags:
             await action.exec('buildah', 'push', f'{self.repository}:{tag}')
+
+    def layer(self, key):
+        layer = copy.deepcopy(self)
+        layer.tags = ['layer-' + key]
+        return layer

shlax/output.py

@@ -1,5 +1,6 @@
 import re
 import sys
+import types
 
 from .colors import colors
 
@@ -25,15 +26,16 @@ class Output:
     def colorize(self, code, content):
         return self.color(code) + content + self.color()
 
-    def colorized(self):
+    def colorized(self, action):
-        if hasattr(self.subject, 'colorized'):
+        if hasattr(action, 'colorized'):
-            return self.subject.colorized(self.colors)
+            return action.colorized(self.colors)
+        elif isinstance(action, types.MethodType):
+            return f'{action.__self__}.{action.__name__}'
         else:
-            return str(self.subject)
+            return str(action)
 
     def __init__(
             self,
-            subject=None,
             prefix=None,
             regexps=None,
             debug='cmd,visit,out',
@@ -41,7 +43,6 @@ class Output:
             flush=None,
             **kwargs
         ):
-        self.subject = subject
         self.prefix = prefix
         self.debug = debug
         self.prefix_length = 0
@@ -114,59 +115,79 @@ class Output:
 
         return line
 
-    def test(self):
+    def test(self, action):
         self(''.join([
             self.colors['purplebold'],
             '!  TEST   ',
             self.colors['reset'],
-            self.colorized(),
+            self.colorized(action),
             '\n',
         ]))
 
-    def clean(self):
+    def clean(self, action):
         if self.debug:
             self(''.join([
                 self.colors['bluebold'],
                 '+  CLEAN  ',
                 self.colors['reset'],
-                self.colorized(),
+                self.colorized(action),
                 '\n',
             ]))
 
-    def start(self):
+    def start(self, action):
         if self.debug is True or 'visit' in str(self.debug):
             self(''.join([
                 self.colors['orangebold'],
                 '⚠  START  ',
                 self.colors['reset'],
-                self.colorized(),
+                self.colorized(action),
                 '\n',
             ]))
 
-    def success(self):
+    def info(self, text):
+        if self.debug is True or 'visit' in str(self.debug):
+            self(''.join([
+                self.colors['cyanbold'],
+                '➤   INFO  ',
+                self.colors['reset'],
+                text,
+                '\n',
+            ]))
+
+    def skip(self, action):
+        if self.debug is True or 'visit' in str(self.debug):
+            self(''.join([
+                self.colors['yellowbold'],
+                '↪️   SKIP  ',
+                self.colors['reset'],
+                self.colorized(action),
+                '\n',
+            ]))
+
+    def success(self, action):
         if self.debug is True or 'visit' in str(self.debug):
             self(''.join([
                 self.colors['greenbold'],
                 '✔ SUCCESS ',
                 self.colors['reset'],
-                self.colorized(),
+                self.colorized(action),
                 '\n',
             ]))
 
-    def fail(self, exception=None):
+    def fail(self, action, exception=None):
         if self.debug is True or 'visit' in str(self.debug):
             self(''.join([
                 self.colors['redbold'],
                 '✘  FAIL   ',
                 self.colors['reset'],
-                self.colorized(),
+                self.colorized(action),
                 '\n',
             ]))
 
-    def results(self):
+    def results(self, action):
         success = 0
         fail = 0
-        for result in self.subject.results:
+        for result in action.results:
             if result.status == 'success':
                 success += 1
             if result.status == 'failure':

shlax/pod.py

@@ -0,0 +1,26 @@
+import cli2
+
+from shlax.targets.base import Target
+from shlax.actions.parallel import Parallel
+
+
+class Pod:
+    """Help text"""
+    def __init__(self, **containers):
+        self.containers = containers
+
+    async def _call(self, target, method, *names):
+        methods = [
+            getattr(container, method)
+            for name, container in self.containers.items()
+            if not names or name in names
+        ]
+        await target(Parallel(*methods))
+
+    async def build(self, target, *names):
+        """Build container images"""
+        await self._call(target, 'build', *names)
+
+    async def start(self, target, *names):
+        """Start container images"""
+        await self._call(target, 'start', *names)

shlax/proc.py

@@ -32,21 +32,21 @@ class PrefixStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol):
     make asynchronous output readable.
     """
 
-    def __init__(self, output, *args, **kwargs):
+    def __init__(self, proc, *args, **kwargs):
-        self.output = output
+        self.proc = proc
         super().__init__(*args, **kwargs)
 
     def pipe_data_received(self, fd, data):
-        if self.output.debug is True or 'out' in str(self.output.debug):
+        if self.proc.output.debug is True or 'out' in str(self.proc.output.debug):
             if fd in (1, 2):
-                self.output(data)
+                self.proc.output(data)
         super().pipe_data_received(fd, data)
 
 
-def protocol_factory(output):
+def protocol_factory(proc):
     def _p():
         return PrefixStreamProtocol(
-            output,
+            proc,
             limit=asyncio.streams._DEFAULT_LIMIT,
             loop=asyncio.events.get_event_loop()
         )
@@ -69,7 +69,10 @@ class Proc:
     """
     test = False
 
-    def __init__(self, *args, prefix=None, raises=True, output=None):
+    def __init__(self, *args, prefix=None, raises=True, output=None, quiet=False):
+        if quiet:
+            self.output = Output(debug=False)
+        else:
             self.output = output or Output()
         self.cmd = ' '.join(args)
         self.args = args
@@ -112,7 +115,7 @@ class Proc:
 
         loop = asyncio.events.get_event_loop()
         transport, protocol = await loop.subprocess_exec(
-            protocol_factory(self.output), *self.args)
+            protocol_factory(self), *self.args)
         self.proc = asyncio.subprocess.Process(transport, protocol, loop)
         self.called = True
 

shlax/shortcuts.py

@@ -3,5 +3,16 @@ from .targets.buildah import Buildah
 from .targets.localhost import Localhost
 from .targets.stub import Stub
 
+from .actions.copy import Copy
 from .actions.packages import Packages
 from .actions.run import Run
+from .actions.pip import Pip
+from .actions.parallel import Parallel
+from .actions.user import User
+
+from .cli import Command, Group
+
+from .container import Container
+from .pod import Pod
+
+from os import getenv, environ

shlax/targets/base.py

@@ -1,5 +1,9 @@
+import asyncio
 import copy
+from pathlib import Path
+import os
 import re
+import sys
 
 from ..output import Output
 from ..proc import Proc
@@ -7,21 +11,29 @@ from ..result import Result, Results
 
 
 class Target:
-    def __init__(self, *actions, **options):
+    def __init__(self, *actions, root=None):
         self.actions = actions
-        self.options = options
         self.results = []
-        self.output = Output(self, **self.options)
+        self.output = Output()
         self.parent = None
+        self.root = root or os.getcwd()
+
+    @property
+    def parent(self):
+        return self._parent or Target()
+
+    @parent.setter
+    def parent(self, value):
+        self._parent = value
 
     @property
     def caller(self):
         """Traverse parents and return the top-levels Target."""
-        if not self.parent:
+        if not self._parent:
             return self
-        caller = self.parent
+        caller = self._parent
-        while caller.parent:
+        while caller._parent:
-            caller = caller.parent
+            caller = caller._parent
         return caller
 
     async def __call__(self, *actions, target=None):
@@ -31,32 +43,35 @@ class Target:
             self.parent = target
 
         for action in actions or self.actions:
-            result = Result(self, action)
+            if await self.action(action, reraise=bool(actions)):
+                break
 
-            self.output = Output(action, **self.options)
+    async def action(self, action, reraise=False):
-            self.output.start()
+        result = Result(self, action)
+        self.output.start(action)
         try:
             await action(target=self)
         except Exception as e:
-                self.output.fail(e)
+            self.output.fail(action, e)
             result.status = 'failure'
             result.exception = e
-                if actions:
+            if reraise:
                 # nested call, re-raise
                 raise
             else:
-                    break
+                import traceback
+                traceback.print_exception(type(e), e, sys.exc_info()[2])
+                return True
         else:
-                self.output.success()
+            self.output.success(action)
             result.status = 'success'
         finally:
             self.caller.results.append(result)
 
             clean = getattr(action, 'clean', None)
             if clean:
-                    action.result = result
+                self.output.clean(action)
-                    self.output.clean()
+                await clean(self, result)
-                    await clean(self)
 
     async def rexec(self, *args, **kwargs):
         kwargs['user'] = 'root'
@@ -76,5 +91,56 @@ class Target:
                 result.append(match.group(1))
         return result
 
+    def shargs(self, *args, **kwargs):
+        user = kwargs.pop('user', None)
+        args = [str(arg) for arg in args if args is not None]
+
+        if args and ' ' in args[0]:
+            if len(args) == 1:
+                args = ['sh', '-euc', args[0]]
+            else:
+                args = ['sh', '-euc'] + list(args)
+
+        if user == 'root':
+            args = ['sudo'] + args
+        elif user:
+            args = ['sudo', '-u', user] + args
+
+        return args, kwargs
+
+        if self.parent:
+            return self.parent.shargs(*args, **kwargs)
+        else:
+            return args, kwargs
+
     async def exec(self, *args, **kwargs):
-        raise NotImplemented()
+        kwargs['output'] = self.output
+        args, kwargs = self.shargs(*args, **kwargs)
+        proc = await Proc(*args, **kwargs)()
+        if kwargs.get('wait', True):
+            await proc.wait()
+        return proc
+
+    @property
+    def root(self):
+        return self._root
+
+    @root.setter
+    def root(self, value):
+        self._root = Path(value or os.getcwd())
+
+    def path(self, path):
+        if str(path).startswith('/'):
+            path = str(path)[1:]
+        return self.root / path
+
+    async def mkdir(self, path):
+        if '_mkdir' not in self.__dict__:
+            self._mkdir = []
+        path = str(path)
+        if path not in self._mkdir:
+            await self.exec('mkdir', '-p', path)
+            self._mkdir.append(path)
+
+    async def copy(self, *args):
+        return await self.exec('cp', '-a', *args)

shlax/targets/buildah.py

@@ -1,3 +1,7 @@
+import asyncio
+import copy
+import hashlib
+import json
 import os
 import sys
 from pathlib import Path
@@ -9,16 +13,17 @@ from ..proc import Proc
 
 
 class Buildah(Target):
+    """Build container image with buildah"""
+
     def __init__(self,
                  *actions,
                  base=None, commit=None,
-                 cmd=None,
+                 cmd=None):
-                 **options):
         self.base = base or 'alpine'
         self.image = Image(commit) if commit else None
 
         self.ctr = None
-        self.mnt = None
+        self.root = None
         self.mounts = dict()
 
         self.config = dict(
@@ -28,7 +33,7 @@ class Buildah(Target):
         # Always consider localhost as parent for now
         self.parent = Target()
 
-        super().__init__(*actions, **options)
+        super().__init__(*actions)
 
     def is_runnable(self):
         return Proc.test or os.getuid() == 0
@@ -36,36 +41,127 @@ class Buildah(Target):
     def __str__(self):
         if not self.is_runnable():
             return 'Replacing with: buildah unshare ' + ' '.join(sys.argv)
-        return 'Buildah image builder'
+        return f'Buildah({self.image})'
 
     async def __call__(self, *actions, target=None):
+        if target:
             self.parent = target
+
         if not self.is_runnable():
             os.execvp('buildah', ['buildah', 'unshare'] + sys.argv)
             # program has been replaced
 
+        layers = await self.layers()
+        keep = await self.cache_setup(layers, *actions)
+        keepnames = [*map(lambda x: 'localhost/' + str(x), keep)]
+        self.invalidate = [name for name in layers if name not in keepnames]
+        if self.invalidate:
+            self.output.info('Invalidating old layers')
+            await self.parent.exec(
+                'buildah', 'rmi', *self.invalidate, raises=False)
+
+        if actions:
+            actions = actions[len(keep):]
+            if not actions:
+                return self.uptodate()
+        else:
+            self.actions = self.actions[len(keep):]
+            if not self.actions:
+                return self.uptodate()
+
         self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out
-        self.mnt = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out)
+        self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out)
-        await super().__call__()
 
-    async def clean(self, target):
+        return await super().__call__(*actions)
+
+    def uptodate(self):
+        self.clean = None
+        self.output.success('Image up to date')
+        return
+
+    async def layers(self):
+        ret = set()
+        results = await self.parent.exec(
+            'buildah images --json',
+            quiet=True,
+        )
+        results = json.loads(results.out)
+
+        prefix = 'localhost/' + self.image.repository + ':layer-'
+        for result in results:
+            if not result.get('names', None):
+                continue
+            for name in result['names']:
+                if name.startswith(prefix):
+                    ret.add(name)
+        return ret
+
+    async def cache_setup(self, layers, *actions):
+        keep = []
+        self.image_previous = Image(self.base)
+        for action in actions or self.actions:
+            action_image = await self.action_image(action)
+            name = 'localhost/' + str(action_image)
+            if name in layers:
+                self.base = self.image_previous = action_image
+                keep.append(action_image)
+                self.output.skip(
+                    f'Found layer for {action}: {action_image.tags[0]}'
+                )
+            else:
+                break
+        return keep
+
+    async def action_image(self, action):
+        prefix = str(self.image_previous)
+        for tag in self.image_previous.tags:
+            if tag.startswith('layer-'):
+                prefix = tag
+                break
+        if hasattr(action, 'cachekey'):
+            action_key = action.cachekey()
+            if asyncio.iscoroutine(action_key):
+                action_key = str(await action_key)
+        else:
+            action_key = str(action)
+        key = prefix + action_key
+        sha1 = hashlib.sha1(key.encode('ascii'))
+        return self.image.layer(sha1.hexdigest())
+
+    async def action(self, action, reraise=False):
+        stop = await super().action(action, reraise)
+        if not stop:
+            action_image = await self.action_image(action)
+            self.output.info(f'Commiting {action_image} for {action}')
+            await self.parent.exec(
+                'buildah',
+                'commit',
+                '--format=' + action_image.format,
+                self.ctr,
+                action_image,
+            )
+            self.image_previous = action_image
+        return stop
+
+    async def clean(self, target, result):
         for src, dst in self.mounts.items():
-            await self.parent.exec('umount', self.mnt / str(dst)[1:])
+            await self.parent.exec('umount', self.root / str(dst)[1:])
 
-        if self.result.status == 'success':
+        if self.root is not None:
-            await self.commit()
-            if os.getenv('BUILDAH_PUSH'):
-                await self.image.push(target)
-
-        if self.mnt is not None:
             await self.parent.exec('buildah', 'umount', self.ctr)
 
         if self.ctr is not None:
+            if result.status == 'success':
+                await self.commit()
+
             await self.parent.exec('buildah', 'rm', self.ctr)
 
+            if result.status == 'success' and os.getenv('BUILDAH_PUSH'):
+                await self.image.push(target)
+
     async def mount(self, src, dst):
         """Mount a host directory into the container."""
-        target = self.mnt / str(dst)[1:]
+        target = self.root / str(dst)[1:]
         await self.parent.exec(f'mkdir -p {src} {target}')
         await self.parent.exec(f'mount -o bind {src} {target}')
         self.mounts[src] = dst
@@ -78,25 +174,51 @@ class Buildah(Target):
         _args += [' '.join([str(a) for a in args])]
         return await self.parent.exec(*_args, **kwargs)
 
-    async def commit(self):
+    async def commit(self, image=None):
-        if not self.image:
+        image = image or self.image
+        if not image:
             return
 
+        if not image:
+            # don't go through that if layer commit
             for key, value in self.config.items():
                 await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}')
 
-        self.sha = (await self.exec(
+        self.sha = (await self.parent.exec(
             'buildah',
             'commit',
-            '--format=' + self.image.format,
+            '--format=' + image.format,
             self.ctr,
-            buildah=False,
         )).out
 
-        if self.image.tags:
+        ENV_TAGS = (
-            tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags]
+            # gitlab
+            'CI_COMMIT_SHORT_SHA',
+            'CI_COMMIT_REF_NAME',
+            'CI_COMMIT_TAG',
+            # CircleCI
+            'CIRCLE_SHA1',
+            'CIRCLE_TAG',
+            'CIRCLE_BRANCH',
+            # contributions welcome here
+        )
+
+        # figure tags from CI vars
+        for name in ENV_TAGS:
+            value = os.getenv(name)
+            if value:
+                self.image.tags.append(value)
+
+        if image.tags:
+            tags = [f'{image.repository}:{tag}' for tag in image.tags]
         else:
-            tags = [self.image.repository]
+            tags = [image.repository]
 
         for tag in tags:
             await self.parent.exec('buildah', 'tag', self.sha, tag)
+
+    async def mkdir(self, path):
+        return await self.parent.mkdir(self.path(path))
+
+    async def copy(self, *args):
+        return await self.parent.copy(*args[:-1], self.path(args[-1]))

shlaxfile.py

@@ -1,12 +1,20 @@
-#!/usr/bin/env shlax
+#!/usr/bin/env python
 """
 Shlaxfile for shlax itself.
 """
 
 from shlax.shortcuts import *
 
+shlax = Container(
     build=Buildah(
-    Run('echo hi'),
+        Packages('python38', 'buildah', 'unzip', 'findutils', upgrade=False),
-    Packages('python38'),
+        Copy('setup.py', 'shlax', '/app'),
+        Pip('/app'),
         base='quay.io/podman/stable',
+        commit='shlax',
+    ),
 )
+
+
+if __name__ == '__main__':
+    print(Group(doc=__doc__).load(shlax).entry_point())

tests/test_image.py

@@ -25,9 +25,3 @@ def test_args(arg, expected):
     im = Image(arg)
     for k, v in expected.items():
         assert getattr(im, k) == v
-
-def test_args_env():
-    os.environ['IMAGE_TEST_ARGS_ENV'] = 'foo'
-    Image.ENV_TAGS = ['IMAGE_TEST_ARGS_ENV']
-    im = Image('re/po:x,y')
-    assert im.tags == ['x', 'y', 'foo']

tests/test_target.py

@@ -68,7 +68,7 @@ async def test_parallel():
 @pytest.mark.asyncio
 async def test_function():
     async def hello(target):
-        target.exec('hello')
+        await target.exec('hello')
     await Stub()(hello)