dnf stuff

See merge request oss/shlax!2

.gitlab-ci.yml

@@ -1,27 +1,14 @@
 build:
   cache:
     key: cache
-    paths: [.cache, /var/lib/containers/]
-  image: yourlabs/buildah
+    paths: [.cache]
+  image: quay.io/buildah/stable
   script:
-  - pip3 install -U --user .[cli]
-  - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA
+  - dnf install -y python3-pip
+  - pip3 install -U --user -e .[cli]
+  - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build
   stage: build
 
-build-itself:
-  cache:
-    key: cache
-    paths: [.cache, /var/lib/containers/]
-  image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA
-  script: python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_REF
-  stage: test
-
-test-exitcode:
-  image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA
-  script:
-  - tests/shlaxfail.py build || [ $? -eq 1 ]
-  - tests/shlaxsuccess.py build
-
 test:
   image: yourlabs/python
   stage: build

README.md

@@ -4,30 +4,6 @@ Shlax is a Python framework for system automation, initially with the purpose
 of replacing docker, docker-compose and ansible with a single tool with the
 purpose of code-reuse made possible by target abstraction.
 
-## Development status: Design state
-
-I got the thing to work with an ugly PoC that I basically brute-forced, I'm
-currently rewriting the codebase with a proper design.
-
-The stories are in development in this order:
-
-- replacing docker build, that's in the state of polishing
-- replacing docker-compose, not in use but the PoC works so far
-- replacing ansible, also working in working PoC state, the shlax command line
-  demonstrates
-
-This project is supposed to unblock me from adding the CI feature to the
-Sentry/GitLab/Portainer implementation I'm doing in pure python on top of
-Django, CRUDLFA+ and Ryzom (isomorphic components in Python to replace
-templates). So, as you can see, I'm really deep in it with a strong
-determination.
-
-Shlax builds its container itself, so check the shlaxfile.py of this repository
-to see what it currently looks like, and check the build job of the CI pipeline
-to see the output.
-
-# Design
-
 The pattern resolves around two moving parts: Actions and Targets.
 
 ## Action

setup.py

@@ -7,7 +7,7 @@ setup(
     setup_requires='setupmeta',
     extras_require=dict(
         cli=[
-            'cli2>=2.3.0',
+            'cli2>=2.2.2',
         ],
         test=[
             'pytest',

shlax/actions/copy.py

@@ -1,32 +1,19 @@
 import asyncio
 import binascii
-import glob
 import os
 
-from ..exceptions import ShlaxException
-
 
 class Copy:
     def __init__(self, *args):
+        self.src = args[:-1]
         self.dst = args[-1]
-        self.src = []
 
-        for src in args[:-1]:
-            if '*' in src:
-                self.src += glob.glob(src)
-            else:
-                self.src.append(src)
-
-    async def listfiles(self, target):
+    def listfiles(self):
         if getattr(self, '_listfiles', None):
             return self._listfiles
 
         result = []
         for src in self.src:
-            if not await target.parent.exists(src):
-                target.output.fail(self)
-                raise ShlaxException(f'File not found {src}')
-
             if os.path.isfile(src):
                 result.append(src)
                 continue
@@ -45,7 +32,7 @@ class Copy:
     async def __call__(self, target):
         await target.mkdir(self.dst)
 
-        for path in await self.listfiles(target):
+        for path in self.listfiles():
             if os.path.isdir(path):
                 await target.mkdir(os.path.join(self.dst, path))
             elif '/' in path:
@@ -61,11 +48,9 @@ class Copy:
     def __str__(self):
         return f'Copy({", ".join(self.src)}, {self.dst})'
 
-    async def cachekey(self, target):
+    async def cachekey(self):
         async def chksum(path):
             with open(path, 'rb') as f:
                 return (path, str(binascii.crc32(f.read())))
-        results = await asyncio.gather(
-            *[chksum(f) for f in await self.listfiles(target)]
-        )
+        results = await asyncio.gather(*[chksum(f) for f in self.listfiles()])
         return {path: chks for path, chks in results}

shlax/actions/packages.py

@@ -27,90 +27,79 @@ class Packages:
             update='apk update',
             upgrade='apk upgrade',
             install='apk add',
-            host=None,
         ),
         apt=dict(
             update='apt-get -y update',
             upgrade='apt-get -y upgrade',
             install='apt-get -y --no-install-recommends install',
-            host=None,
         ),
         pacman=dict(
             update='pacman -Sy',
             upgrade='pacman -Su --noconfirm',
             install='pacman -S --noconfirm',
-            lastupdate='stat -c %Y /var/lib/pacman/sync/core.db',
-            host='/var/lib/pacman',
         ),
         dnf=dict(
             update='dnf makecache --assumeyes',
             upgrade='dnf upgrade --best --assumeyes --skip-broken',  # noqa
             install='dnf install --setopt=install_weak_deps=False --best --assumeyes',  # noqa
-            lastupdate='stat -c %Y /var/cache/dnf/* | head -n1',
-            host=None,
         ),
         yum=dict(
             update='yum update',
             upgrade='yum upgrade',
             install='yum install',
-            host=None,
         ),
     )
 
     installed = []
 
-    def __init__(self, *packages, upgrade=False):
+    def __init__(self, *packages, upgrade=True):
         self.packages = []
         self.upgrade = upgrade
         for package in packages:
             line = dedent(package).strip().replace('\n', ' ')
             self.packages += line.split(' ')
 
-    async def cache_setup(self, target):
-        # Try to use the host cache directory if present rather than home
-        # directory, in cases where host and guest are the same distros
-        hostpath = self.mgrs[self.mgr]['host']
-        if target.exists(hostpath):
-            self.cache_root = hostpath
-
+    @property
+    def cache_root(self):
         if 'CACHE_DIR' in os.environ:
-            self.cache_root = os.path.join(os.getenv('CACHE_DIR'))
+            return os.path.join(os.getenv('CACHE_DIR'))
         else:
-            self.cache_root = os.path.join(await target.parent.getenv('HOME'), '.cache')
-
-        # run pkgmgr_setup functions ie. apk_setup
-        await getattr(self, self.mgr + '_setup')(target)
+            return os.path.join(os.getenv('HOME'), '.cache')
 
     async def update(self, target):
-        # lastupdate = await target.exec(self.cmds['lastupdate'], raises=False)
-        # lastupdate = int(lastupdate.out) if lastupdate.rc == 0 else None
+        # run pkgmgr_setup functions ie. apk_setup
+        cachedir = await getattr(self, self.mgr + '_setup')(target)
+
         lastupdate = None
+        if os.path.exists(cachedir + '/lastupdate'):
+            with open(cachedir + '/lastupdate', 'r') as f:
+                try:
+                    lastupdate = int(f.read().strip())
+                except:
+                    pass
+
+        if not os.path.exists(cachedir):
+            os.makedirs(cachedir)
+
         now = int(datetime.now().strftime('%s'))
-        if not lastupdate or now - lastupdate > 604800:
-            await target.rexec(self.cmds['update'])
-
-        return
-
-        # disabling with the above return call until needed again
-        # might have to rewrite this to not have our own lockfile
-        # or find a better place on the filesystem
-        # also make sure the lockfile is actually needed when running on
-        # targets that don't have isguest=True
+        # cache for a week
         if not lastupdate or now - lastupdate > 604800:
             # crude lockfile implementation, should work against *most*
             # race-conditions ...
             lockfile = cachedir + '/update.lock'
-            if not await target.parent.exists(lockfile):
-                await target.parent.write(lockfile, str(os.getpid()))
+            if not os.path.exists(lockfile):
+                with open(lockfile, 'w+') as f:
+                    f.write(str(os.getpid()))
 
                 try:
                     await target.rexec(self.cmds['update'])
                 finally:
-                    await target.parent.rm(lockfile)
+                    os.unlink(lockfile)
 
-                await target.parent.write(cachedir + '/lastupdate', str(now))
+                with open(cachedir + '/lastupdate', 'w+') as f:
+                    f.write(str(now))
             else:
-                while await target.parent.exists(lockfile):
+                while os.path.exists(lockfile):
                     print(f'{self.target} | Waiting for {lockfile} ...')
                     await asyncio.sleep(1)
 
@@ -127,13 +116,7 @@ class Packages:
             raise Exception('Packages does not yet support this distro')
 
         self.cmds = self.mgrs[self.mgr]
-
-        if target.isguest:
-            # we're going to mount
-            await self.cache_setup(target)
-
         await self.update(target)
-
         if self.upgrade:
             await target.rexec(self.cmds['upgrade'])
 
@@ -164,24 +147,19 @@ class Packages:
         return cachedir
 
     async def apt_setup(self, target):
-        codename = (await target.rexec(
-            f'source /etc/os-release; echo $VERSION_CODENAME'
+        codename = (await self.rexec(
+            f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME'
         )).out
         cachedir = os.path.join(self.cache_root, self.mgr, codename)
         await self.rexec('rm /etc/apt/apt.conf.d/docker-clean')
         cache_archives = os.path.join(cachedir, 'archives')
-        await target.mount(cache_archives, f'/var/cache/apt/archives')
+        await self.mount(cache_archives, f'/var/cache/apt/archives')
         cache_lists = os.path.join(cachedir, 'lists')
-        await target.mount(cache_lists, f'/var/lib/apt/lists')
+        await self.mount(cache_lists, f'/var/lib/apt/lists')
         return cachedir
 
     async def pacman_setup(self, target):
-        cachedir = os.path.join(self.cache_root, self.mgr)
-        await target.mkdir(cachedir + '/cache', cachedir + '/sync')
-        await target.mount(cachedir + '/sync', '/var/lib/pacman/sync')
-        await target.mount(cachedir + '/cache', '/var/cache/pacman')
-        if await target.host.exists('/etc/pacman.d/mirrorlist'):
-            await target.copy('/etc/pacman.d/mirrorlist', '/etc/pacman.d/mirrorlist')
+        return self.cache_root + '/pacman'
 
     def __str__(self):
         return f'Packages({self.packages}, upgrade={self.upgrade})'

shlax/actions/parallel.py

@@ -9,6 +9,3 @@ class Parallel:
         return await asyncio.gather(*[
             target(action) for action in self.actions
         ])
-
-    def __str__(self):
-        return 'Parallel executor'

shlax/actions/pip.py

@@ -19,10 +19,7 @@ class Pip(Action):
             raise Exception('Could not find pip nor python')
 
         # ensure pip module presence
-        result = await target.exec(
-            python, '-m', 'pip',
-            raises=False, quiet=True
-        )
+        result = await target.exec(python, '-m', 'pip', raises=False)
         if result.rc != 0:
             if not os.path.exists('get-pip.py'):
                 req = request.urlopen(

shlax/actions/run.py

@@ -1,15 +1,11 @@
 
 
 class Run:
-    def __init__(self, cmd, root=False):
+    def __init__(self, cmd):
         self.cmd = cmd
-        self.root = root
 
     async def __call__(self, target):
-        if self.root:
-            self.proc = await target.rexec(self.cmd)
-        else:
-            self.proc = await target.exec(self.cmd)
+        self.proc = await target.exec(self.cmd)
 
     def __str__(self):
         return f'Run({self.cmd})'

shlax/actions/user.py

@@ -24,7 +24,7 @@ class User:
         return f'User({self.username}, {self.home}, {self.uid})'
 
     async def __call__(self, target):
-        result = await target.rexec('id', self.uid, raises=False)
+        result = await target.rexec('id', self.uid)
         if result.rc == 0:
             old = re.match('.*\(([^)]*)\).*', result.out).group(1)
             await target.rexec(
@@ -40,5 +40,3 @@ class User:
                 '-u', self.uid,
                 self.username
             )
-        await target.mkdir(self.home)
-        await target.rexec('chown', self.uid, self.home)

shlax/cli.py

@@ -13,8 +13,6 @@ import importlib
 import os
 import sys
 
-from .exceptions import ShlaxException
-
 
 class Group(cli2.Group):
     def __init__(self, *args, **kwargs):
@@ -22,65 +20,22 @@ class Group(cli2.Group):
         self.cmdclass = Command
 
 
-class TargetArgument(cli2.Argument):
-    """
-    Target to execute on: localhost by default, target=@ssh_host for ssh.
-    """
-
-    def __init__(self, cmd, param, doc=None, color=None, default=None):
-        from shlax.targets.base import Target
-        super().__init__(cmd, param, doc=self.__doc__, default=Target())
-        self.alias = ['target', 't']
-
-    def cast(self, value):
-        from shlax.targets.ssh import Ssh
-        user, host = value.split('@')
-        return Ssh(host=host, user=user)
-
-    def match(self, arg):
-        return arg if isinstance(arg, str) and '@' in arg else None
-
-
 class Command(cli2.Command):
-    def setargs(self):
-        super().setargs()
-        if 'target' in self.sig.parameters:
-            self['target'] = TargetArgument(
-                self,
-                self.sig.parameters['target'],
-            )
-        if 'actions' in self:
-            del self['actions']
+    def call(self, *args, **kwargs):
+        return self.shlax_target(self.target)
 
     def __call__(self, *argv):
-        result = None
-
-        try:
-            result = super().__call__(*argv)
-        except ShlaxException as exc:
-            # just output the failure without TB, as command was already
-            # printed anyway
-            self.exit_code = 1
-            self['target'].value.output.fail(exc)
-
-        if self['target'].value.results:
-            if self['target'].value.results[-1].status == 'failure':
-                self.exit_code = 1
-        self['target'].value.output.results(self['target'].value)
+        from shlax.targets.base import Target
+        self.shlax_target = Target()
+        result = super().__call__(*argv)
+        self.shlax_target.output.results(self.shlax_target)
         return result
 
 
-class ActionCommand(cli2.Command):
-    def setargs(self):
-        super().setargs()
-        self['target'] = TargetArgument(
-            self,
-            inspect.Parameter('target', inspect.Parameter.KEYWORD_ONLY),
-        )
-
+class ActionCommand(Command):
     def call(self, *args, **kwargs):
         self.target = self.target(*args, **kwargs)
-        return super().call(self['target'].value)
+        return super().call(*args, **kwargs)
 
 
 class ConsoleScript(Group):

shlax/container.py

@@ -1,19 +1,12 @@
-import copy
 import os
 
-from .podman import Podman
 from .image import Image
 
 
 class Container:
-    def __init__(self, build=None, image=None, env=None, volumes=None):
+    def __init__(self, build=None, image=None):
         self.build = build
-        self.image = image or self.build.image
-        if isinstance(self.image, str):
-            self.image = Image(self.image)
-        self.volumes = volumes or {}
-        self.env = env or {}
-
+        self.image = self.build.image
         prefix = os.getcwd().split('/')[-1]
         repo = self.image.repository.replace('/', '-')
         if prefix == repo:
@@ -21,109 +14,19 @@ class Container:
         else:
             self.name = '-'.join([prefix, repo])
 
-        self.pod = None
-
-    @property
-    def full_name(self):
-        if self.pod:
-            return '-'.join([self.pod.name, self.name])
-        return self.name
-
-    async def up(self, target, *args):
-        """Start the container foreground"""
-        podman = Podman(target)
-        if self.pod:
-            pod = None
-            for _ in await podman.pod.ps():
-                if _['Name'] == self.pod.name:
-                    pod = _
-                    break
-            if not pod:
-                await podman.pod.create('--name', self.pod.name)
-            args = list(args) + ['--pod', self.pod.name]
-
-        # skip if already up
-        for result in await podman.ps('-a'):
-            for name in result['Names']:
-                if name == self.full_name:
-                    if result['State'] == 'running':
-                        target.output.info(f'{self.full_name} already running')
-                        return
-                    elif result['State'] in ('exited', 'configured'):
-                        target.output.info(f'{self.full_name} starting')
-                        startargs = ['podman', 'start']
-                        if '-d' not in args:
-                            startargs.append('--attach')
-                        startargs.append(self.full_name)
-                        await target.exec(*startargs)
-                        return
-
-        cmd = [
+    async def start(self, target):
+        """Start the container"""
+        await target.rexec(
             'podman',
             'run',
-        ] + list(args)
-
-        for src, dest in self.volumes.items():
-            cmd += ['--volume', ':'.join([src, dest])]
-
-        for src, dest in self.env.items():
-            cmd += ['--env', '='.join([src, str(dest)])]
-
-        cmd += [
             '--name',
-            self.full_name,
+            self.name,
             str(self.image),
-        ]
-        await target.exec(*cmd)
-
-    async def start(self, target):
-        """Start the container background"""
-        await self.up(target, '-d')
+        )
 
     async def stop(self, target):
         """Start the container"""
-        await target.exec('podman', 'stop', self.full_name)
-
-    async def inspect(self, target):
-        """Inspect container"""
-        await target.exec('podman', 'inspect', self.full_name)
-
-    async def logs(self, target):
-        """Show container logs"""
-        await target.exec('podman', 'logs', self.full_name)
-
-    async def exec(self, target, cmd=None):
-        """Execute a command in the container"""
-        cmd = cmd or 'bash'
-        if cmd.endswith('sh'):
-            import os
-            os.execvp(
-                '/usr/bin/podman',
-                [
-                    'podman',
-                    'exec',
-                    '-it',
-                    self.full_name,
-                    cmd,
-                ]
-            )
-        result = await target.exec(
-            'podman',
-            'exec',
-            self.full_name,
-            cmd,
-        )
-
-    async def down(self, target):
-        """Start the container"""
-        await target.exec('podman', 'rm', '-f', self.full_name, raises=False)
-
-    async def apply(self, target):
-        """Start the container"""
-        if self.build:
-            await target(self.build)
-        await target(self.down)
-        await target(self.start)
+        await target.rexec('podman', 'stop', self.name)
 
     def __str__(self):
-        return f'Container(name={self.name}, image={self.image}, volumes={self.volumes})'
+        return f'Container(name={self.name}, image={self.image})'

shlax/image.py

@@ -1,52 +1,21 @@
-import json
+import copy
 import os
 import re
 
 
-class Layers(set):
-    def __init__(self, image):
-        self.image = image
-
-    async def ls(self, target):
-        """Fetch layers from localhost"""
-        ret = set()
-        results = await target.parent.exec(
-            'buildah images --json',
-            quiet=True,
-        )
-        results = json.loads(results.out)
-
-        prefix = 'localhost/' + self.image.repository + ':layer-'
-        for result in results:
-            if not result.get('names', None):
-                continue
-            for name in result['names']:
-                if name.startswith(prefix):
-                    self.add(name)
-        return self
-
-    async def rm(self, target, tags=None):
-        """Drop layers for this image"""
-        if tags is None:
-            tags = [layer for layer in await self.ls(target)]
-        await target.exec('podman', 'rmi', *tags, raises=False)
-
-
 class Image:
     PATTERN = re.compile(
         '^((?P<backend>[a-z]*)://)?((?P<registry>[^/]*[.][^/]*)/)?((?P<repository>[^:]+))?(:(?P<tags>.*))?$'  # noqa
         , re.I
     )
 
-    def __init__(self, arg=None, format=None, backend=None, registry=None,
-                 repository=None, tags=None):
+    def __init__(self, arg=None, format=None, backend=None, registry=None, repository=None, tags=None):
         self.arg = arg
         self.format = format
         self.backend = backend
         self.registry = registry
         self.repository = repository
         self.tags = tags or []
-        self.layers = Layers(self)
 
         match = re.match(self.PATTERN, arg)
         if match:
@@ -60,11 +29,9 @@ class Image:
                 setattr(self, k, v)
 
         # docker.io currently has issues with oci format
+        self.format = format or 'oci'
         if self.registry == 'docker.io':
-            self.backend = 'docker'
-
-        if not self.format:
-            self.format = 'docker' if self.backend == 'docker' else 'oci'
+            self.format = 'docker'
 
         # filter out tags which resolved to None
         self.tags = [t for t in self.tags if t]
@@ -76,19 +43,18 @@ class Image:
     def __str__(self):
         return f'{self.repository}:{self.tags[-1]}'
 
-    async def push(self, target, name=None):
-        user = os.getenv('IMAGES_USER', os.getenv('DOCKER_USER'))
-        passwd = os.getenv('IMAGES_PASS', os.getenv('DOCKER_PASS'))
+    async def push(self, *args, **kwargs):
+        user = os.getenv('DOCKER_USER')
+        passwd = os.getenv('DOCKER_PASS')
+        action = kwargs.get('action', self)
         if user and passwd:
-            target.output.cmd('buildah login -u ... -p ...' + self.registry)
-            await target.parent.exec(
-                'buildah', 'login', '-u', user, '-p', passwd,
-                self.registry or 'docker.io', quiet=True)
+            action.output.cmd('buildah login -u ... -p ...' + self.registry)
+            await action.exec('buildah', 'login', '-u', user, '-p', passwd, self.registry or 'docker.io', debug=False)
 
         for tag in self.tags:
-            await target.parent.exec(
-                'buildah',
-                'push',
-                self.repository + ':final',
-                name if isinstance(name, str) else f'{self.registry}/{self.repository}:{tag}'
-            )
+            await action.exec('buildah', 'push', f'{self.repository}:{tag}')
+
+    def layer(self, key):
+        layer = copy.deepcopy(self)
+        layer.tags = ['layer-' + key]
+        return layer

shlax/output.py

@@ -101,11 +101,7 @@ class Output:
         )
 
     def highlight(self, line, highlight=True):
-        try:
-            line = line.decode('utf8') if isinstance(line, bytes) else line
-        except UnicodeDecodeError:
-            highlight = False
-
+        line = line.decode('utf8') if isinstance(line, bytes) else line
         if not highlight or (
             '\x1b[' in line
             or '\033[' in line
@@ -122,7 +118,7 @@ class Output:
     def test(self, action):
         self(''.join([
             self.colors['purplebold'],
-            '! TEST    ',
+            '!  TEST   ',
             self.colors['reset'],
             self.colorized(action),
             '\n',
@@ -132,7 +128,7 @@ class Output:
         if self.debug:
             self(''.join([
                 self.colors['bluebold'],
-                '+ CLEAN   ',
+                '+  CLEAN  ',
                 self.colors['reset'],
                 self.colorized(action),
                 '\n',
@@ -142,7 +138,7 @@ class Output:
         if self.debug is True or 'visit' in str(self.debug):
             self(''.join([
                 self.colors['orangebold'],
-                '⚠ START   ',
+                '⚠  START  ',
                 self.colors['reset'],
                 self.colorized(action),
                 '\n',
@@ -152,7 +148,7 @@ class Output:
         if self.debug is True or 'visit' in str(self.debug):
             self(''.join([
                 self.colors['cyanbold'],
-                '➤ INFO    ',
+                '➤   INFO  ',
                 self.colors['reset'],
                 text,
                 '\n',
@@ -189,8 +185,6 @@ class Output:
             ]))
 
     def results(self, action):
-        if len(action.results) < 2:
-            return
         success = 0
         fail = 0
         for result in action.results:

shlax/pod.py

@@ -1,23 +1,13 @@
 import cli2
-import json
-import os
-import sys
 
 from shlax.targets.base import Target
 from shlax.actions.parallel import Parallel
-from shlax.proc import Proc
-
-from .podman import Podman
 
 
 class Pod:
     """Help text"""
     def __init__(self, **containers):
         self.containers = containers
-        for name, container in self.containers.items():
-            container.pod = self
-            container.name = name
-        self.name = os.getcwd().split('/')[-1]
 
     async def _call(self, target, method, *names):
         methods = [
@@ -29,50 +19,8 @@ class Pod:
 
     async def build(self, target, *names):
         """Build container images"""
-        if not (Proc.test or os.getuid() == 0):
-            os.execvp('buildah', ['buildah', 'unshare'] + sys.argv)
-        else:
-            await self._call(target, 'build', *names)
-
-    async def down(self, target, *names):
-        """Delete container images"""
-        await self._call(target, 'down', *names)
+        await self._call(target, 'build', *names)
 
     async def start(self, target, *names):
         """Start container images"""
         await self._call(target, 'start', *names)
-
-    async def logs(self, target, *names):
-        """Start container images"""
-        await self._call(target, 'logs', *names)
-
-    async def ps(self, target):
-        """Show containers and volumes"""
-        containers = []
-        names = []
-        for container in await Podman(target).ps('-a'):
-            for name in container['Names']:
-                if name.startswith(self.name + '-'):
-                    container['Name'] = name
-                    containers.append(container)
-                    names.append(name)
-
-        for name, container in self.containers.items():
-            full_name = '-'.join([self.name, container.name])
-            if full_name in names:
-                continue
-            containers.append(dict(
-                Name=full_name,
-                State='not created',
-            ))
-
-        cli2.Table(
-            ['Name', 'State'],
-            *[
-                (container['Name'], container['State'])
-                for container in containers
-            ]
-        ).print()
-
-    def __str__(self):
-        return f'Pod({self.name})'

shlax/podman.py

@@ -1,20 +0,0 @@
-import json
-
-
-class Podman(list):
-    def __init__(self, target, *args):
-        self.target = target
-        super().__init__(args or ['podman'])
-
-    def __getattr__(self, command):
-        if command.startswith('_'):
-            return super().__getattr__(command)
-        return Podman(self.target, *self + [command])
-
-    async def __call__(self, *args, **kwargs):
-        cmd = self + list(args) + [
-            f'--{k}={v}' for k, v in kwargs.items()
-        ]
-        if 'ps' in cmd:
-            cmd += ['--format=json']
-        return (await self.target.exec(*cmd, quiet=True)).json

shlax/proc.py

@@ -7,11 +7,10 @@ import os
 import shlex
 import sys
 
-from .exceptions import ShlaxException
 from .output import Output
 
 
-class ProcFailure(ShlaxException):
+class ProcFailure(Exception):
     def __init__(self, proc):
         self.proc = proc
 

shlax/targets/base.py

@@ -6,22 +6,17 @@ import re
 import sys
 
 from ..output import Output
-from ..proc import Proc, ProcFailure
+from ..proc import Proc
 from ..result import Result, Results
 
 
 class Target:
-    isguest = False
-
     def __init__(self, *actions, root=None):
         self.actions = actions
         self.results = []
         self.output = Output()
         self.parent = None
-        self.root = root or ''
-
-    def __str__(self):
-        return 'localhost'
+        self.root = root or os.getcwd()
 
     @property
     def parent(self):
@@ -47,18 +42,10 @@ class Target:
             # the calling target
             self.parent = target
 
-        result = Result(self, self)
-        result.status = 'success'
-
         for action in actions or self.actions:
             if await self.action(action, reraise=bool(actions)):
-                result.status = 'failure'
                 break
 
-        if getattr(self, 'clean', None):
-            self.output.clean(self)
-            await self.clean(self, result)
-
     async def action(self, action, reraise=False):
         result = Result(self, action)
         self.output.start(action)
@@ -68,24 +55,15 @@ class Target:
             self.output.fail(action, e)
             result.status = 'failure'
             result.exception = e
-
-            if not isinstance(e, ProcFailure):
-                # no need to reraise in case of command error
-                # because the command has been printed
-
-                if reraise:
-                    # nested call, re-raise
-                    raise
-                else:
-                    import traceback
-                    traceback.print_exception(type(e), e, sys.exc_info()[2])
-
-            return True  # because it failed
-        else:
-            if getattr(action, 'skipped', False):
-                self.output.skip(action)
+            if reraise:
+                # nested call, re-raise
+                raise
             else:
-                self.output.success(action)
+                import traceback
+                traceback.print_exception(type(e), e, sys.exc_info()[2])
+                return True
+        else:
+            self.output.success(action)
             result.status = 'success'
         finally:
             self.caller.results.append(result)
@@ -149,53 +127,20 @@ class Target:
 
     @root.setter
     def root(self, value):
-        self._root = Path(value) if value else ''
-
-    @property
-    def host(self):
-        current = self
-        while current.isguest:
-            current = self.parent
-        return current
+        self._root = Path(value or os.getcwd())
 
     def path(self, path):
-        if not self.root:
-            return path
         if str(path).startswith('/'):
             path = str(path)[1:]
-        return str(self.root / path)
+        return self.root / path
 
-    async def mkdir(self, *paths):
+    async def mkdir(self, path):
         if '_mkdir' not in self.__dict__:
             self._mkdir = []
-
-        make = [str(path) for path in paths if str(path) not in self._mkdir]
-        if make:
-            await self.exec('mkdir', '-p', *make)
-            self._mkdir += make
+        path = str(path)
+        if path not in self._mkdir:
+            await self.exec('mkdir', '-p', path)
+            self._mkdir.append(path)
 
     async def copy(self, *args):
         return await self.exec('cp', '-a', *args)
-
-    async def exists(self, path):
-        return (await self.exec('ls ' + self.path(path), raises=False)).rc == 0
-
-    async def read(self, path):
-        return (await self.exec('cat', self.path(path))).out
-
-    async def write(self, path, content, **kwargs):
-        return await self.exec(
-            f'cat > {self.path(path)} <<EOF\n'
-            + content
-            + '\nEOF',
-            **kwargs
-        )
-
-    async def rm(self, path):
-        return await self.exec('rm', self.path(path))
-
-    async def getenv(self, key):
-        return (await self.exec('echo $' + key)).out
-
-    async def getcwd(self):
-        return (await self.exec('pwd')).out

shlax/targets/buildah.py

@@ -14,9 +14,11 @@ from ..proc import Proc
 
 class Buildah(Target):
     """Build container image with buildah"""
-    isguest = True
 
-    def __init__(self, *actions, base=None, commit=None):
+    def __init__(self,
+                 *actions,
+                 base=None, commit=None,
+                 cmd=None):
         self.base = base or 'alpine'
         self.image = Image(commit) if commit else None
 
@@ -24,6 +26,10 @@ class Buildah(Target):
         self.root = None
         self.mounts = dict()
 
+        self.config = dict(
+            cmd=cmd or 'sh',
+        )
+
         # Always consider localhost as parent for now
         self.parent = Target()
 
@@ -37,34 +43,59 @@ class Buildah(Target):
             return 'Replacing with: buildah unshare ' + ' '.join(sys.argv)
         return f'Buildah({self.image})'
 
-    async def __call__(self, *actions, target=None, push: str=False):
+    async def __call__(self, *actions, target=None):
         if target:
             self.parent = target
 
-        self.push = push
-
         if not self.is_runnable():
             os.execvp('buildah', ['buildah', 'unshare'] + sys.argv)
-            return  # process has been replaced
+            # program has been replaced
 
-        layers = await self.image.layers.ls(self)
-        keep = await self.cache_setup(self.image.layers, *actions)
+        layers = await self.layers()
+        keep = await self.cache_setup(layers, *actions)
         keepnames = [*map(lambda x: 'localhost/' + str(x), keep)]
-        self.invalidate = [name for name in self.image.layers if name not in keepnames]
+        self.invalidate = [name for name in layers if name not in keepnames]
         if self.invalidate:
             self.output.info('Invalidating old layers')
-            await self.image.layers.rm(self.parent, self.invalidate)
+            await self.parent.exec(
+                'buildah', 'rmi', *self.invalidate, raises=False)
 
         if actions:
             actions = actions[len(keep):]
+            if not actions:
+                return self.uptodate()
         else:
             self.actions = self.actions[len(keep):]
+            if not self.actions:
+                return self.uptodate()
 
         self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out
         self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out)
 
         return await super().__call__(*actions)
 
+    def uptodate(self):
+        self.clean = None
+        self.output.success('Image up to date')
+        return
+
+    async def layers(self):
+        ret = set()
+        results = await self.parent.exec(
+            'buildah images --json',
+            quiet=True,
+        )
+        results = json.loads(results.out)
+
+        prefix = 'localhost/' + self.image.repository + ':layer-'
+        for result in results:
+            if not result.get('names', None):
+                continue
+            for name in result['names']:
+                if name.startswith(prefix):
+                    ret.add(name)
+        return ret
+
     async def cache_setup(self, layers, *actions):
         keep = []
         self.image_previous = Image(self.base)
@@ -88,16 +119,14 @@ class Buildah(Target):
                 prefix = tag
                 break
         if hasattr(action, 'cachekey'):
-            action_key = action.cachekey(self)
+            action_key = action.cachekey()
             if asyncio.iscoroutine(action_key):
                 action_key = str(await action_key)
         else:
             action_key = str(action)
         key = prefix + action_key
         sha1 = hashlib.sha1(key.encode('ascii'))
-        action_image = copy.deepcopy(self.image)
-        action_image.tags = ['layer-' + sha1.hexdigest()]
-        return action_image
+        return self.image.layer(sha1.hexdigest())
 
     async def action(self, action, reraise=False):
         stop = await super().action(action, reraise)
@@ -107,6 +136,7 @@ class Buildah(Target):
             await self.parent.exec(
                 'buildah',
                 'commit',
+                '--format=' + action_image.format,
                 self.ctr,
                 action_image,
             )
@@ -114,19 +144,21 @@ class Buildah(Target):
         return stop
 
     async def clean(self, target, result):
-        if self.ctr is not None:
-            for src, dst in self.mounts.items():
-                await self.parent.exec('umount', self.root / str(dst)[1:])
+        for src, dst in self.mounts.items():
+            await self.parent.exec('umount', self.root / str(dst)[1:])
+
+        if self.root is not None:
             await self.parent.exec('buildah', 'umount', self.ctr)
 
-        if result.status == 'success' and self.ctr:
-            await self.commit()
-            if self.push:
-                await self.image.push(target, self.push)
-
         if self.ctr is not None:
+            if result.status == 'success':
+                await self.commit()
+
             await self.parent.exec('buildah', 'rm', self.ctr)
 
+            if result.status == 'success' and os.getenv('BUILDAH_PUSH'):
+                await self.image.push(target)
+
     async def mount(self, src, dst):
         """Mount a host directory into the container."""
         target = self.root / str(dst)[1:]
@@ -142,21 +174,22 @@ class Buildah(Target):
         _args += [' '.join([str(a) for a in args])]
         return await self.parent.exec(*_args, **kwargs)
 
-    async def commit(self):
-        await self.parent.exec(
+    async def commit(self, image=None):
+        image = image or self.image
+        if not image:
+            return
+
+        if not image:
+            # don't go through that if layer commit
+            for key, value in self.config.items():
+                await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}')
+
+        self.sha = (await self.parent.exec(
             'buildah',
             'commit',
-            f'--format={self.image.format}',
+            '--format=' + image.format,
             self.ctr,
-            f'{self.image.repository}:final',
-        )
-        if self.image.backend == 'docker':
-            await self.parent.exec(
-                'buildah',
-                'push',
-                f'{self.image.repository}:final',
-                f'docker-daemon:{self.image.repository}:latest'
-            )
+        )).out
 
         ENV_TAGS = (
             # gitlab
@@ -176,56 +209,16 @@ class Buildah(Target):
             if value:
                 self.image.tags.append(value)
 
-        if self.image.tags:
-            tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags]
+        if image.tags:
+            tags = [f'{image.repository}:{tag}' for tag in image.tags]
         else:
-            tags = [self.image.repository]
+            tags = [image.repository]
 
-        await self.parent.exec('buildah', 'tag', self.image.repository + ':final', *tags)
+        for tag in tags:
+            await self.parent.exec('buildah', 'tag', self.sha, tag)
 
-    async def mkdir(self, *paths):
-        return await self.parent.mkdir(*[self.path(path) for path in paths])
+    async def mkdir(self, path):
+        return await self.parent.mkdir(self.path(path))
 
     async def copy(self, *args):
-        return await self.parent.exec('buildah', 'copy', self.ctr, *args)
-
-    async def write(self, path, content):
-        return await self.write(path, content)
-
-    async def write(self, path, content, **kwargs):
-        return await self.exec(
-            f'cat > {path} <<EOF\n'
-            + content
-            + '\nEOF',
-            **kwargs
-        )
-
-    class Config:
-        def __init__(self, **config):
-            self.config = config
-
-        async def __call__(self, target):
-            for key, value in self.config.items():
-                await target.parent.exec(
-                    f'buildah config --{key} "{value}" {target.ctr}'
-                )
-
-        def __str__(self):
-            return f'Buildah.Config({self.config})'
-
-    class Env:
-        def __init__(self, **env):
-            self.env = env
-
-        async def __call__(self, target):
-            for key, value in self.env.items():
-                await target.parent.exec(
-                    'buildah',
-                    'config',
-                    '--env',
-                    f'{key}={value}',
-                    target.ctr,
-                )
-
-        def __str__(self):
-            return f'Buildah.Env({self.env})'
+        return await self.parent.copy(*args[:-1], self.path(args[-1]))

shlax/targets/ssh.py

@@ -1,15 +0,0 @@
-from .base import Target
-
-
-class Ssh(Target):
-    def __init__(self, *actions, host, user=None):
-        self.host = host
-        self.user = user
-        super().__init__(*actions)
-
-    async def exec(self, *args, user=None, **kwargs):
-        _args = ['ssh', self.host]
-        if user == 'root':
-            _args += ['sudo']
-        _args += [' '.join([str(a) for a in args])]
-        return await self.parent.exec(*_args, **kwargs)

shlaxfile.py

@@ -9,9 +9,9 @@ shlax = Container(
     build=Buildah(
         Packages('python38', 'buildah', 'unzip', 'findutils', upgrade=False),
         Copy('setup.py', 'shlax', '/app'),
-        Pip('/app[cli]'),
-        base='quay.io/buildah/stable',
-        commit='docker://docker.io/yourlabs/shlax',
+        Pip('/app'),
+        base='quay.io/podman/stable',
+        commit='shlax',
     ),
 )
 

tests/shlaxfail.py

@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-"""
-Shlaxfile for shlax itself.
-"""
-
-from shlax.shortcuts import *
-
-shlax = Container(
-    build=Buildah(
-        Packages('prout', upgrade=False),
-        base='alpine',
-        commit='shlaxfail',
-    ),
-)
-
-
-if __name__ == '__main__':
-    print(Group(doc=__doc__).load(shlax).entry_point())

tests/shlaxsuccess.py

@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-"""
-Shlaxfile for shlax itself.
-"""
-
-from shlax.shortcuts import *
-
-shlax = Container(
-    build=Buildah(
-        base='alpine',
-        commit='shlaxsuccess',
-    ),
-)
-
-
-if __name__ == '__main__':
-    print(Group(doc=__doc__).load(shlax).entry_point())

tests/test_target.py

@@ -72,37 +72,6 @@ async def test_function():
     await Stub()(hello)
 
 
-@pytest.mark.asyncio
-async def test_action_clean():
-    class Example:
-        def __init__(self):
-            self.was_called = False
-        async def clean(self, target, result):
-            self.was_called = True
-        async def __call__(self, target):
-            raise Exception('lol')
-
-    action = Example()
-    target = Stub()
-    with pytest.raises(Exception):
-        await target(action)
-    assert action.was_called
-
-
-@pytest.mark.asyncio
-async def test_target_clean():
-    class Example(Stub):
-        def __init__(self, action):
-            self.was_called = False
-            super().__init__(action)
-        async def clean(self, target, result):
-            self.was_called = True
-
-    target = Example(Error())
-    await target()
-    assert target.was_called
-
-
 @pytest.mark.asyncio
 async def test_method():
     class Example: