Improve exception handling

Complete core rewrite, with documentation

See merge request oss/shlax!1

.gitlab-ci.yml

@@ -1,21 +1,27 @@
 build:
   cache:
     key: cache
-    paths: [.cache]
+    paths: [.cache, /var/lib/containers/]
   image: yourlabs/buildah
   script:
-  - pip3 install -U --user -e .[cli]
-  - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push
+  - pip3 install -U --user .[cli]
+  - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA
   stage: build
 
 build-itself:
   cache:
     key: cache
-    paths: [.cache]
-  image: quay.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA
-  script: python3 ./shlaxfile.py build
+    paths: [.cache, /var/lib/containers/]
+  image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA
+  script: python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_REF
   stage: test
 
+test-exitcode:
+  image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA
+  script:
+  - tests/shlaxfail.py build || [ $? -eq 1 ]
+  - tests/shlaxsuccess.py build
+
 test:
   image: yourlabs/python
   stage: build

shlax/actions/copy.py

@@ -1,19 +1,32 @@
 import asyncio
 import binascii
+import glob
 import os
 
+from ..exceptions import ShlaxException
+
 
 class Copy:
     def __init__(self, *args):
-        self.src = args[:-1]
         self.dst = args[-1]
+        self.src = []
 
-    def listfiles(self):
+        for src in args[:-1]:
+            if '*' in src:
+                self.src += glob.glob(src)
+            else:
+                self.src.append(src)
+
+    async def listfiles(self, target):
         if getattr(self, '_listfiles', None):
             return self._listfiles
 
         result = []
         for src in self.src:
+            if not await target.parent.exists(src):
+                target.output.fail(self)
+                raise ShlaxException(f'File not found {src}')
+
             if os.path.isfile(src):
                 result.append(src)
                 continue
@@ -32,7 +45,7 @@ class Copy:
     async def __call__(self, target):
         await target.mkdir(self.dst)
 
-        for path in self.listfiles():
+        for path in await self.listfiles(target):
             if os.path.isdir(path):
                 await target.mkdir(os.path.join(self.dst, path))
             elif '/' in path:
@@ -48,9 +61,11 @@ class Copy:
     def __str__(self):
         return f'Copy({", ".join(self.src)}, {self.dst})'
 
-    async def cachekey(self):
+    async def cachekey(self, target):
         async def chksum(path):
             with open(path, 'rb') as f:
                 return (path, str(binascii.crc32(f.read())))
-        results = await asyncio.gather(*[chksum(f) for f in self.listfiles()])
+        results = await asyncio.gather(
+            *[chksum(f) for f in await self.listfiles(target)]
+        )
         return {path: chks for path, chks in results}

shlax/actions/packages.py

@@ -27,28 +27,33 @@ class Packages:
             update='apk update',
             upgrade='apk upgrade',
             install='apk add',
+            host=None,
         ),
         apt=dict(
             update='apt-get -y update',
             upgrade='apt-get -y upgrade',
             install='apt-get -y --no-install-recommends install',
+            host=None,
         ),
         pacman=dict(
             update='pacman -Sy',
             upgrade='pacman -Su --noconfirm',
             install='pacman -S --noconfirm',
             lastupdate='stat -c %Y /var/lib/pacman/sync/core.db',
+            host='/var/lib/pacman',
         ),
         dnf=dict(
             update='dnf makecache --assumeyes',
             upgrade='dnf upgrade --best --assumeyes --skip-broken',  # noqa
             install='dnf install --setopt=install_weak_deps=False --best --assumeyes',  # noqa
             lastupdate='stat -c %Y /var/cache/dnf/* | head -n1',
+            host=None,
         ),
         yum=dict(
             update='yum update',
             upgrade='yum upgrade',
             install='yum install',
+            host=None,
         ),
     )
 
@@ -62,6 +67,12 @@ class Packages:
             self.packages += line.split(' ')
 
     async def cache_setup(self, target):
+        # Try to use the host cache directory if present rather than home
+        # directory, in cases where host and guest are the same distros
+        hostpath = self.mgrs[self.mgr]['host']
+        if target.exists(hostpath):
+            self.cache_root = hostpath
+
         if 'CACHE_DIR' in os.environ:
             self.cache_root = os.path.join(os.getenv('CACHE_DIR'))
         else:

shlax/cli.py

@@ -13,7 +13,7 @@ import importlib
 import os
 import sys
 
-from .proc import ProcFailure
+from .exceptions import ShlaxException
 
 
 class Group(cli2.Group):
@@ -57,10 +57,15 @@ class Command(cli2.Command):
 
         try:
             result = super().__call__(*argv)
-        except ProcFailure:
+        except ShlaxException as exc:
             # just output the failure without TB, as command was already
             # printed anyway
-            pass
+            self.exit_code = 1
+            self['target'].value.output.fail(exc)
+
+        if self['target'].value.results:
+            if self['target'].value.results[-1].status == 'failure':
+                self.exit_code = 1
         self['target'].value.output.results(self['target'].value)
         return result
 

shlax/image.py

@@ -60,9 +60,11 @@ class Image:
                 setattr(self, k, v)
 
         # docker.io currently has issues with oci format
-        self.format = format or 'oci'
         if self.registry == 'docker.io':
-            self.format = 'docker'
+            self.backend = 'docker'
+
+        if not self.format:
+            self.format = 'docker' if self.backend == 'docker' else 'oci'
 
         # filter out tags which resolved to None
         self.tags = [t for t in self.tags if t]
@@ -74,19 +76,19 @@ class Image:
     def __str__(self):
         return f'{self.repository}:{self.tags[-1]}'
 
-    async def push(self, target):
-        user = os.getenv('IMAGES_USER')
-        passwd = os.getenv('IMAGES_PASS')
+    async def push(self, target, name=None):
+        user = os.getenv('IMAGES_USER', os.getenv('DOCKER_USER'))
+        passwd = os.getenv('IMAGES_PASS', os.getenv('DOCKER_PASS'))
         if user and passwd:
             target.output.cmd('buildah login -u ... -p ...' + self.registry)
             await target.parent.exec(
                 'buildah', 'login', '-u', user, '-p', passwd,
-                self.registry or 'docker.io', debug=False)
+                self.registry or 'docker.io', quiet=True)
 
         for tag in self.tags:
             await target.parent.exec(
                 'buildah',
                 'push',
                 self.repository + ':final',
-                f'{self.registry}/{self.repository}:{tag}'
+                name if isinstance(name, str) else f'{self.registry}/{self.repository}:{tag}'
             )

shlax/output.py

@@ -101,7 +101,11 @@ class Output:
         )
 
     def highlight(self, line, highlight=True):
-        line = line.decode('utf8') if isinstance(line, bytes) else line
+        try:
+            line = line.decode('utf8') if isinstance(line, bytes) else line
+        except UnicodeDecodeError:
+            highlight = False
+
         if not highlight or (
             '\x1b[' in line
             or '\033[' in line

shlax/proc.py

@@ -7,10 +7,11 @@ import os
 import shlex
 import sys
 
+from .exceptions import ShlaxException
 from .output import Output
 
 
-class ProcFailure(Exception):
+class ProcFailure(ShlaxException):
     def __init__(self, proc):
         self.proc = proc
 

shlax/targets/base.py

@@ -6,7 +6,7 @@ import re
 import sys
 
 from ..output import Output
-from ..proc import Proc
+from ..proc import Proc, ProcFailure
 from ..result import Result, Results
 
 
@@ -68,13 +68,19 @@ class Target:
             self.output.fail(action, e)
             result.status = 'failure'
             result.exception = e
-            if reraise:
-                # nested call, re-raise
-                raise
-            else:
-                import traceback
-                traceback.print_exception(type(e), e, sys.exc_info()[2])
-                return True
+
+            if not isinstance(e, ProcFailure):
+                # no need to reraise in case of command error
+                # because the command has been printed
+
+                if reraise:
+                    # nested call, re-raise
+                    raise
+                else:
+                    import traceback
+                    traceback.print_exception(type(e), e, sys.exc_info()[2])
+
+            return True  # because it failed
         else:
             if getattr(action, 'skipped', False):
                 self.output.skip(action)

shlax/targets/buildah.py

@@ -37,7 +37,7 @@ class Buildah(Target):
             return 'Replacing with: buildah unshare ' + ' '.join(sys.argv)
         return f'Buildah({self.image})'
 
-    async def __call__(self, *actions, target=None, push: bool=False):
+    async def __call__(self, *actions, target=None, push: str=False):
         if target:
             self.parent = target
 
@@ -57,12 +57,8 @@ class Buildah(Target):
 
         if actions:
             actions = actions[len(keep):]
-            if not actions:
-                return self.output.success('Image up to date')
         else:
             self.actions = self.actions[len(keep):]
-            if not self.actions:
-                return self.output.success('Image up to date')
 
         self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out
         self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out)
@@ -92,7 +88,7 @@ class Buildah(Target):
                 prefix = tag
                 break
         if hasattr(action, 'cachekey'):
-            action_key = action.cachekey()
+            action_key = action.cachekey(self)
             if asyncio.iscoroutine(action_key):
                 action_key = str(await action_key)
         else:
@@ -111,7 +107,6 @@ class Buildah(Target):
             await self.parent.exec(
                 'buildah',
                 'commit',
-                '--format=' + action_image.format,
                 self.ctr,
                 action_image,
             )
@@ -127,7 +122,7 @@ class Buildah(Target):
         if result.status == 'success' and self.ctr:
             await self.commit()
             if self.push:
-                await self.image.push(target)
+                await self.image.push(target, self.push)
 
         if self.ctr is not None:
             await self.parent.exec('buildah', 'rm', self.ctr)
@@ -149,8 +144,19 @@ class Buildah(Target):
 
     async def commit(self):
         await self.parent.exec(
-            f'buildah commit {self.ctr} {self.image.repository}:final'
+            'buildah',
+            'commit',
+            f'--format={self.image.format}',
+            self.ctr,
+            f'{self.image.repository}:final',
         )
+        if self.image.backend == 'docker':
+            await self.parent.exec(
+                'buildah',
+                'push',
+                f'{self.image.repository}:final',
+                f'docker-daemon:{self.image.repository}:latest'
+            )
 
         ENV_TAGS = (
             # gitlab
@@ -181,7 +187,7 @@ class Buildah(Target):
         return await self.parent.mkdir(*[self.path(path) for path in paths])
 
     async def copy(self, *args):
-        return await self.parent.copy(*args[:-1], self.path(args[-1]))
+        return await self.parent.exec('buildah', 'copy', self.ctr, *args)
 
     async def write(self, path, content):
         return await self.write(path, content)

shlaxfile.py

@@ -9,9 +9,9 @@ shlax = Container(
     build=Buildah(
         Packages('python38', 'buildah', 'unzip', 'findutils', upgrade=False),
         Copy('setup.py', 'shlax', '/app'),
-        Pip('/app'),
-        base='quay.io/podman/stable',
-        commit='quay.io/yourlabs/shlax',
+        Pip('/app[cli]'),
+        base='quay.io/buildah/stable',
+        commit='docker://docker.io/yourlabs/shlax',
     ),
 )
 

tests/shlaxfail.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+"""
+Shlaxfile for shlax itself.
+"""
+
+from shlax.shortcuts import *
+
+shlax = Container(
+    build=Buildah(
+        Packages('prout', upgrade=False),
+        base='alpine',
+        commit='shlaxfail',
+    ),
+)
+
+
+if __name__ == '__main__':
+    print(Group(doc=__doc__).load(shlax).entry_point())

tests/shlaxsuccess.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+"""
+Shlaxfile for shlax itself.
+"""
+
+from shlax.shortcuts import *
+
+shlax = Container(
+    build=Buildah(
+        base='alpine',
+        commit='shlaxsuccess',
+    ),
+)
+
+
+if __name__ == '__main__':
+    print(Group(doc=__doc__).load(shlax).entry_point())