Fixing push

... when action level cleaning was added

Test both while we're at it, as this is pretty critical for actions to
clean up properly

.gitlab-ci.yml

@@ -1,27 +1,21 @@
 build:
   cache:
     key: cache
-    paths: [.cache, /var/lib/containers/]
+    paths: [.cache]
   image: yourlabs/buildah
   script:
-  - pip3 install -U --user .[cli]
+  - pip3 install -U --user -e .[cli]
-  - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA
+  - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push
   stage: build
 
 build-itself:
   cache:
     key: cache
-    paths: [.cache, /var/lib/containers/]
+    paths: [.cache]
-  image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA
+  image: quay.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA
-  script: python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_REF
+  script: python3 ./shlaxfile.py build
   stage: test
 
-test-exitcode:
-  image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA
-  script:
-  - tests/shlaxfail.py build || [ $? -eq 1 ]
-  - tests/shlaxsuccess.py build
-
 test:
   image: yourlabs/python
   stage: build

shlax/actions/copy.py

@@ -1,32 +1,19 @@
 import asyncio
 import binascii
-import glob
 import os
 
-from ..exceptions import ShlaxException
-
 
 class Copy:
     def __init__(self, *args):
+        self.src = args[:-1]
         self.dst = args[-1]
-        self.src = []
 
-        for src in args[:-1]:
+    def listfiles(self):
-            if '*' in src:
-                self.src += glob.glob(src)
-            else:
-                self.src.append(src)
-
-    async def listfiles(self, target):
         if getattr(self, '_listfiles', None):
             return self._listfiles
 
         result = []
         for src in self.src:
-            if not await target.parent.exists(src):
-                target.output.fail(self)
-                raise ShlaxException(f'File not found {src}')
-
             if os.path.isfile(src):
                 result.append(src)
                 continue
@@ -45,7 +32,7 @@ class Copy:
     async def __call__(self, target):
         await target.mkdir(self.dst)
 
-        for path in await self.listfiles(target):
+        for path in self.listfiles():
             if os.path.isdir(path):
                 await target.mkdir(os.path.join(self.dst, path))
             elif '/' in path:
@@ -61,11 +48,9 @@ class Copy:
     def __str__(self):
         return f'Copy({", ".join(self.src)}, {self.dst})'
 
-    async def cachekey(self, target):
+    async def cachekey(self):
         async def chksum(path):
             with open(path, 'rb') as f:
                 return (path, str(binascii.crc32(f.read())))
-        results = await asyncio.gather(
+        results = await asyncio.gather(*[chksum(f) for f in self.listfiles()])
-            *[chksum(f) for f in await self.listfiles(target)]
-        )
         return {path: chks for path, chks in results}

shlax/actions/packages.py

@@ -27,33 +27,28 @@ class Packages:
             update='apk update',
             upgrade='apk upgrade',
             install='apk add',
-            host=None,
         ),
         apt=dict(
             update='apt-get -y update',
             upgrade='apt-get -y upgrade',
             install='apt-get -y --no-install-recommends install',
-            host=None,
         ),
         pacman=dict(
             update='pacman -Sy',
             upgrade='pacman -Su --noconfirm',
             install='pacman -S --noconfirm',
             lastupdate='stat -c %Y /var/lib/pacman/sync/core.db',
-            host='/var/lib/pacman',
         ),
         dnf=dict(
             update='dnf makecache --assumeyes',
             upgrade='dnf upgrade --best --assumeyes --skip-broken',  # noqa
             install='dnf install --setopt=install_weak_deps=False --best --assumeyes',  # noqa
             lastupdate='stat -c %Y /var/cache/dnf/* | head -n1',
-            host=None,
         ),
         yum=dict(
             update='yum update',
             upgrade='yum upgrade',
             install='yum install',
-            host=None,
         ),
     )
 
@@ -67,12 +62,6 @@ class Packages:
             self.packages += line.split(' ')
 
     async def cache_setup(self, target):
-        # Try to use the host cache directory if present rather than home
-        # directory, in cases where host and guest are the same distros
-        hostpath = self.mgrs[self.mgr]['host']
-        if target.exists(hostpath):
-            self.cache_root = hostpath
-
         if 'CACHE_DIR' in os.environ:
             self.cache_root = os.path.join(os.getenv('CACHE_DIR'))
         else:

shlax/cli.py

@@ -13,7 +13,7 @@ import importlib
 import os
 import sys
 
-from .exceptions import ShlaxException
+from .proc import ProcFailure
 
 
 class Group(cli2.Group):
@@ -57,15 +57,10 @@ class Command(cli2.Command):
 
         try:
             result = super().__call__(*argv)
-        except ShlaxException as exc:
+        except ProcFailure:
             # just output the failure without TB, as command was already
             # printed anyway
-            self.exit_code = 1
+            pass
-            self['target'].value.output.fail(exc)
-
-        if self['target'].value.results:
-            if self['target'].value.results[-1].status == 'failure':
-                self.exit_code = 1
         self['target'].value.output.results(self['target'].value)
         return result
 

shlax/image.py

@@ -60,11 +60,9 @@ class Image:
                 setattr(self, k, v)
 
         # docker.io currently has issues with oci format
+        self.format = format or 'oci'
         if self.registry == 'docker.io':
-            self.backend = 'docker'
+            self.format = 'docker'
-
-        if not self.format:
-            self.format = 'docker' if self.backend == 'docker' else 'oci'
 
         # filter out tags which resolved to None
         self.tags = [t for t in self.tags if t]
@@ -76,19 +74,19 @@ class Image:
     def __str__(self):
         return f'{self.repository}:{self.tags[-1]}'
 
-    async def push(self, target, name=None):
+    async def push(self, target):
-        user = os.getenv('IMAGES_USER', os.getenv('DOCKER_USER'))
+        user = os.getenv('IMAGES_USER')
-        passwd = os.getenv('IMAGES_PASS', os.getenv('DOCKER_PASS'))
+        passwd = os.getenv('IMAGES_PASS')
         if user and passwd:
             target.output.cmd('buildah login -u ... -p ...' + self.registry)
             await target.parent.exec(
                 'buildah', 'login', '-u', user, '-p', passwd,
-                self.registry or 'docker.io', quiet=True)
+                self.registry or 'docker.io', debug=False)
 
         for tag in self.tags:
             await target.parent.exec(
                 'buildah',
                 'push',
                 self.repository + ':final',
-                name if isinstance(name, str) else f'{self.registry}/{self.repository}:{tag}'
+                f'{self.registry}/{self.repository}:{tag}'
             )

shlax/output.py

@@ -101,11 +101,7 @@ class Output:
         )
 
     def highlight(self, line, highlight=True):
-        try:
         line = line.decode('utf8') if isinstance(line, bytes) else line
-        except UnicodeDecodeError:
-            highlight = False
-
         if not highlight or (
             '\x1b[' in line
             or '\033[' in line

shlax/proc.py

@@ -7,11 +7,10 @@ import os
 import shlex
 import sys
 
-from .exceptions import ShlaxException
 from .output import Output
 
 
-class ProcFailure(ShlaxException):
+class ProcFailure(Exception):
     def __init__(self, proc):
         self.proc = proc
 

shlax/targets/base.py

@@ -6,7 +6,7 @@ import re
 import sys
 
 from ..output import Output
-from ..proc import Proc, ProcFailure
+from ..proc import Proc
 from ..result import Result, Results
 
 
@@ -68,19 +68,13 @@ class Target:
             self.output.fail(action, e)
             result.status = 'failure'
             result.exception = e
-
-            if not isinstance(e, ProcFailure):
-                # no need to reraise in case of command error
-                # because the command has been printed
-
             if reraise:
                 # nested call, re-raise
                 raise
             else:
                 import traceback
                 traceback.print_exception(type(e), e, sys.exc_info()[2])
-
+                return True
-            return True  # because it failed
         else:
             if getattr(action, 'skipped', False):
                 self.output.skip(action)

shlax/targets/buildah.py

@@ -37,7 +37,7 @@ class Buildah(Target):
             return 'Replacing with: buildah unshare ' + ' '.join(sys.argv)
         return f'Buildah({self.image})'
 
-    async def __call__(self, *actions, target=None, push: str=False):
+    async def __call__(self, *actions, target=None, push: bool=False):
         if target:
             self.parent = target
 
@@ -57,8 +57,12 @@ class Buildah(Target):
 
         if actions:
             actions = actions[len(keep):]
+            if not actions:
+                return self.output.success('Image up to date')
         else:
             self.actions = self.actions[len(keep):]
+            if not self.actions:
+                return self.output.success('Image up to date')
 
         self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out
         self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out)
@@ -88,7 +92,7 @@ class Buildah(Target):
                 prefix = tag
                 break
         if hasattr(action, 'cachekey'):
-            action_key = action.cachekey(self)
+            action_key = action.cachekey()
             if asyncio.iscoroutine(action_key):
                 action_key = str(await action_key)
         else:
@@ -107,6 +111,7 @@ class Buildah(Target):
             await self.parent.exec(
                 'buildah',
                 'commit',
+                '--format=' + action_image.format,
                 self.ctr,
                 action_image,
             )
@@ -122,7 +127,7 @@ class Buildah(Target):
         if result.status == 'success' and self.ctr:
             await self.commit()
             if self.push:
-                await self.image.push(target, self.push)
+                await self.image.push(target)
 
         if self.ctr is not None:
             await self.parent.exec('buildah', 'rm', self.ctr)
@@ -144,18 +149,7 @@ class Buildah(Target):
 
     async def commit(self):
         await self.parent.exec(
-            'buildah',
+            f'buildah commit {self.ctr} {self.image.repository}:final'
-            'commit',
-            f'--format={self.image.format}',
-            self.ctr,
-            f'{self.image.repository}:final',
-        )
-        if self.image.backend == 'docker':
-            await self.parent.exec(
-                'buildah',
-                'push',
-                f'{self.image.repository}:final',
-                f'docker-daemon:{self.image.repository}:latest'
         )
 
         ENV_TAGS = (
@@ -187,7 +181,7 @@ class Buildah(Target):
         return await self.parent.mkdir(*[self.path(path) for path in paths])
 
     async def copy(self, *args):
-        return await self.parent.exec('buildah', 'copy', self.ctr, *args)
+        return await self.parent.copy(*args[:-1], self.path(args[-1]))
 
     async def write(self, path, content):
         return await self.write(path, content)

shlaxfile.py

@@ -9,9 +9,9 @@ shlax = Container(
     build=Buildah(
         Packages('python38', 'buildah', 'unzip', 'findutils', upgrade=False),
         Copy('setup.py', 'shlax', '/app'),
-        Pip('/app[cli]'),
+        Pip('/app'),
-        base='quay.io/buildah/stable',
+        base='quay.io/podman/stable',
-        commit='docker://docker.io/yourlabs/shlax',
+        commit='quay.io/yourlabs/shlax',
     ),
 )
 

tests/shlaxfail.py

@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-"""
-Shlaxfile for shlax itself.
-"""
-
-from shlax.shortcuts import *
-
-shlax = Container(
-    build=Buildah(
-        Packages('prout', upgrade=False),
-        base='alpine',
-        commit='shlaxfail',
-    ),
-)
-
-
-if __name__ == '__main__':
-    print(Group(doc=__doc__).load(shlax).entry_point())

tests/shlaxsuccess.py

@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-"""
-Shlaxfile for shlax itself.
-"""
-
-from shlax.shortcuts import *
-
-shlax = Container(
-    build=Buildah(
-        base='alpine',
-        commit='shlaxsuccess',
-    ),
-)
-
-
-if __name__ == '__main__':
-    print(Group(doc=__doc__).load(shlax).entry_point())