Merge branch 'mostadvanced' into 'master'

Complete core rewrite, with documentation

See merge request oss/shlax!1

.gitlab-ci.yml

@@ -1,21 +1,27 @@
 build:
   cache:
     key: cache
-    paths: [.cache]
+    paths: [.cache, /var/lib/containers/]
   image: yourlabs/buildah
   script:
-  - pip3 install -U --user -e .[cli]
+  - pip3 install -U --user .[cli]
-  - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push
+  - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA
   stage: build
 
 build-itself:
   cache:
     key: cache
-    paths: [.cache]
+    paths: [.cache, /var/lib/containers/]
-  image: quay.io/yourlabs/shlax:$CI_COMMIT_SHORT_SHA
+  image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA
-  script: python3 ./shlaxfile.py build
+  script: python3 ./shlaxfile.py build push=docker://docker.io/yourlabs/shlax:$CI_COMMIT_REF
   stage: test
 
+test-exitcode:
+  image: yourlabs/shlax:$CI_COMMIT_SHORT_SHA
+  script:
+  - tests/shlaxfail.py build || [ $? -eq 1 ]
+  - tests/shlaxsuccess.py build
+
 test:
   image: yourlabs/python
   stage: build

shlax/actions/copy.py

@@ -1,12 +1,19 @@
 import asyncio
 import binascii
+import glob
 import os
 
 
 class Copy:
     def __init__(self, *args):
-        self.src = args[:-1]
         self.dst = args[-1]
+        self.src = []
+
+        for src in args[:-1]:
+            if '*' in src:
+                self.src += glob.glob(src)
+            else:
+                self.src.append(src)
 
     def listfiles(self):
         if getattr(self, '_listfiles', None):

shlax/cli.py

@@ -61,6 +61,10 @@ class Command(cli2.Command):
             # just output the failure without TB, as command was already
             # printed anyway
             pass
+
+        if self['target'].value.results:
+            if self['target'].value.results[-1].status == 'failure':
+                self.exit_code = 1
         self['target'].value.output.results(self['target'].value)
         return result
 

shlax/image.py

@@ -60,9 +60,11 @@ class Image:
                 setattr(self, k, v)
 
         # docker.io currently has issues with oci format
-        self.format = format or 'oci'
         if self.registry == 'docker.io':
-            self.format = 'docker'
+            self.backend = 'docker'
+
+        if not self.format:
+            self.format = 'docker' if self.backend == 'docker' else 'oci'
 
         # filter out tags which resolved to None
         self.tags = [t for t in self.tags if t]
@@ -74,19 +76,19 @@ class Image:
     def __str__(self):
         return f'{self.repository}:{self.tags[-1]}'
 
-    async def push(self, target):
+    async def push(self, target, name=None):
-        user = os.getenv('IMAGES_USER')
+        user = os.getenv('IMAGES_USER', os.getenv('DOCKER_USER'))
-        passwd = os.getenv('IMAGES_PASS')
+        passwd = os.getenv('IMAGES_PASS', os.getenv('DOCKER_PASS'))
         if user and passwd:
             target.output.cmd('buildah login -u ... -p ...' + self.registry)
             await target.parent.exec(
                 'buildah', 'login', '-u', user, '-p', passwd,
-                self.registry or 'docker.io', debug=False)
+                self.registry or 'docker.io', quiet=True)
 
         for tag in self.tags:
             await target.parent.exec(
                 'buildah',
                 'push',
                 self.repository + ':final',
-                f'{self.registry}/{self.repository}:{tag}'
+                name if isinstance(name, str) else f'{self.registry}/{self.repository}:{tag}'
             )

shlax/output.py

@@ -101,7 +101,11 @@ class Output:
         )
 
     def highlight(self, line, highlight=True):
-        line = line.decode('utf8') if isinstance(line, bytes) else line
+        try:
+            line = line.decode('utf8') if isinstance(line, bytes) else line
+        except UnicodeDecodeError:
+            highlight = False
+
         if not highlight or (
             '\x1b[' in line
             or '\033[' in line

shlax/targets/base.py

@@ -6,7 +6,7 @@ import re
 import sys
 
 from ..output import Output
-from ..proc import Proc
+from ..proc import Proc, ProcFailure
 from ..result import Result, Results
 
 
@@ -68,13 +68,19 @@ class Target:
             self.output.fail(action, e)
             result.status = 'failure'
             result.exception = e
-            if reraise:
+
-                # nested call, re-raise
+            if not isinstance(e, ProcFailure):
-                raise
+                # no need to reraise in case of command error
-            else:
+                # because the command has been printed
-                import traceback
+
-                traceback.print_exception(type(e), e, sys.exc_info()[2])
+                if reraise:
-                return True
+                    # nested call, re-raise
+                    raise
+                else:
+                    import traceback
+                    traceback.print_exception(type(e), e, sys.exc_info()[2])
+
+            return True  # because it failed
         else:
             if getattr(action, 'skipped', False):
                 self.output.skip(action)

shlax/targets/buildah.py

@@ -37,7 +37,7 @@ class Buildah(Target):
             return 'Replacing with: buildah unshare ' + ' '.join(sys.argv)
         return f'Buildah({self.image})'
 
-    async def __call__(self, *actions, target=None, push: bool=False):
+    async def __call__(self, *actions, target=None, push: str=False):
         if target:
             self.parent = target
 
@@ -57,12 +57,8 @@ class Buildah(Target):
 
         if actions:
             actions = actions[len(keep):]
-            if not actions:
-                return self.output.success('Image up to date')
         else:
             self.actions = self.actions[len(keep):]
-            if not self.actions:
-                return self.output.success('Image up to date')
 
         self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out
         self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out)
@@ -111,7 +107,6 @@ class Buildah(Target):
             await self.parent.exec(
                 'buildah',
                 'commit',
-                '--format=' + action_image.format,
                 self.ctr,
                 action_image,
             )
@@ -127,7 +122,7 @@ class Buildah(Target):
         if result.status == 'success' and self.ctr:
             await self.commit()
             if self.push:
-                await self.image.push(target)
+                await self.image.push(target, self.push)
 
         if self.ctr is not None:
             await self.parent.exec('buildah', 'rm', self.ctr)
@@ -149,8 +144,19 @@ class Buildah(Target):
 
     async def commit(self):
         await self.parent.exec(
-            f'buildah commit {self.ctr} {self.image.repository}:final'
+            'buildah',
+            'commit',
+            f'--format={self.image.format}',
+            self.ctr,
+            f'{self.image.repository}:final',
         )
+        if self.image.backend == 'docker':
+            await self.parent.exec(
+                'buildah',
+                'push',
+                f'{self.image.repository}:final',
+                f'docker-daemon:{self.image.repository}:latest'
+            )
 
         ENV_TAGS = (
             # gitlab
@@ -181,7 +187,7 @@ class Buildah(Target):
         return await self.parent.mkdir(*[self.path(path) for path in paths])
 
     async def copy(self, *args):
-        return await self.parent.copy(*args[:-1], self.path(args[-1]))
+        return await self.parent.exec('buildah', 'copy', self.ctr, *args)
 
     async def write(self, path, content):
         return await self.write(path, content)

shlaxfile.py

@@ -9,9 +9,9 @@ shlax = Container(
     build=Buildah(
         Packages('python38', 'buildah', 'unzip', 'findutils', upgrade=False),
         Copy('setup.py', 'shlax', '/app'),
-        Pip('/app'),
+        Pip('/app[cli]'),
-        base='quay.io/podman/stable',
+        base='quay.io/buildah/stable',
-        commit='quay.io/yourlabs/shlax',
+        commit='docker://docker.io/yourlabs/shlax',
     ),
 )
 

tests/shlaxfail.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+"""
+Shlaxfile for shlax itself.
+"""
+
+from shlax.shortcuts import *
+
+shlax = Container(
+    build=Buildah(
+        Packages('prout', upgrade=False),
+        base='alpine',
+        commit='shlaxfail',
+    ),
+)
+
+
+if __name__ == '__main__':
+    print(Group(doc=__doc__).load(shlax).entry_point())

tests/shlaxsuccess.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+"""
+Shlaxfile for shlax itself.
+"""
+
+from shlax.shortcuts import *
+
+shlax = Container(
+    build=Buildah(
+        base='alpine',
+        commit='shlaxsuccess',
+    ),
+)
+
+
+if __name__ == '__main__':
+    print(Group(doc=__doc__).load(shlax).entry_point())