From 363bdb1493be3d81d8021b38605320602abc8458 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 03:53:53 +0200 Subject: [PATCH] Fix CI command to build --- .gitlab-ci.yml | 6 +++--- shlax/actions/user.py | 10 ++++++++++ shlax/container.py | 32 ++++++++++++++++++++++++++++++++ shlax/image.py | 18 ------------------ shlax/pod.py | 26 ++++++++++++++++++++++++++ shlax/targets/buildah.py | 32 +++++++++++++++++++++++++++----- shlaxfile.py | 3 +-- tests/test_image.py | 6 ------ 8 files changed, 99 insertions(+), 34 deletions(-) create mode 100644 shlax/container.py create mode 100644 shlax/pod.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6640159..5c62b09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,9 +6,9 @@ build: script: - dnf install -y curl python38 - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - - python3.8 get-pip.py - - pip3.8 install -U --user -e .[cli] - - CACHE_DIR=$(pwd)/.cache ~/.local/bin/shlax ./shlaxfile.py build + - python3 get-pip.py + - pip3 install -U --user -e .[cli] + - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build stage: build test: diff --git a/shlax/actions/user.py b/shlax/actions/user.py index f919852..f52c20c 100644 --- a/shlax/actions/user.py +++ b/shlax/actions/user.py @@ -5,6 +5,16 @@ from .packages import Packages class User: + """ + Create a user. + + Example: + + User('app', '/app', getenv('_CONTAINERS_ROOTLESS_UID', 1000)), + + _CONTAINERS_ROOTLESS_UID allows to get your UID during build, which happens + in buildah unshare. + """ def __init__(self, username, home, uid): self.username = username self.home = home diff --git a/shlax/container.py b/shlax/container.py new file mode 100644 index 0000000..fcc074b --- /dev/null +++ b/shlax/container.py @@ -0,0 +1,32 @@ +import os + +from .image import Image + + +class Container: + def __init__(self, build=None, image=None): + self.build = build + self.image = self.build.image + prefix = os.getcwd().split('/')[-1] + repo = self.image.repository.replace('/', '-') + if prefix == repo: + self.name = repo + else: + self.name = '-'.join([prefix, repo]) + + async def start(self, target): + """Start the container""" + await target.rexec( + 'podman', + 'run', + '--name', + self.name, + str(self.image), + ) + + async def stop(self, target): + """Start the container""" + await target.rexec('podman', 'stop', self.name) + + def __str__(self): + return f'Container(name={self.name}, image={self.image})' diff --git a/shlax/image.py b/shlax/image.py index 4ae7567..f21737b 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -4,18 +4,6 @@ import re class Image: - ENV_TAGS = ( - # gitlab - 'CI_COMMIT_SHORT_SHA', - 'CI_COMMIT_REF_NAME', - 'CI_COMMIT_TAG', - # CircleCI - 'CIRCLE_SHA1', - 'CIRCLE_TAG', - 'CIRCLE_BRANCH', - # contributions welcome here - ) - PATTERN = re.compile( '^((?P[a-z]*)://)?((?P[^/]*[.][^/]*)/)?((?P[^:]+))?(:(?P.*))?$' # noqa , re.I @@ -45,12 +33,6 @@ class Image: if self.registry == 'docker.io': self.format = 'docker' - # figure tags from CI vars - for name in self.ENV_TAGS: - value = os.getenv(name) - if value: - self.tags.append(value) - # filter out tags which resolved to None self.tags = [t for t in self.tags if t] diff --git a/shlax/pod.py b/shlax/pod.py new file mode 100644 index 0000000..82da2af --- /dev/null +++ b/shlax/pod.py @@ -0,0 +1,26 @@ +import cli2 + +from shlax.targets.base import Target +from shlax.actions.parallel import Parallel + + +class Pod: + """Help text""" + def __init__(self, **containers): + self.containers = containers + + async def _call(self, target, method, *names): + methods = [ + getattr(container, method) + for name, container in self.containers.items() + if not names or name in names + ] + await target(Parallel(*methods)) + + async def build(self, target, *names): + """Build container images""" + await self._call(target, 'build', *names) + + async def start(self, target, *names): + """Start container images""" + await self._call(target, 'start', *names) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 4564f02..927fb26 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -105,16 +105,19 @@ class Buildah(Target): if name in layers: self.base = self.image_previous = action_image keep.append(action_image) - self.output.skip(f'Found valid cached layer for {action}') + self.output.skip( + f'Found layer for {action}: {action_image.tags[0]}' + ) else: break return keep async def action_image(self, action): - if self.image_previous: - prefix = self.image_previous.tags[0] - else: - prefix = self.base + prefix = str(self.image_previous) + for tag in self.image_previous.tags: + if tag.startswith('layer-'): + prefix = tag + break if hasattr(action, 'cachekey'): action_key = action.cachekey() if asyncio.iscoroutine(action_key): @@ -129,6 +132,7 @@ class Buildah(Target): stop = await super().action(action, reraise) if not stop: action_image = await self.action_image(action) + self.output.info(f'Commiting {action_image} for {action}') await self.parent.exec( 'buildah', 'commit', @@ -187,6 +191,24 @@ class Buildah(Target): self.ctr, )).out + ENV_TAGS = ( + # gitlab + 'CI_COMMIT_SHORT_SHA', + 'CI_COMMIT_REF_NAME', + 'CI_COMMIT_TAG', + # CircleCI + 'CIRCLE_SHA1', + 'CIRCLE_TAG', + 'CIRCLE_BRANCH', + # contributions welcome here + ) + + # figure tags from CI vars + for name in ENV_TAGS: + value = os.getenv(name) + if value: + self.image.tags.append(value) + if image.tags: tags = [f'{image.repository}:{tag}' for tag in image.tags] else: diff --git a/shlaxfile.py b/shlaxfile.py index f3dfa7e..7a87b23 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -7,8 +7,7 @@ from shlax.shortcuts import * shlax = Container( build=Buildah( - Packages('python38', 'buildah', 'unzip', 'findutils'), - User('app', '/app', getenv('_CONTAINERS_ROOTLESS_UID')), + Packages('python38', 'buildah', 'unzip', 'findutils', upgrade=False), Copy('setup.py', 'shlax', '/app'), Pip('/app'), base='quay.io/podman/stable', diff --git a/tests/test_image.py b/tests/test_image.py index 257f591..df25f72 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -25,9 +25,3 @@ def test_args(arg, expected): im = Image(arg) for k, v in expected.items(): assert getattr(im, k) == v - -def test_args_env(): - os.environ['IMAGE_TEST_ARGS_ENV'] = 'foo' - Image.ENV_TAGS = ['IMAGE_TEST_ARGS_ENV'] - im = Image('re/po:x,y') - assert im.tags == ['x', 'y', 'foo']