From 4366ee63f98e6f9514dedcac5c001c261cd4734d Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 13 Feb 2020 05:42:04 +0100 Subject: [PATCH] wip --- .gitlab-ci.yml | 9 ++++++-- examples/simple/pod.py | 29 ----------------------- podctl/build.py | 45 ++++++++++++++++++++++++++++++++++-- podctl/console_script.py | 14 ++++++++++- podctl/container.py | 1 - podctl/script.py | 19 --------------- podctl/visitors/base.py | 3 ++- podctl/visitors/commit.py | 8 +++++-- tests/test_build_copy.sh | 19 --------------- tests/test_build_empty.sh | 15 ------------ tests/test_build_packages.sh | 30 ------------------------ tests/test_build_run.sh | 27 ---------------------- tests/test_build_user.sh | 42 --------------------------------- tests/test_commit.py | 19 +++++++++++++++ 14 files changed, 90 insertions(+), 190 deletions(-) delete mode 100644 tests/test_build_copy.sh delete mode 100644 tests/test_build_empty.sh delete mode 100644 tests/test_build_packages.sh delete mode 100644 tests/test_build_run.sh delete mode 100644 tests/test_build_user.sh create mode 100644 tests/test_commit.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c3de0d4..4817888 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,10 +7,15 @@ build: - .cache key: cache -test: +py-test: stage: test image: yourlabs/python - script: pip install -e . && cd examples && podctl test -d simple + script: pip install -e . && py.test -s + +pod-test: + stage: test + image: yourlabs/python + script: pip install -e . && examples/simple && podctl test -d . pypi: stage: deploy diff --git a/examples/simple/pod.py b/examples/simple/pod.py index a3ad737..0325925 100644 --- a/examples/simple/pod.py +++ b/examples/simple/pod.py @@ -20,34 +20,5 @@ async def test_pod_story2(pod): async def test_pod_story(pod): await pod.script('down')() - await pod.script('build')('ex') await pod.script('up')() await pod.script('down')() - - -async def aoeutest_podctl(host): - from podctl.console_script import console_script - console_script.options['debug'] = 'visit' - console_script.options['debug'] = True - - - from podctl.podfile import Podfile - pod = Podfile.factory(__file__).pod - - from podctl.proc import Proc - - #await Proc('podctl', 'down')() - #await Proc('podctl', 'build', 'ex')() - #await Proc('podctl', '-d=cmd,out,visit', 'up', 'ex')() - #assert host.podman('simple-ex').is_running - ##import time; time.sleep(5) - #await Proc('podctl', 'down')() - #assert 'simple-ex' not in [c.name for c in host.podman.get_containers()] - #return - - await pod.script('down')() - await pod.script('build')('ex') - await pod.script('up')('ex') - assert host.podman('ex').is_running - await pod.script('down')() - assert 'simple-ex' not in [c.name for c in host.podman.get_containers()] diff --git a/podctl/build.py b/podctl/build.py index 9fbc9e4..dd13fe2 100644 --- a/podctl/build.py +++ b/podctl/build.py @@ -4,9 +4,10 @@ import asyncio import signal import shlex import subprocess +import sys import textwrap -from .proc import Proc +from .proc import Proc, output from .script import Script @@ -20,6 +21,8 @@ class Build(Script): def __init__(self): super().__init__() self.mounts = dict() + self.ctr = None + self.mnt = None async def config(self, line): """Run buildah config.""" @@ -56,7 +59,8 @@ class Build(Script): async def umount(self): """Unmount the buildah container with buildah unmount.""" - await self.exec(f'buildah unmount {self.ctr}') + if self.ctr: + await self.exec(f'buildah unmount {self.ctr}') async def paths(self): """Return the list of $PATH directories""" @@ -79,3 +83,40 @@ class Build(Script): def __repr__(self): return f'Build' + + async def run(self, *args, **kwargs): + if os.getuid() == 0: + return await super().run(*args, **kwargs) + + from podctl.console_script import console_script + # restart under buildah unshare environment ! + argv = [ + 'buildah', 'unshare', + sys.argv[0], # current podctl location + ] + if console_script.options.get('debug') is True: + argv.append('-d') + elif isinstance(console_script.options.get('debug'), str): + argv.append('-d=' + console_script.options.get('debug')) + argv += [ + type(self).__name__.lower(), # script name ? + ] + output(' '.join(argv), 'EXECUTION') + + proc = await asyncio.create_subprocess_shell( + shlex.join(argv), + stderr=sys.stderr, + stdin=sys.stdin, + stdout=sys.stdout, + ) + await proc.communicate() + console_script.exit_code = proc.returncode + return + pp = subprocess.Popen( + argv, + stderr=sys.stderr, + stdin=sys.stdin, + stdout=sys.stdout, + ) + pp.communicate() + console_script.exit_code = pp.returncode diff --git a/podctl/console_script.py b/podctl/console_script.py index 15ae830..7458e77 100644 --- a/podctl/console_script.py +++ b/podctl/console_script.py @@ -35,7 +35,19 @@ async def test(*args, **kwargs): '\n\x1b[1;38;5;160;48;5;118m BUILD START \x1b[0m' + ' ' + podfile.path + '\n' ) - podfile.pod.script('build')() + + old_exit_code = console_script.exit_code + console_script.exit_code = 0 + try: + await podfile.pod.script('build')() + except Exception as e: + report.append(('build ' + candidate, False)) + continue + + if console_script.exit_code != 0: + report.append(('build ' + candidate, False)) + continue + console_script.exit_code = old_exit_code for name, test in podfile.tests.items(): name = '::'.join([podfile.path, name]) diff --git a/podctl/container.py b/podctl/container.py index 9513f0e..374b9c0 100644 --- a/podctl/container.py +++ b/podctl/container.py @@ -43,7 +43,6 @@ class Container(Visitable): await script.exec('podman', 'inspect', self.container_name) except WrongResult as ee: output('Container creating', self.name) - breakpoint() await script.exec( 'podman', 'run', '-d', '--name', self.container_name, self.image_name, diff --git a/podctl/script.py b/podctl/script.py index b1297a3..7a34616 100644 --- a/podctl/script.py +++ b/podctl/script.py @@ -104,25 +104,6 @@ class Script: sys.stdout.flush() async def run(self, *args, **kwargs): - if self.unshare and os.getuid() != 0: - from podctl.console_script import console_script - # restart under buildah unshare environment ! - argv = [ - 'buildah', 'unshare', - sys.argv[0], # current podctl location - ] + console_script.parser.argv + [ - type(self).__name__.lower() # script name ? - ] + list(args) - print('Executing', ' '.join(argv)) - pp = subprocess.Popen( - argv, - stderr=sys.stderr, - stdin=sys.stdin, - stdout=sys.stdout, - ) - pp.communicate() - return pp.returncode - for key, value in kwargs.items(): setattr(self, key, value) diff --git a/podctl/visitors/base.py b/podctl/visitors/base.py index bdc0456..4a4cd23 100644 --- a/podctl/visitors/base.py +++ b/podctl/visitors/base.py @@ -12,7 +12,8 @@ class Base: async def clean_build(self, script): await script.umounts() await script.umount() - proc = await script.exec('buildah', 'rm', script.ctr, raises=False) + if script.ctr: + proc = await script.exec('buildah', 'rm', script.ctr, raises=False) def __repr__(self): return f'Base({self.base})' diff --git a/podctl/visitors/commit.py b/podctl/visitors/commit.py index 208b69f..cedc6f1 100644 --- a/podctl/visitors/commit.py +++ b/podctl/visitors/commit.py @@ -20,20 +20,24 @@ class Commit: self.repo = repo self.registry = registry or 'localhost' self.push = push or os.getenv('CI') + self.tags = tags or [] # figure out registry host if '/' in self.repo and not registry: first = self.repo.split('/')[0] if '.' in first or ':' in first: self.registry = self.repo.split('/')[0] + self.repo = '/'.join(self.repo.split('/')[1:]) + + if ':' in self.repo and not tags: + self.tags = [self.repo.split(':')[1]] + self.repo = self.repo.split(':')[0] # docker.io currently has issues with oci format self.format = format or 'oci' if self.registry == 'docker.io': self.format = 'docker' - self.tags = tags or [] - # figure tags from CI vars if not self.tags: for name in CI_VARS: diff --git a/tests/test_build_copy.sh b/tests/test_build_copy.sh deleted file mode 100644 index 84c46dc..0000000 --- a/tests/test_build_copy.sh +++ /dev/null @@ -1,19 +0,0 @@ -#/usr/bin/env bash -base="alpine" -repo="None" -mounts=() -umounts() { - for i in "${mounts[@]}"; do - umount $i - mounts=("${mounts[@]/$i}") - done - buildah unmount $ctr - trap - 0 -} -trap umounts 0 -ctr=$(buildah from $base) -mnt=$(buildah mount $ctr) -echo "Copy.init_build" -echo "Copy.build" -buildah run --user root $ctr -- mkdir -p /app -cp -a /test $mnt/app \ No newline at end of file diff --git a/tests/test_build_empty.sh b/tests/test_build_empty.sh deleted file mode 100644 index d0f4556..0000000 --- a/tests/test_build_empty.sh +++ /dev/null @@ -1,15 +0,0 @@ -#/usr/bin/env bash -base="alpine" -repo="None" -mounts=() -umounts() { - for i in "${mounts[@]}"; do - umount $i - mounts=("${mounts[@]/$i}") - done - buildah unmount $ctr - trap - 0 -} -trap umounts 0 -ctr=$(buildah from $base) -mnt=$(buildah mount $ctr) \ No newline at end of file diff --git a/tests/test_build_packages.sh b/tests/test_build_packages.sh deleted file mode 100644 index 5d3fa0b..0000000 --- a/tests/test_build_packages.sh +++ /dev/null @@ -1,30 +0,0 @@ -#/usr/bin/env bash -base="alpine" -repo="None" -mounts=() -umounts() { - for i in "${mounts[@]}"; do - umount $i - mounts=("${mounts[@]/$i}") - done - buildah unmount $ctr - trap - 0 -} -trap umounts 0 -ctr=$(buildah from $base) -mnt=$(buildah mount $ctr) -echo "Packages.pre_build" -echo "Packages.build" -buildah run --user root $ctr -- mkdir -p /var/cache/apk -mkdir -p /test/apk -mount -o bind /test/apk $mnt/var/cache/apk -mounts=("$mnt/var/cache/apk" "${mounts[@]}") -buildah run $ctr -- ln -s /var/cache/apk /etc/apk/cache -old="$(find .cache/apk/ -name APKINDEX.* -mtime +3)" -if [ -n "$old" ] || ! ls .cache/apk/APKINDEX.*; then - buildah run --user root $ctr -- apk update -else - echo Cache recent enough, skipping index update. -fi -buildah run --user root $ctr -- apk upgrade -buildah run --user root $ctr -- apk add bash \ No newline at end of file diff --git a/tests/test_build_run.sh b/tests/test_build_run.sh deleted file mode 100644 index d0aea7f..0000000 --- a/tests/test_build_run.sh +++ /dev/null @@ -1,27 +0,0 @@ -#/usr/bin/env bash -base="alpine" -repo="None" -mounts=() -umounts() { - for i in "${mounts[@]}"; do - umount $i - mounts=("${mounts[@]/$i}") - done - buildah unmount $ctr - trap - 0 -} -trap umounts 0 -ctr=$(buildah from $base) -mnt=$(buildah mount $ctr) -echo "Run.build" -buildah run $ctr -- foo -echo "Run.build" -buildah run --user root $ctr -- bar -echo "Run.build" -buildah run --user root $ctr -- bash -eux <<__EOF - bar > test -__EOF -echo "Run.build" -buildah run $ctr -- bash -eux <<__EOF - bar -__EOF \ No newline at end of file diff --git a/tests/test_build_user.sh b/tests/test_build_user.sh deleted file mode 100644 index bee2438..0000000 --- a/tests/test_build_user.sh +++ /dev/null @@ -1,42 +0,0 @@ -#/usr/bin/env bash -base="alpine" -repo="None" -mounts=() -umounts() { - for i in "${mounts[@]}"; do - umount $i - mounts=("${mounts[@]/$i}") - done - buildah unmount $ctr - trap - 0 -} -trap umounts 0 -ctr=$(buildah from $base) -mnt=$(buildah mount $ctr) -echo "User.init_build" -echo "User.init_build" -echo "Packages.pre_build" -echo "User.pre_build" -echo "Packages.build" -buildah run --user root $ctr -- mkdir -p /var/cache/apk -mkdir -p /test/apk -mount -o bind /test/apk $mnt/var/cache/apk -mounts=("$mnt/var/cache/apk" "${mounts[@]}") -buildah run $ctr -- ln -s /var/cache/apk /etc/apk/cache -old="$(find .cache/apk/ -name APKINDEX.* -mtime +3)" -if [ -n "$old" ] || ! ls .cache/apk/APKINDEX.*; then - buildah run --user root $ctr -- apk update -else - echo Cache recent enough, skipping index update. -fi -buildah run --user root $ctr -- apk upgrade -buildah run --user root $ctr -- apk add shadow -echo "User.build" -if buildah run $ctr -- id 1000; then - i=$(buildah run $ctr -- id -gn 1000) - buildah run $ctr -- usermod -d /app -l app $i -else - buildah run $ctr -- useradd -d /app -u 1000 app -fi -echo "User.post_build" -buildah config --user app $ctr \ No newline at end of file diff --git a/tests/test_commit.py b/tests/test_commit.py new file mode 100644 index 0000000..38740e3 --- /dev/null +++ b/tests/test_commit.py @@ -0,0 +1,19 @@ +from podctl.visitors.commit import Commit + + +def test_name_parse(): + commit = Commit('foo.ee/bar/test:y') + assert commit.registry == 'foo.ee' + assert commit.repo == 'bar/test' + assert commit.tags == ['y'] + + commit = Commit('foo.ee/bar/test') + assert commit.registry == 'foo.ee' + assert commit.repo == 'bar/test' + + commit = Commit('bar/test') + assert commit.repo == 'bar/test' + + commit = Commit('bar/test:y') + assert commit.repo == 'bar/test' + assert commit.tags == ['y']