diff --git a/examples/django.py b/examples/django.py deleted file mode 100644 index 4893289..0000000 --- a/examples/django.py +++ /dev/null @@ -1,65 +0,0 @@ -import os - -from podctl import * - - -class Django(Container): - tag = 'yourlabs/crudlfap' - base = 'alpine' - packages = [ - 'bash', - 'python3', - switch(dev='vim'), - ] - env = dict(FOO='bar') - annotations = dict(test='foo') - labels = dict(foo='test') - cmd = 'bash' - entrypoint = ['bash', '-v'] - ports = [1234] - user = dict( - shell='/bin/bash', - name='app', - home='/app', - id=os.getenv('SUDO_ID', os.getenv('UID')), - ) - volumes = [ - '/bydir', - 'byname:/byname', - switch(dev='.:/app'), - ] - build = [ - 'sudo pip3 install --upgrade pip', - 'pip3 install --user -e /app', - ] - workdir = '/app' - -django = Container( - Base('alpine'), - Packages('bash', switch(dev='vim')), - User( - uid=1000, - home='/app', - directories=('log', 'spooler', 'static') - ), - switch(default=Mount('.', '/app'), production=Copy('.', '/app')), - Npm('build'), - Env('PATH', 'PATH=/app/node_modules/.bin:$PATH'), - Pip('requirements.txt'), - Env('PATH', 'PATH=$HOME/.local/bin:$PATH'), - switch(dev=Run(''' - manage.py collectstatic --noinput --clear - find frontend/static/dist/css -type f | xargs gzip -f -k -9 - find frontend/static/dist/js -type f | xargs gzip -f -k -9 - ''')), - Expose(8000), - Tag('equisafe'), -) - -pod = Pod( - Service('django', django, restart='unless-stopped'), - Service('db', - Container(Tag('postgresql:latest')), - restart='unless-stopped' - ), -) diff --git a/podctl/build.py b/podctl/build.py index 08cecf1..e9c9472 100644 --- a/podctl/build.py +++ b/podctl/build.py @@ -2,16 +2,14 @@ import asyncio import os import asyncio import signal +import shlex import subprocess import textwrap +from .proc import Proc from .script import Script -class WrongResult(Exception): - pass - - class Build(Script): def __init__(self, container): super().__init__() @@ -19,95 +17,40 @@ class Build(Script): self.log = [] self.mounts = dict() - async def cmd(self, line): - log = dict(cmd=line) - self.log.append(log) - line = self.unshare(line) - print(self.container.name + ' | ' + line) - def protocol_factory(): - from .console_script import BuildStreamProtocol - return BuildStreamProtocol( - self.container, - limit=asyncio.streams._DEFAULT_LIMIT, - loop=self.loop, - ) - transport, protocol = await self.loop.subprocess_shell( - protocol_factory, - line, - ) - proc = asyncio.subprocess.Process( - transport, - protocol, - self.loop, - ) - result = await proc.wait() - if result: - raise WrongResult(proc) - return proc - ''' - proc = await asyncio.create_subprocess_shell( - line, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - ''' - - async def append(self, value): - for line in value.split('\n'): - if line.startswith('#') or not line.strip(): - continue - log = dict(proc=await self.cmd(line)) - log['stdout'], log['stderr'] = await log['proc'].communicate() - - return log['stdout'] - - def unshare(self, line): - return 'buildah unshare ' + line - async def config(self, line): return await self.append(f'buildah config {line} {self.ctr}') - def _run(self, cmd, inject=False): - user = self.container.variable('username') - _cmd = textwrap.dedent(cmd) - if cmd.startswith('sudo '): - _cmd = _cmd[5:] - - heredoc = False - for i in ('\n', '>', '<', '|', '&'): - if i in _cmd: - heredoc = True - break - - if heredoc: - _cmd = ''.join(['bash -eux <<__EOF\n', _cmd.strip(), '\n__EOF']) - - if cmd.startswith('sudo '): - return f'buildah run --user root {self.ctr} -- {_cmd}' - elif user and self.container.variable('user_created'): - return f'buildah run --user {user} {self.ctr} -- {_cmd}' - else: - return f'buildah run {self.ctr} -- {_cmd}' - - async def run(self, cmd): - return await self.append(self._run(cmd)) - async def copy(self, src, dst): return await self.append(f'buildah copy {self.ctr} {src} {dst}') + async def exec(self, *args, **kwargs): + kwargs.setdefault('prefix', self.container.name) + proc = await Proc(*args, **kwargs)() + if kwargs.get('wait', True): + await proc.wait() + return proc + + async def cexec(self, *args, user=None, **kwargs): + _args = ['buildah', 'run', self.ctr] + if user: + _args += ['--user', user] + _args += ['--', 'sh', '-euc'] + return await self.exec(*(_args + list(args))) + async def mount(self, src, dst): - await self.run('sudo mkdir -p ' + dst) - await self.append('mkdir -p ' + src) - await self.append(f'mount -o bind {src} {self.mnt}{dst}') - #await self.append('mounts=("$mnt' + dst + '" "${mounts[@]}")') - self.mounts[dst] = src + target = self.mnt / str(dst)[1:] + await self.exec(f'mkdir -p {src} {target}') + await self.exec(f'mount -o bind {src} {target}') + self.mounts[src] = dst async def umounts(self): - for src, dst in self.mounts: - await self.append('buildah unmount ' + dst) - await self.append('buildah unmount ' + self.ctr) + for src, dst in self.mounts.items(): + await self.exec('umount', self.mnt / str(dst)[1:]) + + async def umount(self): + await self.exec(f'buildah unmount {self.ctr}') def which(self, cmd): for path in self.container.paths: - if os.path.exists(os.path.join(path, cmd)): + if os.path.exists(os.path.join(self.mnt, path[1:], cmd)): return True diff --git a/podctl/console_script.py b/podctl/console_script.py index 3aa4793..72ebadf 100644 --- a/podctl/console_script.py +++ b/podctl/console_script.py @@ -10,29 +10,20 @@ import sys from .container import Container from .pod import Pod +from .proc import WrongResult from .service import Service -class BuildStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): - def __init__(self, service, *args, **kwargs): - self.service = service - super().__init__(*args, **kwargs) - - def pipe_data_received(self, fd, data): - if fd in (1, 2): - for line in data.split(b'\n'): - if not line: - continue - sys.stdout.buffer.write( - self.service.name.encode('utf8') + b' | ' + line + b'\n' - ) - sys.stdout.flush() - super().pipe_data_received(fd, data) - - @cli2.option('debug', help='Print debug output', color=cli2.GREEN, alias='d') -async def build(*services, **kwargs): - procs = [] +async def build(*services_or_flags, **kwargs): + flags = [] + services = [] + for arg in services_or_flags: + if arg.startswith('-') or arg.startswith('+'): + flags.append(arg) + else: + services.append(arg) + if services: services = { k: v @@ -42,14 +33,17 @@ async def build(*services, **kwargs): else: services = console_script.pod.services - a = [] - loop = asyncio.events.get_event_loop() + procs = [] + asyncio.events.get_event_loop() for name, service in services.items(): service.container.name = name - a.append(service.container.script('build', loop)) + service.container.flags = flags + procs.append(service.container.script('build', flags)) - result = await asyncio.gather(*a, return_exceptions=False) - print(result) + try: + result = await asyncio.gather(*procs) + except WrongResult: + sys.exit(1) class ConsoleScript(cli2.ConsoleScript): diff --git a/podctl/container.py b/podctl/container.py index 59420ce..f9ba5c0 100644 --- a/podctl/container.py +++ b/podctl/container.py @@ -1,36 +1,11 @@ import asyncio import os +import shlex from .build import Build from .visitable import Visitable -class BuildStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): - def __init__(self, service, *args, **kwargs): - self.service = service - super().__init__(*args, **kwargs) - - def pipe_data_received(self, fd, data): - if fd in (1, 2): - for line in data.split(b'\n'): - if not line: - continue - sys.stdout.buffer.write( - self.service.name.encode('utf8') + b' | ' + line + b'\n' - ) - sys.stdout.flush() - super().pipe_data_received(fd, data) - - -def protocol_factory(): - loop = asyncio.events.get_event_loop() - return BuildStreamProtocol( - service, - limit=asyncio.streams._DEFAULT_LIMIT, - loop=loop, - ) - - class Container(Visitable): default_scripts = dict( build=Build, @@ -42,22 +17,14 @@ class Container(Visitable): '/usr/sbin', ] - async def script(self, name, loop): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.log = [] + + async def script(self, name, flags, loop=None): + self.loop = loop or asyncio.events.get_event_loop() self.packages = [] for visitor in self.visitors: self.packages += getattr(visitor, 'packages', []) result = await super().script(name, loop) return result - - def script_run(self, name, debug): - script = f'.podctl_build_{name}.sh' - with open(script, 'w+') as f: - f.write(str(self.script('build'))) - - if os.getenv('BUILDAH_ISOLATION') == 'chroot': - prefix = '' - else: - prefix = 'buildah unshare ' - - x = 'x' if debug else '' - return prefix + f'bash -eu{x} {script}' diff --git a/podctl/script.py b/podctl/script.py index 297e63e..11cd3e1 100644 --- a/podctl/script.py +++ b/podctl/script.py @@ -1,13 +1,5 @@ import textwrap -class Script(list): - def __str__(self): - if not getattr(self, '_postconfig', False): - if hasattr(self, 'post_config'): - self.post_config() - self._postconfig = True - return '\n'.join([ - textwrap.dedent(line.lstrip('\n')).strip() - for line in self - ]) +class Script: + pass diff --git a/podctl/visitable.py b/podctl/visitable.py index a62f77b..1b0533e 100644 --- a/podctl/visitable.py +++ b/podctl/visitable.py @@ -15,7 +15,14 @@ class Visitable: script.loop = loop results = [] - for prefix in ('init_', 'pre_', '', 'post_'): + async def clean(): + for visitor in self.visitors: + if hasattr(visitor, 'clean_' + name): + result = getattr(visitor, 'clean_' + name)(self) + if result: + await result + + for prefix in ('init_', 'pre_', '', 'post_', 'clean_'): method = prefix + name for visitor in self.visitors: if not hasattr(visitor, method): @@ -25,8 +32,11 @@ class Visitable: print(self.name + ' | ', type(visitor).__name__, method, rep) result = getattr(visitor, method)(script) if result: - await result - + try: + await result + except Exception as e: + await clean() + raise def visitor(self, name): for visitor in self.visitors: diff --git a/podctl/visitors/base.py b/podctl/visitors/base.py index 1601328..aa97f20 100644 --- a/podctl/visitors/base.py +++ b/podctl/visitors/base.py @@ -1,3 +1,4 @@ +from pathlib import Path class Base: @@ -5,9 +6,8 @@ class Base: self.base = base async def init_build(self, script): - ctr = await script.cmd('buildah from ' + self.base) - stdout, stderr = await ctr.communicate() - script.ctr = stdout.decode('utf8').strip() - mnt = await script.cmd('buildah mount ' + script.ctr) - stdout, stderr = await mnt.communicate() - script.mnt = stdout.decode('utf8').strip() + script.ctr = Path((await script.exec('buildah', 'from', self.base)).out) + script.mnt = Path((await script.exec('buildah', 'mount', script.ctr)).out) + + async def post_build(self, script): + await script.umounts() diff --git a/podctl/visitors/commit.py b/podctl/visitors/commit.py index 3aa9f27..a35bd45 100644 --- a/podctl/visitors/commit.py +++ b/podctl/visitors/commit.py @@ -43,27 +43,34 @@ class Commit: self.tags = [t for t in self.tags if t is not None] async def post_build(self, script): - await script.append(f''' - umounts - buildah commit --format={self.format} $ctr {self.repo} - ''') + await script.exec( + 'buildah', + 'commit', + '--format=' + self.format, + script.ctr, + ) if 'master' in self.tags: self.tags.append('latest') if self.tags: tags = ' '.join([f'{self.repo}:{tag}' for tag in self.tags]) - await script.append(f'buildah tag {self.repo} {tags}') + await script.run('buildah', 'tag', self.repo, ' '.join(tags)) if self.push: user = os.getenv('DOCKER_USER') passwd = os.getenv('DOCKER_PASS') if user and passwd and os.getenv('CI') and self.registry: - subprocess.check_call([ - 'podman', 'login', - '-u', user, '-p', passwd, - self.registry - ]) + await script.exec( + 'podman', + 'login', + '-u', + user, + '-p', + passwd, + self.registry, + ) for tag in self.tags: - await script.append(f'podman push {self.repo}:{tag}') + await script.run('podman', 'push', f'{self.repo}:{tag}') + await script.umount() diff --git a/podctl/visitors/packages.py b/podctl/visitors/packages.py index 522a5b9..242fa92 100644 --- a/podctl/visitors/packages.py +++ b/podctl/visitors/packages.py @@ -1,3 +1,7 @@ +import asyncio + +from datetime import datetime +from glob import glob import os import subprocess from textwrap import dedent @@ -6,24 +10,24 @@ from textwrap import dedent class Packages: mgrs = dict( apk=dict( - update='sudo apk update', - upgrade='sudo apk upgrade', - install='sudo apk add', + update='apk update', + upgrade='apk upgrade', + install='apk add', ), apt=dict( - update='sudo apt-get -y update', - upgrade='sudo apt-get -y upgrade', - install='sudo apt-get -y --no-install-recommends install', + update='apt-get -y update', + upgrade='apt-get -y upgrade', + install='apt-get -y --no-install-recommends install', ), dnf=dict( - update='sudo dnf update', - upgrade='sudo dnf upgrade --exclude container-selinux --best --assumeyes', # noqa - install='sudo dnf install --exclude container-selinux --setopt=install_weak_deps=False --best --assumeyes', # noqa + update='dnf update', + upgrade='dnf upgrade --exclude container-selinux --best --assumeyes', # noqa + install='dnf install --exclude container-selinux --setopt=install_weak_deps=False --best --assumeyes', # noqa ), yum=dict( - update='sudo yum update', - upgrade='sudo yum upgrade', - install='sudo yum install', + update='yum update', + upgrade='yum upgrade', + install='yum install', ), ) @@ -32,25 +36,22 @@ class Packages: dedent(l).strip().replace('\n', ' ') for l in packages ]) self.mgr = kwargs.pop('mgr') if 'mgr' in kwargs else None - if 'CACHE_DIR' in os.environ: - self.cache = os.path.join(os.getenv('CACHE_DIR'), self.mgr) - else: - self.cache = os.path.join(os.getenv('HOME'), '.cache', self.mgr) - async def pre_build(self, script): - base = script.container.variable('base') - if self.mgr: - self.cmds = self.mgrs[self.mgr] + @property + def cache(self): + if 'CACHE_DIR' in os.environ: + return os.path.join(os.getenv('CACHE_DIR'), self.mgr) else: - for mgr, cmds in self.mgrs.items(): - cmd = ['podman', 'run', base, 'sh', '-c', f'type {mgr}'] - try: - subprocess.check_call(cmd) + return os.path.join(os.getenv('HOME'), '.cache', self.mgr) + + async def init_build(self, script): + paths = ('bin', 'sbin', 'usr/bin', 'usr/sbin') + for mgr, cmds in self.mgrs.items(): + for path in paths: + if (script.mnt / path / mgr).exists(): self.mgr = mgr self.cmds = cmds break - except subprocess.CalledProcessError: - continue if not self.mgr: raise Exception('Packages does not yet support this distro') @@ -60,35 +61,61 @@ class Packages: await getattr(self, self.mgr + '_setup')(script) # first run on container means inject visitor packages self.packages += script.container.packages - await script.run(self.cmds['upgrade']) + await script.cexec(self.cmds['upgrade']) script.container._packages_upgraded = True - await script.run(' '.join([self.cmds['install']] + self.packages)) + await script.cexec(' '.join([self.cmds['install']] + self.packages)) async def apk_setup(self, script): await script.mount(self.cache, f'/var/cache/{self.mgr}') # special step to enable apk cache - await script.run('ln -s /var/cache/apk /etc/apk/cache') - await script.append(f''' - old="$(find {self.cache} -name APKINDEX.* -mtime +3)" - if [ -n "$old" ] || ! ls .cache/apk/APKINDEX.*; then - {script._run(self.cmds['update'])} - else - echo Cache recent enough, skipping index update. - fi - ''') + await script.cexec('ln -s /var/cache/apk /etc/apk/cache') + + # do we have to update ? + update = False + for f in glob(self.cache + '/APKINDEX*'): + mtime = os.stat(f).st_mtime + now = int(datetime.now().strftime('%s')) + # expect hacker to have internet at least once a week + if now - mtime > 604800: + update = True + break + else: + update = True + + if update: + await self.apk_update(script) + + async def apk_update(self, script): + while os.path.exists(self.cache + '/update'): + print(f'{script.container.name} | Waiting for update ...') + await asyncio.sleep(1) + return # update was done by another job + + with open(self.cache + '/update', 'w+') as f: + f.write(str(os.getpid())) + + try: + await script.cexec(self.cmds['update']) + except: + raise + finally: + os.unlink(self.cache + '/update') async def dnf_setup(self, script): await script.mount(self.cache, f'/var/cache/{self.mgr}') - await script.run('sh -c "echo keepcache=True >> /etc/dnf/dnf.conf"') + await script.run('echo keepcache=True >> /etc/dnf/dnf.conf') async def apt_setup(self, script): cache = self.cache + '/$(source $mnt/etc/os-release; echo $VERSION_CODENAME)/' # noqa - await script.run('sudo rm /etc/apt/apt.conf.d/docker-clean') + await script.run('rm /etc/apt/apt.conf.d/docker-clean') cache_archives = os.path.join(self.cache, 'archives') await script.mount(cache_archives, f'/var/cache/apt/archives') cache_lists = os.path.join(self.cache, 'lists') await script.mount(cache_lists, f'/var/lib/apt/lists') + + await script.run(self.cmds['update']) + """ await script.append(f''' old="$(find {cache_lists} -name lastup -mtime +3)" if [ -n "$old" ] || ! ls {cache_lists}/lastup; then @@ -99,3 +126,4 @@ class Packages: echo Cache recent enough, skipping index update. fi ''') + """ diff --git a/podctl/visitors/user.py b/podctl/visitors/user.py index a5b4993..e0be081 100644 --- a/podctl/visitors/user.py +++ b/podctl/visitors/user.py @@ -14,7 +14,7 @@ class User: self.user_created = False async def build(self, script): - await script.append(f''' + await script.run(f''' if {script._run('id ' + str(self.uid))}; then i=$({script._run('id -gn ' + str(self.uid))}) {script._run('usermod -d ' + self.home + ' -l ' + self.username + ' $i')} diff --git a/tests/test_build.py b/tests/test_build.py index f6d613d..6b34b45 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -3,7 +3,7 @@ import os import sys from podctl.container import Container -from podctl.build import BuildScript +from podctl.build import Build from podctl.visitors import ( Base, Copy,