Migrate the whole thing from bash script generation to async processes

examples/django.py

@@ -1,65 +0,0 @@
-import os
-
-from podctl import *
-
-
-class Django(Container):
-    tag = 'yourlabs/crudlfap'
-    base = 'alpine'
-    packages = [
-        'bash',
-        'python3',
-        switch(dev='vim'),
-    ]
-    env = dict(FOO='bar')
-    annotations = dict(test='foo')
-    labels = dict(foo='test')
-    cmd = 'bash'
-    entrypoint = ['bash', '-v']
-    ports = [1234]
-    user = dict(
-        shell='/bin/bash',
-        name='app',
-        home='/app',
-        id=os.getenv('SUDO_ID', os.getenv('UID')),
-    )
-    volumes = [
-        '/bydir',
-        'byname:/byname',
-        switch(dev='.:/app'),
-    ]
-    build = [
-        'sudo pip3 install --upgrade pip',
-        'pip3 install --user -e /app',
-    ]
-    workdir = '/app'
-
-django = Container(
-    Base('alpine'),
-    Packages('bash', switch(dev='vim')),
-    User(
-        uid=1000,
-        home='/app',
-        directories=('log', 'spooler', 'static')
-    ),
-    switch(default=Mount('.', '/app'), production=Copy('.', '/app')),
-    Npm('build'),
-    Env('PATH', 'PATH=/app/node_modules/.bin:$PATH'),
-    Pip('requirements.txt'),
-    Env('PATH', 'PATH=$HOME/.local/bin:$PATH'),
-    switch(dev=Run('''
-        manage.py collectstatic --noinput --clear
-        find frontend/static/dist/css -type f | xargs gzip -f -k -9
-        find frontend/static/dist/js -type f | xargs gzip -f -k -9
-    ''')),
-    Expose(8000),
-    Tag('equisafe'),
-)
-
-pod = Pod(
-    Service('django', django, restart='unless-stopped'),
-    Service('db',
-        Container(Tag('postgresql:latest')),
-        restart='unless-stopped'
-    ),
-)

podctl/build.py

@@ -2,16 +2,14 @@ import asyncio
 import os
 import asyncio
 import signal
+import shlex
 import subprocess
 import textwrap
 
+from .proc import Proc
 from .script import Script
 
 
-class WrongResult(Exception):
-    pass
-
-
 class Build(Script):
     def __init__(self, container):
         super().__init__()
@@ -19,95 +17,40 @@ class Build(Script):
         self.log = []
         self.mounts = dict()
 
-    async def cmd(self, line):
-        log = dict(cmd=line)
-        self.log.append(log)
-        line = self.unshare(line)
-        print(self.container.name + ' | ' + line)
-        def protocol_factory():
-            from .console_script import BuildStreamProtocol
-            return BuildStreamProtocol(
-                self.container,
-                limit=asyncio.streams._DEFAULT_LIMIT,
-                loop=self.loop,
-            )
-        transport, protocol = await self.loop.subprocess_shell(
-            protocol_factory,
-            line,
-        )
-        proc = asyncio.subprocess.Process(
-            transport,
-            protocol,
-            self.loop,
-        )
-        result = await proc.wait()
-        if result:
-            raise WrongResult(proc)
-        return proc
-        '''
-        proc = await asyncio.create_subprocess_shell(
-            line,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
-        '''
-
-    async def append(self, value):
-        for line in value.split('\n'):
-            if line.startswith('#') or not line.strip():
-                continue
-            log = dict(proc=await self.cmd(line))
-            log['stdout'], log['stderr'] = await log['proc'].communicate()
-
-        return log['stdout']
-
-    def unshare(self, line):
-        return 'buildah unshare ' + line
-
     async def config(self, line):
         return await self.append(f'buildah config {line} {self.ctr}')
 
-    def _run(self, cmd, inject=False):
-        user = self.container.variable('username')
-        _cmd = textwrap.dedent(cmd)
-        if cmd.startswith('sudo '):
-            _cmd = _cmd[5:]
-
-        heredoc = False
-        for i in ('\n', '>', '<', '|', '&'):
-            if i in _cmd:
-                heredoc = True
-                break
-
-        if heredoc:
-            _cmd = ''.join(['bash -eux <<__EOF\n', _cmd.strip(), '\n__EOF'])
-
-        if cmd.startswith('sudo '):
-            return f'buildah run --user root {self.ctr} -- {_cmd}'
-        elif user and self.container.variable('user_created'):
-            return f'buildah run --user {user} {self.ctr} -- {_cmd}'
-        else:
-            return f'buildah run {self.ctr} -- {_cmd}'
-
-    async def run(self, cmd):
-        return await self.append(self._run(cmd))
-
     async def copy(self, src, dst):
         return await self.append(f'buildah copy {self.ctr} {src} {dst}')
 
+    async def exec(self, *args, **kwargs):
+        kwargs.setdefault('prefix', self.container.name)
+        proc = await Proc(*args, **kwargs)()
+        if kwargs.get('wait', True):
+            await proc.wait()
+        return proc
+
+    async def cexec(self, *args, user=None, **kwargs):
+        _args = ['buildah', 'run', self.ctr]
+        if user:
+            _args += ['--user', user]
+        _args += ['--', 'sh', '-euc']
+        return await self.exec(*(_args + list(args)))
+
     async def mount(self, src, dst):
-        await self.run('sudo mkdir -p ' + dst)
+        target = self.mnt / str(dst)[1:]
-        await self.append('mkdir -p ' + src)
+        await self.exec(f'mkdir -p {src} {target}')
-        await self.append(f'mount -o bind {src} {self.mnt}{dst}')
+        await self.exec(f'mount -o bind {src} {target}')
-        #await self.append('mounts=("$mnt' + dst + '" "${mounts[@]}")')
+        self.mounts[src] = dst
-        self.mounts[dst] = src
 
     async def umounts(self):
-        for src, dst in self.mounts:
+        for src, dst in self.mounts.items():
-            await self.append('buildah unmount ' + dst)
+            await self.exec('umount', self.mnt / str(dst)[1:])
-        await self.append('buildah unmount ' + self.ctr)
+
+    async def umount(self):
+        await self.exec(f'buildah unmount {self.ctr}')
 
     def which(self, cmd):
         for path in self.container.paths:
-            if os.path.exists(os.path.join(path, cmd)):
+            if os.path.exists(os.path.join(self.mnt, path[1:], cmd)):
                 return True

podctl/console_script.py

@@ -10,29 +10,20 @@ import sys
 
 from .container import Container
 from .pod import Pod
+from .proc import WrongResult
 from .service import Service
 
 
-class BuildStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol):
-    def __init__(self, service, *args, **kwargs):
-        self.service = service
-        super().__init__(*args, **kwargs)
-
-    def pipe_data_received(self, fd, data):
-        if fd in (1, 2):
-            for line in data.split(b'\n'):
-                if not line:
-                    continue
-                sys.stdout.buffer.write(
-                    self.service.name.encode('utf8') + b' | ' + line + b'\n'
-                )
-            sys.stdout.flush()
-        super().pipe_data_received(fd, data)
-
-
 @cli2.option('debug', help='Print debug output', color=cli2.GREEN, alias='d')
-async def build(*services, **kwargs):
+async def build(*services_or_flags, **kwargs):
-    procs = []
+    flags = []
+    services = []
+    for arg in services_or_flags:
+        if arg.startswith('-') or arg.startswith('+'):
+            flags.append(arg)
+        else:
+            services.append(arg)
+
     if services:
         services = {
             k: v
@@ -42,14 +33,17 @@ async def build(*services, **kwargs):
     else:
         services = console_script.pod.services
 
-    a = []
+    procs = []
-    loop = asyncio.events.get_event_loop()
+    asyncio.events.get_event_loop()
     for name, service in services.items():
         service.container.name = name
-        a.append(service.container.script('build', loop))
+        service.container.flags = flags
+        procs.append(service.container.script('build', flags))
 
-    result = await asyncio.gather(*a, return_exceptions=False)
+    try:
-    print(result)
+        result = await asyncio.gather(*procs)
+    except WrongResult:
+        sys.exit(1)
 
 
 class ConsoleScript(cli2.ConsoleScript):

podctl/container.py

@@ -1,36 +1,11 @@
 import asyncio
 import os
+import shlex
 
 from .build import Build
 from .visitable import Visitable
 
 
-class BuildStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol):
-    def __init__(self, service, *args, **kwargs):
-        self.service = service
-        super().__init__(*args, **kwargs)
-
-    def pipe_data_received(self, fd, data):
-        if fd in (1, 2):
-            for line in data.split(b'\n'):
-                if not line:
-                    continue
-                sys.stdout.buffer.write(
-                    self.service.name.encode('utf8') + b' | ' + line + b'\n'
-                )
-            sys.stdout.flush()
-        super().pipe_data_received(fd, data)
-
-
-def protocol_factory():
-    loop = asyncio.events.get_event_loop()
-    return BuildStreamProtocol(
-        service,
-        limit=asyncio.streams._DEFAULT_LIMIT,
-        loop=loop,
-    )
-
-
 class Container(Visitable):
     default_scripts = dict(
         build=Build,
@@ -42,22 +17,14 @@ class Container(Visitable):
         '/usr/sbin',
     ]
 
-    async def script(self, name, loop):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.log = []
+
+    async def script(self, name, flags, loop=None):
+        self.loop = loop or asyncio.events.get_event_loop()
         self.packages = []
         for visitor in self.visitors:
             self.packages += getattr(visitor, 'packages', [])
         result = await super().script(name, loop)
         return result
-
-    def script_run(self, name, debug):
-        script = f'.podctl_build_{name}.sh'
-        with open(script, 'w+') as f:
-            f.write(str(self.script('build')))
-
-        if os.getenv('BUILDAH_ISOLATION') == 'chroot':
-            prefix = ''
-        else:
-            prefix = 'buildah unshare '
-
-        x = 'x' if debug else ''
-        return prefix + f'bash -eu{x} {script}'

podctl/script.py

@@ -1,13 +1,5 @@
 import textwrap
 
 
-class Script(list):
+class Script:
-    def __str__(self):
+    pass
-        if not getattr(self, '_postconfig', False):
-            if hasattr(self, 'post_config'):
-                self.post_config()
-            self._postconfig = True
-        return '\n'.join([
-            textwrap.dedent(line.lstrip('\n')).strip()
-            for line in self
-        ])

podctl/visitable.py

@@ -15,7 +15,14 @@ class Visitable:
         script.loop = loop
         results = []
 
-        for prefix in ('init_', 'pre_', '', 'post_'):
+        async def clean():
+            for visitor in self.visitors:
+                if hasattr(visitor, 'clean_' + name):
+                    result = getattr(visitor, 'clean_' + name)(self)
+                    if result:
+                        await result
+
+        for prefix in ('init_', 'pre_', '', 'post_', 'clean_'):
             method = prefix + name
             for visitor in self.visitors:
                 if not hasattr(visitor, method):
@@ -25,8 +32,11 @@ class Visitable:
                 print(self.name + ' | ', type(visitor).__name__, method, rep)
                 result = getattr(visitor, method)(script)
                 if result:
-                    await result
+                    try:
-
+                        await result
+                    except Exception as e:
+                        await clean()
+                        raise
 
     def visitor(self, name):
         for visitor in self.visitors:

podctl/visitors/base.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 
 
 class Base:
@@ -5,9 +6,8 @@ class Base:
         self.base = base
 
     async def init_build(self, script):
-        ctr = await script.cmd('buildah from ' + self.base)
+        script.ctr = Path((await script.exec('buildah', 'from', self.base)).out)
-        stdout, stderr = await ctr.communicate()
+        script.mnt = Path((await script.exec('buildah', 'mount', script.ctr)).out)
-        script.ctr = stdout.decode('utf8').strip()
+
-        mnt = await script.cmd('buildah mount ' + script.ctr)
+    async def post_build(self, script):
-        stdout, stderr = await mnt.communicate()
+        await script.umounts()
-        script.mnt = stdout.decode('utf8').strip()

podctl/visitors/commit.py

@@ -43,27 +43,34 @@ class Commit:
         self.tags = [t for t in self.tags if t is not None]
 
     async def post_build(self, script):
-        await script.append(f'''
+        await script.exec(
-            umounts
+            'buildah',
-            buildah commit --format={self.format} $ctr {self.repo}
+            'commit',
-        ''')
+            '--format=' + self.format,
+            script.ctr,
+        )
 
         if 'master' in self.tags:
             self.tags.append('latest')
 
         if self.tags:
             tags = ' '.join([f'{self.repo}:{tag}' for tag in self.tags])
-            await script.append(f'buildah tag {self.repo} {tags}')
+            await script.run('buildah', 'tag', self.repo, ' '.join(tags))
 
             if self.push:
                 user = os.getenv('DOCKER_USER')
                 passwd = os.getenv('DOCKER_PASS')
                 if user and passwd and os.getenv('CI') and self.registry:
-                    subprocess.check_call([
+                    await script.exec(
-                        'podman', 'login',
+                        'podman',
-                        '-u', user, '-p', passwd,
+                        'login',
-                        self.registry
+                        '-u',
-                    ])
+                        user,
+                        '-p',
+                        passwd,
+                        self.registry,
+                    )
 
                 for tag in self.tags:
-                    await script.append(f'podman push {self.repo}:{tag}')
+                    await script.run('podman', 'push', f'{self.repo}:{tag}')
+        await script.umount()

podctl/visitors/packages.py

@@ -1,3 +1,7 @@
+import asyncio
+
+from datetime import datetime
+from glob import glob
 import os
 import subprocess
 from textwrap import dedent
@@ -6,24 +10,24 @@ from textwrap import dedent
 class Packages:
     mgrs = dict(
         apk=dict(
-            update='sudo apk update',
+            update='apk update',
-            upgrade='sudo apk upgrade',
+            upgrade='apk upgrade',
-            install='sudo apk add',
+            install='apk add',
         ),
         apt=dict(
-            update='sudo apt-get -y update',
+            update='apt-get -y update',
-            upgrade='sudo apt-get -y upgrade',
+            upgrade='apt-get -y upgrade',
-            install='sudo apt-get -y --no-install-recommends install',
+            install='apt-get -y --no-install-recommends install',
         ),
         dnf=dict(
-            update='sudo dnf update',
+            update='dnf update',
-            upgrade='sudo dnf upgrade --exclude container-selinux --best --assumeyes',  # noqa
+            upgrade='dnf upgrade --exclude container-selinux --best --assumeyes',  # noqa
-            install='sudo dnf install --exclude container-selinux --setopt=install_weak_deps=False --best --assumeyes',  # noqa
+            install='dnf install --exclude container-selinux --setopt=install_weak_deps=False --best --assumeyes',  # noqa
         ),
         yum=dict(
-            update='sudo yum update',
+            update='yum update',
-            upgrade='sudo yum upgrade',
+            upgrade='yum upgrade',
-            install='sudo yum install',
+            install='yum install',
         ),
     )
 
@@ -32,25 +36,22 @@ class Packages:
             dedent(l).strip().replace('\n', ' ') for l in packages
         ])
         self.mgr = kwargs.pop('mgr') if 'mgr' in kwargs else None
-        if 'CACHE_DIR' in os.environ:
-            self.cache = os.path.join(os.getenv('CACHE_DIR'), self.mgr)
-        else:
-            self.cache = os.path.join(os.getenv('HOME'), '.cache', self.mgr)
 
-    async def pre_build(self, script):
+    @property
-        base = script.container.variable('base')
+    def cache(self):
-        if self.mgr:
+        if 'CACHE_DIR' in os.environ:
-            self.cmds = self.mgrs[self.mgr]
+            return os.path.join(os.getenv('CACHE_DIR'), self.mgr)
         else:
-            for mgr, cmds in self.mgrs.items():
+            return os.path.join(os.getenv('HOME'), '.cache', self.mgr)
-                cmd = ['podman', 'run', base, 'sh', '-c', f'type {mgr}']
+
-                try:
+    async def init_build(self, script):
-                    subprocess.check_call(cmd)
+        paths = ('bin', 'sbin', 'usr/bin', 'usr/sbin')
+        for mgr, cmds in self.mgrs.items():
+            for path in paths:
+                if (script.mnt / path / mgr).exists():
                     self.mgr = mgr
                     self.cmds = cmds
                     break
-                except subprocess.CalledProcessError:
-                    continue
         if not self.mgr:
             raise Exception('Packages does not yet support this distro')
 
@@ -60,35 +61,61 @@ class Packages:
             await getattr(self, self.mgr + '_setup')(script)
             # first run on container means inject visitor packages
             self.packages += script.container.packages
-            await script.run(self.cmds['upgrade'])
+            await script.cexec(self.cmds['upgrade'])
             script.container._packages_upgraded = True
 
-        await script.run(' '.join([self.cmds['install']] + self.packages))
+        await script.cexec(' '.join([self.cmds['install']] + self.packages))
 
     async def apk_setup(self, script):
         await script.mount(self.cache, f'/var/cache/{self.mgr}')
         # special step to enable apk cache
-        await script.run('ln -s /var/cache/apk /etc/apk/cache')
+        await script.cexec('ln -s /var/cache/apk /etc/apk/cache')
-        await script.append(f'''
+
-        old="$(find {self.cache} -name APKINDEX.* -mtime +3)"
+        # do we have to update ?
-        if [ -n "$old" ] || ! ls .cache/apk/APKINDEX.*; then
+        update = False
-            {script._run(self.cmds['update'])}
+        for f in glob(self.cache + '/APKINDEX*'):
-        else
+            mtime = os.stat(f).st_mtime
-            echo Cache recent enough, skipping index update.
+            now = int(datetime.now().strftime('%s'))
-        fi
+            # expect hacker to have internet at least once a week
-        ''')
+            if now - mtime > 604800:
+                update = True
+                break
+        else:
+            update = True
+
+        if update:
+            await self.apk_update(script)
+
+    async def apk_update(self, script):
+        while os.path.exists(self.cache + '/update'):
+            print(f'{script.container.name} | Waiting for update ...')
+            await asyncio.sleep(1)
+            return  # update was done by another job
+
+        with open(self.cache + '/update', 'w+') as f:
+            f.write(str(os.getpid()))
+
+        try:
+            await script.cexec(self.cmds['update'])
+        except:
+            raise
+        finally:
+            os.unlink(self.cache + '/update')
 
     async def dnf_setup(self, script):
         await script.mount(self.cache, f'/var/cache/{self.mgr}')
-        await script.run('sh -c "echo keepcache=True >> /etc/dnf/dnf.conf"')
+        await script.run('echo keepcache=True >> /etc/dnf/dnf.conf')
 
     async def apt_setup(self, script):
         cache = self.cache + '/$(source $mnt/etc/os-release; echo $VERSION_CODENAME)/'  # noqa
-        await script.run('sudo rm /etc/apt/apt.conf.d/docker-clean')
+        await script.run('rm /etc/apt/apt.conf.d/docker-clean')
         cache_archives = os.path.join(self.cache, 'archives')
         await script.mount(cache_archives, f'/var/cache/apt/archives')
         cache_lists = os.path.join(self.cache, 'lists')
         await script.mount(cache_lists, f'/var/lib/apt/lists')
+
+        await script.run(self.cmds['update'])
+        """
         await script.append(f'''
         old="$(find {cache_lists} -name lastup -mtime +3)"
         if [ -n "$old" ] || ! ls {cache_lists}/lastup; then
@@ -99,3 +126,4 @@ class Packages:
             echo Cache recent enough, skipping index update.
         fi
         ''')
+        """

podctl/visitors/user.py

@@ -14,7 +14,7 @@ class User:
         self.user_created = False
 
     async def build(self, script):
-        await script.append(f'''
+        await script.run(f'''
             if {script._run('id ' + str(self.uid))}; then
                 i=$({script._run('id -gn ' + str(self.uid))})
                 {script._run('usermod -d ' + self.home + ' -l ' + self.username + ' $i')}

tests/test_build.py

@@ -3,7 +3,7 @@ import os
 import sys
 
 from podctl.container import Container
-from podctl.build import BuildScript
+from podctl.build import Build
 from podctl.visitors import (
     Base,
     Copy,