Build script suppotrs packages

podctl/__init__.py

@@ -0,0 +1,2 @@
+from .container import Container, switch
+from .pod import Pod

podctl/build.py

@@ -0,0 +1,245 @@
+from glob import glob
+import inspect
+import os
+import subprocess
+import time
+
+from .script import Script
+
+
+class BuildScript(Script):
+    export = ('base', 'repo', 'tag', 'image')
+
+    def __init__(self, container):
+        super().__init__()
+        self.container = container
+
+        for var in self.export:
+            if var in self.container:
+                self.append(f'{var}="{self.container[var]}"')
+
+        self.append('''
+            mounts=()
+            umounts() {
+                for i in "${mounts[@]}"; do
+                    umount $i
+                done
+            }
+            trap umounts 0
+            ctr=$(buildah from $base)
+            mnt=$(buildah mount $ctr)
+            mounts=("$mnt" "${mounts[@]}")
+        ''')
+
+        if self.container.get('packages'):
+            self.container.packages_install(self)
+
+    def config(self, line):
+        self.append(f'buildah config {line} $ctr')
+
+    def run(self, cmd):
+        self.append('buildah run $ctr -- ' + cmd)
+
+    def copy(self, src, dst):
+        self.append(f'buildah copy $ctr {src} {dst}')
+
+    def mount(self, src, dst):
+        self.run('mkdir -p ' + dst)
+        self.append('mkdir -p ' + src)
+        self.append(f'mount -o bind {src} $mnt{dst}')
+        # for unmount trap
+        self.append('mounts=("$mnt%s" "${mounts[@]}")' % dst)
+
+
+class Plugins(dict):
+    def __init__(self, *plugins):
+        default_plugins = [
+            PkgPlugin(),
+            UserPlugin(),
+            FsPlugin(),
+            BuildPlugin(),
+            ConfigPlugin(),
+        ]
+
+        super().__init__()
+        for plugin in plugins or default_plugins:
+            self.add(plugin)
+
+    def add(self, plugin):
+        plugin.plugins = self
+        self[plugin.name] = plugin
+
+    def __call__(self, method, *args, **kwargs):
+        hook = f'pre_{method}'
+        for plugin in self.values():
+            plugin(hook, *args, **kwargs)
+
+        for plugin in self.values():
+            if hasattr(plugin, method):
+                plugin(method, *args, **kwargs)
+
+        hook = f'post_{method}'
+        for plugin in self.values():
+            plugin(hook, *args, **kwargs)
+
+
+class Plugin:
+    @property
+    def name(self):
+        return type(self).__name__.replace('Plugin', '').lower()
+
+    def bubble(self, hook, *args, **kwargs):
+        for plugin in self.plugins.values():
+            if plugin is self:
+                continue
+            plugin(hook, _bubble=False, *args, **kwargs)
+
+    def __call__(self, method, *args, **kwargs):
+        _bubble = kwargs.pop('_bubble', True)
+        if _bubble:
+            self.bubble(f'pre_{self.name}_{method}', *args, **kwargs)
+
+        if hasattr(self, method):
+            meth = getattr(self, method)
+            argspec = inspect.getargspec(meth)
+            if argspec.varargs and argspec.keywords:
+                meth(*args, **kwargs)
+            else:
+                numargs = len(argspec.args) - 1 - len(argspec.defaults or [])
+                args = args[:numargs]
+                kwargs = {
+                    k: v
+                    for k, v in kwargs.items()
+                    if k in argspec.args
+                }
+                meth(*args, **kwargs)
+
+        if _bubble:
+            self.bubble(f'post_{self.name}_{method}', *args, **kwargs)
+
+
+class BuildPlugin(Plugin):
+    def build(self, script):
+        user = script.service.get('user', None)
+
+        for cmd in script.service['build']:
+            if cmd.startswith('sudo'):
+                if user:
+                    script.config(f'--user root')
+                script.run(cmd[5:])
+            else:
+                script.config(f'--user {script.service["user"]}')
+                script.run(cmd)
+
+
+class FsPlugin(Plugin):
+    def build(self, script):
+        for key, value in script.service.items():
+            if not key.startswith('/'):
+                continue
+
+            parts = key.split(':')
+            dst = parts.pop(0)
+            mode = parts.pop(0) if parts else '0500'
+            script.run(f'mkdir -p {dst}')
+            script.run(f'chmod {mode} $mnt{dst}')
+
+            if value and isinstance(value, list):
+                for item in value:
+                    if isinstance(item, str):
+                        script.run(f'cp -a {item} $mnt{dst}')
+
+                    if not isinstance(item, dict):
+                        pass
+
+
+class PkgPlugin(Plugin):
+    mgrs = dict(
+        apk=dict(
+            update='apk update',
+            upgrade='apk upgrade',
+            install='apk add',
+        ),
+    )
+
+    def pre_build(self, script):
+        for mgr, cmds in self.mgrs.items():
+            cmd = [
+                'podman',
+                'run',
+                script.service['base'],
+                'which',
+                mgr
+            ]
+            print('+ ' + ' '.join(cmd))
+            try:
+                subprocess.check_call(cmd)
+                script.service.mgr = mgr
+                script.service.cmds = cmds
+                break
+            except subprocess.CalledProcessError:
+                continue
+
+    def build(self, script):
+        cache = f'.cache/{script.service.mgr}'
+        script.mount(
+            '$(pwd)/' + cache,
+            f'/var/cache/{script.service.mgr}'
+        )
+
+        cached = False
+        if script.service.mgr == 'apk':
+            # special step to enable apk cache
+            script.run('ln -s /var/cache/apk /etc/apk/cache')
+            for index in glob(cache + '/APKINDEX*'):
+                if time.time() - os.stat(index).st_mtime < 3600:
+                    cached = True
+                    break
+
+        if not cached:
+            script.run(script.service.cmds['update'])
+
+        script.run(script.service.cmds['upgrade'])
+        script.run(' '.join([
+            script.service.cmds['install'],
+            ' '.join(script.service.get('packages', []))
+        ]))
+
+
+class ConfigPlugin(Plugin):
+    def post_build(self, script):
+        for value in script.service['ports']:
+            script.config(f'--port {value}')
+
+        for key, value in script.service['env'].items():
+            script.config(f'--env {key}={value}')
+
+        for key, value in script.service['labels'].items():
+            script.config(f'--label {key}={value}')
+
+        for key, value in script.service['annotations'].items():
+            script.config(f'--annotation {key}={value}')
+
+        for volume in script.service['volumes']:
+            if ':' in volume:
+                continue  # it's a runtime volume
+            script.config(f'--volume {volume}')
+
+        if 'workdir' in script.service:
+            script.config(f'--workingdir {script.service["workdir"]}')
+
+
+class UserPlugin(Plugin):
+    def pre_pkg_build(self, script):
+        if script.service.mgr == 'apk':
+            script.service['packages'].append('shadow')
+
+    def build(self, script):
+        script.append(f'''
+            if buildah run $ctr -- id {script.service['user']['id']}; then
+                i=$(buildah run $ctr -- id -n {script.service['user']['id']})
+                buildah run $ctr -- usermod -d {script.service['user']['home']} -l {script.service['user']['id']} $i
+            else
+                buildah run $ctr -- useradd -d {script.service['user']['home']} {script.service['user']['id']}
+            fi
+        ''')

podctl/console_script.py

@@ -0,0 +1,122 @@
+'''
+docker & docker-compose frustrated me, podctl unfrustrates me.
+'''
+
+import asyncio
+import cli2
+import os
+import subprocess
+import sys
+import textwrap
+
+from .pod import Pod
+
+
+class BuildStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol):
+    def __init__(self, service, *args, **kwargs):
+        self.service = service
+        super().__init__(*args, **kwargs)
+
+    def pipe_data_received(self, fd, data):
+        if fd in (1, 2):
+            for line in data.split(b'\n'):
+                if not line:
+                    continue
+                sys.stdout.buffer.write(
+                    self.service.name.encode('utf8') + b' | ' + line + b'\n'
+                )
+            sys.stdout.flush()
+        super().pipe_data_received(fd, data)
+
+
+@cli2.option('debug', help='Print debug output', color=cli2.GREEN, alias='d')
+async def build(service=None, **kwargs):
+    procs = []
+    for name, container in console_script.pod.items():
+        if not container.base:
+            continue
+
+        script = f'.podctl_build_{name}.sh'
+        with open(script, 'w+') as f:
+            f.write(str(container.script_build()))
+
+        loop = asyncio.events.get_event_loop()
+        protocol_factory = lambda: BuildStreamProtocol(
+            container=container,
+            limit=asyncio.streams._DEFAULT_LIMIT,
+            loop=loop,
+        )
+        transport, protocol = await loop.subprocess_shell(
+            protocol_factory,
+            f'buildah unshare bash -eux {script}',
+        )
+        procs.append(asyncio.subprocess.Process(
+            transport,
+            protocol,
+            loop,
+        ))
+
+    for proc in procs:
+        await proc.communicate()
+
+
+@cli2.option('debug', help='Print debug output', color=cli2.GREEN, alias='d')
+async def up(service=None, **kwargs):
+    procs = []
+    for name, service in console_script.pod.services.items():
+        if 'base' not in service:
+            continue
+
+        script = f'.podctl_up_{name}.sh'
+        with open(script, 'w+') as f:
+            f.write(str(service.build()))
+
+        loop = asyncio.events.get_event_loop()
+        protocol_factory = lambda: BuildStreamProtocol(
+            service=service,
+            limit=asyncio.streams._DEFAULT_LIMIT,
+            loop=loop,
+        )
+        transport, protocol = await loop.subprocess_shell(
+            protocol_factory,
+            f'bash -eux {script}',
+        )
+        procs.append(asyncio.subprocess.Process(
+            transport,
+            protocol,
+            loop,
+        ))
+
+    for proc in procs:
+        await proc.communicate()
+
+
+class ConsoleScript(cli2.ConsoleScript):
+    def __setitem__(self, name, cb):
+        if name != 'help':
+            cli2.option(
+                'file',
+                alias='f',
+                help='Path to pod definition (default: pod.py)',
+                color=cli2.YELLOW,
+                default='pod.py',
+            )(cb.target)
+            cli2.option(
+                'home',
+                alias='h',
+                help=f'Pod home (default is cwd: {os.getcwd()})',
+                color=cli2.YELLOW,
+                default=os.getcwd(),
+            )(cb.target)
+        super().__setitem__(name, cb)
+
+    def call(self, command):
+        if command.name != 'help':
+            self.path = self.parser.options['file']
+            self.home = self.parser.options['home']
+            with open(self.path) as f:
+                self.pod = Pod.factory(self.path)
+        return super().call(command)
+
+
+console_script = ConsoleScript(__doc__).add_module('podctl.console_script')

podctl/container.py

@@ -0,0 +1,109 @@
+import collections
+import copy
+from glob import glob
+import subprocess
+
+from .build import BuildScript
+
+
+PACKAGE_MANAGERS = dict(
+    apk=dict(
+        update='apk update',
+        upgrade='apk upgrade',
+        install='apk add',
+    ),
+)
+
+
+class Container(collections.UserDict):
+    cfg = dict()
+
+    def __init__(self, profile=None, **cfg):
+        newcfg = copy.deepcopy(self.cfg)
+        newcfg.update(cfg)
+        super().__init__(**newcfg)
+        self.profile = profile or 'default'
+
+    def __getitem__(self, name, type=None):
+        try:
+            result = super().__getitem__(name)
+        except KeyError:
+            if hasattr(self, name + '_get'):
+                result = self[name] = getattr(self, name + '_get')()
+            else:
+                raise
+        else:
+            if isinstance(result, (dict, list, tuple, switch)):
+                return self.switch_value(result)
+        return result
+
+    def switch_value(self, value):
+        _switch = lambda v: v.value(self) if isinstance(v, switch) else v
+
+        if isinstance(value, dict):
+            return {
+                k: self.switch_value(v)
+                for k, v in value.items()
+                if self.switch_value(v) is not None
+            }
+        elif isinstance(value, (list, tuple)):
+            return [
+                self.switch_value(i)
+                for i in value
+                if self.switch_value(i) is not None
+            ]
+        else:
+            return _switch(value)
+
+    def script_build(self):
+        return BuildScript(self)
+
+    def package_manager_get(self):
+        for mgr in PACKAGE_MANAGERS.keys():
+            cmd = ['podman', 'run', self['base'], 'which', mgr]
+            try:
+                subprocess.check_call(cmd)
+                return mgr
+                break
+            except subprocess.CalledProcessError:
+                continue
+        raise Exception('Package manager not supported yet')
+
+    def package_manager_cmd(self, cmd):
+        return PACKAGE_MANAGERS[self['package_manager']][cmd]
+
+    def packages_install(self, script):
+        cache = f'.cache/{self["package_manager"]}'
+        script.mount(
+            '$(pwd)/' + cache,
+            f'/var/cache/{self["package_manager"]}'
+        )
+
+        cached = False
+        if self['package_manager'] == 'apk':
+            # special step to enable apk cache
+            script.run('ln -s /var/cache/apk /etc/apk/cache')
+            script.append(f'''
+            if [ -n "$(find .cache/apk/ -name APKINDEX.* -mtime +3)" ]; then
+                buildah run $ctr -- {self.package_manager_cmd("update")}
+            fi
+            ''')
+
+        script.run(self.package_manager_cmd('upgrade'))
+        script.run(' '.join([
+            self.package_manager_cmd('install'),
+            ' '.join(self.get('packages', []))
+        ]))
+
+
+class switch:
+    def __init__(self, **values):
+        """Instanciate a switch to vary values based on container profile."""
+        self.values = values
+
+    def value(self, container):
+        """Return value from container profile or default."""
+        return self.values.get(
+            container.profile,
+            self.values.get('default', None)
+        )

podctl/pod.py

@@ -0,0 +1,11 @@
+import collections
+import importlib.util
+
+
+class Pod(collections.UserDict):
+    @classmethod
+    def factory(cls, path):
+        spec = importlib.util.spec_from_file_location('pod', path)
+        pod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(pod)
+        return pod.pod

podctl/script.py

@@ -0,0 +1,17 @@
+import textwrap
+
+
+class Script(list):
+    def __init__(self, shebang=None):
+        super().__init__()
+        self.append(shebang or '#/usr/bin/env bash')
+
+    def __str__(self):
+        if not getattr(self, '_postconfig', False):
+            if hasattr(self, 'post_config'):
+                self.post_config()
+            self._postconfig = True
+        return '\n'.join([
+            textwrap.dedent(line.lstrip('\n')).strip()
+            for line in self
+        ])

setup.py

@@ -0,0 +1,28 @@
+from setuptools import setup
+
+
+setup(
+    name='podctl',
+    versioning='dev',
+    setup_requires='setupmeta',
+    install_requires=['cli2'],
+    extras_require=dict(
+        test=[
+            'freezegun',
+            'pytest',
+            'pytest-cov',
+        ],
+    ),
+    author='James Pic',
+    author_email='jamespic@gmail.com',
+    url='https://yourlabs.io/oss/podctl',
+    include_package_data=True,
+    license='MIT',
+    keywords='cli',
+    python_requires='>=3',
+    entry_points={
+        'console_scripts': [
+            'podctl = podctl.console_script:console_script',
+        ],
+    },
+)

tests/test_build.py

@@ -0,0 +1,40 @@
+import difflib
+import os
+import sys
+
+from podctl.container import Container
+from podctl.build import BuildScript
+
+
+def script_test(name, result):
+    path = os.path.join(
+        os.path.dirname(__file__),
+        f'test_{name}.sh',
+    )
+
+    if not os.path.exists(path):
+        with open(path, 'w+') as f:
+            f.write(result)
+        raise Exception('Fixture created test_build_packages.sh')
+    with open(path, 'r') as f:
+        expected = f.read()
+    result = difflib.unified_diff(
+        expected,
+        result,
+        fromfile='expected',
+        tofile='result'
+    )
+    assert not list(result), sys.stdout.writelines(result)
+
+
+def test_build_empty():
+    result = str(BuildScript(Container()))
+    script_test('build_empty', result)
+
+
+def test_build_packages():
+    result = str(BuildScript(Container(
+        base='alpine',
+        packages=['bash'],
+    )))
+    script_test('build_packages', result)

tests/test_build_empty.sh

@@ -0,0 +1,11 @@
+#/usr/bin/env bash
+mounts=()
+umounts() {
+    for i in "${mounts[@]}"; do
+        umount $i
+    done
+}
+trap umounts 0
+ctr=$(buildah from $base)
+mnt=$(buildah mount $ctr)
+mounts=("$mnt" "${mounts[@]}")

tests/test_build_packages.sh

@@ -0,0 +1,22 @@
+#/usr/bin/env bash
+base="alpine"
+mounts=()
+umounts() {
+    for i in "${mounts[@]}"; do
+        umount $i
+    done
+}
+trap umounts 0
+ctr=$(buildah from $base)
+mnt=$(buildah mount $ctr)
+mounts=("$mnt" "${mounts[@]}")
+buildah run $ctr -- mkdir -p /var/cache/apk
+mkdir -p $(pwd)/.cache/apk
+mount -o bind $(pwd)/.cache/apk $mnt/var/cache/apk
+mounts=("$mnt/var/cache/apk" "${mounts[@]}")
+buildah run $ctr -- ln -s /var/cache/apk /etc/apk/cache
+if [ -n "$(find .cache/apk/ -name APKINDEX.* -mtime +3)" ]; then
+    buildah run $ctr -- apk update
+fi
+buildah run $ctr -- apk upgrade
+buildah run $ctr -- apk add bash

tests/test_container.py

@@ -0,0 +1,56 @@
+from .container import Container, switch
+
+
+def test_container_configuration():
+    '''Attributes should be passable to constructor or as class attributes'''
+    assert Container(a='b')['a'] == 'b'
+    class Test(Container):
+        cfg = dict(a='b')
+    assert Test()['a'] == 'b'
+
+
+def test_switch_simple():
+    assert Container(a=switch(default='expected'))['a'] == 'expected'
+    assert Container(a=switch(noise='noise'))['a'] == None
+    fixture = Container(
+        'test',
+        a=switch(default='noise', test='expected')
+    )
+    assert fixture['a'] == 'expected'
+    assert [*fixture.values()][0] == 'expected'
+    assert [*fixture.items()][0][1] == 'expected'
+
+
+def test_switch_iterable():
+    class TContainer(Container):
+        cfg = dict(
+            a=switch(dev='test')
+        )
+    assert TContainer()['a'] is None
+    assert TContainer('dev')['a'] == 'test'
+    assert TContainer('dev', a=[switch(dev='y')])['a'] == ['y']
+    assert TContainer('dev', a=[switch(default='y')])['a'] == ['y']
+
+
+def test_switch_value_list():
+    assert Container('test').switch_value(
+        [switch(default='noise', test=False)]
+    ) == [False]
+
+    assert Container('none').switch_value(
+        [switch(noise='noise')]
+    ) == []
+
+
+def test_switch_value_dict():
+    assert Container('foo').switch_value(
+        dict(i=switch(default='expected', noise='noise'))
+    ) == dict(i='expected')
+
+    assert Container('test').switch_value(
+        dict(i=switch(default='noise', test='expected'))
+    ) == dict(i='expected')
+
+    assert Container('none').switch_value(
+        dict(i=switch(noise='noise'), j=dict(e=switch(none=1)))
+    ) == dict(j=dict(e=1))

tests/test_pod.py

@@ -0,0 +1,10 @@
+import os
+from pathlib import Path
+
+from pod import Pod
+
+
+def test_pod_file():
+    path = Path(os.path.dirname(__file__)) / '..' / 'pod.py'
+    pod = Pod.factory(path)
+    assert pod['podctl']