From a866ba5a0df0c26d125527734d74f94a8748ccc4 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 25 Jan 2020 16:44:29 +0100 Subject: [PATCH] Refactor into visitor pattern --- .gitignore | 10 + .gitlab-ci.yml | 15 ++ examples/django.py | 65 +++++ pod.py | 17 ++ podctl/__init__.py | 5 +- podctl/build.py | 222 ++---------------- podctl/console_script.py | 76 +++--- podctl/container.py | 111 +-------- podctl/pod.py | 13 +- podctl/service.py | 5 + podctl/visitable.py | 32 +++ podctl/visitors/__init__.py | 8 + podctl/visitors/base.py | 5 + podctl/visitors/config.py | 7 + podctl/visitors/copy.py | 26 ++ podctl/visitors/packages.py | 54 +++++ podctl/visitors/pip.py | 25 ++ podctl/visitors/run.py | 7 + podctl/visitors/tag.py | 6 + podctl/visitors/user.py | 37 +++ ...ython-38-pytest-5.3.3.dev45+g622995a50.pyc | Bin 1645 -> 0 bytes tests/test_build.py | 49 +++- tests/test_build_copy.sh | 18 ++ tests/test_build_empty.sh | 7 + tests/test_build_packages.sh | 19 +- tests/test_build_user.sh | 38 +++ tests/test_container.py | 56 ----- tests/test_pod.py | 10 - tests/test_visitable.py | 91 +++++++ 29 files changed, 590 insertions(+), 444 deletions(-) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 examples/django.py create mode 100644 pod.py create mode 100644 podctl/service.py create mode 100644 podctl/visitable.py create mode 100644 podctl/visitors/__init__.py create mode 100644 podctl/visitors/base.py create mode 100644 podctl/visitors/config.py create mode 100644 podctl/visitors/copy.py create mode 100644 podctl/visitors/packages.py create mode 100644 podctl/visitors/pip.py create mode 100644 podctl/visitors/run.py create mode 100644 podctl/visitors/tag.py create mode 100644 podctl/visitors/user.py delete mode 100644 tests/__pycache__/test_build.cpython-38-pytest-5.3.3.dev45+g622995a50.pyc create mode 100644 tests/test_build_copy.sh create mode 100644 tests/test_build_user.sh delete mode 100644 tests/test_container.py delete mode 100644 tests/test_pod.py create mode 100644 tests/test_visitable.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9e213dc --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.pyc +__pycache__ +.cache/ +.coverage +.eggs/ +.podctl_build_django.sh +.podctl_build_podctl.sh +.setupmeta.version +.testmondata +*.egg-info diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..88bc0be --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,15 @@ +qa: + stage: test + image: yourlabs/python + script: flake8 podctl + +test: + stage: test + image: yourlabs/python + script: pip install -e . && py.test -v tests + +pypi: + stage: deploy + image: yourlabs/python + script: pypi-release + only: [tags] diff --git a/examples/django.py b/examples/django.py new file mode 100644 index 0000000..4893289 --- /dev/null +++ b/examples/django.py @@ -0,0 +1,65 @@ +import os + +from podctl import * + + +class Django(Container): + tag = 'yourlabs/crudlfap' + base = 'alpine' + packages = [ + 'bash', + 'python3', + switch(dev='vim'), + ] + env = dict(FOO='bar') + annotations = dict(test='foo') + labels = dict(foo='test') + cmd = 'bash' + entrypoint = ['bash', '-v'] + ports = [1234] + user = dict( + shell='/bin/bash', + name='app', + home='/app', + id=os.getenv('SUDO_ID', os.getenv('UID')), + ) + volumes = [ + '/bydir', + 'byname:/byname', + switch(dev='.:/app'), + ] + build = [ + 'sudo pip3 install --upgrade pip', + 'pip3 install --user -e /app', + ] + workdir = '/app' + +django = Container( + Base('alpine'), + Packages('bash', switch(dev='vim')), + User( + uid=1000, + home='/app', + directories=('log', 'spooler', 'static') + ), + switch(default=Mount('.', '/app'), production=Copy('.', '/app')), + Npm('build'), + Env('PATH', 'PATH=/app/node_modules/.bin:$PATH'), + Pip('requirements.txt'), + Env('PATH', 'PATH=$HOME/.local/bin:$PATH'), + switch(dev=Run(''' + manage.py collectstatic --noinput --clear + find frontend/static/dist/css -type f | xargs gzip -f -k -9 + find frontend/static/dist/js -type f | xargs gzip -f -k -9 + ''')), + Expose(8000), + Tag('equisafe'), +) + +pod = Pod( + Service('django', django, restart='unless-stopped'), + Service('db', + Container(Tag('postgresql:latest')), + restart='unless-stopped' + ), +) diff --git a/pod.py b/pod.py new file mode 100644 index 0000000..5ccbfe5 --- /dev/null +++ b/pod.py @@ -0,0 +1,17 @@ +""" +Basic pod to contain the podctl command. + +For advanced examples, check the examples sub-directory of the git repository. +""" +from podctl import * + + +podctl = Container( + Base('alpine'), + Packages('bash', 'python3'), + User('app', 1000, '/app'), + Copy(['setup.py', 'podctl'], '/app'), + Pip('/app'), + Config(cmd='podctl'), + Tag('yourlabs/podctl'), +) diff --git a/podctl/__init__.py b/podctl/__init__.py index aab674e..d54078a 100644 --- a/podctl/__init__.py +++ b/podctl/__init__.py @@ -1,2 +1,3 @@ -from .container import Container, switch -from .pod import Pod +from .container import Container # noqa +from .pod import Pod # noqa +from .visitors import * # noqa diff --git a/podctl/build.py b/podctl/build.py index ed6c201..ec7141f 100644 --- a/podctl/build.py +++ b/podctl/build.py @@ -1,9 +1,3 @@ -from glob import glob -import inspect -import os -import subprocess -import time - from .script import Script @@ -15,14 +9,16 @@ class BuildScript(Script): self.container = container for var in self.export: - if var in self.container: - self.append(f'{var}="{self.container[var]}"') + self.append(f'{var}="{container.variable(var)}"') self.append(''' mounts=() umounts() { for i in "${mounts[@]}"; do umount $i + echo $mounts + mounts=("${mounts[@]/$i}") + echo $mounts done } trap umounts 0 @@ -31,215 +27,27 @@ class BuildScript(Script): mounts=("$mnt" "${mounts[@]}") ''') - if self.container.get('packages'): - self.container.packages_install(self) - def config(self, line): self.append(f'buildah config {line} $ctr') + def _run(self, cmd): + user = self.container.variable('username') + if cmd.startswith('sudo '): + return f'buildah run --user root $ctr -- {cmd[5:]}' + elif user and self.container.variable('user_created'): + return f'buildah run --user {user} $ctr -- {cmd}' + else: + return f'buildah run $ctr -- {cmd}' + def run(self, cmd): - self.append('buildah run $ctr -- ' + cmd) + self.append(self._run(cmd)) def copy(self, src, dst): self.append(f'buildah copy $ctr {src} {dst}') def mount(self, src, dst): - self.run('mkdir -p ' + dst) + self.run('sudo mkdir -p ' + dst) self.append('mkdir -p ' + src) self.append(f'mount -o bind {src} $mnt{dst}') # for unmount trap self.append('mounts=("$mnt%s" "${mounts[@]}")' % dst) - - -class Plugins(dict): - def __init__(self, *plugins): - default_plugins = [ - PkgPlugin(), - UserPlugin(), - FsPlugin(), - BuildPlugin(), - ConfigPlugin(), - ] - - super().__init__() - for plugin in plugins or default_plugins: - self.add(plugin) - - def add(self, plugin): - plugin.plugins = self - self[plugin.name] = plugin - - def __call__(self, method, *args, **kwargs): - hook = f'pre_{method}' - for plugin in self.values(): - plugin(hook, *args, **kwargs) - - for plugin in self.values(): - if hasattr(plugin, method): - plugin(method, *args, **kwargs) - - hook = f'post_{method}' - for plugin in self.values(): - plugin(hook, *args, **kwargs) - - -class Plugin: - @property - def name(self): - return type(self).__name__.replace('Plugin', '').lower() - - def bubble(self, hook, *args, **kwargs): - for plugin in self.plugins.values(): - if plugin is self: - continue - plugin(hook, _bubble=False, *args, **kwargs) - - def __call__(self, method, *args, **kwargs): - _bubble = kwargs.pop('_bubble', True) - if _bubble: - self.bubble(f'pre_{self.name}_{method}', *args, **kwargs) - - if hasattr(self, method): - meth = getattr(self, method) - argspec = inspect.getargspec(meth) - if argspec.varargs and argspec.keywords: - meth(*args, **kwargs) - else: - numargs = len(argspec.args) - 1 - len(argspec.defaults or []) - args = args[:numargs] - kwargs = { - k: v - for k, v in kwargs.items() - if k in argspec.args - } - meth(*args, **kwargs) - - if _bubble: - self.bubble(f'post_{self.name}_{method}', *args, **kwargs) - - -class BuildPlugin(Plugin): - def build(self, script): - user = script.service.get('user', None) - - for cmd in script.service['build']: - if cmd.startswith('sudo'): - if user: - script.config(f'--user root') - script.run(cmd[5:]) - else: - script.config(f'--user {script.service["user"]}') - script.run(cmd) - - -class FsPlugin(Plugin): - def build(self, script): - for key, value in script.service.items(): - if not key.startswith('/'): - continue - - parts = key.split(':') - dst = parts.pop(0) - mode = parts.pop(0) if parts else '0500' - script.run(f'mkdir -p {dst}') - script.run(f'chmod {mode} $mnt{dst}') - - if value and isinstance(value, list): - for item in value: - if isinstance(item, str): - script.run(f'cp -a {item} $mnt{dst}') - - if not isinstance(item, dict): - pass - - -class PkgPlugin(Plugin): - mgrs = dict( - apk=dict( - update='apk update', - upgrade='apk upgrade', - install='apk add', - ), - ) - - def pre_build(self, script): - for mgr, cmds in self.mgrs.items(): - cmd = [ - 'podman', - 'run', - script.service['base'], - 'which', - mgr - ] - print('+ ' + ' '.join(cmd)) - try: - subprocess.check_call(cmd) - script.service.mgr = mgr - script.service.cmds = cmds - break - except subprocess.CalledProcessError: - continue - - def build(self, script): - cache = f'.cache/{script.service.mgr}' - script.mount( - '$(pwd)/' + cache, - f'/var/cache/{script.service.mgr}' - ) - - cached = False - if script.service.mgr == 'apk': - # special step to enable apk cache - script.run('ln -s /var/cache/apk /etc/apk/cache') - for index in glob(cache + '/APKINDEX*'): - if time.time() - os.stat(index).st_mtime < 3600: - cached = True - break - - if not cached: - script.run(script.service.cmds['update']) - - script.run(script.service.cmds['upgrade']) - script.run(' '.join([ - script.service.cmds['install'], - ' '.join(script.service.get('packages', [])) - ])) - - -class ConfigPlugin(Plugin): - def post_build(self, script): - for value in script.service['ports']: - script.config(f'--port {value}') - - for key, value in script.service['env'].items(): - script.config(f'--env {key}={value}') - - for key, value in script.service['labels'].items(): - script.config(f'--label {key}={value}') - - for key, value in script.service['annotations'].items(): - script.config(f'--annotation {key}={value}') - - for volume in script.service['volumes']: - if ':' in volume: - continue # it's a runtime volume - script.config(f'--volume {volume}') - - if 'workdir' in script.service: - script.config(f'--workingdir {script.service["workdir"]}') - - -class UserPlugin(Plugin): - def pre_pkg_build(self, script): - if script.service.mgr == 'apk': - script.service['packages'].append('shadow') - - def build(self, script): - script.append(f''' - if buildah run $ctr -- id {script.service['user']['id']}; then - i=$(buildah run $ctr -- id -n {script.service['user']['id']}) - buildah run $ctr -- usermod -d {script.service['user']['home']} -l {script.service['user']['id']} $i - else - buildah run $ctr -- useradd -d {script.service['user']['home']} {script.service['user']['id']} - fi - ''') diff --git a/podctl/console_script.py b/podctl/console_script.py index 10aa709..382be41 100644 --- a/podctl/console_script.py +++ b/podctl/console_script.py @@ -4,12 +4,13 @@ docker & docker-compose frustrated me, podctl unfrustrates me. import asyncio import cli2 +import importlib import os -import subprocess import sys -import textwrap +from .container import Container from .pod import Pod +from .service import Service class BuildStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): @@ -32,20 +33,23 @@ class BuildStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): @cli2.option('debug', help='Print debug output', color=cli2.GREEN, alias='d') async def build(service=None, **kwargs): procs = [] - for name, container in console_script.pod.items(): - if not container.base: + for name, service in console_script.pod.services.items(): + container = service.container + if not container.variable('base'): continue script = f'.podctl_build_{name}.sh' with open(script, 'w+') as f: - f.write(str(container.script_build())) + f.write(str(container.script('build'))) loop = asyncio.events.get_event_loop() - protocol_factory = lambda: BuildStreamProtocol( - container=container, - limit=asyncio.streams._DEFAULT_LIMIT, - loop=loop, - ) + + def protocol_factory(): + return BuildStreamProtocol( + service, + limit=asyncio.streams._DEFAULT_LIMIT, + loop=loop, + ) transport, protocol = await loop.subprocess_shell( protocol_factory, f'buildah unshare bash -eux {script}', @@ -60,37 +64,6 @@ async def build(service=None, **kwargs): await proc.communicate() -@cli2.option('debug', help='Print debug output', color=cli2.GREEN, alias='d') -async def up(service=None, **kwargs): - procs = [] - for name, service in console_script.pod.services.items(): - if 'base' not in service: - continue - - script = f'.podctl_up_{name}.sh' - with open(script, 'w+') as f: - f.write(str(service.build())) - - loop = asyncio.events.get_event_loop() - protocol_factory = lambda: BuildStreamProtocol( - service=service, - limit=asyncio.streams._DEFAULT_LIMIT, - loop=loop, - ) - transport, protocol = await loop.subprocess_shell( - protocol_factory, - f'bash -eux {script}', - ) - procs.append(asyncio.subprocess.Process( - transport, - protocol, - loop, - )) - - for proc in procs: - await proc.communicate() - - class ConsoleScript(cli2.ConsoleScript): def __setitem__(self, name, cb): if name != 'help': @@ -114,8 +87,25 @@ class ConsoleScript(cli2.ConsoleScript): if command.name != 'help': self.path = self.parser.options['file'] self.home = self.parser.options['home'] - with open(self.path) as f: - self.pod = Pod.factory(self.path) + self.containers = dict() + self.pods = dict() + self.pod = None + spec = importlib.util.spec_from_file_location('pod', self.path) + pod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(pod) + for name, value in pod.__dict__.items(): + if isinstance(value, Container): + self.containers[name] = value + elif isinstance(value, Pod): + self.pods[name] = value + + if 'pod' in self.pods: + self.pod = self.pods['pod'] + if not self.pod: + self.pod = Pod(*[ + Service(name, value, restart='no') + for name, value in self.containers.items() + ]) return super().call(command) diff --git a/podctl/container.py b/podctl/container.py index 85a90d5..37eaecc 100644 --- a/podctl/container.py +++ b/podctl/container.py @@ -1,109 +1,8 @@ -import collections -import copy -from glob import glob -import subprocess - from .build import BuildScript +from .visitable import Visitable -PACKAGE_MANAGERS = dict( - apk=dict( - update='apk update', - upgrade='apk upgrade', - install='apk add', - ), -) - - -class Container(collections.UserDict): - cfg = dict() - - def __init__(self, profile=None, **cfg): - newcfg = copy.deepcopy(self.cfg) - newcfg.update(cfg) - super().__init__(**newcfg) - self.profile = profile or 'default' - - def __getitem__(self, name, type=None): - try: - result = super().__getitem__(name) - except KeyError: - if hasattr(self, name + '_get'): - result = self[name] = getattr(self, name + '_get')() - else: - raise - else: - if isinstance(result, (dict, list, tuple, switch)): - return self.switch_value(result) - return result - - def switch_value(self, value): - _switch = lambda v: v.value(self) if isinstance(v, switch) else v - - if isinstance(value, dict): - return { - k: self.switch_value(v) - for k, v in value.items() - if self.switch_value(v) is not None - } - elif isinstance(value, (list, tuple)): - return [ - self.switch_value(i) - for i in value - if self.switch_value(i) is not None - ] - else: - return _switch(value) - - def script_build(self): - return BuildScript(self) - - def package_manager_get(self): - for mgr in PACKAGE_MANAGERS.keys(): - cmd = ['podman', 'run', self['base'], 'which', mgr] - try: - subprocess.check_call(cmd) - return mgr - break - except subprocess.CalledProcessError: - continue - raise Exception('Package manager not supported yet') - - def package_manager_cmd(self, cmd): - return PACKAGE_MANAGERS[self['package_manager']][cmd] - - def packages_install(self, script): - cache = f'.cache/{self["package_manager"]}' - script.mount( - '$(pwd)/' + cache, - f'/var/cache/{self["package_manager"]}' - ) - - cached = False - if self['package_manager'] == 'apk': - # special step to enable apk cache - script.run('ln -s /var/cache/apk /etc/apk/cache') - script.append(f''' - if [ -n "$(find .cache/apk/ -name APKINDEX.* -mtime +3)" ]; then - buildah run $ctr -- {self.package_manager_cmd("update")} - fi - ''') - - script.run(self.package_manager_cmd('upgrade')) - script.run(' '.join([ - self.package_manager_cmd('install'), - ' '.join(self.get('packages', [])) - ])) - - -class switch: - def __init__(self, **values): - """Instanciate a switch to vary values based on container profile.""" - self.values = values - - def value(self, container): - """Return value from container profile or default.""" - return self.values.get( - container.profile, - self.values.get('default', None) - ) +class Container(Visitable): + default_scripts = dict( + build=BuildScript, + ) diff --git a/podctl/pod.py b/podctl/pod.py index 9dd3b6f..642775e 100644 --- a/podctl/pod.py +++ b/podctl/pod.py @@ -1,11 +1,6 @@ -import collections -import importlib.util -class Pod(collections.UserDict): - @classmethod - def factory(cls, path): - spec = importlib.util.spec_from_file_location('pod', path) - pod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(pod) - return pod.pod +class Pod: + def __init__(self, *services, **scripts): + self.scripts = scripts + self.services = {s.name: s for s in services} diff --git a/podctl/service.py b/podctl/service.py new file mode 100644 index 0000000..132c931 --- /dev/null +++ b/podctl/service.py @@ -0,0 +1,5 @@ +class Service: + def __init__(self, name, container, restart=None): + self.name = name + self.container = container + self.restart = restart diff --git a/podctl/visitable.py b/podctl/visitable.py new file mode 100644 index 0000000..526ab5e --- /dev/null +++ b/podctl/visitable.py @@ -0,0 +1,32 @@ +from copy import copy + + +class Visitable: + default_scripts = dict() + + def __init__(self, *visitors, **scripts): + self.visitors = list(visitors) + self.scripts = scripts or { + k: v(self) for k, v in self.default_scripts.items() + } + + def script(self, name): + script = copy(self.scripts[name]) + + for prefix in ('init_', 'pre_', '', 'post_'): + method = prefix + name + for visitor in self.visitors: + if hasattr(visitor, method): + getattr(visitor, method)(script) + + return script + + def visitor(self, name): + for visitor in self.visitors: + if name.lower() == type(visitor).__name__.lower(): + return visitor + + def variable(self, name): + for visitor in self.visitors: + if getattr(visitor, name, None) is not None: + return getattr(visitor, name) diff --git a/podctl/visitors/__init__.py b/podctl/visitors/__init__.py new file mode 100644 index 0000000..45a7dc8 --- /dev/null +++ b/podctl/visitors/__init__.py @@ -0,0 +1,8 @@ +from .base import Base # noqa +from .config import Config # noqa +from .copy import Copy # noqa +from .packages import Packages # noqa +from .pip import Pip # noqa +from .run import Run # noqa +from .tag import Tag # noqa +from .user import User # noqa diff --git a/podctl/visitors/base.py b/podctl/visitors/base.py new file mode 100644 index 0000000..46c7aa9 --- /dev/null +++ b/podctl/visitors/base.py @@ -0,0 +1,5 @@ + + +class Base: + def __init__(self, base): + self.base = base diff --git a/podctl/visitors/config.py b/podctl/visitors/config.py new file mode 100644 index 0000000..06bf479 --- /dev/null +++ b/podctl/visitors/config.py @@ -0,0 +1,7 @@ +class Config: + def __init__(self, **values): + self.values = values + + def post_build(self, script): + for key, value in self.values.items(): + script.config(f'--{key} {value}') diff --git a/podctl/visitors/copy.py b/podctl/visitors/copy.py new file mode 100644 index 0000000..cbc9e6a --- /dev/null +++ b/podctl/visitors/copy.py @@ -0,0 +1,26 @@ +class Copy: + def __init__(self, src, dst): + self.src = src + self.dst = dst + + def init_build(self, script): + count = self.dst.count(':') + self.mode = None + self.owner = None + if count == 2: + self.dst, self.mode, self.owner = self.dst.split(':') + elif count == 1: + self.dst, self.mode = self.dst.split(':') + self.owner = script.variable('user') + + def build(self, script): + if isinstance(self.src, list): + script.run(f'sudo mkdir -p {self.dst}') + for item in self.src: + script.append(f'cp -a {item} $mnt{self.dst}') + + if self.mode: + script.run(f'sudo chmod {self.mode} $mnt{self.dst}') + + if self.owner: + script.run(f'sudo chown -R {self.owner} $mnt{self.dst}') diff --git a/podctl/visitors/packages.py b/podctl/visitors/packages.py new file mode 100644 index 0000000..ae01472 --- /dev/null +++ b/podctl/visitors/packages.py @@ -0,0 +1,54 @@ +import subprocess + + +class Packages: + mgrs = dict( + apk=dict( + update='sudo apk update', + upgrade='sudo apk upgrade', + install='sudo apk add', + ), + ) + + def __init__(self, *packages): + self.packages = list(packages) + + def pre_build(self, script): + for mgr, cmds in self.mgrs.items(): + cmd = [ + 'podman', + 'run', + script.container.variable('base'), + 'which', + mgr + ] + print('+ ' + ' '.join(cmd)) + try: + subprocess.check_call(cmd) + self.mgr = mgr + self.cmds = cmds + break + except subprocess.CalledProcessError: + continue + + def build(self, script): + cache = f'.cache/{self.mgr}' + script.mount( + '$(pwd)/' + cache, + f'/var/cache/{self.mgr}' + ) + + if self.mgr == 'apk': + # special step to enable apk cache + script.run('ln -s /var/cache/apk /etc/apk/cache') + script.append(f''' + old="$(find .cache/apk/ -name APKINDEX.* -mtime +3)" + if [ -n "$old" ] || ! ls .cache/apk/APKINDEX.*; then + {script._run(self.cmds['update'])} + else + echo Cache recent enough, skipping index update. + fi + ''') + + script.run(self.cmds['upgrade']) + script.run(' '.join([self.cmds['install']] + self.packages)) diff --git a/podctl/visitors/pip.py b/podctl/visitors/pip.py new file mode 100644 index 0000000..3bee27b --- /dev/null +++ b/podctl/visitors/pip.py @@ -0,0 +1,25 @@ +class Pip: + def __init__(self, *pip_packages): + self.pip_packages = pip_packages + + def build(self, script): + script.append(f''' + if {script._run("bash -c 'type pip3'")}; then + _pip=pip3 + elif {script._run("bash -c 'type pip'")}; then + _pip=pip + elif {script._run("bash -c 'type pip2'")}; then + _pip=pip2 + fi + ''') + script.mount('.cache/pip', '/root/.cache/pip') + script.run('sudo $_pip install --upgrade pip') + source = [p for p in self.pip_packages if p.startswith('/')] + if source: + script.run( + f'sudo $_pip install --upgrade --editable {" ".join(source)}' + ) + + nonsource = [p for p in self.pip_packages if not p.startswith('/')] + if nonsource: + script.run(f'sudo $_pip install --upgrade {" ".join(source)}') diff --git a/podctl/visitors/run.py b/podctl/visitors/run.py new file mode 100644 index 0000000..6e16577 --- /dev/null +++ b/podctl/visitors/run.py @@ -0,0 +1,7 @@ +class Run: + def __init__(self, *commands): + self.commands = commands + + def build(self, script): + for command in self.commands: + script.run(command) diff --git a/podctl/visitors/tag.py b/podctl/visitors/tag.py new file mode 100644 index 0000000..a0a346b --- /dev/null +++ b/podctl/visitors/tag.py @@ -0,0 +1,6 @@ +class Tag: + def __init__(self, tag): + self.tag = tag + + def post_build(self, script): + script.append(f'umounts && trap - 0 && buildah commit $ctr {self.tag}') diff --git a/podctl/visitors/user.py b/podctl/visitors/user.py new file mode 100644 index 0000000..e8651c7 --- /dev/null +++ b/podctl/visitors/user.py @@ -0,0 +1,37 @@ +from .packages import Packages + + +class User: + """Secure the image with a user""" + + def __init__(self, username, uid, home): + self.username = username + self.uid = uid + self.home = home + self.user_created = False + + def init_build(self, script): + """Inject the Packages visitor if necessary.""" + packages = script.container.visitor('packages') + if not packages: + index = script.container.visitors.index(self) + script.container.visitors.insert(index, Packages()) + + def pre_build(self, script): + """Inject the shadow package for the usermod command""" + if script.container.variable('mgr') == 'apk': + script.container.variable('packages').append('shadow') + + def build(self, script): + script.append(f''' + if buildah run $ctr -- id {self.uid}; then + i=$(buildah run $ctr -- id -n {self.uid}) + buildah run $ctr -- usermod --home-dir {self.home} --no-log-init {self.uid} $i + else + buildah run $ctr -- useradd --home-dir {self.home} --uid {self.uid} {self.username} + fi + ''') # noqa + self.user_created = True + + def post_build(self, script): + script.config(f'--user {self.username}') diff --git a/tests/__pycache__/test_build.cpython-38-pytest-5.3.3.dev45+g622995a50.pyc b/tests/__pycache__/test_build.cpython-38-pytest-5.3.3.dev45+g622995a50.pyc deleted file mode 100644 index 567a8dc37c29bb7952ddafd34df4820fb5be35d2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1645 zcmZWpOK;mo5Z>iW6h$kJox~1W6bAZ0SQk=aC$~08Yotd5O;EJpp#;rcS#(H|*j*u( zBVD3igZADBtfT)$&;1E|Y)=LH3yLD>%+fN{gal`1XJ@$Ie4OFeYimKUzx&;QDg zc|g7*Psq1K`OKNqm(;|$5Z5(v?FE@RD^bBoXX;ETR>S;P&Wx&XO6PRuPTi2qys7si zm7!WY@2YDj>*v?;6hPJ9A}mlF=bH=i%#**t{tXo_sQ4YMn`n(J7O>V;#2*-ex#Dk( zKno&T#&+R-ouBPnWY$u*z_T@NEdaqw_*M>?xik1Oz-)LJyNXMdoY_4&JhPtfo_+LX zeyYZT?_`3finATFe}q&gb(%d(kGX_cO@;RPY0Wc;(VpOPT&T#>{ygi(%qm zP~=BC9FOur&RK#Efo-FMNA<*1oeF(D8C0T7RnpW@%42QIiPWA{tQspFnD-aZl2V6| zmmpIrA3;P>WoaSxb|Qz>7}yiM7Rhl@9i>HLFt-z#4mi4X_j5xL;Cn1YCG_Ug9c=Kk07eB!}9BwZqQ zAwI*TJCS*yu~1N18)@C5>A z-n{>UHk@E}rbO{ESbBY>bxrnr;FS&tLc3J-LBE9zxjleKegL9D#wfFnDYIW$R|uUI zgzjPNr9}-L5IaYy9KsNwhL&88OWghc_y{CX^iaHkcyk5uGW&f*6M#V4E$X}u9I*$h zXkGiuW)oPMzMV9E+iN-jTBaxXW&dBfbnDV5ZbOdPLa~P89T4$)(+j<9`JXi;$Kj>h u)X|WS-_=lPz^oKvN;3TWpOzIH7yKc9a=8J*CmY0ve(^2n9NWL?yZ-@=+`Bgb diff --git a/tests/test_build.py b/tests/test_build.py index e38ba63..0e0972f 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -4,9 +4,16 @@ import sys from podctl.container import Container from podctl.build import BuildScript +from podctl.visitors import ( + Base, + Copy, + Packages, + User, +) -def script_test(name, result): +def script_test(name, *visitors): + result = str(Container(*visitors).script('build')) path = os.path.join( os.path.dirname(__file__), f'test_{name}.sh', @@ -15,7 +22,7 @@ def script_test(name, result): if not os.path.exists(path): with open(path, 'w+') as f: f.write(result) - raise Exception('Fixture created test_build_packages.sh') + raise Exception(f'Fixture created test_{name}.sh') with open(path, 'r') as f: expected = f.read() result = difflib.unified_diff( @@ -28,13 +35,43 @@ def script_test(name, result): def test_build_empty(): - result = str(BuildScript(Container())) - script_test('build_empty', result) + script_test( + 'build_empty', + Base('alpine'), + ) def test_build_packages(): + script_test( + 'build_packages', + Base('alpine'), + Packages('bash'), + ) + + +def test_build_user(): + script_test( + 'build_user', + Base('alpine'), + User('app', 1000, '/app'), + ) + + +def test_build_copy(): + script_test( + 'build_copy', + Base('alpine'), + Copy(os.path.dirname(__file__), '/app'), + ) + + +''' + +def test_build_files(): result = str(BuildScript(Container( base='alpine', - packages=['bash'], + files=[ + Directory('/app', '0500').add('setup.py', 'podctl'), + ] ))) - script_test('build_packages', result) + ''' diff --git a/tests/test_build_copy.sh b/tests/test_build_copy.sh new file mode 100644 index 0000000..ee9fa08 --- /dev/null +++ b/tests/test_build_copy.sh @@ -0,0 +1,18 @@ +#/usr/bin/env bash +base="alpine" +repo="None" +tag="None" +image="None" +mounts=() +umounts() { + for i in "${mounts[@]}"; do + umount $i + echo $mounts + mounts=("${mounts[@]/$i}") + echo $mounts + done +} +trap umounts 0 +ctr=$(buildah from $base) +mnt=$(buildah mount $ctr) +mounts=("$mnt" "${mounts[@]}") \ No newline at end of file diff --git a/tests/test_build_empty.sh b/tests/test_build_empty.sh index 570722c..ee9fa08 100644 --- a/tests/test_build_empty.sh +++ b/tests/test_build_empty.sh @@ -1,8 +1,15 @@ #/usr/bin/env bash +base="alpine" +repo="None" +tag="None" +image="None" mounts=() umounts() { for i in "${mounts[@]}"; do umount $i + echo $mounts + mounts=("${mounts[@]/$i}") + echo $mounts done } trap umounts 0 diff --git a/tests/test_build_packages.sh b/tests/test_build_packages.sh index 01e20e7..194501f 100644 --- a/tests/test_build_packages.sh +++ b/tests/test_build_packages.sh @@ -1,22 +1,31 @@ #/usr/bin/env bash base="alpine" +repo="None" +tag="None" +image="None" mounts=() umounts() { for i in "${mounts[@]}"; do umount $i + echo $mounts + mounts=("${mounts[@]/$i}") + echo $mounts done } trap umounts 0 ctr=$(buildah from $base) mnt=$(buildah mount $ctr) mounts=("$mnt" "${mounts[@]}") -buildah run $ctr -- mkdir -p /var/cache/apk +buildah run --user root $ctr -- mkdir -p /var/cache/apk mkdir -p $(pwd)/.cache/apk mount -o bind $(pwd)/.cache/apk $mnt/var/cache/apk mounts=("$mnt/var/cache/apk" "${mounts[@]}") buildah run $ctr -- ln -s /var/cache/apk /etc/apk/cache -if [ -n "$(find .cache/apk/ -name APKINDEX.* -mtime +3)" ]; then - buildah run $ctr -- apk update +old="$(find .cache/apk/ -name APKINDEX.* -mtime +3)" +if [ -n "$old" ] || ! ls .cache/apk/APKINDEX.*; then + buildah run --user root $ctr -- apk update +else + echo Cache recent enough, skipping index update. fi -buildah run $ctr -- apk upgrade -buildah run $ctr -- apk add bash \ No newline at end of file +buildah run --user root $ctr -- apk upgrade +buildah run --user root $ctr -- apk add bash \ No newline at end of file diff --git a/tests/test_build_user.sh b/tests/test_build_user.sh new file mode 100644 index 0000000..8bebad4 --- /dev/null +++ b/tests/test_build_user.sh @@ -0,0 +1,38 @@ +#/usr/bin/env bash +base="alpine" +repo="None" +tag="None" +image="None" +mounts=() +umounts() { + for i in "${mounts[@]}"; do + umount $i + echo $mounts + mounts=("${mounts[@]/$i}") + echo $mounts + done +} +trap umounts 0 +ctr=$(buildah from $base) +mnt=$(buildah mount $ctr) +mounts=("$mnt" "${mounts[@]}") +buildah run --user root $ctr -- mkdir -p /var/cache/apk +mkdir -p $(pwd)/.cache/apk +mount -o bind $(pwd)/.cache/apk $mnt/var/cache/apk +mounts=("$mnt/var/cache/apk" "${mounts[@]}") +buildah run $ctr -- ln -s /var/cache/apk /etc/apk/cache +old="$(find .cache/apk/ -name APKINDEX.* -mtime +3)" +if [ -n "$old" ] || ! ls .cache/apk/APKINDEX.*; then + buildah run --user root $ctr -- apk update +else + echo Cache recent enough, skipping index update. +fi +buildah run --user root $ctr -- apk upgrade +buildah run --user root $ctr -- apk add shadow +if buildah run $ctr -- id 1000; then + i=$(buildah run $ctr -- id -n 1000) + buildah run $ctr -- usermod --home-dir /app --no-log-init 1000 $i +else + buildah run $ctr -- useradd --home-dir /app --uid 1000 app +fi +buildah config --user app $ctr \ No newline at end of file diff --git a/tests/test_container.py b/tests/test_container.py deleted file mode 100644 index c66dd14..0000000 --- a/tests/test_container.py +++ /dev/null @@ -1,56 +0,0 @@ -from .container import Container, switch - - -def test_container_configuration(): - '''Attributes should be passable to constructor or as class attributes''' - assert Container(a='b')['a'] == 'b' - class Test(Container): - cfg = dict(a='b') - assert Test()['a'] == 'b' - - -def test_switch_simple(): - assert Container(a=switch(default='expected'))['a'] == 'expected' - assert Container(a=switch(noise='noise'))['a'] == None - fixture = Container( - 'test', - a=switch(default='noise', test='expected') - ) - assert fixture['a'] == 'expected' - assert [*fixture.values()][0] == 'expected' - assert [*fixture.items()][0][1] == 'expected' - - -def test_switch_iterable(): - class TContainer(Container): - cfg = dict( - a=switch(dev='test') - ) - assert TContainer()['a'] is None - assert TContainer('dev')['a'] == 'test' - assert TContainer('dev', a=[switch(dev='y')])['a'] == ['y'] - assert TContainer('dev', a=[switch(default='y')])['a'] == ['y'] - - -def test_switch_value_list(): - assert Container('test').switch_value( - [switch(default='noise', test=False)] - ) == [False] - - assert Container('none').switch_value( - [switch(noise='noise')] - ) == [] - - -def test_switch_value_dict(): - assert Container('foo').switch_value( - dict(i=switch(default='expected', noise='noise')) - ) == dict(i='expected') - - assert Container('test').switch_value( - dict(i=switch(default='noise', test='expected')) - ) == dict(i='expected') - - assert Container('none').switch_value( - dict(i=switch(noise='noise'), j=dict(e=switch(none=1))) - ) == dict(j=dict(e=1)) diff --git a/tests/test_pod.py b/tests/test_pod.py deleted file mode 100644 index a2f235d..0000000 --- a/tests/test_pod.py +++ /dev/null @@ -1,10 +0,0 @@ -import os -from pathlib import Path - -from pod import Pod - - -def test_pod_file(): - path = Path(os.path.dirname(__file__)) / '..' / 'pod.py' - pod = Pod.factory(path) - assert pod['podctl'] diff --git a/tests/test_visitable.py b/tests/test_visitable.py new file mode 100644 index 0000000..7c84473 --- /dev/null +++ b/tests/test_visitable.py @@ -0,0 +1,91 @@ +from unittest.mock import MagicMock + +from podctl.script import Script +from podctl.visitable import Visitable + + +class Visitor0: + def __init__(self, name=None): + self.name = name or 'visit0' + + +class Visitor1: + def pre_build(self, script): + script.append('pre_build') + def build(self, script): + script.append('build') + def post_build(self, script): + script.append('post_build') + + +def test_visitable_visitor(): + visitable = Visitable(Visitor0(), Visitor1(), build=Script()) + script = visitable.script('build') + assert 'pre_build' in script + assert 'build' in script + assert 'post_build' in script + + +def test_visitable_visitor(): + x = Visitor0() + assert Visitable(x).visitor('visitor0') is x + + +def test_visitable_variable(): + assert Visitable(Visitor0('foo')).variable('name') == 'foo' + +# +# +#def test_visitable_configuration(): +# '''Attributes should be passable to constructor or as class attributes''' +# assert Container(a='b')['a'] == 'b' +# class Test(Container): +# cfg = dict(a='b') +# assert Test()['a'] == 'b' +# +# +#def test_switch_simple(): +# assert Container(a=switch(default='expected'))['a'] == 'expected' +# assert Container(a=switch(noise='noise'))['a'] == None +# fixture = Container( +# 'test', +# a=switch(default='noise', test='expected') +# ) +# assert fixture['a'] == 'expected' +# assert [*fixture.values()][0] == 'expected' +# assert [*fixture.items()][0][1] == 'expected' +# +# +#def test_switch_iterable(): +# class TContainer(Container): +# cfg = dict( +# a=switch(dev='test') +# ) +# assert TContainer()['a'] is None +# assert TContainer('dev')['a'] == 'test' +# assert TContainer('dev', a=[switch(dev='y')])['a'] == ['y'] +# assert TContainer('dev', a=[switch(default='y')])['a'] == ['y'] +# +# +#def test_switch_value_list(): +# assert Container('test').switch_value( +# [switch(default='noise', test=False)] +# ) == [False] +# +# assert Container('none').switch_value( +# [switch(noise='noise')] +# ) == [] +# +# +#def test_switch_value_dict(): +# assert Container('foo').switch_value( +# dict(i=switch(default='expected', noise='noise')) +# ) == dict(i='expected') +# +# assert Container('test').switch_value( +# dict(i=switch(default='noise', test='expected')) +# ) == dict(i='expected') +# +# assert Container('none').switch_value( +# dict(i=switch(noise='noise'), j=dict(e=switch(none=1))) +# ) == dict(j=dict(e=1))