From 7753175d6c27d294ee009bd3e0771b7d154a2f4e Mon Sep 17 00:00:00 2001 From: jpic Date: Mon, 3 Feb 2020 02:17:45 +0100 Subject: [PATCH] Porting to subprocess --- podctl/build.py | 61 ++++++++++++++++++++++------------------ podctl/console_script.py | 42 +++++++-------------------- podctl/container.py | 36 ++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 61 deletions(-) diff --git a/podctl/build.py b/podctl/build.py index 36764b7..9331bcb 100644 --- a/podctl/build.py +++ b/podctl/build.py @@ -1,35 +1,36 @@ +import asyncio +import os +import subprocess import textwrap from .script import Script -class BuildScript(Script): - export = ('base', 'repo') - +class Build(Script): def __init__(self, container): super().__init__() self.container = container - for var in self.export: - self.append(f'{var}="{container.variable(var)}"') + def append(self, value): + res = [] + for line in value.split('\n'): + if line.startswith('#') or not line.strip(): + continue + res.append(self.unshare(line)) + return '\n'.join(res) - self.append(''' - mounts=() - umounts() { - for i in "${mounts[@]}"; do - umount $i - mounts=("${mounts[@]/$i}") - done - buildah unmount $ctr - trap - 0 - } - trap umounts 0 - ctr=$(buildah from $base) - mnt=$(buildah mount $ctr) - ''') + async def unshare(self, line): + print('+ buildah unshare ' + line) + proc = await asyncio.create_subprocess_shell( + 'buildah unshare ' + line, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ).decode('utf8') + stdout, stderr = await proc.communicate() + return stdout def config(self, line): - self.append(f'buildah config {line} $ctr') + self.append(f'buildah config {line} {self.ctr}') def _run(self, cmd, inject=False): user = self.container.variable('username') @@ -44,23 +45,29 @@ class BuildScript(Script): break if heredoc: - _cmd = ' '.join(['bash -eux <<__EOF\n', _cmd.strip(), '\n__EOF']) + _cmd = ''.join(['bash -eux <<__EOF\n', _cmd.strip(), '\n__EOF']) if cmd.startswith('sudo '): - return f'buildah run --user root $ctr -- {_cmd}' + return f'buildah run --user root {self.ctr} -- {_cmd}' elif user and self.container.variable('user_created'): - return f'buildah run --user {user} $ctr -- {_cmd}' + return f'buildah run --user {user} {self.ctr} -- {_cmd}' else: - return f'buildah run $ctr -- {_cmd}' + return f'buildah run {self.ctr} -- {_cmd}' def run(self, cmd): self.append(self._run(cmd)) def copy(self, src, dst): - self.append(f'buildah copy $ctr {src} {dst}') + self.append(f'buildah copy {self.ctr} {src} {dst}') def mount(self, src, dst): self.run('sudo mkdir -p ' + dst) self.append('mkdir -p ' + src) - self.append(f'mount -o bind {src} $mnt{dst}') - self.append('mounts=("$mnt' + dst + '" "${mounts[@]}")') + self.append(f'mount -o bind {src} {self.mnt}{dst}') + #self.append('mounts=("$mnt' + dst + '" "${mounts[@]}")') + self.mounts.append((src, dst)) + + def umounts(self): + for src, dst in self.mounts: + self.append('buildah unmount ' + dst) + self.append('buildah unmount ' + self.ctr) diff --git a/podctl/console_script.py b/podctl/console_script.py index dbc09c6..f4ab7ad 100644 --- a/podctl/console_script.py +++ b/podctl/console_script.py @@ -42,42 +42,20 @@ async def build(*services, **kwargs): else: services = console_script.pod.services + loop = asyncio.events.get_event_loop() + debug = console_script.parser.options.get('debug', False) + def protocol_factory(): + return BuildStreamProtocol( + service, + limit=asyncio.streams._DEFAULT_LIMIT, + loop=loop, + ) + for name, service in services.items(): container = service.container if not container.variable('base'): continue - - script = f'.podctl_build_{name}.sh' - with open(script, 'w+') as f: - f.write(str(container.script('build'))) - - loop = asyncio.events.get_event_loop() - - def protocol_factory(): - return BuildStreamProtocol( - service, - limit=asyncio.streams._DEFAULT_LIMIT, - loop=loop, - ) - - if os.getenv('BUILDAH_ISOLATION') == 'chroot': - prefix = '' - else: - prefix = 'buildah unshare ' - x = 'x' if console_script.parser.options.get('debug', False) else '' - transport, protocol = await loop.subprocess_shell( - protocol_factory, - prefix + f'bash -eu{x} {script}', - ) - print('+ ' + prefix + f' bash -eux {script}') - procs.append(asyncio.subprocess.Process( - transport, - protocol, - loop, - )) - - for proc in procs: - await proc.communicate() + await container.build(loop, protocol_factory) for proc in procs: if proc.returncode != 0: diff --git a/podctl/container.py b/podctl/container.py index 37eaecc..a3b4281 100644 --- a/podctl/container.py +++ b/podctl/container.py @@ -1,8 +1,40 @@ -from .build import BuildScript +import os + +from .build import Build from .visitable import Visitable class Container(Visitable): default_scripts = dict( - build=BuildScript, + build=Build, ) + + def script(self, name): + self.packages = [] + for visitor in self.visitors: + self.packages += getattr(visitor, 'packages', []) + return super().script(name) + + def script_run(self, name, debug): + script = f'.podctl_build_{name}.sh' + with open(script, 'w+') as f: + f.write(str(self.script('build'))) + + if os.getenv('BUILDAH_ISOLATION') == 'chroot': + prefix = '' + else: + prefix = 'buildah unshare ' + + x = 'x' if debug else '' + return prefix + f'bash -eu{x} {script}' + + async def build(self, loop, protocol_factory): + transport, protocol = await loop.subprocess_shell( + protocol_factory, + cmd, + ) + await asyncio.subprocess.Process( + transport, + protocol, + loop, + ).communicate()