sam/shlax
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate the whole thing from bash script generation to async processes

Browse Source
This commit is contained in:
jpic 2020-02-10 17:49:14 +01:00
parent dc92b484e8
commit f52cc8971a
11 changed files with 159 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
examples/django.py
View File

@ -1,65 +0,0 @@
import os
from podctl import *
class Django(Container):
tag = 'yourlabs/crudlfap'
base = 'alpine'
packages = [
'bash',
'python3',
switch(dev='vim'),
]
env = dict(FOO='bar')
annotations = dict(test='foo')
labels = dict(foo='test')
cmd = 'bash'
entrypoint = ['bash', '-v']
ports = [1234]
user = dict(
shell='/bin/bash',
name='app',
home='/app',
id=os.getenv('SUDO_ID', os.getenv('UID')),
)
volumes = [
'/bydir',
'byname:/byname',
switch(dev='.:/app'),
]
build = [
'sudo pip3 install --upgrade pip',
'pip3 install --user -e /app',
]
workdir = '/app'
django = Container(
Base('alpine'),
Packages('bash', switch(dev='vim')),
User(
uid=1000,
home='/app',
directories=('log', 'spooler', 'static')
),
switch(default=Mount('.', '/app'), production=Copy('.', '/app')),
Npm('build'),
Env('PATH', 'PATH=/app/node_modules/.bin:$PATH'),
Pip('requirements.txt'),
Env('PATH', 'PATH=$HOME/.local/bin:$PATH'),
switch(dev=Run('''
manage.py collectstatic --noinput --clear
find frontend/static/dist/css -type f | xargs gzip -f -k -9
find frontend/static/dist/js -type f | xargs gzip -f -k -9
''')),
Expose(8000),
Tag('equisafe'),
)
pod = Pod(
Service('django', django, restart='unless-stopped'),
Service('db',
Container(Tag('postgresql:latest')),
restart='unless-stopped'
),
)

109
podctl/build.py
View File

@ -2,16 +2,14 @@ import asyncio
import os
import asyncio
import signal
import shlex
import subprocess
import textwrap
from .proc import Proc
from .script import Script
class WrongResult(Exception):
pass
class Build(Script):
def __init__(self, container):
super().__init__()
@ -19,95 +17,40 @@ class Build(Script):
self.log = []
self.mounts = dict()
async def cmd(self, line):
log = dict(cmd=line)
self.log.append(log)
line = self.unshare(line)
print(self.container.name + ' | ' + line)
def protocol_factory():
from .console_script import BuildStreamProtocol
return BuildStreamProtocol(
self.container,
limit=asyncio.streams._DEFAULT_LIMIT,
loop=self.loop,
)
transport, protocol = await self.loop.subprocess_shell(
protocol_factory,
line,
)
proc = asyncio.subprocess.Process(
transport,
protocol,
self.loop,
)
result = await proc.wait()
if result:
raise WrongResult(proc)
return proc
'''
proc = await asyncio.create_subprocess_shell(
line,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
'''
async def append(self, value):
for line in value.split('\n'):
if line.startswith('#') or not line.strip():
continue
log = dict(proc=await self.cmd(line))
log['stdout'], log['stderr'] = await log['proc'].communicate()
return log['stdout']
def unshare(self, line):
return 'buildah unshare ' + line
async def config(self, line):
return await self.append(f'buildah config {line} {self.ctr}')
def _run(self, cmd, inject=False):
user = self.container.variable('username')
_cmd = textwrap.dedent(cmd)
if cmd.startswith('sudo '):
_cmd = _cmd[5:]
heredoc = False
for i in ('\n', '>', '<', '|', '&'):
if i in _cmd:
heredoc = True
break
if heredoc:
_cmd = ''.join(['bash -eux <<__EOF\n', _cmd.strip(), '\n__EOF'])
if cmd.startswith('sudo '):
return f'buildah run --user root {self.ctr} -- {_cmd}'
elif user and self.container.variable('user_created'):
return f'buildah run --user {user} {self.ctr} -- {_cmd}'
else:
return f'buildah run {self.ctr} -- {_cmd}'
async def run(self, cmd):
return await self.append(self._run(cmd))
async def copy(self, src, dst):
return await self.append(f'buildah copy {self.ctr} {src} {dst}')
async def exec(self, *args, **kwargs):
kwargs.setdefault('prefix', self.container.name)
proc = await Proc(*args, **kwargs)()
if kwargs.get('wait', True):
await proc.wait()
return proc
async def cexec(self, *args, user=None, **kwargs):
_args = ['buildah', 'run', self.ctr]
if user:
_args += ['--user', user]
_args += ['--', 'sh', '-euc']
return await self.exec(*(_args + list(args)))
async def mount(self, src, dst):
await self.run('sudo mkdir -p ' + dst)
await self.append('mkdir -p ' + src)
await self.append(f'mount -o bind {src} {self.mnt}{dst}')
#await self.append('mounts=("$mnt' + dst + '" "${mounts[@]}")')
self.mounts[dst] = src
target = self.mnt / str(dst)[1:]
await self.exec(f'mkdir -p {src} {target}')
await self.exec(f'mount -o bind {src} {target}')
self.mounts[src] = dst
async def umounts(self):
for src, dst in self.mounts:
await self.append('buildah unmount ' + dst)
await self.append('buildah unmount ' + self.ctr)
for src, dst in self.mounts.items():
await self.exec('umount', self.mnt / str(dst)[1:])
async def umount(self):
await self.exec(f'buildah unmount {self.ctr}')
def which(self, cmd):
for path in self.container.paths:
if os.path.exists(os.path.join(path, cmd)):
if os.path.exists(os.path.join(self.mnt, path[1:], cmd)):
return True

42
podctl/console_script.py
View File

@ -10,29 +10,20 @@ import sys
from .container import Container
from .pod import Pod
from .proc import WrongResult
from .service import Service
class BuildStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol):
def __init__(self, service, *args, **kwargs):
self.service = service
super().__init__(*args, **kwargs)
def pipe_data_received(self, fd, data):
if fd in (1, 2):
for line in data.split(b'\n'):
if not line:
continue
sys.stdout.buffer.write(
self.service.name.encode('utf8') + b' | ' + line + b'\n'
)
sys.stdout.flush()
super().pipe_data_received(fd, data)
@cli2.option('debug', help='Print debug output', color=cli2.GREEN, alias='d')
async def build(*services, **kwargs):
procs = []
async def build(*services_or_flags, **kwargs):
flags = []
services = []
for arg in services_or_flags:
if arg.startswith('-') or arg.startswith('+'):
flags.append(arg)
else:
services.append(arg)
if services:
services = {
k: v
@ -42,14 +33,17 @@ async def build(*services, **kwargs):
else:
services = console_script.pod.services
a = []
loop = asyncio.events.get_event_loop()
procs = []
asyncio.events.get_event_loop()
for name, service in services.items():
service.container.name = name
a.append(service.container.script('build', loop))
service.container.flags = flags
procs.append(service.container.script('build', flags))
result = await asyncio.gather(*a, return_exceptions=False)
print(result)
try:
result = await asyncio.gather(*procs)
except WrongResult:
sys.exit(1)
class ConsoleScript(cli2.ConsoleScript):

47
podctl/container.py
View File

@ -1,36 +1,11 @@
import asyncio
import os
import shlex
from .build import Build
from .visitable import Visitable
class BuildStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol):
def __init__(self, service, *args, **kwargs):
self.service = service
super().__init__(*args, **kwargs)
def pipe_data_received(self, fd, data):
if fd in (1, 2):
for line in data.split(b'\n'):
if not line:
continue
sys.stdout.buffer.write(
self.service.name.encode('utf8') + b' | ' + line + b'\n'
)
sys.stdout.flush()
super().pipe_data_received(fd, data)
def protocol_factory():
loop = asyncio.events.get_event_loop()
return BuildStreamProtocol(
service,
limit=asyncio.streams._DEFAULT_LIMIT,
loop=loop,
)
class Container(Visitable):
default_scripts = dict(
build=Build,
@ -42,22 +17,14 @@ class Container(Visitable):
'/usr/sbin',
]
async def script(self, name, loop):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.log = []
async def script(self, name, flags, loop=None):
self.loop = loop or asyncio.events.get_event_loop()
self.packages = []
for visitor in self.visitors:
self.packages += getattr(visitor, 'packages', [])
result = await super().script(name, loop)
return result
def script_run(self, name, debug):
script = f'.podctl_build_{name}.sh'
with open(script, 'w+') as f:
f.write(str(self.script('build')))
if os.getenv('BUILDAH_ISOLATION') == 'chroot':
prefix = ''
else:
prefix = 'buildah unshare '
x = 'x' if debug else ''
return prefix + f'bash -eu{x} {script}'

12
podctl/script.py
View File

@ -1,13 +1,5 @@
import textwrap
class Script(list):
def __str__(self):
if not getattr(self, '_postconfig', False):
if hasattr(self, 'post_config'):
self.post_config()
self._postconfig = True
return '\n'.join([
textwrap.dedent(line.lstrip('\n')).strip()
for line in self
])
class Script:
pass

14
podctl/visitable.py
View File

@ -15,7 +15,14 @@ class Visitable:
script.loop = loop
results = []
for prefix in ('init_', 'pre_', '', 'post_'):
async def clean():
for visitor in self.visitors:
if hasattr(visitor, 'clean_' + name):
result = getattr(visitor, 'clean_' + name)(self)
if result:
await result
for prefix in ('init_', 'pre_', '', 'post_', 'clean_'):
method = prefix + name
for visitor in self.visitors:
if not hasattr(visitor, method):
@ -25,8 +32,11 @@ class Visitable:
print(self.name + ' | ', type(visitor).__name__, method, rep)
result = getattr(visitor, method)(script)
if result:
try:
await result
except Exception as e:
await clean()
raise
def visitor(self, name):
for visitor in self.visitors:

12
podctl/visitors/base.py
View File

@ -1,3 +1,4 @@
from pathlib import Path
class Base:
@ -5,9 +6,8 @@ class Base:
self.base = base
async def init_build(self, script):
ctr = await script.cmd('buildah from ' + self.base)
stdout, stderr = await ctr.communicate()
script.ctr = stdout.decode('utf8').strip()
mnt = await script.cmd('buildah mount ' + script.ctr)
stdout, stderr = await mnt.communicate()
script.mnt = stdout.decode('utf8').strip()
script.ctr = Path((await script.exec('buildah', 'from', self.base)).out)
script.mnt = Path((await script.exec('buildah', 'mount', script.ctr)).out)
async def post_build(self, script):
await script.umounts()

29
podctl/visitors/commit.py
View File

@ -43,27 +43,34 @@ class Commit:
self.tags = [t for t in self.tags if t is not None]
async def post_build(self, script):
await script.append(f'''
umounts
buildah commit --format={self.format} $ctr {self.repo}
''')
await script.exec(
'buildah',
'commit',
'--format=' + self.format,
script.ctr,
)
if 'master' in self.tags:
self.tags.append('latest')
if self.tags:
tags = ' '.join([f'{self.repo}:{tag}' for tag in self.tags])
await script.append(f'buildah tag {self.repo} {tags}')
await script.run('buildah', 'tag', self.repo, ' '.join(tags))
if self.push:
user = os.getenv('DOCKER_USER')
passwd = os.getenv('DOCKER_PASS')
if user and passwd and os.getenv('CI') and self.registry:
subprocess.check_call([
'podman', 'login',
'-u', user, '-p', passwd,
self.registry
])
await script.exec(
'podman',
'login',
'-u',
user,
'-p',
passwd,
self.registry,
)
for tag in self.tags:
await script.append(f'podman push {self.repo}:{tag}')
await script.run('podman', 'push', f'{self.repo}:{tag}')
await script.umount()

104
podctl/visitors/packages.py
View File

@ -1,3 +1,7 @@
import asyncio
from datetime import datetime
from glob import glob
import os
import subprocess
from textwrap import dedent
@ -6,24 +10,24 @@ from textwrap import dedent
class Packages:
mgrs = dict(
apk=dict(
update='sudo apk update',
upgrade='sudo apk upgrade',
install='sudo apk add',
update='apk update',
upgrade='apk upgrade',
install='apk add',
),
apt=dict(
update='sudo apt-get -y update',
upgrade='sudo apt-get -y upgrade',
install='sudo apt-get -y --no-install-recommends install',
update='apt-get -y update',
upgrade='apt-get -y upgrade',
install='apt-get -y --no-install-recommends install',
),
dnf=dict(
update='sudo dnf update',
upgrade='sudo dnf upgrade --exclude container-selinux --best --assumeyes', # noqa
install='sudo dnf install --exclude container-selinux --setopt=install_weak_deps=False --best --assumeyes', # noqa
update='dnf update',
upgrade='dnf upgrade --exclude container-selinux --best --assumeyes', # noqa
install='dnf install --exclude container-selinux --setopt=install_weak_deps=False --best --assumeyes', # noqa
),
yum=dict(
update='sudo yum update',
upgrade='sudo yum upgrade',
install='sudo yum install',
update='yum update',
upgrade='yum upgrade',
install='yum install',
),
)
@ -32,25 +36,22 @@ class Packages:
dedent(l).strip().replace('\n', ' ') for l in packages
])
self.mgr = kwargs.pop('mgr') if 'mgr' in kwargs else None
if 'CACHE_DIR' in os.environ:
self.cache = os.path.join(os.getenv('CACHE_DIR'), self.mgr)
else:
self.cache = os.path.join(os.getenv('HOME'), '.cache', self.mgr)
async def pre_build(self, script):
base = script.container.variable('base')
if self.mgr:
self.cmds = self.mgrs[self.mgr]
@property
def cache(self):
if 'CACHE_DIR' in os.environ:
return os.path.join(os.getenv('CACHE_DIR'), self.mgr)
else:
return os.path.join(os.getenv('HOME'), '.cache', self.mgr)
async def init_build(self, script):
paths = ('bin', 'sbin', 'usr/bin', 'usr/sbin')
for mgr, cmds in self.mgrs.items():
cmd = ['podman', 'run', base, 'sh', '-c', f'type {mgr}']
try:
subprocess.check_call(cmd)
for path in paths:
if (script.mnt / path / mgr).exists():
self.mgr = mgr
self.cmds = cmds
break
except subprocess.CalledProcessError:
continue
if not self.mgr:
raise Exception('Packages does not yet support this distro')
@ -60,35 +61,61 @@ class Packages:
await getattr(self, self.mgr + '_setup')(script)
# first run on container means inject visitor packages
self.packages += script.container.packages
await script.run(self.cmds['upgrade'])
await script.cexec(self.cmds['upgrade'])
script.container._packages_upgraded = True
await script.run(' '.join([self.cmds['install']] + self.packages))
await script.cexec(' '.join([self.cmds['install']] + self.packages))
async def apk_setup(self, script):
await script.mount(self.cache, f'/var/cache/{self.mgr}')
# special step to enable apk cache
await script.run('ln -s /var/cache/apk /etc/apk/cache')
await script.append(f'''
old="$(find {self.cache} -name APKINDEX.* -mtime +3)"
if [ -n "$old" ] || ! ls .cache/apk/APKINDEX.*; then
{script._run(self.cmds['update'])}
else
echo Cache recent enough, skipping index update.
fi
''')
await script.cexec('ln -s /var/cache/apk /etc/apk/cache')
# do we have to update ?
update = False
for f in glob(self.cache + '/APKINDEX*'):
mtime = os.stat(f).st_mtime
now = int(datetime.now().strftime('%s'))
# expect hacker to have internet at least once a week
if now - mtime > 604800:
update = True
break
else:
update = True
if update:
await self.apk_update(script)
async def apk_update(self, script):
while os.path.exists(self.cache + '/update'):
print(f'{script.container.name} | Waiting for update ...')
await asyncio.sleep(1)
return # update was done by another job
with open(self.cache + '/update', 'w+') as f:
f.write(str(os.getpid()))
try:
await script.cexec(self.cmds['update'])
except:
raise
finally:
os.unlink(self.cache + '/update')
async def dnf_setup(self, script):
await script.mount(self.cache, f'/var/cache/{self.mgr}')
await script.run('sh -c "echo keepcache=True >> /etc/dnf/dnf.conf"')
await script.run('echo keepcache=True >> /etc/dnf/dnf.conf')
async def apt_setup(self, script):
cache = self.cache + '/$(source $mnt/etc/os-release; echo $VERSION_CODENAME)/' # noqa
await script.run('sudo rm /etc/apt/apt.conf.d/docker-clean')
await script.run('rm /etc/apt/apt.conf.d/docker-clean')
cache_archives = os.path.join(self.cache, 'archives')
await script.mount(cache_archives, f'/var/cache/apt/archives')
cache_lists = os.path.join(self.cache, 'lists')
await script.mount(cache_lists, f'/var/lib/apt/lists')
await script.run(self.cmds['update'])
"""
await script.append(f'''
old="$(find {cache_lists} -name lastup -mtime +3)"
if [ -n "$old" ] || ! ls {cache_lists}/lastup; then
@ -99,3 +126,4 @@ class Packages:
echo Cache recent enough, skipping index update.
fi
''')
"""

2
podctl/visitors/user.py
View File

@ -14,7 +14,7 @@ class User:
self.user_created = False
async def build(self, script):
await script.append(f'''
await script.run(f'''
if {script._run('id ' + str(self.uid))}; then
i=$({script._run('id -gn ' + str(self.uid))})
{script._run('usermod -d ' + self.home + ' -l ' + self.username + ' $i')}

2
tests/test_build.py
View File

@ -3,7 +3,7 @@ import os
import sys
from podctl.container import Container
from podctl.build import BuildScript
from podctl.build import Build
from podctl.visitors import (
Base,
Copy,