sam/shlax
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate the whole thing from bash script generation to async processes

Browse Source
This commit is contained in:
jpic
2020-02-10 17:49:14 +01:00
parent dc92b484e8
commit f52cc8971a
11 changed files with 159 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files
podctl/visitors/base.py
+6 -6
View File
@@ -1,3 +1,4 @@
from pathlib import Path
class Base:
@@ -5,9 +6,8 @@ class Base:
self.base = base
async def init_build(self, script):
ctr = await script.cmd('buildah from ' + self.base)
stdout, stderr = await ctr.communicate()
script.ctr = stdout.decode('utf8').strip()
mnt = await script.cmd('buildah mount ' + script.ctr)
stdout, stderr = await mnt.communicate()
script.mnt = stdout.decode('utf8').strip()
script.ctr = Path((await script.exec('buildah', 'from', self.base)).out)
script.mnt = Path((await script.exec('buildah', 'mount', script.ctr)).out)
async def post_build(self, script):
await script.umounts()
podctl/visitors/commit.py
+18 -11
View File
@@ -43,27 +43,34 @@ class Commit:
self.tags = [t for t in self.tags if t is not None]
async def post_build(self, script):
await script.append(f'''
umounts
buildah commit --format={self.format} $ctr {self.repo}
''')
await script.exec(
'buildah',
'commit',
'--format=' + self.format,
script.ctr,
)
if 'master' in self.tags:
self.tags.append('latest')
if self.tags:
tags = ' '.join([f'{self.repo}:{tag}' for tag in self.tags])
await script.append(f'buildah tag {self.repo} {tags}')
await script.run('buildah', 'tag', self.repo, ' '.join(tags))
if self.push:
user = os.getenv('DOCKER_USER')
passwd = os.getenv('DOCKER_PASS')
if user and passwd and os.getenv('CI') and self.registry:
subprocess.check_call([
'podman', 'login',
'-u', user, '-p', passwd,
self.registry
])
await script.exec(
'podman',
'login',
'-u',
user,
'-p',
passwd,
self.registry,
)
for tag in self.tags:
await script.append(f'podman push {self.repo}:{tag}')
await script.run('podman', 'push', f'{self.repo}:{tag}')
await script.umount()
podctl/visitors/packages.py
+67 -39
View File
@@ -1,3 +1,7 @@
import asyncio
from datetime import datetime
from glob import glob
import os
import subprocess
from textwrap import dedent
@@ -6,24 +10,24 @@ from textwrap import dedent
class Packages:
mgrs = dict(
apk=dict(
update='sudo apk update',
upgrade='sudo apk upgrade',
install='sudo apk add',
update='apk update',
upgrade='apk upgrade',
install='apk add',
),
apt=dict(
update='sudo apt-get -y update',
upgrade='sudo apt-get -y upgrade',
install='sudo apt-get -y --no-install-recommends install',
update='apt-get -y update',
upgrade='apt-get -y upgrade',
install='apt-get -y --no-install-recommends install',
),
dnf=dict(
update='sudo dnf update',
upgrade='sudo dnf upgrade --exclude container-selinux --best --assumeyes', # noqa
install='sudo dnf install --exclude container-selinux --setopt=install_weak_deps=False --best --assumeyes', # noqa
update='dnf update',
upgrade='dnf upgrade --exclude container-selinux --best --assumeyes', # noqa
install='dnf install --exclude container-selinux --setopt=install_weak_deps=False --best --assumeyes', # noqa
),
yum=dict(
update='sudo yum update',
upgrade='sudo yum upgrade',
install='sudo yum install',
update='yum update',
upgrade='yum upgrade',
install='yum install',
),
)
@@ -32,25 +36,22 @@ class Packages:
dedent(l).strip().replace('\n', ' ') for l in packages
])
self.mgr = kwargs.pop('mgr') if 'mgr' in kwargs else None
if 'CACHE_DIR' in os.environ:
self.cache = os.path.join(os.getenv('CACHE_DIR'), self.mgr)
else:
self.cache = os.path.join(os.getenv('HOME'), '.cache', self.mgr)
async def pre_build(self, script):
base = script.container.variable('base')
if self.mgr:
self.cmds = self.mgrs[self.mgr]
@property
def cache(self):
if 'CACHE_DIR' in os.environ:
return os.path.join(os.getenv('CACHE_DIR'), self.mgr)
else:
for mgr, cmds in self.mgrs.items():
cmd = ['podman', 'run', base, 'sh', '-c', f'type {mgr}']
try:
subprocess.check_call(cmd)
return os.path.join(os.getenv('HOME'), '.cache', self.mgr)
async def init_build(self, script):
paths = ('bin', 'sbin', 'usr/bin', 'usr/sbin')
for mgr, cmds in self.mgrs.items():
for path in paths:
if (script.mnt / path / mgr).exists():
self.mgr = mgr
self.cmds = cmds
break
except subprocess.CalledProcessError:
continue
if not self.mgr:
raise Exception('Packages does not yet support this distro')
@@ -60,35 +61,61 @@ class Packages:
await getattr(self, self.mgr + '_setup')(script)
# first run on container means inject visitor packages
self.packages += script.container.packages
await script.run(self.cmds['upgrade'])
await script.cexec(self.cmds['upgrade'])
script.container._packages_upgraded = True
await script.run(' '.join([self.cmds['install']] + self.packages))
await script.cexec(' '.join([self.cmds['install']] + self.packages))
async def apk_setup(self, script):
await script.mount(self.cache, f'/var/cache/{self.mgr}')
# special step to enable apk cache
await script.run('ln -s /var/cache/apk /etc/apk/cache')
await script.append(f'''
old="$(find {self.cache} -name APKINDEX.* -mtime +3)"
if [ -n "$old" ] || ! ls .cache/apk/APKINDEX.*; then
{script._run(self.cmds['update'])}
else
echo Cache recent enough, skipping index update.
fi
''')
await script.cexec('ln -s /var/cache/apk /etc/apk/cache')
# do we have to update ?
update = False
for f in glob(self.cache + '/APKINDEX*'):
mtime = os.stat(f).st_mtime
now = int(datetime.now().strftime('%s'))
# expect hacker to have internet at least once a week
if now - mtime > 604800:
update = True
break
else:
update = True
if update:
await self.apk_update(script)
async def apk_update(self, script):
while os.path.exists(self.cache + '/update'):
print(f'{script.container.name} | Waiting for update ...')
await asyncio.sleep(1)
return # update was done by another job
with open(self.cache + '/update', 'w+') as f:
f.write(str(os.getpid()))
try:
await script.cexec(self.cmds['update'])
except:
raise
finally:
os.unlink(self.cache + '/update')
async def dnf_setup(self, script):
await script.mount(self.cache, f'/var/cache/{self.mgr}')
await script.run('sh -c "echo keepcache=True >> /etc/dnf/dnf.conf"')
await script.run('echo keepcache=True >> /etc/dnf/dnf.conf')
async def apt_setup(self, script):
cache = self.cache + '/$(source $mnt/etc/os-release; echo $VERSION_CODENAME)/' # noqa
await script.run('sudo rm /etc/apt/apt.conf.d/docker-clean')
await script.run('rm /etc/apt/apt.conf.d/docker-clean')
cache_archives = os.path.join(self.cache, 'archives')
await script.mount(cache_archives, f'/var/cache/apt/archives')
cache_lists = os.path.join(self.cache, 'lists')
await script.mount(cache_lists, f'/var/lib/apt/lists')
await script.run(self.cmds['update'])
"""
await script.append(f'''
old="$(find {cache_lists} -name lastup -mtime +3)"
if [ -n "$old" ] || ! ls {cache_lists}/lastup; then
@@ -99,3 +126,4 @@ class Packages:
echo Cache recent enough, skipping index update.
fi
''')
"""
podctl/visitors/user.py
+1 -1
View File
@@ -14,7 +14,7 @@ class User:
self.user_created = False
async def build(self, script):
await script.append(f'''
await script.run(f'''
if {script._run('id ' + str(self.uid))}; then
i=$({script._run('id -gn ' + str(self.uid))})
{script._run('usermod -d ' + self.home + ' -l ' + self.username + ' $i')}