wip

podctl/visitors/__init__.py

@@ -9,5 +9,6 @@ from .mount import Mount  # noqa
 from .packages import Packages  # noqa
 from .pip import Pip  # noqa
 from .run import Run  # noqa
-from .template import Template  # noqa
+from .template import Append, Template  # noqa
 from .user import User  # noqa
+from .uwsgi import uWSGI  # noqa

podctl/visitors/base.py

@@ -9,5 +9,7 @@ class Base:
         script.ctr = Path((await script.exec('buildah', 'from', self.base)).out)
         script.mnt = Path((await script.exec('buildah', 'mount', script.ctr)).out)
 
-    async def post_build(self, script):
+    async def clean_build(self, script):
         await script.umounts()
+        await script.umount()
+        proc = await script.exec('buildah', 'rm', script.ctr, raises=False)

podctl/visitors/commit.py

@@ -42,20 +42,24 @@ class Commit:
         # filter out tags which resolved to None
         self.tags = [t for t in self.tags if t is not None]
 
+        # default tag by default ...
+        if not self.tags:
+            self.tags = ['latest']
+
     async def post_build(self, script):
-        await script.exec(
+        self.sha = (await script.exec(
             'buildah',
             'commit',
             '--format=' + self.format,
             script.ctr,
-        )
+        )).out
 
         if 'master' in self.tags:
             self.tags.append('latest')
 
         if self.tags:
             tags = ' '.join([f'{self.repo}:{tag}' for tag in self.tags])
-            await script.run('buildah', 'tag', self.repo, ' '.join(tags))
+            await script.exec('buildah', 'tag', self.sha, self.repo, tags)
 
             if self.push:
                 user = os.getenv('DOCKER_USER')
@@ -72,5 +76,12 @@ class Commit:
                     )
 
                 for tag in self.tags:
-                    await script.run('podman', 'push', f'{self.repo}:{tag}')
+                    await script.exec('podman', 'push', f'{self.repo}:{tag}')
         await script.umount()
+
+    async def run(self, script):
+        await script.exec(
+            'podman', 'run', '-d',
+            '--name', script.container.name,
+            ':'.join((self.repo, self.tags[0])),
+        )

podctl/visitors/packages.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 
 from datetime import datetime
 from glob import glob
@@ -8,6 +9,14 @@ from textwrap import dedent
 
 
 class Packages:
+    """
+    The Packages visitor wraps around the container's package manager.
+
+    It's a central piece of the build process, and does iterate over other
+    container visitors in order to pick up packages. For example, the Pip
+    visitor will declare ``self.packages = dict(apt=['python3-pip'])``, and the
+    Packages visitor will pick it up.
+    """
     mgrs = dict(
         apk=dict(
             update='apk update',
@@ -31,99 +40,113 @@ class Packages:
         ),
     )
 
+    installed = []
+
     def __init__(self, *packages, **kwargs):
-        self.packages = list([
-            dedent(l).strip().replace('\n', ' ') for l in packages
-        ])
+        self.packages = []
+
+        for package in packages:
+            line = dedent(package).strip().replace('\n', ' ')
+            self.packages += line.split(' ')
+
         self.mgr = kwargs.pop('mgr') if 'mgr' in kwargs else None
 
     @property
-    def cache(self):
+    def cache_root(self):
         if 'CACHE_DIR' in os.environ:
-            return os.path.join(os.getenv('CACHE_DIR'), self.mgr)
+            return os.path.join(os.getenv('CACHE_DIR'))
         else:
-            return os.path.join(os.getenv('HOME'), '.cache', self.mgr)
+            return os.path.join(os.getenv('HOME'), '.cache')
 
     async def init_build(self, script):
-        paths = ('bin', 'sbin', 'usr/bin', 'usr/sbin')
-        for mgr, cmds in self.mgrs.items():
-            for path in paths:
-                if (script.mnt / path / mgr).exists():
+        cached = script.container.variable('mgr')
+        if cached:
+            self.mgr = cached
+        else:
+            for mgr, cmds in self.mgrs.items():
+                if await script.which(mgr):
                     self.mgr = mgr
-                    self.cmds = cmds
                     break
+
         if not self.mgr:
             raise Exception('Packages does not yet support this distro')
 
+        self.cmds = self.mgrs[self.mgr]
+
+    async def update(self, script):
+        # run pkgmgr_setup functions ie. apk_setup
+        cachedir = await getattr(self, self.mgr + '_setup')(script)
+
+        lastupdate = None
+        if os.path.exists(cachedir + '/lastupdate'):
+            with open(cachedir + '/lastupdate', 'r') as f:
+                try:
+                    lastupdate = int(f.read().strip())
+                except:
+                    pass
+
+        now = int(datetime.now().strftime('%s'))
+        # cache for a week
+        if not lastupdate or now - lastupdate > 604800:
+            # crude lockfile implementation, should work against *most*
+            # race-conditions ...
+            lockfile = cachedir + '/update.lock'
+            if not os.path.exists(lockfile):
+                with open(lockfile, 'w+') as f:
+                    f.write(str(os.getpid()))
+
+                try:
+                    await script.cexec(self.cmds['update'])
+                finally:
+                    os.unlink(lockfile)
+
+                with open(cachedir + '/lastupdate', 'w+') as f:
+                    f.write(str(now))
+            else:
+                while os.path.exists(lockfile):
+                    print(f'{script.container.name} | Waiting for update ...')
+                    await asyncio.sleep(1)
+
     async def build(self, script):
         if not getattr(script.container, '_packages_upgraded', None):
-            # run pkgmgr_setup functions ie. apk_setup
-            await getattr(self, self.mgr + '_setup')(script)
-            # first run on container means inject visitor packages
-            self.packages += script.container.packages
+            await self.update(script)
             await script.cexec(self.cmds['upgrade'])
-            script.container._packages_upgraded = True
 
-        await script.cexec(' '.join([self.cmds['install']] + self.packages))
+            # first run on container means inject visitor packages
+            packages = []
+            for visitor in script.container.visitors:
+                pp = getattr(visitor, 'packages', None)
+                if pp:
+                    if isinstance(pp, list):
+                        packages += pp
+                    elif self.mgr in pp:
+                        packages += pp[self.mgr]
+
+            script.container._packages_upgraded = True
+        else:
+            packages = self.packages
+
+        await script.crexec(*self.cmds['install'].split(' ') + packages)
 
     async def apk_setup(self, script):
-        await script.mount(self.cache, f'/var/cache/{self.mgr}')
+        cachedir = os.path.join(self.cache_root, self.mgr)
+        await script.mount(cachedir, '/var/cache/apk')
         # special step to enable apk cache
         await script.cexec('ln -s /var/cache/apk /etc/apk/cache')
-
-        # do we have to update ?
-        update = False
-        for f in glob(self.cache + '/APKINDEX*'):
-            mtime = os.stat(f).st_mtime
-            now = int(datetime.now().strftime('%s'))
-            # expect hacker to have internet at least once a week
-            if now - mtime > 604800:
-                update = True
-                break
-        else:
-            update = True
-
-        if update:
-            await self.apk_update(script)
-
-    async def apk_update(self, script):
-        while os.path.exists(self.cache + '/update'):
-            print(f'{script.container.name} | Waiting for update ...')
-            await asyncio.sleep(1)
-            return  # update was done by another job
-
-        with open(self.cache + '/update', 'w+') as f:
-            f.write(str(os.getpid()))
-
-        try:
-            await script.cexec(self.cmds['update'])
-        except:
-            raise
-        finally:
-            os.unlink(self.cache + '/update')
+        return cachedir
 
     async def dnf_setup(self, script):
         await script.mount(self.cache, f'/var/cache/{self.mgr}')
         await script.run('echo keepcache=True >> /etc/dnf/dnf.conf')
 
     async def apt_setup(self, script):
-        cache = self.cache + '/$(source $mnt/etc/os-release; echo $VERSION_CODENAME)/'  # noqa
-        await script.run('rm /etc/apt/apt.conf.d/docker-clean')
-        cache_archives = os.path.join(self.cache, 'archives')
+        codename = (await script.exec(
+            f'source {script.mnt}/etc/os-release; echo $VERSION_CODENAME'
+        )).out
+        cachedir = os.path.join(self.cache_root, self.mgr, codename)
+        await script.cexec('rm /etc/apt/apt.conf.d/docker-clean')
+        cache_archives = os.path.join(cachedir, 'archives')
         await script.mount(cache_archives, f'/var/cache/apt/archives')
-        cache_lists = os.path.join(self.cache, 'lists')
+        cache_lists = os.path.join(cachedir, 'lists')
         await script.mount(cache_lists, f'/var/lib/apt/lists')
-
-        await script.run(self.cmds['update'])
-        """
-        await script.append(f'''
-        old="$(find {cache_lists} -name lastup -mtime +3)"
-        if [ -n "$old" ] || ! ls {cache_lists}/lastup; then
-            until [ -z $(lsof /var/lib/dpkg/lock) ]; do sleep 1; done
-            {script._run(self.cmds['update'])}
-            touch {cache_lists}/lastup
-        else
-            echo Cache recent enough, skipping index update.
-        fi
-        ''')
-        """
+        return cachedir

podctl/visitors/pip.py

@@ -3,16 +3,19 @@ import os
 
 
 class Pip:
+    packages = dict(
+        apt=['python3-pip'],
+    )
+
     def __init__(self, *pip_packages, pip=None, requirements=None):
         self.pip_packages = pip_packages
-        self.pip = pip
+        #self.pip = pip
         self.requirements = requirements
 
     async def build(self, script):
-        for pip in ('pip3', 'pip', 'pip2'):
-            if script.which(pip):
-                self.pip = pip
-                break
+        self.pip = await script.which(('pip3', 'pip', 'pip2'))
+        if not self.pip:
+            raise Exception('Could not find pip command')
 
         if 'CACHE_DIR' in os.environ:
             cache = os.path.join(os.getenv('CACHE_DIR'), 'pip')
@@ -20,16 +23,27 @@ class Pip:
             cache = os.path.join(os.getenv('HOME'), '.cache', 'pip')
 
         await script.mount(cache, '/root/.cache/pip')
-        await script.run(f'sudo {self.pip} install --upgrade pip')
-        source = [p for p in self.pip_packages if p.startswith('/')]
+        await script.crexec(f'{self.pip} install --upgrade pip')
+
+        # https://github.com/pypa/pip/issues/5599
+        self.pip = 'python3 -m pip'
+
+        pip_packages = []
+        for visitor in script.container.visitors:
+            pp = getattr(visitor, 'pip_packages', None)
+            if not pp:
+                continue
+            pip_packages += pip_packages
+
+        source = [p for p in pip_packages if p.startswith('/')]
         if source:
-            await script.run(
-                f'sudo {self.pip} install --upgrade --editable {" ".join(source)}'
+            await script.crexec(
+                f'{self.pip} install --upgrade --editable {" ".join(source)}'
             )
 
-        nonsource = [p for p in self.pip_packages if not p.startswith('/')]
+        nonsource = [p for p in pip_packages if not p.startswith('/')]
         if nonsource:
-            await script.run(f'sudo {self.pip} install --upgrade {" ".join(source)}')
+            await script.crexec(f'{self.pip} install --upgrade {" ".join(nonsource)}')
 
         if self.requirements:
-            await script.run(f'sudo {self.pip} install --upgrade -r {self.requirements}')
+            await script.crexec(f'{self.pip} install --upgrade -r {self.requirements}')

podctl/visitors/template.py

@@ -1,11 +1,13 @@
 from textwrap import dedent
 
 
-CMD = '''cat <<EOF > {target}
-{script}
-EOF'''
-
 class Template:
+    CMD = dedent(
+        '''cat <<EOF > {target}
+        {script}
+        EOF'''
+    )
+
     def __init__(self, target, *lines, **variables):
         self.target = target
         self.lines = lines
@@ -15,6 +17,14 @@ class Template:
         self.script = '\n'.join([
             dedent(l).strip() for l in self.lines
         ]).format(**self.variables)
-        await script.run(CMD.strip().format(**self.__dict__))
+        await script.cexec(self.CMD.strip().format(**self.__dict__))
         if self.script.startswith('#!'):
-            await script.run('sudo chmod +x ' + self.target)
+            await script.cexec('chmod +x ' + self.target, user='root')
+
+
+class Append(Template):
+    CMD = dedent(
+        '''cat <<EOF >> {target}
+        {script}
+        EOF'''
+    )

podctl/visitors/user.py

@@ -3,25 +3,25 @@ from .packages import Packages
 
 class User:
     """Secure the image with a user"""
-    packages = [
-        'shadow',
-    ]
+    packages = dict(
+        apk=['shadow'],
+    )
 
-    def __init__(self, username, uid, home):
+    def __init__(self, username, uid, home, directories=None):
         self.username = username
         self.uid = uid
         self.home = home
         self.user_created = False
+        self.directories = directories
 
     async def build(self, script):
-        await script.run(f'''
-            if {script._run('id ' + str(self.uid))}; then
-                i=$({script._run('id -gn ' + str(self.uid))})
-                {script._run('usermod -d ' + self.home + ' -l ' + self.username + ' $i')}
-            else
-                {script._run('useradd -d ' + self.home + ' -u ' + str(self.uid) + ' ' + self.username)}
-            fi
-        ''')  # noqa
+        try:
+            await script.cexec('id', self.uid)
+        except:
+            await script.cexec('useradd', '-d', self.home, '-u', self.uid, ' ',
+                    self.username)
+        else:
+            await script.cexec('id', '-gn', self.uid)
         self.user_created = True
 
     def post_build(self, script):