Package action working both on SSH host and Buildah guest now

shlax/actions/packages.py

@@ -37,11 +37,13 @@ class Packages:
             update='pacman -Sy',
             upgrade='pacman -Su --noconfirm',
             install='pacman -S --noconfirm',
+            lastupdate='stat -c %Y /var/lib/pacman/sync/core.db',
         ),
         dnf=dict(
             update='dnf makecache --assumeyes',
             upgrade='dnf upgrade --best --assumeyes --skip-broken',  # noqa
             install='dnf install --setopt=install_weak_deps=False --best --assumeyes',  # noqa
+            lastupdate='stat -c %Y /var/cache/dnf/* | head -n1',
         ),
         yum=dict(
             update='yum update',
@@ -52,57 +54,52 @@ class Packages:
 
     installed = []
 
-    def __init__(self, *packages, upgrade=True):
+    def __init__(self, *packages, upgrade=False):
         self.packages = []
         self.upgrade = upgrade
         for package in packages:
             line = dedent(package).strip().replace('\n', ' ')
             self.packages += line.split(' ')
 
-    @property
+    async def cache_setup(self, target):
-    def cache_root(self):
         if 'CACHE_DIR' in os.environ:
-            return os.path.join(os.getenv('CACHE_DIR'))
+            self.cache_root = os.path.join(os.getenv('CACHE_DIR'))
         else:
-            return os.path.join(os.getenv('HOME'), '.cache')
+            self.cache_root = os.path.join(await target.parent.getenv('HOME'), '.cache')
-
-    async def update(self, target):
-        if not target.islocal:
-            return await target.rexec(self.cmds['update'])
 
         # run pkgmgr_setup functions ie. apk_setup
-        cachedir = await getattr(self, self.mgr + '_setup')(target)
+        await getattr(self, self.mgr + '_setup')(target)
 
+    async def update(self, target):
+        # lastupdate = await target.exec(self.cmds['lastupdate'], raises=False)
+        # lastupdate = int(lastupdate.out) if lastupdate.rc == 0 else None
         lastupdate = None
-        if os.path.exists(cachedir + '/lastupdate'):
-            with open(cachedir + '/lastupdate', 'r') as f:
-                try:
-                    lastupdate = int(f.read().strip())
-                except:
-                    pass
-
-        if not os.path.exists(cachedir):
-            os.makedirs(cachedir)
-
         now = int(datetime.now().strftime('%s'))
-        # cache for a week
+        if not lastupdate or now - lastupdate > 604800:
+            await target.rexec(self.cmds['update'])
+
+        return
+
+        # disabling with the above return call until needed again
+        # might have to rewrite this to not have our own lockfile
+        # or find a better place on the filesystem
+        # also make sure the lockfile is actually needed when running on
+        # targets that don't have isguest=True
         if not lastupdate or now - lastupdate > 604800:
             # crude lockfile implementation, should work against *most*
             # race-conditions ...
             lockfile = cachedir + '/update.lock'
-            if not os.path.exists(lockfile):
+            if not await target.parent.exists(lockfile):
-                with open(lockfile, 'w+') as f:
+                await target.parent.write(lockfile, str(os.getpid()))
-                    f.write(str(os.getpid()))
 
                 try:
                     await target.rexec(self.cmds['update'])
                 finally:
-                    os.unlink(lockfile)
+                    await target.parent.rm(lockfile)
 
-                with open(cachedir + '/lastupdate', 'w+') as f:
+                await target.parent.write(cachedir + '/lastupdate', str(now))
-                    f.write(str(now))
             else:
-                while os.path.exists(lockfile):
+                while await target.parent.exists(lockfile):
                     print(f'{self.target} | Waiting for {lockfile} ...')
                     await asyncio.sleep(1)
 
@@ -119,7 +116,13 @@ class Packages:
             raise Exception('Packages does not yet support this distro')
 
         self.cmds = self.mgrs[self.mgr]
+
+        if target.isguest:
+            # we're going to mount
+            await self.cache_setup(target)
+
         await self.update(target)
+
         if self.upgrade:
             await target.rexec(self.cmds['upgrade'])
 
@@ -150,19 +153,24 @@ class Packages:
         return cachedir
 
     async def apt_setup(self, target):
-        codename = (await self.rexec(
+        codename = (await target.rexec(
-            f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME'
+            f'source /etc/os-release; echo $VERSION_CODENAME'
         )).out
         cachedir = os.path.join(self.cache_root, self.mgr, codename)
         await self.rexec('rm /etc/apt/apt.conf.d/docker-clean')
         cache_archives = os.path.join(cachedir, 'archives')
-        await self.mount(cache_archives, f'/var/cache/apt/archives')
+        await target.mount(cache_archives, f'/var/cache/apt/archives')
         cache_lists = os.path.join(cachedir, 'lists')
-        await self.mount(cache_lists, f'/var/lib/apt/lists')
+        await target.mount(cache_lists, f'/var/lib/apt/lists')
         return cachedir
 
     async def pacman_setup(self, target):
-        return self.cache_root + '/pacman'
+        cachedir = os.path.join(self.cache_root, self.mgr)
+        await target.mkdir(cachedir + '/cache', cachedir + '/sync')
+        await target.mount(cachedir + '/sync', '/var/lib/pacman/sync')
+        await target.mount(cachedir + '/cache', '/var/cache/pacman')
+        if await target.host.exists('/etc/pacman.d/mirrorlist'):
+            await target.copy('/etc/pacman.d/mirrorlist', '/etc/pacman.d/mirrorlist')
 
     def __str__(self):
         return f'Packages({self.packages}, upgrade={self.upgrade})'

shlax/targets/base.py

@@ -11,13 +11,14 @@ from ..result import Result, Results
 
 
 class Target:
+    isguest = False
+
     def __init__(self, *actions, root=None):
         self.actions = actions
         self.results = []
         self.output = Output()
         self.parent = None
-        self.root = root or os.getcwd()
+        self.root = root or ''
-        self.islocal = getattr(self, 'islocal', True)
 
     def __str__(self):
         return 'localhost'
@@ -131,20 +132,48 @@ class Target:
 
     @root.setter
     def root(self, value):
-        self._root = Path(value or os.getcwd())
+        self._root = Path(value) if value else ''
+
+    @property
+    def host(self):
+        current = self
+        while current.isguest:
+            current = self.parent
+        return current
 
     def path(self, path):
+        if not self.root:
+            return path
         if str(path).startswith('/'):
             path = str(path)[1:]
-        return self.root / path
+        return str(self.root / path)
 
-    async def mkdir(self, path):
+    async def mkdir(self, *paths):
         if '_mkdir' not in self.__dict__:
             self._mkdir = []
-        path = str(path)
+
-        if path not in self._mkdir:
+        make = [str(path) for path in paths if str(path) not in self._mkdir]
-            await self.exec('mkdir', '-p', path)
+        if make:
-            self._mkdir.append(path)
+            await self.exec('mkdir', '-p', *make)
+            self._mkdir += make
 
     async def copy(self, *args):
         return await self.exec('cp', '-a', *args)
+
+    async def exists(self, path):
+        return (await self.exec('ls ' + self.path(path), raises=False)).rc == 0
+
+    async def read(self, path):
+        return (await self.exec('cat', self.path(path))).out
+
+    async def write(self, path, content):
+        return await self.exec('echo ' + content + ' > ' + self.path(path))
+
+    async def rm(self, path):
+        return await self.exec('rm', self.path(path))
+
+    async def getenv(self, key):
+        return (await self.exec('echo $' + key)).out
+
+    async def getcwd(self):
+        return (await self.exec('pwd')).out

shlax/targets/buildah.py

@@ -14,6 +14,7 @@ from ..proc import Proc
 
 class Buildah(Target):
     """Build container image with buildah"""
+    isguest = True
 
     def __init__(self,
                  *actions,
@@ -200,8 +201,8 @@ class Buildah(Target):
 
         await self.parent.exec('buildah', 'tag', self.image_previous, *tags)
 
-    async def mkdir(self, path):
+    async def mkdir(self, *paths):
-        return await self.parent.mkdir(self.path(path))
+        return await self.parent.mkdir(*[self.path(path) for path in paths])
 
     async def copy(self, *args):
         return await self.parent.copy(*args[:-1], self.path(args[-1]))

shlax/targets/ssh.py

@@ -5,7 +5,6 @@ class Ssh(Target):
     def __init__(self, *actions, host, user=None):
         self.host = host
         self.user = user
-        self.islocal = False
         super().__init__(*actions)
 
     async def exec(self, *args, user=None, **kwargs):