Proper cache invalidation

shlax/image.py

@@ -74,5 +74,5 @@ class Image:
 
     def layer(self, key):
         layer = copy.deepcopy(self)
-        layer.tags = [key]
+        layer.tags = ['layer-' + key]
         return layer

shlax/output.py

@@ -144,11 +144,21 @@ class Output:
                 '\n',
             ]))
 
+    def info(self, text):
+        if self.debug is True or 'visit' in str(self.debug):
+            self(''.join([
+                self.colors['cyanbold'],
+                '➤   INFO  ',
+                self.colors['reset'],
+                text,
+                '\n',
+            ]))
+
     def skip(self, action):
         if self.debug is True or 'visit' in str(self.debug):
             self(''.join([
                 self.colors['yellowbold'],
-                '↪️ SKIP    ',
+                '↪️   SKIP  ',
                 self.colors['reset'],
                 self.colorized(action),
                 '\n',

shlax/targets/base.py

@@ -1,3 +1,4 @@
+import asyncio
 import copy
 from pathlib import Path
 import os

shlax/targets/buildah.py

@@ -1,3 +1,4 @@
+import asyncio
 import copy
 import hashlib
 import json
@@ -50,54 +51,74 @@ class Buildah(Target):
             os.execvp('buildah', ['buildah', 'unshare'] + sys.argv)
             # program has been replaced
 
-        actions_done = await self.cache_load(*actions)
+        layers = await self.layers()
+        keep = await self.cache_setup(layers, *actions)
+        keepnames = [*map(lambda x: 'localhost/' + str(x), keep)]
+        self.invalidate = [name for name in layers if name not in keepnames]
+        if self.invalidate:
+            self.output.info('Invalidating old layers')
+            await self.parent.exec(
+                'buildah', 'rmi', *self.invalidate, raises=False)
 
         if actions:
-            actions = actions[len(actions_done):]
+            actions = actions[len(keep):]
             if not actions:
-                self.clean = None
+                return self.uptodate()
-                self.output.success('Image up to date')
-                return
         else:
-            self.actions = self.actions[len(actions_done):]
+            self.actions = self.actions[len(keep):]
             if not self.actions:
-                self.clean = None
+                return self.uptodate()
-                self.output.success('Image up to date')
-                return
 
         self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out
         self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out)
-        await super().__call__(*actions)
 
-    async def images(self):
+        return await super().__call__(*actions)
-        result = await self.parent.exec(
+
-            'podman image list --format json',
+    def uptodate(self):
+        self.clean = None
+        self.output.success('Image up to date')
+        return
+
+    async def layers(self):
+        ret = set()
+        results = await self.parent.exec(
+            'buildah images --json',
             quiet=True,
         )
-        result = json.loads(result.out)
+        results = json.loads(results.out)
-        return [item for sublist in result for item in sublist['History']]
 
-    async def cache_load(self, *actions):
+        prefix = 'localhost/' + self.image.repository + ':layer-'
-        actions_done = []
+        for result in results:
+            if not result.get('names', None):
+                continue
+            for name in result['names']:
+                if name.startswith(prefix):
+                    ret.add(name)
+        return ret
+
+    async def cache_setup(self, layers, *actions):
+        keep = []
         self.image_previous = Image(self.base)
-        images = await self.images()
         for action in actions or self.actions:
-            action_image = self.action_image(action)
+            action_image = await self.action_image(action)
-            if 'localhost/' + str(action_image) in images:
+            name = 'localhost/' + str(action_image)
+            if name in layers:
                 self.base = self.image_previous = action_image
-                actions_done.append(action)
+                keep.append(action_image)
                 self.output.skip(f'Found valid cached layer for {action}')
             else:
                 break
-        return actions_done
+        return keep
 
-    def action_image(self, action):
+    async def action_image(self, action):
         if self.image_previous:
             prefix = self.image_previous.tags[0]
         else:
             prefix = self.base
-        if hasattr(action, 'cachehash'):
+        if hasattr(action, 'cachekey'):
-            action_key = action.cachehash()
+            action_key = action.cachekey()
+            if asyncio.iscoroutine(action_key):
+                action_key = str(await action_key)
         else:
             action_key = str(action)
         key = prefix + action_key
@@ -106,8 +127,14 @@ class Buildah(Target):
 
     async def action(self, action, reraise=False):
         result = await super().action(action, reraise)
-        action_image = self.action_image(action)
+        action_image = await self.action_image(action)
-        await self.commit(action_image)
+        await self.parent.exec(
+            'buildah',
+            'commit',
+            '--format=' + action_image.format,
+            self.ctr,
+            action_image,
+        )
         self.image_previous = action_image
         return result