Copy action: refactor, caching, filtering

shlax/actions/copy.py

@@ -1,19 +1,56 @@
+import asyncio
+import binascii
+import os
+
+
 class Copy:
     def __init__(self, *args):
         self.src = args[:-1]
         self.dst = args[-1]
 
-    @property
+    def listfiles(self):
-    def files(self):
+        if getattr(self, '_listfiles', None):
-        for root, dirs, files in os.walk(self.dst):
+            return self._listfiles
-            pass
 
+        result = []
+        for src in self.src:
+            if os.path.isfile(src):
+                result.append(src)
+                continue
+
+            for root, dirs, files in os.walk(src):
+                if '__pycache__' in root:
+                    continue
+                result += [
+                    os.path.join(root, f)
+                    for f in files
+                    if not f.endswith('.pyc')
+                ]
+        self._listfiles = result
+        return result
 
     async def __call__(self, target):
-        await target.copy(*self.args)
+        await target.mkdir(self.dst)
+
+        for path in self.listfiles():
+            if os.path.isdir(path):
+                await target.mkdir(os.path.join(self.dst, path))
+            elif '/' in path:
+                dirname = os.path.join(
+                    self.dst,
+                    '/'.join(path.split('/')[:-1])
+                )
+                await target.mkdir(dirname)
+                await target.copy(path, dirname)
+            else:
+                await target.copy(path, self.dst)
 
     def __str__(self):
-        return f'Copy(*{self.src}, {self.dst})'
+        return f'Copy({", ".join(self.src)}, {self.dst})'
 
-    def cachehash(self):
+    async def cachekey(self):
-        return str(self)
+        async def chksum(path):
+            with open(path, 'rb') as f:
+                return (path, str(binascii.crc32(f.read())))
+        results = await asyncio.gather(*[chksum(f) for f in self.listfiles()])
+        return {path: chks for path, chks in results}

shlax/targets/base.py

@@ -134,13 +134,12 @@ class Target:
         return self.root / path
 
     async def mkdir(self, path):
-        return await self.exec('mkdir -p ' + str(path))
+        if '_mkdir' not in self.__dict__:
+            self._mkdir = []
+        path = str(path)
+        if path not in self._mkdir:
+            await self.exec('mkdir', '-p', path)
+            self._mkdir.append(path)
 
     async def copy(self, *args):
-        src = args[:-1]
+        return await self.exec('cp', '-a', *args)
-        dst = self.path(args[-1])
-        await self.mkdir(dst)
-        return await self.exec(
-            *('cp', '-av') + src,
-            dst
-        )

shlax/targets/buildah.py

@@ -169,9 +169,7 @@ class Buildah(Target):
             await self.parent.exec('buildah', 'tag', self.sha, tag)
 
     async def mkdir(self, path):
-        return await self.parent.mkdir(self.root / path)
+        return await self.parent.mkdir(self.path(path))
 
     async def copy(self, *args):
-        args = list(args)
+        return await self.parent.copy(*args[:-1], self.path(args[-1]))
-        args[-1] = self.path(args[-1])
-        return await self.parent.copy(*args)