diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 43615bd..7d0e9f2 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -37,11 +37,13 @@ class Packages: update='pacman -Sy', upgrade='pacman -Su --noconfirm', install='pacman -S --noconfirm', + lastupdate='stat -c %Y /var/lib/pacman/sync/core.db', ), dnf=dict( update='dnf makecache --assumeyes', upgrade='dnf upgrade --best --assumeyes --skip-broken', # noqa install='dnf install --setopt=install_weak_deps=False --best --assumeyes', # noqa + lastupdate='stat -c %Y /var/cache/dnf/* | head -n1', ), yum=dict( update='yum update', @@ -52,57 +54,52 @@ class Packages: installed = [] - def __init__(self, *packages, upgrade=True): + def __init__(self, *packages, upgrade=False): self.packages = [] self.upgrade = upgrade for package in packages: line = dedent(package).strip().replace('\n', ' ') self.packages += line.split(' ') - @property - def cache_root(self): + async def cache_setup(self, target): if 'CACHE_DIR' in os.environ: - return os.path.join(os.getenv('CACHE_DIR')) + self.cache_root = os.path.join(os.getenv('CACHE_DIR')) else: - return os.path.join(os.getenv('HOME'), '.cache') - - async def update(self, target): - if not target.islocal: - return await target.rexec(self.cmds['update']) + self.cache_root = os.path.join(await target.parent.getenv('HOME'), '.cache') # run pkgmgr_setup functions ie. apk_setup - cachedir = await getattr(self, self.mgr + '_setup')(target) + await getattr(self, self.mgr + '_setup')(target) + async def update(self, target): + # lastupdate = await target.exec(self.cmds['lastupdate'], raises=False) + # lastupdate = int(lastupdate.out) if lastupdate.rc == 0 else None lastupdate = None - if os.path.exists(cachedir + '/lastupdate'): - with open(cachedir + '/lastupdate', 'r') as f: - try: - lastupdate = int(f.read().strip()) - except: - pass - - if not os.path.exists(cachedir): - os.makedirs(cachedir) - now = int(datetime.now().strftime('%s')) - # cache for a week + if not lastupdate or now - lastupdate > 604800: + await target.rexec(self.cmds['update']) + + return + + # disabling with the above return call until needed again + # might have to rewrite this to not have our own lockfile + # or find a better place on the filesystem + # also make sure the lockfile is actually needed when running on + # targets that don't have isguest=True if not lastupdate or now - lastupdate > 604800: # crude lockfile implementation, should work against *most* # race-conditions ... lockfile = cachedir + '/update.lock' - if not os.path.exists(lockfile): - with open(lockfile, 'w+') as f: - f.write(str(os.getpid())) + if not await target.parent.exists(lockfile): + await target.parent.write(lockfile, str(os.getpid())) try: await target.rexec(self.cmds['update']) finally: - os.unlink(lockfile) + await target.parent.rm(lockfile) - with open(cachedir + '/lastupdate', 'w+') as f: - f.write(str(now)) + await target.parent.write(cachedir + '/lastupdate', str(now)) else: - while os.path.exists(lockfile): + while await target.parent.exists(lockfile): print(f'{self.target} | Waiting for {lockfile} ...') await asyncio.sleep(1) @@ -119,7 +116,13 @@ class Packages: raise Exception('Packages does not yet support this distro') self.cmds = self.mgrs[self.mgr] + + if target.isguest: + # we're going to mount + await self.cache_setup(target) + await self.update(target) + if self.upgrade: await target.rexec(self.cmds['upgrade']) @@ -150,19 +153,24 @@ class Packages: return cachedir async def apt_setup(self, target): - codename = (await self.rexec( - f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME' + codename = (await target.rexec( + f'source /etc/os-release; echo $VERSION_CODENAME' )).out cachedir = os.path.join(self.cache_root, self.mgr, codename) await self.rexec('rm /etc/apt/apt.conf.d/docker-clean') cache_archives = os.path.join(cachedir, 'archives') - await self.mount(cache_archives, f'/var/cache/apt/archives') + await target.mount(cache_archives, f'/var/cache/apt/archives') cache_lists = os.path.join(cachedir, 'lists') - await self.mount(cache_lists, f'/var/lib/apt/lists') + await target.mount(cache_lists, f'/var/lib/apt/lists') return cachedir async def pacman_setup(self, target): - return self.cache_root + '/pacman' + cachedir = os.path.join(self.cache_root, self.mgr) + await target.mkdir(cachedir + '/cache', cachedir + '/sync') + await target.mount(cachedir + '/sync', '/var/lib/pacman/sync') + await target.mount(cachedir + '/cache', '/var/cache/pacman') + if await target.host.exists('/etc/pacman.d/mirrorlist'): + await target.copy('/etc/pacman.d/mirrorlist', '/etc/pacman.d/mirrorlist') def __str__(self): return f'Packages({self.packages}, upgrade={self.upgrade})' diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 1e8a843..37a2856 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -11,13 +11,14 @@ from ..result import Result, Results class Target: + isguest = False + def __init__(self, *actions, root=None): self.actions = actions self.results = [] self.output = Output() self.parent = None - self.root = root or os.getcwd() - self.islocal = getattr(self, 'islocal', True) + self.root = root or '' def __str__(self): return 'localhost' @@ -131,20 +132,48 @@ class Target: @root.setter def root(self, value): - self._root = Path(value or os.getcwd()) + self._root = Path(value) if value else '' + + @property + def host(self): + current = self + while current.isguest: + current = self.parent + return current def path(self, path): + if not self.root: + return path if str(path).startswith('/'): path = str(path)[1:] - return self.root / path + return str(self.root / path) - async def mkdir(self, path): + async def mkdir(self, *paths): if '_mkdir' not in self.__dict__: self._mkdir = [] - path = str(path) - if path not in self._mkdir: - await self.exec('mkdir', '-p', path) - self._mkdir.append(path) + + make = [str(path) for path in paths if str(path) not in self._mkdir] + if make: + await self.exec('mkdir', '-p', *make) + self._mkdir += make async def copy(self, *args): return await self.exec('cp', '-a', *args) + + async def exists(self, path): + return (await self.exec('ls ' + self.path(path), raises=False)).rc == 0 + + async def read(self, path): + return (await self.exec('cat', self.path(path))).out + + async def write(self, path, content): + return await self.exec('echo ' + content + ' > ' + self.path(path)) + + async def rm(self, path): + return await self.exec('rm', self.path(path)) + + async def getenv(self, key): + return (await self.exec('echo $' + key)).out + + async def getcwd(self): + return (await self.exec('pwd')).out diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 1451b95..15a09e7 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -14,6 +14,7 @@ from ..proc import Proc class Buildah(Target): """Build container image with buildah""" + isguest = True def __init__(self, *actions, @@ -200,8 +201,8 @@ class Buildah(Target): await self.parent.exec('buildah', 'tag', self.image_previous, *tags) - async def mkdir(self, path): - return await self.parent.mkdir(self.path(path)) + async def mkdir(self, *paths): + return await self.parent.mkdir(*[self.path(path) for path in paths]) async def copy(self, *args): return await self.parent.copy(*args[:-1], self.path(args[-1])) diff --git a/shlax/targets/ssh.py b/shlax/targets/ssh.py index d3dd707..c0b9e57 100644 --- a/shlax/targets/ssh.py +++ b/shlax/targets/ssh.py @@ -5,7 +5,6 @@ class Ssh(Target): def __init__(self, *actions, host, user=None): self.host = host self.user = user - self.islocal = False super().__init__(*actions) async def exec(self, *args, user=None, **kwargs):