diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c62b09..187f2d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,15 +2,20 @@ build: cache: key: cache paths: [.cache] - image: quay.io/buildah/stable + image: yourlabs/buildah script: - - dnf install -y curl python38 - - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - - python3 get-pip.py - pip3 install -U --user -e .[cli] - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build stage: build +#build-itself: +# cache: +# key: cache +# paths: [.cache] +# image: shlax:$CI_COMMIT_SHORT_SHA +# script: python3 ./shlaxfile.py build +# stage: test + test: image: yourlabs/python stage: build diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 814da6a..7d0e9f2 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -37,11 +37,13 @@ class Packages: update='pacman -Sy', upgrade='pacman -Su --noconfirm', install='pacman -S --noconfirm', + lastupdate='stat -c %Y /var/lib/pacman/sync/core.db', ), dnf=dict( update='dnf makecache --assumeyes', upgrade='dnf upgrade --best --assumeyes --skip-broken', # noqa install='dnf install --setopt=install_weak_deps=False --best --assumeyes', # noqa + lastupdate='stat -c %Y /var/cache/dnf/* | head -n1', ), yum=dict( update='yum update', @@ -52,54 +54,52 @@ class Packages: installed = [] - def __init__(self, *packages, upgrade=True): + def __init__(self, *packages, upgrade=False): self.packages = [] self.upgrade = upgrade for package in packages: line = dedent(package).strip().replace('\n', ' ') self.packages += line.split(' ') - @property - def cache_root(self): + async def cache_setup(self, target): if 'CACHE_DIR' in os.environ: - return os.path.join(os.getenv('CACHE_DIR')) + self.cache_root = os.path.join(os.getenv('CACHE_DIR')) else: - return os.path.join(os.getenv('HOME'), '.cache') + self.cache_root = os.path.join(await target.parent.getenv('HOME'), '.cache') + + # run pkgmgr_setup functions ie. apk_setup + await getattr(self, self.mgr + '_setup')(target) async def update(self, target): - # run pkgmgr_setup functions ie. apk_setup - cachedir = await getattr(self, self.mgr + '_setup')(target) - + # lastupdate = await target.exec(self.cmds['lastupdate'], raises=False) + # lastupdate = int(lastupdate.out) if lastupdate.rc == 0 else None lastupdate = None - if os.path.exists(cachedir + '/lastupdate'): - with open(cachedir + '/lastupdate', 'r') as f: - try: - lastupdate = int(f.read().strip()) - except: - pass - - if not os.path.exists(cachedir): - os.makedirs(cachedir) - now = int(datetime.now().strftime('%s')) - # cache for a week + if not lastupdate or now - lastupdate > 604800: + await target.rexec(self.cmds['update']) + + return + + # disabling with the above return call until needed again + # might have to rewrite this to not have our own lockfile + # or find a better place on the filesystem + # also make sure the lockfile is actually needed when running on + # targets that don't have isguest=True if not lastupdate or now - lastupdate > 604800: # crude lockfile implementation, should work against *most* # race-conditions ... lockfile = cachedir + '/update.lock' - if not os.path.exists(lockfile): - with open(lockfile, 'w+') as f: - f.write(str(os.getpid())) + if not await target.parent.exists(lockfile): + await target.parent.write(lockfile, str(os.getpid())) try: await target.rexec(self.cmds['update']) finally: - os.unlink(lockfile) + await target.parent.rm(lockfile) - with open(cachedir + '/lastupdate', 'w+') as f: - f.write(str(now)) + await target.parent.write(cachedir + '/lastupdate', str(now)) else: - while os.path.exists(lockfile): + while await target.parent.exists(lockfile): print(f'{self.target} | Waiting for {lockfile} ...') await asyncio.sleep(1) @@ -116,7 +116,13 @@ class Packages: raise Exception('Packages does not yet support this distro') self.cmds = self.mgrs[self.mgr] + + if target.isguest: + # we're going to mount + await self.cache_setup(target) + await self.update(target) + if self.upgrade: await target.rexec(self.cmds['upgrade']) @@ -147,19 +153,24 @@ class Packages: return cachedir async def apt_setup(self, target): - codename = (await self.rexec( - f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME' + codename = (await target.rexec( + f'source /etc/os-release; echo $VERSION_CODENAME' )).out cachedir = os.path.join(self.cache_root, self.mgr, codename) await self.rexec('rm /etc/apt/apt.conf.d/docker-clean') cache_archives = os.path.join(cachedir, 'archives') - await self.mount(cache_archives, f'/var/cache/apt/archives') + await target.mount(cache_archives, f'/var/cache/apt/archives') cache_lists = os.path.join(cachedir, 'lists') - await self.mount(cache_lists, f'/var/lib/apt/lists') + await target.mount(cache_lists, f'/var/lib/apt/lists') return cachedir async def pacman_setup(self, target): - return self.cache_root + '/pacman' + cachedir = os.path.join(self.cache_root, self.mgr) + await target.mkdir(cachedir + '/cache', cachedir + '/sync') + await target.mount(cachedir + '/sync', '/var/lib/pacman/sync') + await target.mount(cachedir + '/cache', '/var/cache/pacman') + if await target.host.exists('/etc/pacman.d/mirrorlist'): + await target.copy('/etc/pacman.d/mirrorlist', '/etc/pacman.d/mirrorlist') def __str__(self): return f'Packages({self.packages}, upgrade={self.upgrade})' diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py index 7d8e8c7..949442b 100644 --- a/shlax/actions/pip.py +++ b/shlax/actions/pip.py @@ -19,7 +19,10 @@ class Pip(Action): raise Exception('Could not find pip nor python') # ensure pip module presence - result = await target.exec(python, '-m', 'pip', raises=False) + result = await target.exec( + python, '-m', 'pip', + raises=False, quiet=True + ) if result.rc != 0: if not os.path.exists('get-pip.py'): req = request.urlopen( diff --git a/shlax/cli.py b/shlax/cli.py index feae747..770e920 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -20,22 +20,51 @@ class Group(cli2.Group): self.cmdclass = Command +class TargetArgument(cli2.Argument): + """DSN of the target to execute on, localhost by default, TBI""" + + def __init__(self, cmd, param, doc=None, color=None, default=None): + from shlax.targets.base import Target + super().__init__(cmd, param, doc=self.__doc__, default=Target()) + self.alias = ['target', 't'] + + def cast(self, value): + from shlax.targets.ssh import Ssh + if '@' in value: + user, host = value.split('@') + return Ssh(host=host, user=user) + + class Command(cli2.Command): + def setargs(self): + super().setargs() + self['target'] = TargetArgument( + self, + self.sig.parameters['target'], + ) + if 'actions' in self: + del self['actions'] + def call(self, *args, **kwargs): + self.shlax_target = self['target'].value return self.shlax_target(self.target) def __call__(self, *argv): - from shlax.targets.base import Target - self.shlax_target = Target() - result = super().__call__(*argv) + super().__call__(*argv) self.shlax_target.output.results(self.shlax_target) - return result -class ActionCommand(Command): +class ActionCommand(cli2.Command): + def setargs(self): + super().setargs() + self['target'] = TargetArgument( + self, + inspect.Parameter('target', inspect.Parameter.KEYWORD_ONLY), + ) + def call(self, *args, **kwargs): self.target = self.target(*args, **kwargs) - return super().call(*args, **kwargs) + return super().call(self['target'].value) class ConsoleScript(Group): diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 04f5306..37a2856 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -11,12 +11,17 @@ from ..result import Result, Results class Target: + isguest = False + def __init__(self, *actions, root=None): self.actions = actions self.results = [] self.output = Output() self.parent = None - self.root = root or os.getcwd() + self.root = root or '' + + def __str__(self): + return 'localhost' @property def parent(self): @@ -127,20 +132,48 @@ class Target: @root.setter def root(self, value): - self._root = Path(value or os.getcwd()) + self._root = Path(value) if value else '' + + @property + def host(self): + current = self + while current.isguest: + current = self.parent + return current def path(self, path): + if not self.root: + return path if str(path).startswith('/'): path = str(path)[1:] - return self.root / path + return str(self.root / path) - async def mkdir(self, path): + async def mkdir(self, *paths): if '_mkdir' not in self.__dict__: self._mkdir = [] - path = str(path) - if path not in self._mkdir: - await self.exec('mkdir', '-p', path) - self._mkdir.append(path) + + make = [str(path) for path in paths if str(path) not in self._mkdir] + if make: + await self.exec('mkdir', '-p', *make) + self._mkdir += make async def copy(self, *args): return await self.exec('cp', '-a', *args) + + async def exists(self, path): + return (await self.exec('ls ' + self.path(path), raises=False)).rc == 0 + + async def read(self, path): + return (await self.exec('cat', self.path(path))).out + + async def write(self, path, content): + return await self.exec('echo ' + content + ' > ' + self.path(path)) + + async def rm(self, path): + return await self.exec('rm', self.path(path)) + + async def getenv(self, key): + return (await self.exec('echo $' + key)).out + + async def getcwd(self): + return (await self.exec('pwd')).out diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 927fb26..0073d3c 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -14,6 +14,7 @@ from ..proc import Proc class Buildah(Target): """Build container image with buildah""" + isguest = True def __init__(self, *actions, @@ -63,22 +64,17 @@ class Buildah(Target): if actions: actions = actions[len(keep):] if not actions: - return self.uptodate() + return self.output.success('Image up to date') else: self.actions = self.actions[len(keep):] if not self.actions: - return self.uptodate() + return self.output.success('Image up to date') self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) return await super().__call__(*actions) - def uptodate(self): - self.clean = None - self.output.success('Image up to date') - return - async def layers(self): ret = set() results = await self.parent.exec( @@ -144,21 +140,19 @@ class Buildah(Target): return stop async def clean(self, target, result): - for src, dst in self.mounts.items(): - await self.parent.exec('umount', self.root / str(dst)[1:]) - - if self.root is not None: + if self.ctr is not None: + for src, dst in self.mounts.items(): + await self.parent.exec('umount', self.root / str(dst)[1:]) await self.parent.exec('buildah', 'umount', self.ctr) - if self.ctr is not None: - if result.status == 'success': - await self.commit() - - await self.parent.exec('buildah', 'rm', self.ctr) - - if result.status == 'success' and os.getenv('BUILDAH_PUSH'): + if result.status == 'success': + await self.commit() + if os.getenv('BUILDAH_PUSH'): await self.image.push(target) + if self.ctr is not None: + await self.parent.exec('buildah', 'rm', self.ctr) + async def mount(self, src, dst): """Mount a host directory into the container.""" target = self.root / str(dst)[1:] @@ -174,22 +168,13 @@ class Buildah(Target): _args += [' '.join([str(a) for a in args])] return await self.parent.exec(*_args, **kwargs) - async def commit(self, image=None): - image = image or self.image - if not image: - return + async def commit(self): + for key, value in self.config.items(): + await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') - if not image: - # don't go through that if layer commit - for key, value in self.config.items(): - await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') - - self.sha = (await self.parent.exec( - 'buildah', - 'commit', - '--format=' + image.format, - self.ctr, - )).out + await self.parent.exec( + f'buildah commit {self.ctr} {self.image.repository}:final' + ) ENV_TAGS = ( # gitlab @@ -209,16 +194,15 @@ class Buildah(Target): if value: self.image.tags.append(value) - if image.tags: - tags = [f'{image.repository}:{tag}' for tag in image.tags] + if self.image.tags: + tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags] else: - tags = [image.repository] + tags = [self.image.repository] - for tag in tags: - await self.parent.exec('buildah', 'tag', self.sha, tag) + await self.parent.exec('buildah', 'tag', self.image.repository + ':final', *tags) - async def mkdir(self, path): - return await self.parent.mkdir(self.path(path)) + async def mkdir(self, *paths): + return await self.parent.mkdir(*[self.path(path) for path in paths]) async def copy(self, *args): return await self.parent.copy(*args[:-1], self.path(args[-1])) diff --git a/shlax/targets/ssh.py b/shlax/targets/ssh.py new file mode 100644 index 0000000..c0b9e57 --- /dev/null +++ b/shlax/targets/ssh.py @@ -0,0 +1,15 @@ +from .base import Target + + +class Ssh(Target): + def __init__(self, *actions, host, user=None): + self.host = host + self.user = user + super().__init__(*actions) + + async def exec(self, *args, user=None, **kwargs): + _args = ['ssh', self.host] + if user == 'root': + _args += ['sudo'] + _args += [' '.join([str(a) for a in args])] + return await self.parent.exec(*_args, **kwargs)