From 28f555bcf3c1b3217d91ced7f7cd1e6d2e8339f6 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 12:23:48 +0200 Subject: [PATCH 01/10] Try more build --- .gitlab-ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c62b09..d9fb414 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,20 @@ build: - cache: - key: cache - paths: [.cache] - image: quay.io/buildah/stable - script: - dnf install -y curl python38 - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - python3 get-pip.py + script: - pip3 install -U --user -e .[cli] - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build stage: build +build-itself: + cache: + key: cache + paths: [.cache] + image: shlax:$CI_COMMIT_SHORT_SHA + script: python3 ./shlaxfile.py build + stage: test + test: image: yourlabs/python stage: build From 833cef7527b3014875fee47b0c47a5ee6eafd828 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 12:28:49 +0200 Subject: [PATCH 02/10] Typo in ci --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d9fb414..bb4f630 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,8 @@ build: - - dnf install -y curl python38 - - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - - python3 get-pip.py + cache: + key: cache + paths: [.cache] + image: yourlabs/buildah script: - pip3 install -U --user -e .[cli] - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build From 0ec287c9776042dda1d7f6feaed41ea03a9453f7 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 12:48:06 +0200 Subject: [PATCH 03/10] Tag/build/push refactor --- shlax/targets/buildah.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 927fb26..1451b95 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -63,22 +63,17 @@ class Buildah(Target): if actions: actions = actions[len(keep):] if not actions: - return self.uptodate() + return self.output.success('Image up to date') else: self.actions = self.actions[len(keep):] if not self.actions: - return self.uptodate() + return self.output.success('Image up to date') self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out) return await super().__call__(*actions) - def uptodate(self): - self.clean = None - self.output.success('Image up to date') - return - async def layers(self): ret = set() results = await self.parent.exec( @@ -144,19 +139,15 @@ class Buildah(Target): return stop async def clean(self, target, result): - for src, dst in self.mounts.items(): - await self.parent.exec('umount', self.root / str(dst)[1:]) - - if self.root is not None: - await self.parent.exec('buildah', 'umount', self.ctr) - if self.ctr is not None: - if result.status == 'success': - await self.commit() - + for src, dst in self.mounts.items(): + await self.parent.exec('umount', self.root / str(dst)[1:]) + await self.parent.exec('buildah', 'umount', self.ctr) await self.parent.exec('buildah', 'rm', self.ctr) - if result.status == 'success' and os.getenv('BUILDAH_PUSH'): + if result.status == 'success': + await self.commit() + if os.getenv('BUILDAH_PUSH'): await self.image.push(target) async def mount(self, src, dst): @@ -184,13 +175,6 @@ class Buildah(Target): for key, value in self.config.items(): await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') - self.sha = (await self.parent.exec( - 'buildah', - 'commit', - '--format=' + image.format, - self.ctr, - )).out - ENV_TAGS = ( # gitlab 'CI_COMMIT_SHORT_SHA', @@ -214,8 +198,7 @@ class Buildah(Target): else: tags = [image.repository] - for tag in tags: - await self.parent.exec('buildah', 'tag', self.sha, tag) + await self.parent.exec('buildah', 'tag', self.image_previous, *tags) async def mkdir(self, path): return await self.parent.mkdir(self.path(path)) From e733694f2dc45221958e91e1b1ca3f9849f1eff4 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 13:21:09 +0200 Subject: [PATCH 04/10] A bit of work on the command line --- shlax/cli.py | 31 ++++++++++++++++++++++--------- shlax/targets/base.py | 3 +++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/shlax/cli.py b/shlax/cli.py index feae747..a40b741 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -20,22 +20,35 @@ class Group(cli2.Group): self.cmdclass = Command +class TargetArgument(cli2.Argument): + """DSN of the target to execute on, localhost by default, TBI""" + + def __init__(self, cmd, param, doc=None, color=None, default=None): + from shlax.targets.base import Target + super().__init__(cmd, param, doc=self.__doc__, default=Target()) + self.alias = ['target', 't'] + + class Command(cli2.Command): - def call(self, *args, **kwargs): - return self.shlax_target(self.target) + def setargs(self): + super().setargs() + self['target'] = TargetArgument( + self, + self.sig.parameters['target'], + ) + if 'actions' in self: + del self['actions'] def __call__(self, *argv): - from shlax.targets.base import Target - self.shlax_target = Target() - result = super().__call__(*argv) - self.shlax_target.output.results(self.shlax_target) - return result + super().__call__(*argv) + self['target'].value.output.results(self['target'].value) -class ActionCommand(Command): +class ActionCommand(cli2.Command): def call(self, *args, **kwargs): self.target = self.target(*args, **kwargs) - return super().call(*args, **kwargs) + from shlax.targets.base import Target + return super().call(Target()) class ConsoleScript(Group): diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 04f5306..0b555b3 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -18,6 +18,9 @@ class Target: self.parent = None self.root = root or os.getcwd() + def __str__(self): + return 'localhost' + @property def parent(self): return self._parent or Target() From 879bb6f79eb7cac10802f6cfade7d08a4dd347a5 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 31 May 2020 13:24:31 +0200 Subject: [PATCH 05/10] Silence pip output --- shlax/actions/pip.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py index 7d8e8c7..949442b 100644 --- a/shlax/actions/pip.py +++ b/shlax/actions/pip.py @@ -19,7 +19,10 @@ class Pip(Action): raise Exception('Could not find pip nor python') # ensure pip module presence - result = await target.exec(python, '-m', 'pip', raises=False) + result = await target.exec( + python, '-m', 'pip', + raises=False, quiet=True + ) if result.rc != 0: if not os.path.exists('get-pip.py'): req = request.urlopen( From 3b6a4f3fc4d8ce890d303b2b0686f82857ea35a3 Mon Sep 17 00:00:00 2001 From: jpic Date: Mon, 1 Jun 2020 15:41:22 +0200 Subject: [PATCH 06/10] Added ssh target --- shlax/actions/packages.py | 3 +++ shlax/cli.py | 16 ++++++++++++++-- shlax/targets/base.py | 1 + shlax/targets/ssh.py | 16 ++++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 shlax/targets/ssh.py diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 814da6a..43615bd 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -67,6 +67,9 @@ class Packages: return os.path.join(os.getenv('HOME'), '.cache') async def update(self, target): + if not target.islocal: + return await target.rexec(self.cmds['update']) + # run pkgmgr_setup functions ie. apk_setup cachedir = await getattr(self, self.mgr + '_setup')(target) diff --git a/shlax/cli.py b/shlax/cli.py index a40b741..4fac0fb 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -28,6 +28,12 @@ class TargetArgument(cli2.Argument): super().__init__(cmd, param, doc=self.__doc__, default=Target()) self.alias = ['target', 't'] + def cast(self, value): + from shlax.targets.ssh import Ssh + if '@' in value: + user, host = value.split('@') + return Ssh(host=host, user=user) + class Command(cli2.Command): def setargs(self): @@ -45,10 +51,16 @@ class Command(cli2.Command): class ActionCommand(cli2.Command): + def setargs(self): + super().setargs() + self['target'] = TargetArgument( + self, + inspect.Parameter('target', inspect.Parameter.KEYWORD_ONLY), + ) + def call(self, *args, **kwargs): self.target = self.target(*args, **kwargs) - from shlax.targets.base import Target - return super().call(Target()) + return super().call(self['target'].value) class ConsoleScript(Group): diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 0b555b3..1e8a843 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -17,6 +17,7 @@ class Target: self.output = Output() self.parent = None self.root = root or os.getcwd() + self.islocal = getattr(self, 'islocal', True) def __str__(self): return 'localhost' diff --git a/shlax/targets/ssh.py b/shlax/targets/ssh.py new file mode 100644 index 0000000..d3dd707 --- /dev/null +++ b/shlax/targets/ssh.py @@ -0,0 +1,16 @@ +from .base import Target + + +class Ssh(Target): + def __init__(self, *actions, host, user=None): + self.host = host + self.user = user + self.islocal = False + super().__init__(*actions) + + async def exec(self, *args, user=None, **kwargs): + _args = ['ssh', self.host] + if user == 'root': + _args += ['sudo'] + _args += [' '.join([str(a) for a in args])] + return await self.parent.exec(*_args, **kwargs) From 5f451a786f87836a4ceb49bb10850d131ed500cb Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 03:19:01 +0200 Subject: [PATCH 07/10] Package action working both on SSH host and Buildah guest now --- shlax/actions/packages.py | 74 ++++++++++++++++++++++----------------- shlax/targets/base.py | 47 ++++++++++++++++++++----- shlax/targets/buildah.py | 5 +-- shlax/targets/ssh.py | 1 - 4 files changed, 82 insertions(+), 45 deletions(-) diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 43615bd..7d0e9f2 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -37,11 +37,13 @@ class Packages: update='pacman -Sy', upgrade='pacman -Su --noconfirm', install='pacman -S --noconfirm', + lastupdate='stat -c %Y /var/lib/pacman/sync/core.db', ), dnf=dict( update='dnf makecache --assumeyes', upgrade='dnf upgrade --best --assumeyes --skip-broken', # noqa install='dnf install --setopt=install_weak_deps=False --best --assumeyes', # noqa + lastupdate='stat -c %Y /var/cache/dnf/* | head -n1', ), yum=dict( update='yum update', @@ -52,57 +54,52 @@ class Packages: installed = [] - def __init__(self, *packages, upgrade=True): + def __init__(self, *packages, upgrade=False): self.packages = [] self.upgrade = upgrade for package in packages: line = dedent(package).strip().replace('\n', ' ') self.packages += line.split(' ') - @property - def cache_root(self): + async def cache_setup(self, target): if 'CACHE_DIR' in os.environ: - return os.path.join(os.getenv('CACHE_DIR')) + self.cache_root = os.path.join(os.getenv('CACHE_DIR')) else: - return os.path.join(os.getenv('HOME'), '.cache') - - async def update(self, target): - if not target.islocal: - return await target.rexec(self.cmds['update']) + self.cache_root = os.path.join(await target.parent.getenv('HOME'), '.cache') # run pkgmgr_setup functions ie. apk_setup - cachedir = await getattr(self, self.mgr + '_setup')(target) + await getattr(self, self.mgr + '_setup')(target) + async def update(self, target): + # lastupdate = await target.exec(self.cmds['lastupdate'], raises=False) + # lastupdate = int(lastupdate.out) if lastupdate.rc == 0 else None lastupdate = None - if os.path.exists(cachedir + '/lastupdate'): - with open(cachedir + '/lastupdate', 'r') as f: - try: - lastupdate = int(f.read().strip()) - except: - pass - - if not os.path.exists(cachedir): - os.makedirs(cachedir) - now = int(datetime.now().strftime('%s')) - # cache for a week + if not lastupdate or now - lastupdate > 604800: + await target.rexec(self.cmds['update']) + + return + + # disabling with the above return call until needed again + # might have to rewrite this to not have our own lockfile + # or find a better place on the filesystem + # also make sure the lockfile is actually needed when running on + # targets that don't have isguest=True if not lastupdate or now - lastupdate > 604800: # crude lockfile implementation, should work against *most* # race-conditions ... lockfile = cachedir + '/update.lock' - if not os.path.exists(lockfile): - with open(lockfile, 'w+') as f: - f.write(str(os.getpid())) + if not await target.parent.exists(lockfile): + await target.parent.write(lockfile, str(os.getpid())) try: await target.rexec(self.cmds['update']) finally: - os.unlink(lockfile) + await target.parent.rm(lockfile) - with open(cachedir + '/lastupdate', 'w+') as f: - f.write(str(now)) + await target.parent.write(cachedir + '/lastupdate', str(now)) else: - while os.path.exists(lockfile): + while await target.parent.exists(lockfile): print(f'{self.target} | Waiting for {lockfile} ...') await asyncio.sleep(1) @@ -119,7 +116,13 @@ class Packages: raise Exception('Packages does not yet support this distro') self.cmds = self.mgrs[self.mgr] + + if target.isguest: + # we're going to mount + await self.cache_setup(target) + await self.update(target) + if self.upgrade: await target.rexec(self.cmds['upgrade']) @@ -150,19 +153,24 @@ class Packages: return cachedir async def apt_setup(self, target): - codename = (await self.rexec( - f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME' + codename = (await target.rexec( + f'source /etc/os-release; echo $VERSION_CODENAME' )).out cachedir = os.path.join(self.cache_root, self.mgr, codename) await self.rexec('rm /etc/apt/apt.conf.d/docker-clean') cache_archives = os.path.join(cachedir, 'archives') - await self.mount(cache_archives, f'/var/cache/apt/archives') + await target.mount(cache_archives, f'/var/cache/apt/archives') cache_lists = os.path.join(cachedir, 'lists') - await self.mount(cache_lists, f'/var/lib/apt/lists') + await target.mount(cache_lists, f'/var/lib/apt/lists') return cachedir async def pacman_setup(self, target): - return self.cache_root + '/pacman' + cachedir = os.path.join(self.cache_root, self.mgr) + await target.mkdir(cachedir + '/cache', cachedir + '/sync') + await target.mount(cachedir + '/sync', '/var/lib/pacman/sync') + await target.mount(cachedir + '/cache', '/var/cache/pacman') + if await target.host.exists('/etc/pacman.d/mirrorlist'): + await target.copy('/etc/pacman.d/mirrorlist', '/etc/pacman.d/mirrorlist') def __str__(self): return f'Packages({self.packages}, upgrade={self.upgrade})' diff --git a/shlax/targets/base.py b/shlax/targets/base.py index 1e8a843..37a2856 100644 --- a/shlax/targets/base.py +++ b/shlax/targets/base.py @@ -11,13 +11,14 @@ from ..result import Result, Results class Target: + isguest = False + def __init__(self, *actions, root=None): self.actions = actions self.results = [] self.output = Output() self.parent = None - self.root = root or os.getcwd() - self.islocal = getattr(self, 'islocal', True) + self.root = root or '' def __str__(self): return 'localhost' @@ -131,20 +132,48 @@ class Target: @root.setter def root(self, value): - self._root = Path(value or os.getcwd()) + self._root = Path(value) if value else '' + + @property + def host(self): + current = self + while current.isguest: + current = self.parent + return current def path(self, path): + if not self.root: + return path if str(path).startswith('/'): path = str(path)[1:] - return self.root / path + return str(self.root / path) - async def mkdir(self, path): + async def mkdir(self, *paths): if '_mkdir' not in self.__dict__: self._mkdir = [] - path = str(path) - if path not in self._mkdir: - await self.exec('mkdir', '-p', path) - self._mkdir.append(path) + + make = [str(path) for path in paths if str(path) not in self._mkdir] + if make: + await self.exec('mkdir', '-p', *make) + self._mkdir += make async def copy(self, *args): return await self.exec('cp', '-a', *args) + + async def exists(self, path): + return (await self.exec('ls ' + self.path(path), raises=False)).rc == 0 + + async def read(self, path): + return (await self.exec('cat', self.path(path))).out + + async def write(self, path, content): + return await self.exec('echo ' + content + ' > ' + self.path(path)) + + async def rm(self, path): + return await self.exec('rm', self.path(path)) + + async def getenv(self, key): + return (await self.exec('echo $' + key)).out + + async def getcwd(self): + return (await self.exec('pwd')).out diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 1451b95..15a09e7 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -14,6 +14,7 @@ from ..proc import Proc class Buildah(Target): """Build container image with buildah""" + isguest = True def __init__(self, *actions, @@ -200,8 +201,8 @@ class Buildah(Target): await self.parent.exec('buildah', 'tag', self.image_previous, *tags) - async def mkdir(self, path): - return await self.parent.mkdir(self.path(path)) + async def mkdir(self, *paths): + return await self.parent.mkdir(*[self.path(path) for path in paths]) async def copy(self, *args): return await self.parent.copy(*args[:-1], self.path(args[-1])) diff --git a/shlax/targets/ssh.py b/shlax/targets/ssh.py index d3dd707..c0b9e57 100644 --- a/shlax/targets/ssh.py +++ b/shlax/targets/ssh.py @@ -5,7 +5,6 @@ class Ssh(Target): def __init__(self, *actions, host, user=None): self.host = host self.user = user - self.islocal = False super().__init__(*actions) async def exec(self, *args, user=None, **kwargs): From cfe75d28730198ac42fc8afe630a88e0f0625805 Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 03:19:13 +0200 Subject: [PATCH 08/10] Clean on KeyboardInterrupt, proper call in shlax Command --- shlax/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shlax/cli.py b/shlax/cli.py index 4fac0fb..770e920 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -45,9 +45,13 @@ class Command(cli2.Command): if 'actions' in self: del self['actions'] + def call(self, *args, **kwargs): + self.shlax_target = self['target'].value + return self.shlax_target(self.target) + def __call__(self, *argv): super().__call__(*argv) - self['target'].value.output.results(self['target'].value) + self.shlax_target.output.results(self.shlax_target) class ActionCommand(cli2.Command): From 62f1f5d162c3705983a0ed20bb8cda129e980251 Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 17:37:09 +0200 Subject: [PATCH 09/10] Commiting build-itself job until i re-implement docker commiting --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb4f630..187f2d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,13 +8,13 @@ build: - CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build stage: build -build-itself: - cache: - key: cache - paths: [.cache] - image: shlax:$CI_COMMIT_SHORT_SHA - script: python3 ./shlaxfile.py build - stage: test +#build-itself: +# cache: +# key: cache +# paths: [.cache] +# image: shlax:$CI_COMMIT_SHORT_SHA +# script: python3 ./shlaxfile.py build +# stage: test test: image: yourlabs/python From d2784cc87b4f4897aaf9e6f02de1084e4def4c61 Mon Sep 17 00:00:00 2001 From: jpic Date: Thu, 11 Jun 2020 17:37:30 +0200 Subject: [PATCH 10/10] Fixing container config support --- shlax/targets/buildah.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 15a09e7..0073d3c 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -144,13 +144,15 @@ class Buildah(Target): for src, dst in self.mounts.items(): await self.parent.exec('umount', self.root / str(dst)[1:]) await self.parent.exec('buildah', 'umount', self.ctr) - await self.parent.exec('buildah', 'rm', self.ctr) if result.status == 'success': await self.commit() if os.getenv('BUILDAH_PUSH'): await self.image.push(target) + if self.ctr is not None: + await self.parent.exec('buildah', 'rm', self.ctr) + async def mount(self, src, dst): """Mount a host directory into the container.""" target = self.root / str(dst)[1:] @@ -166,15 +168,13 @@ class Buildah(Target): _args += [' '.join([str(a) for a in args])] return await self.parent.exec(*_args, **kwargs) - async def commit(self, image=None): - image = image or self.image - if not image: - return + async def commit(self): + for key, value in self.config.items(): + await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') - if not image: - # don't go through that if layer commit - for key, value in self.config.items(): - await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}') + await self.parent.exec( + f'buildah commit {self.ctr} {self.image.repository}:final' + ) ENV_TAGS = ( # gitlab @@ -194,12 +194,12 @@ class Buildah(Target): if value: self.image.tags.append(value) - if image.tags: - tags = [f'{image.repository}:{tag}' for tag in image.tags] + if self.image.tags: + tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags] else: - tags = [image.repository] + tags = [self.image.repository] - await self.parent.exec('buildah', 'tag', self.image_previous, *tags) + await self.parent.exec('buildah', 'tag', self.image.repository + ':final', *tags) async def mkdir(self, *paths): return await self.parent.mkdir(*[self.path(path) for path in paths])