From 6924d39590f268a1f15957d12628a40ee356a6bb Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 13:47:33 +0100 Subject: [PATCH 01/27] Add Docker target, recursive calls --- .gitlab-ci.yml | 10 ++--- shlax/actions/base.py | 34 ++++++++++++++++- shlax/actions/packages.py | 2 +- shlax/actions/pip.py | 40 +++++++++---------- shlax/actions/run.py | 14 ++++++- shlax/cli.py | 5 ++- shlax/contrib/gitlab.py | 11 +++++- shlax/image.py | 7 ++++ shlax/output.py | 58 +++++++++++++++------------- shlax/targets/__init__.py | 1 + shlax/targets/buildah.py | 37 +++++++++--------- shlax/targets/docker.py | 78 ++++++++++++++++++++++++++++++++++++++ shlax/targets/localhost.py | 19 +++++++--- shlaxfile.py | 54 +++++++++++++++----------- 14 files changed, 268 insertions(+), 102 deletions(-) create mode 100644 shlax/targets/docker.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cca2778..d71d9db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,14 @@ build: image: yourlabs/shlax - script: pip install -U .[test] && py.test -svv tests - stage: test + script: ./shlaxfile.py build + stage: build pypi: image: yourlabs/python only: - tags - script: pypi-release + script: ./shlaxfile.py pypi stage: deploy test: - image: yourlabs/python - script: pip install -U .[test] && py.test -svv tests + image: yourlabs/shlax + script: ./shlaxfile.py test stage: test diff --git a/shlax/actions/base.py b/shlax/actions/base.py index 1b337aa..d2e0e76 100644 --- a/shlax/actions/base.py +++ b/shlax/actions/base.py @@ -1,4 +1,6 @@ +import functools import inspect +import importlib import sys from ..output import Output @@ -164,16 +166,46 @@ class Action: def callable(self): from ..targets import Localhost async def cb(*a, **k): - return await Localhost(self, quiet=True)(*a, **k) + from shlax.cli import cli + script = Localhost(self, quiet=True) + result = await script(*a, **k) + + success = functools.reduce( + lambda a, b: a + b, + [1 for c in script.children() if c.status == 'success'] or [0]) + if success: + script.output.success(f'{success} PASS') + + failures = functools.reduce( + lambda a, b: a + b, + [1 for c in script.children() if c.status == 'fail'] or [0]) + if failures: + script.output.fail(f'{failures} FAIL') + cli.exit_code = failures + return result return cb def kwargs_output(self): return self.kwargs def action(self, action, *args, **kwargs): + if isinstance(action, str): + import cli2 + a = cli2.Callable.factory(action).target + if not a: + a = cli2.Callable.factory( + '.'.join(['shlax', action]) + ).target + if a: + action = a + p = action(*args, **kwargs) for parent in self.parents(): if hasattr(parent, 'actions'): break p.parent = parent + if 'actions' not in self.__dict__: + # "mutate" to Strategy + from ..strategies.script import Actions + self.actions = Actions(self, [p]) return p diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index c509a5a..37f6d4b 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -113,7 +113,7 @@ class Packages(Action): else: mgr = await self.which(*self.mgrs.keys()) if mgr: - self.mgr = mgr.split('/')[-1] + self.mgr = mgr[0].split('/')[-1] if not self.mgr: raise Exception('Packages does not yet support this distro') diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py index 8fa1342..fcbae98 100644 --- a/shlax/actions/pip.py +++ b/shlax/actions/pip.py @@ -5,25 +5,27 @@ from .base import Action class Pip(Action): - packages = dict( - apt=['python3-pip'], - ) - def __init__(self, *pip_packages, pip=None, requirements=None): - self.pip_packages = pip_packages self.requirements = requirements super().__init__(*pip_packages, pip=pip, requirements=requirements) async def call(self, *args, **kwargs): - self.pip = await self.which('pip3', 'pip', 'pip2') - if not self.pip: + pip = await self.which('pip3', 'pip', 'pip2') + if pip: + pip = pip[0] + else: from .packages import Packages - action = self.action(Packages, 'python3,apk', 'python3-pip,apt', args=args, kwargs=kwargs) + action = self.action( + Packages, + 'python3,apk', 'python3-pip,apt', + args=args, kwargs=kwargs + ) await action(*args, **kwargs) - - self.pip = await self.which('pip3', 'pip', 'pip2') - if not self.pip: - raise Exception('Could not install a pip command') + pip = await self.which('pip3', 'pip', 'pip2') + if not pip: + raise Exception('Could not install a pip command') + else: + pip = pip[0] if 'CACHE_DIR' in os.environ: cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') @@ -33,20 +35,20 @@ class Pip(Action): if getattr(self, 'mount', None): # we are in a target which shares a mount command await self.mount(cache, '/root/.cache/pip') - await self.exec(f'{self.pip} install --upgrade pip') + await self.exec(f'{pip} install --upgrade pip') # https://github.com/pypa/pip/issues/5599 - self.pip = 'python3 -m pip' + pip = 'python3 -m pip' - source = [p for p in self.pip_packages if p.startswith('/')] + source = [p for p in self.args if p.startswith('/') or p.startswith('.')] if source: await self.exec( - f'{self.pip} install --upgrade --editable {" ".join(source)}' + f'{pip} install --upgrade --editable {" ".join(source)}' ) - nonsource = [p for p in self.pip_packages if not p.startswith('/')] + nonsource = [p for p in self.args if not p.startswith('/')] if nonsource: - await self.exec(f'{self.pip} install --upgrade {" ".join(nonsource)}') + await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}') if self.requirements: - await self.exec(f'{self.pip} install --upgrade -r {self.requirements}') + await self.exec(f'{pip} install --upgrade -r {self.requirements}') diff --git a/shlax/actions/run.py b/shlax/actions/run.py index 83ea901..e3f8730 100644 --- a/shlax/actions/run.py +++ b/shlax/actions/run.py @@ -1,6 +1,18 @@ +from ..targets.buildah import Buildah +from ..targets.docker import Docker + from .base import Action class Run(Action): async def call(self, *args, **kwargs): - return await self.exec(*self.args, **self.kwargs) + image = self.kwargs.get('image', None) + if not image: + return await self.exec(*self.args, **self.kwargs) + if isinstance(image, Buildah): + breakpoint() + result = await self.action(image, *args, **kwargs) + + return await Docker( + image=image, + ).exec(*args, **kwargs) diff --git a/shlax/cli.py b/shlax/cli.py index fc43129..fa1767f 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -8,6 +8,7 @@ shlax is a micro-framework to orchestrate commands. import asyncio import cli2 +import copy import inspect import os import sys @@ -130,8 +131,10 @@ class ConsoleScript(cli2.ConsoleScript): return super().__call__(*args, **kwargs) def call(self, command): + kwargs = copy.copy(self.parser.funckwargs) + kwargs.update(self.parser.options) try: - return super().call(command) + return command(*self.parser.funcargs, **kwargs) except WrongResult as e: print(e) self.exit_code = e.proc.rc diff --git a/shlax/contrib/gitlab.py b/shlax/contrib/gitlab.py index e3b8891..f292d47 100644 --- a/shlax/contrib/gitlab.py +++ b/shlax/contrib/gitlab.py @@ -3,9 +3,16 @@ import yaml from shlax import * -class GitLabCIConfig(Script): +class GitLabCI(Script): async def call(self, *args, write=True, **kwargs): - output = yaml.dump(self.kwargs) + output = dict() + for key, value in self.kwargs.items(): + if isinstance(value, dict): + output[key] = value + output[key]['script'] = './shlaxfile.py ' + key + else: + output[key] = value + output = yaml.dump(output) if kwargs['debug'] is True: self.output(output) if write: diff --git a/shlax/image.py b/shlax/image.py index 9ba27c8..173b8fa 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -55,3 +55,10 @@ class Image: # default tag by default ... if not self.tags: self.tags = ['latest'] + + async def __call__(self, action, *args, **kwargs): + args = list(args) + return await action.exec(*args, **self.kwargs) + + def __str__(self): + return self.repository diff --git a/shlax/output.py b/shlax/output.py index c9109ca..ee38572 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -33,8 +33,8 @@ class Output: self.write = write or sys.stdout.buffer.write self.flush = flush or sys.stdout.flush - def __call__(self, line, highlight=True, flush=True): - if self.prefix and self.prefix not in self.prefixes: + def prefix_line(self): + if self.prefix not in self.prefixes: self.prefixes[self.prefix] = self.prefix_colors[len(self.prefixes)] if len(self.prefix) > self.prefix_length: self.prefix_length = len(self.prefix) @@ -44,24 +44,24 @@ class Output: if prefix_padding: prefix_padding = ' ' + prefix_padding + ' ' - self.write(( - ( - prefix_color - + prefix_padding - + self.prefix - + ' ' - + self.colors['reset'] - + '| ' - if self.prefix - else '' - ) - + self.highlight(line, highlight) - + self.colors['reset'] - ).encode('utf8')) + return [ + prefix_color, + prefix_padding, + self.prefix, + ' ', + self.colors['reset'], + '| ' + ] + + def __call__(self, line, highlight=True, flush=True): + line = [self.highlight(line) if highlight else line] + if self.prefix: + line = self.prefix_line() + line + line = ''.join(line) + + self.write(line.encode('utf8')) if flush: - if not line.endswith('\n'): - self.write(b'\n') self.flush() def cmd(self, line): @@ -70,7 +70,8 @@ class Output: + '\x1b[1;38;5;15m' + ' ' + self.highlight(line, 'bash') - + self.colors['reset'], + + self.colors['reset'] + + '\n', highlight=False ) @@ -92,25 +93,28 @@ class Output: for regexp, colors in self.regexps.items(): line = re.sub(regexp, colors.format(**self.colors), line) + line = line + self.colors['reset'] return line def clean(self, action): - if self.debug is True or 'visit' in str(self.debug): + if self.debug is True: self(''.join([ self.colors['bluebold'], - '+ CLEAN ', + '+ CLEAN ', self.colors['reset'], action.colorized(), + '\n', ])) def start(self, action): - if self.debug is True or 'visit' in str(self.debug): + if self.debug is True: self(''.join([ self.colors['orangebold'], - '⚠ START ', + '⚠ START ', self.colors['reset'], action.colorized(), + '\n', ])) def success(self, action): @@ -119,14 +123,16 @@ class Output: self.colors['greenbold'], '✔ SUCCESS ', self.colors['reset'], - action.colorized(), + action.colorized() if hasattr(action, 'colorized') else str(action), + '\n', ])) def fail(self, action, exception=None): if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['redbold'], - '✘ FAIL ', + '✘ FAIL ', self.colors['reset'], - action.colorized(), + action.colorized() if hasattr(action, 'colorized') else str(action), + '\n', ])) diff --git a/shlax/targets/__init__.py b/shlax/targets/__init__.py index c7fb36f..3f7353f 100644 --- a/shlax/targets/__init__.py +++ b/shlax/targets/__init__.py @@ -1,3 +1,4 @@ from .buildah import Buildah +from .docker import Docker from .localhost import Localhost from .ssh import Ssh diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index df55e92..bfde5bb 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -8,6 +8,8 @@ import subprocess import sys import textwrap +from ..actions.base import Action +from ..exceptions import Mistake from ..proc import Proc from ..image import Image from .localhost import Localhost @@ -20,7 +22,10 @@ class Buildah(Localhost): """ contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] - def __init__(self, base, *args, commit=None, push=False, **kwargs): + def __init__(self, base, *args, commit=None, push=False, cmd=None, **kwargs): + if isinstance(base, Action): + args = [base] + list(args) + base = 'alpine' # default selection in case of mistake super().__init__(*args, **kwargs) self.base = base self.mounts = dict() @@ -28,6 +33,9 @@ class Buildah(Localhost): self.mnt = None self.image = Image(commit) if commit else None self.push = push or os.getenv('CI') + self.config= dict( + cmd=cmd or 'sh', + ) def shargs(self, *args, user=None, buildah=True, **kwargs): if not buildah or args[0].startswith('buildah'): @@ -52,9 +60,6 @@ class Buildah(Localhost): """Run buildah config.""" return await self.exec(f'buildah config {line} {self.ctr}', buildah=False) - async def mkdir(self, *dirs): - return await self.exec(*['mkdir', '-p'] + list(dirs)) - async def copy(self, *args): """Run buildah copy to copy a file from host into container.""" src = args[:-1] @@ -81,19 +86,6 @@ class Buildah(Localhost): await self.exec(f'mount -o bind {src} {target}', buildah=False) self.mounts[src] = dst - async def which(self, *cmd): - """ - Return the first path to the cmd in the container. - - If cmd argument is a list then it will try all commands. - """ - paths = (await self.env('PATH')).split(':') - for path in paths: - for c in cmd: - p = os.path.join(self.mnt, path[1:], c) - if os.path.exists(p): - return p[len(str(self.mnt)):] - def is_wrapper(self): return not ( Proc.test @@ -101,7 +93,6 @@ class Buildah(Localhost): or getattr(self.parent, 'parent', None) ) - async def call(self, *args, **kwargs): if not self.is_wrapper(): self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out @@ -124,7 +115,11 @@ class Buildah(Localhost): argv += [ cli.parser.command.name, # script name ? ] - self.output(' '.join(argv), 'EXECUTION', flush=True) + + await self.exec(*argv) + ''' + if debug is True or 'cmd' in str(debug): + self.output.cmd(' '.join(argv)) proc = await asyncio.create_subprocess_shell( shlex.join(argv), @@ -134,11 +129,15 @@ class Buildah(Localhost): ) await proc.communicate() cli.exit_code = await proc.wait() + ''' async def commit(self): if not self.image: return + for key, value in self.config.items(): + await self.exec(f'buildah config --{key} "{value}" {self.ctr}') + self.sha = (await self.exec( 'buildah', 'commit', diff --git a/shlax/targets/docker.py b/shlax/targets/docker.py new file mode 100644 index 0000000..80db9c5 --- /dev/null +++ b/shlax/targets/docker.py @@ -0,0 +1,78 @@ +import asyncio +from pathlib import Path +import os + +from ..image import Image +from .localhost import Localhost + + +class Docker(Localhost): + contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] + + def __init__(self, *args, **kwargs): + self.image = Image(kwargs.get('image', 'alpine')) + super().__init__(*args, **kwargs) + self.context['ctr'] = None + + def shargs(self, *args, daemon=False, **kwargs): + if args[0] == 'docker': + return args, kwargs + + extra = [] + if 'user' in kwargs: + extra += ['--user', kwargs.pop('user')] + + args, kwargs = super().shargs(*args, **kwargs) + + if self.context['ctr']: + executor = 'exec' + extra = [self.context['ctr']] + return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + list(args), kwargs + + executor = 'run' + cwd = os.getcwd() + if daemon: + extra += ['-d'] + extra = extra + ['-v', f'{cwd}:{cwd}', '-w', f'{cwd}'] + return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + [str(self.image)] + list(args), kwargs + + async def call(self, *args, **kwargs): + name = kwargs.get('name', os.getcwd()).split('/')[-1] + self.context['ctr'] = ( + await self.exec( + 'docker', 'ps', '-aq', '--filter', + 'name=' + name, + raises=False + ) + ).out.split('\n')[0] + + if 'recreate' in args and self.context['ctr']: + await self.exec('docker', 'rm', '-f', self.context['ctr']) + self.context['ctr'] = None + + if self.context['ctr']: + self.context['ctr'] = (await self.exec('docker', 'start', name)).out + else: + self.context['ctr'] = ( + await self.exec('sleep', '120', daemon=True) + ).out + return await super().call(*args, **kwargs) + + async def copy(self, *args): + src = args[:-1] + dst = args[-1] + await self.mkdir(dst) + + procs = [] + for s in src: + ''' + if Path(s).is_dir(): + await self.mkdir(s) + args = ['docker', 'copy', self.ctr, s, Path(dst) / s] + else: + args = ['docker', 'copy', self.ctr, s, dst] + ''' + args = ['docker', 'cp', s, self.context['ctr'] + ':' + dst] + procs.append(self.exec(*args)) + + return await asyncio.gather(*procs) diff --git a/shlax/targets/localhost.py b/shlax/targets/localhost.py index 892383f..8bc0294 100644 --- a/shlax/targets/localhost.py +++ b/shlax/targets/localhost.py @@ -1,4 +1,5 @@ import os +import re from shlax.proc import Proc @@ -50,12 +51,20 @@ class Localhost(Script): If cmd argument is a list then it will try all commands. """ - for path in (await self.env('PATH')).split(':'): - for c in cmd: - p = os.path.join(self.root, path[1:], c) - if os.path.exists(p): - return p[len(str(self.root)):] + proc = await self.exec('type ' + ' '.join(cmd), raises=False) + result = [] + for res in proc.out.split('\n'): + match = re.match('([^ ]+) is ([^ ]+)$', res.strip()) + if match: + result.append(match.group(1)) + return result async def copy(self, *args): args = ['cp', '-ra'] + list(args) return await self.exec(*args) + + async def mount(self, *dirs): + pass + + async def mkdir(self, *dirs): + return await self.exec(*['mkdir', '-p'] + list(dirs)) diff --git a/shlaxfile.py b/shlaxfile.py index 4ae04e5..556a02a 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -3,32 +3,42 @@ from shlax.contrib.gitlab import * PYTEST = 'py.test -svv tests' -build = Buildah('alpine', +build = Buildah( Copy('shlax/', 'setup.py', '/app'), Pip('/app'), commit='yourlabs/shlax', -) - -gitlabci = GitLabCIConfig( - build=dict( - stage='test', - image='yourlabs/shlax', - script='pip install -U .[test] && ' + PYTEST, - ), - test=dict( - stage='test', - image='yourlabs/python', - script='pip install -U .[test] && ' + PYTEST, - ), - pypi=dict( - stage='deploy', - image='yourlabs/python', - script='pypi-release', - only=['tags'] - ), + workdir='/app', ) test = Script( - gitlabci, - Run('gitlab-runner exec docker test'), + Pip('.[test]'), + Run(PYTEST), +) + +buildtest = Docker( + *test.actions, + mount={'.': '/app'}, + workdir='/app', +) + +pypi = Run( + 'pypi-release', + stage='deploy', + image='yourlabs/python', +) + +gitlabci = GitLabCI( + build=dict( + stage='build', + image='yourlabs/shlax', + ), + test=dict( + stage='test', + image='yourlabs/shlax', + ), + pypi=dict( + stage='deploy', + only=['tags'], + image='yourlabs/python', + ), ) From 2ea37fb01dfe16e3f90dbbf36567a7c6f8a58697 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 14:36:03 +0100 Subject: [PATCH 02/27] GitLabCI can now execute the same jobs it generates for gitlab --- .gitlab-ci.yml | 14 ++++---------- shlax/contrib/gitlab.py | 22 +++++++++++++++++++++- shlax/image.py | 2 +- shlaxfile.py | 2 +- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d71d9db..f141d1c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,8 @@ -build: - image: yourlabs/shlax - script: ./shlaxfile.py build - stage: build +build: {image: yourlabs/shlax, script: ./shlaxfile.py build, stage: build} pypi: image: yourlabs/python - only: - - tags + only: [tags] script: ./shlaxfile.py pypi stage: deploy -test: - image: yourlabs/shlax - script: ./shlaxfile.py test - stage: test +test: {image: 'yourlabs/shlax:$CI_COMMIT_SHORT_SHA', script: ./shlaxfile.py test, + stage: test} diff --git a/shlax/contrib/gitlab.py b/shlax/contrib/gitlab.py index f292d47..c125cac 100644 --- a/shlax/contrib/gitlab.py +++ b/shlax/contrib/gitlab.py @@ -1,3 +1,4 @@ +from copy import deepcopy import yaml from shlax import * @@ -8,10 +9,14 @@ class GitLabCI(Script): output = dict() for key, value in self.kwargs.items(): if isinstance(value, dict): - output[key] = value + output[key] = deepcopy(value) output[key]['script'] = './shlaxfile.py ' + key + image = output[key].get('image', 'alpine') + if hasattr(image, 'image'): + output[key]['image'] = image.image.repository + ':$CI_COMMIT_SHORT_SHA' else: output[key] = value + output = yaml.dump(output) if kwargs['debug'] is True: self.output(output) @@ -19,5 +24,20 @@ class GitLabCI(Script): with open('.gitlab-ci.yml', 'w+') as f: f.write(output) + from shlax.cli import cli + for arg in args: + job = self.kwargs[arg] + _args = [] + if not isinstance(job['image'], str): + await job['image'](**kwargs) + image = str(job['image'].image) + _args.append('recreate') + else: + image = job['image'] + await Docker( + *cli.shlaxfile.actions[arg].actions, + image=image + )(*_args, **kwargs) + def colorized(self): return type(self).__name__ diff --git a/shlax/image.py b/shlax/image.py index 173b8fa..bee742f 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -61,4 +61,4 @@ class Image: return await action.exec(*args, **self.kwargs) def __str__(self): - return self.repository + return f'{self.repository}:{self.tags[-1]}' diff --git a/shlaxfile.py b/shlaxfile.py index 556a02a..9a8d576 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -34,7 +34,7 @@ gitlabci = GitLabCI( ), test=dict( stage='test', - image='yourlabs/shlax', + image=build, ), pypi=dict( stage='deploy', From 9736c1bb2bf5f7c0b57b41448f0091da45c3e27b Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 14:42:48 +0100 Subject: [PATCH 03/27] Add python3 yaml package --- shlax/actions/packages.py | 2 +- shlaxfile.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 37f6d4b..e9f0497 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -141,7 +141,7 @@ class Packages(Action): cachedir = os.path.join(self.cache_root, self.mgr) await self.mount(cachedir, '/var/cache/apk') # special step to enable apk cache - await self.rexec('ln -s /var/cache/apk /etc/apk/cache') + await self.rexec('ln -sf /var/cache/apk /etc/apk/cache') return cachedir async def dnf_setup(self): diff --git a/shlaxfile.py b/shlaxfile.py index 9a8d576..ec57e48 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -4,6 +4,7 @@ from shlax.contrib.gitlab import * PYTEST = 'py.test -svv tests' build = Buildah( + Packages('py3-yaml'), Copy('shlax/', 'setup.py', '/app'), Pip('/app'), commit='yourlabs/shlax', From e0c902eb4736015f2583d981abc85e52d9eede0d Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 14:47:03 +0100 Subject: [PATCH 04/27] Buildah: support push arg --- shlax/targets/buildah.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index bfde5bb..ceabe60 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -169,17 +169,19 @@ class Buildah(Localhost): await self.exec('podman', 'push', f'{self.image.repository}:{tag}', buildah=False) async def clean(self, *args, **kwargs): - if self.is_wrapper(): - return + if not self.is_wrapper(): + for src, dst in self.mounts.items(): + await self.exec('umount', self.mnt / str(dst)[1:], buildah=False) - for src, dst in self.mounts.items(): - await self.exec('umount', self.mnt / str(dst)[1:], buildah=False) + if self.status == 'success': + await self.commit() - if self.status == 'success': - await self.commit() + if self.mnt is not None: + await self.exec('buildah', 'umount', self.ctr, buildah=False) - if self.mnt is not None: - await self.exec('buildah', 'umount', self.ctr, buildah=False) + if self.ctr is not None: + await self.exec('buildah', 'rm', self.ctr, buildah=False) - if self.ctr is not None: - await self.exec('buildah', 'rm', self.ctr, buildah=False) + if 'push' in args: + for tag in self.image.tags: + await self.exec('podman', 'push', f'{self.image.repository}:{tag}', buildah=False) From e20946a0f60e9be37b7d2de103a23add06afece6 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 15:03:27 +0100 Subject: [PATCH 05/27] Optional upgrade, build async --- shlax/actions/packages.py | 5 +++-- shlax/strategies/asyn.py | 2 +- shlaxfile.py | 15 +++++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index e9f0497..fc11109 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -121,7 +121,8 @@ class Packages(Action): self.cmds = self.mgrs[self.mgr] if not getattr(self, '_packages_upgraded', None): await self.update() - await self.rexec(self.cmds['upgrade']) + if self.kwargs.get('upgrade', True): + await self.rexec(self.cmds['upgrade']) self._packages_upgraded = True packages = [] @@ -147,7 +148,7 @@ class Packages(Action): async def dnf_setup(self): cachedir = os.path.join(self.cache_root, self.mgr) await self.mount(cachedir, f'/var/cache/{self.mgr}') - await self.run('echo keepcache=True >> /etc/dnf/dnf.conf') + await self.rexec('echo keepcache=True >> /etc/dnf/dnf.conf') return cachedir async def apt_setup(self): diff --git a/shlax/strategies/asyn.py b/shlax/strategies/asyn.py index ee45d87..e7944f2 100644 --- a/shlax/strategies/asyn.py +++ b/shlax/strategies/asyn.py @@ -6,6 +6,6 @@ from .script import Script class Async(Script): async def call(self, *args, **kwargs): return asyncio.gather(*[ - procs.append(action(*args, **kwargs)) + action(*args, **kwargs) for action in self.actions ]) diff --git a/shlaxfile.py b/shlaxfile.py index ec57e48..f67184d 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -4,8 +4,19 @@ from shlax.contrib.gitlab import * PYTEST = 'py.test -svv tests' build = Buildah( - Packages('py3-yaml'), - Copy('shlax/', 'setup.py', '/app'), + 'quay.io/podman/stable', + Packages('python38', 'buildah', 'unzip', 'findutils', 'python3-yaml', upgrade=False), + Async( + # python3.8 on centos with pip dance ... + Run(''' + curl -o setuptools.zip https://files.pythonhosted.org/packages/42/3e/2464120172859e5d103e5500315fb5555b1e908c0dacc73d80d35a9480ca/setuptools-45.1.0.zip + unzip setuptools.zip + mkdir -p /usr/local/lib/python3.8/site-packages/ + sh -c "cd setuptools-* && python3.8 setup.py install" + easy_install-3.8 pip + '''), + Copy('shlax/', 'setup.py', '/app'), + ), Pip('/app'), commit='yourlabs/shlax', workdir='/app', From 440761fea2c57682673dd29f8954c0a8e3a7db9a Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 15:15:54 +0100 Subject: [PATCH 06/27] Python3.8 on centos dance --- shlax/actions/pip.py | 33 ++++++++++++++++++--------------- shlaxfile.py | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py index fcbae98..c74815c 100644 --- a/shlax/actions/pip.py +++ b/shlax/actions/pip.py @@ -10,22 +10,24 @@ class Pip(Action): super().__init__(*pip_packages, pip=pip, requirements=requirements) async def call(self, *args, **kwargs): - pip = await self.which('pip3', 'pip', 'pip2') - if pip: - pip = pip[0] - else: - from .packages import Packages - action = self.action( - Packages, - 'python3,apk', 'python3-pip,apt', - args=args, kwargs=kwargs - ) - await action(*args, **kwargs) + pip = self.kwargs.get('pip', None) + if not pip: pip = await self.which('pip3', 'pip', 'pip2') - if not pip: - raise Exception('Could not install a pip command') - else: + if pip: pip = pip[0] + else: + from .packages import Packages + action = self.action( + Packages, + 'python3,apk', 'python3-pip,apt', + args=args, kwargs=kwargs + ) + await action(*args, **kwargs) + pip = await self.which('pip3', 'pip', 'pip2') + if not pip: + raise Exception('Could not install a pip command') + else: + pip = pip[0] if 'CACHE_DIR' in os.environ: cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') @@ -38,7 +40,8 @@ class Pip(Action): await self.exec(f'{pip} install --upgrade pip') # https://github.com/pypa/pip/issues/5599 - pip = 'python3 -m pip' + if 'pip' not in self.kwargs: + pip = 'python3 -m pip' source = [p for p in self.args if p.startswith('/') or p.startswith('.')] if source: diff --git a/shlaxfile.py b/shlaxfile.py index f67184d..6c0fb1c 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -17,7 +17,7 @@ build = Buildah( '''), Copy('shlax/', 'setup.py', '/app'), ), - Pip('/app'), + Pip('/app', pip='python3.8 -m pip'), commit='yourlabs/shlax', workdir='/app', ) From a0cba02c25aa7fbd1067450ec5604dbffa106e01 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 15:16:04 +0100 Subject: [PATCH 07/27] Await asyncio jobs --- shlax/strategies/asyn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/strategies/asyn.py b/shlax/strategies/asyn.py index e7944f2..06213e0 100644 --- a/shlax/strategies/asyn.py +++ b/shlax/strategies/asyn.py @@ -5,7 +5,7 @@ from .script import Script class Async(Script): async def call(self, *args, **kwargs): - return asyncio.gather(*[ + return await asyncio.gather(*[ action(*args, **kwargs) for action in self.actions ]) From 933c81b7a80d544f44abe0e171545fa2c8e3ca44 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 15:34:44 +0100 Subject: [PATCH 08/27] Break script on error --- shlax/strategies/script.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shlax/strategies/script.py b/shlax/strategies/script.py index 24ac09b..bd8b0d7 100644 --- a/shlax/strategies/script.py +++ b/shlax/strategies/script.py @@ -29,4 +29,6 @@ class Script(Action): async def call(self, *args, **kwargs): for action in self.actions: - await action(*args, **kwargs) + result = await action(*args, **kwargs) + if action.status != 'success': + break From 5b11100e0af7804961b3d799bec4071e56aa0dd2 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 15:35:02 +0100 Subject: [PATCH 09/27] Rename is_wrapper with is_runnable, rely on uid check --- shlax/targets/buildah.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index ceabe60..a271a94 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -86,15 +86,14 @@ class Buildah(Localhost): await self.exec(f'mount -o bind {src} {target}', buildah=False) self.mounts[src] = dst - def is_wrapper(self): - return not ( + def is_runnable(self): + return ( Proc.test or os.getuid() == 0 - or getattr(self.parent, 'parent', None) ) async def call(self, *args, **kwargs): - if not self.is_wrapper(): + if self.is_runnable(): self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out self.mnt = Path((await self.exec('buildah', 'mount', self.ctr, buildah=False)).out) result = await super().call(*args, **kwargs) @@ -169,7 +168,7 @@ class Buildah(Localhost): await self.exec('podman', 'push', f'{self.image.repository}:{tag}', buildah=False) async def clean(self, *args, **kwargs): - if not self.is_wrapper(): + if self.is_runnable(): for src, dst in self.mounts.items(): await self.exec('umount', self.mnt / str(dst)[1:], buildah=False) From 8b145e6378e803066e6f1ef820403db06e75e427 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 16:12:12 +0100 Subject: [PATCH 10/27] Add automatic build test --- shlax/output.py | 10 ++++++++++ shlax/targets/buildah.py | 14 +++++++++++++- shlax/targets/docker.py | 4 +++- shlaxfile.py | 21 +++++++++------------ 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/shlax/output.py b/shlax/output.py index ee38572..21dda39 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -97,6 +97,16 @@ class Output: return line + def test(self, action): + if self.debug is True: + self(''.join([ + self.colors['purplebold'], + '! TEST ', + self.colors['reset'], + action.colorized(), + '\n', + ])) + def clean(self, action): if self.debug is True: self(''.join([ diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index a271a94..ee54592 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -20,7 +20,7 @@ class Buildah(Localhost): The build script iterates over visitors and runs the build functions, it also provides wrappers around the buildah command. """ - contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] + contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount', 'image'] def __init__(self, base, *args, commit=None, push=False, cmd=None, **kwargs): if isinstance(base, Action): @@ -56,6 +56,18 @@ class Buildah(Localhost): def __repr__(self): return f'Base({self.base})' + async def __call__(self, *args, **kwargs): + result = await super().__call__(*args, **kwargs) + if 'test' in self.kwargs and self.is_runnable(): + self.output.test(self) + await self.action('Docker', + *self.kwargs['test'].actions, + image=self.image, + mount={'.': '/app'}, + workdir='/app', + )(*args, **kwargs) + return result + async def config(self, line): """Run buildah config.""" return await self.exec(f'buildah config {line} {self.ctr}', buildah=False) diff --git a/shlax/targets/docker.py b/shlax/targets/docker.py index 80db9c5..450fe19 100644 --- a/shlax/targets/docker.py +++ b/shlax/targets/docker.py @@ -10,7 +10,9 @@ class Docker(Localhost): contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount'] def __init__(self, *args, **kwargs): - self.image = Image(kwargs.get('image', 'alpine')) + self.image = kwargs.get('image', 'alpine') + if not isinstance(self.image, Image): + self.image = Image(image) super().__init__(*args, **kwargs) self.context['ctr'] = None diff --git a/shlaxfile.py b/shlaxfile.py index 6c0fb1c..3554b5f 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -3,6 +3,11 @@ from shlax.contrib.gitlab import * PYTEST = 'py.test -svv tests' +test = Script( + Pip('.[test]'), + Run(PYTEST), +) + build = Buildah( 'quay.io/podman/stable', Packages('python38', 'buildah', 'unzip', 'findutils', 'python3-yaml', upgrade=False), @@ -14,23 +19,15 @@ build = Buildah( mkdir -p /usr/local/lib/python3.8/site-packages/ sh -c "cd setuptools-* && python3.8 setup.py install" easy_install-3.8 pip + echo python3.8 -m pip > /usr/bin/pip + chmod +x /usr/bin/pip '''), Copy('shlax/', 'setup.py', '/app'), ), - Pip('/app', pip='python3.8 -m pip'), + Pip('/app'), commit='yourlabs/shlax', workdir='/app', -) - -test = Script( - Pip('.[test]'), - Run(PYTEST), -) - -buildtest = Docker( - *test.actions, - mount={'.': '/app'}, - workdir='/app', + test=test, ) pypi = Run( From a720e22a501abd9aa146da5e53b9b17a44aa3659 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 16:20:56 +0100 Subject: [PATCH 11/27] Made ./shlaxfile.py build test push to do the right thing --- shlax/targets/buildah.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index ee54592..fc110ec 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -58,14 +58,6 @@ class Buildah(Localhost): async def __call__(self, *args, **kwargs): result = await super().__call__(*args, **kwargs) - if 'test' in self.kwargs and self.is_runnable(): - self.output.test(self) - await self.action('Docker', - *self.kwargs['test'].actions, - image=self.image, - mount={'.': '/app'}, - workdir='/app', - )(*args, **kwargs) return result async def config(self, line): @@ -185,6 +177,14 @@ class Buildah(Localhost): await self.exec('umount', self.mnt / str(dst)[1:], buildah=False) if self.status == 'success': + if 'test' in self.kwargs and 'test' in args: + self.output.test(self) + await self.action('Docker', + *self.kwargs['test'].actions, + image=self.image, + mount={'.': '/app'}, + workdir='/app', + )(*args, **kwargs) await self.commit() if self.mnt is not None: From b771e9bd1a6018623a99447c7362110a2b2bf954 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 17:45:04 +0100 Subject: [PATCH 12/27] Added Container strategy --- shlax/strategies/__init__.py | 1 + shlax/strategies/pod.py | 42 ++++++++++++++++++++++++++++++++++++ shlaxfile.py | 4 ++++ 3 files changed, 47 insertions(+) create mode 100644 shlax/strategies/pod.py diff --git a/shlax/strategies/__init__.py b/shlax/strategies/__init__.py index 3ee13c2..1712bfe 100644 --- a/shlax/strategies/__init__.py +++ b/shlax/strategies/__init__.py @@ -1,2 +1,3 @@ from .asyn import Async from .script import Script +from .pod import Pod, Container diff --git a/shlax/strategies/pod.py b/shlax/strategies/pod.py new file mode 100644 index 0000000..ce7ca39 --- /dev/null +++ b/shlax/strategies/pod.py @@ -0,0 +1,42 @@ +import os +from .script import Script + + +class Container(Script): + async def call(self, *args, **kwargs): + if not args or 'build' in args: + await self.kwargs['build'](*args, **kwargs) + self.image = self.kwargs['build'].image + + if not args or 'test' in args: + self.output.test(self) + await self.action('Docker', + *self.kwargs['test'].actions, + image=self.image, + mount={'.': '/app'}, + workdir='/app', + )(*args, **kwargs) + + if not args or 'push' in args: + user = os.getenv('DOCKER_USER') + passwd = os.getenv('DOCKER_PASS') + if user and passwd and os.getenv('CI') and self.registry: + await self.exec( + 'podman', + 'login', + '-u', + user, + '-p', + passwd, + self.registry, + buildah=False, + ) + + for tag in self.image.tags: + await self.exec('podman', 'push', f'{self.image.repository}:{tag}') + + #name = kwargs.get('name', os.getcwd()).split('/')[-1] + + +class Pod(Script): + pass diff --git a/shlaxfile.py b/shlaxfile.py index 3554b5f..f7bd3df 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -27,6 +27,10 @@ build = Buildah( Pip('/app'), commit='yourlabs/shlax', workdir='/app', +) + +shlax = Container( + build=build, test=test, ) From 42595bfd10220c2efc9d43aaf6049c4b561d223e Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 18:12:18 +0100 Subject: [PATCH 13/27] Still sorting out test part --- setup.py | 3 +++ shlax/contrib/gitlab.py | 7 +------ shlax/targets/docker.py | 6 +----- shlaxfile.py | 13 +++++-------- 4 files changed, 10 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 078df4b..b15babf 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,9 @@ setup( setup_requires='setupmeta', install_requires=['cli2'], extras_require=dict( + full=[ + 'pyyaml', + ], test=[ 'pytest', 'pytest-cov', diff --git a/shlax/contrib/gitlab.py b/shlax/contrib/gitlab.py index c125cac..1e7ce35 100644 --- a/shlax/contrib/gitlab.py +++ b/shlax/contrib/gitlab.py @@ -29,15 +29,10 @@ class GitLabCI(Script): job = self.kwargs[arg] _args = [] if not isinstance(job['image'], str): - await job['image'](**kwargs) image = str(job['image'].image) - _args.append('recreate') else: image = job['image'] - await Docker( - *cli.shlaxfile.actions[arg].actions, - image=image - )(*_args, **kwargs) + await self.action('Docker', Run(job['script']), image=image)(*_args, **kwargs) def colorized(self): return type(self).__name__ diff --git a/shlax/targets/docker.py b/shlax/targets/docker.py index 450fe19..331c8b9 100644 --- a/shlax/targets/docker.py +++ b/shlax/targets/docker.py @@ -12,7 +12,7 @@ class Docker(Localhost): def __init__(self, *args, **kwargs): self.image = kwargs.get('image', 'alpine') if not isinstance(self.image, Image): - self.image = Image(image) + self.image = Image(self.image) super().__init__(*args, **kwargs) self.context['ctr'] = None @@ -54,10 +54,6 @@ class Docker(Localhost): if self.context['ctr']: self.context['ctr'] = (await self.exec('docker', 'start', name)).out - else: - self.context['ctr'] = ( - await self.exec('sleep', '120', daemon=True) - ).out return await super().call(*args, **kwargs) async def copy(self, *args): diff --git a/shlaxfile.py b/shlaxfile.py index f7bd3df..df287da 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -24,34 +24,31 @@ build = Buildah( '''), Copy('shlax/', 'setup.py', '/app'), ), - Pip('/app'), + Pip('/app[full]'), commit='yourlabs/shlax', workdir='/app', ) shlax = Container( build=build, - test=test, -) - -pypi = Run( - 'pypi-release', - stage='deploy', - image='yourlabs/python', + test=Script(Run('./shlaxfile.py -d test')), ) gitlabci = GitLabCI( build=dict( stage='build', image='yourlabs/shlax', + script='./shlaxfile.py shlax build', ), test=dict( stage='test', + script='./shlaxfile.py -d test', image=build, ), pypi=dict( stage='deploy', only=['tags'], image='yourlabs/python', + script='pypi-release', ), ) From 8be4a5c84fb2c56f108abf09a4a26d76cf5e42a0 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 18:15:05 +0100 Subject: [PATCH 14/27] Debug output for shlax build --- shlaxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlaxfile.py b/shlaxfile.py index df287da..995ea8d 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -38,7 +38,7 @@ gitlabci = GitLabCI( build=dict( stage='build', image='yourlabs/shlax', - script='./shlaxfile.py shlax build', + script='./shlaxfile.py -d shlax build', ), test=dict( stage='test', From 82f98b41b4c9cf0694874717bfef63d0ebf38762 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 18:18:07 +0100 Subject: [PATCH 15/27] Enforce docker format --- shlaxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlaxfile.py b/shlaxfile.py index 995ea8d..15f6e5d 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -25,7 +25,7 @@ build = Buildah( Copy('shlax/', 'setup.py', '/app'), ), Pip('/app[full]'), - commit='yourlabs/shlax', + commit='docker.io/yourlabs/shlax', workdir='/app', ) From 3f4738f4323bf1ef4ad937b001784a071d7d2c0f Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 18:21:04 +0100 Subject: [PATCH 16/27] Gitlab cache --- .gitlab-ci.yml | 8 +++++++- shlaxfile.py | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f141d1c..eb40472 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,10 @@ -build: {image: yourlabs/shlax, script: ./shlaxfile.py build, stage: build} +build: + cache: + key: cache + paths: [.cache] + image: yourlabs/shlax + script: ./shlaxfile.py build + stage: build pypi: image: yourlabs/python only: [tags] diff --git a/shlaxfile.py b/shlaxfile.py index 15f6e5d..26f7a36 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -38,7 +38,8 @@ gitlabci = GitLabCI( build=dict( stage='build', image='yourlabs/shlax', - script='./shlaxfile.py -d shlax build', + script='CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push', + cache=dict(paths=['.cache'], key='cache'), ), test=dict( stage='test', From a5f6c4fe74a2596a4d581d3c99430cc2ef679fb9 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 18:23:22 +0100 Subject: [PATCH 17/27] Better to define script --- .gitlab-ci.yml | 6 +++--- shlax/contrib/gitlab.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb40472..46054b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,12 +3,12 @@ build: key: cache paths: [.cache] image: yourlabs/shlax - script: ./shlaxfile.py build + script: CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push stage: build pypi: image: yourlabs/python only: [tags] - script: ./shlaxfile.py pypi + script: pypi-release stage: deploy -test: {image: 'yourlabs/shlax:$CI_COMMIT_SHORT_SHA', script: ./shlaxfile.py test, +test: {image: 'yourlabs/shlax:$CI_COMMIT_SHORT_SHA', script: ./shlaxfile.py -d test, stage: test} diff --git a/shlax/contrib/gitlab.py b/shlax/contrib/gitlab.py index 1e7ce35..000efc0 100644 --- a/shlax/contrib/gitlab.py +++ b/shlax/contrib/gitlab.py @@ -10,7 +10,6 @@ class GitLabCI(Script): for key, value in self.kwargs.items(): if isinstance(value, dict): output[key] = deepcopy(value) - output[key]['script'] = './shlaxfile.py ' + key image = output[key].get('image', 'alpine') if hasattr(image, 'image'): output[key]['image'] = image.image.repository + ':$CI_COMMIT_SHORT_SHA' From 3152f839716104de61aae924de25eb965772197d Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 18:34:15 +0100 Subject: [PATCH 18/27] Cleanup success code --- shlax/targets/buildah.py | 45 +++++++++++++--------------------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index fc110ec..f5f3571 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -32,7 +32,6 @@ class Buildah(Localhost): self.ctr = None self.mnt = None self.image = Image(commit) if commit else None - self.push = push or os.getenv('CI') self.config= dict( cmd=cmd or 'sh', ) @@ -150,26 +149,20 @@ class Buildah(Localhost): )).out if self.image.tags: - tags = ' '.join([f'{self.image.repository}:{tag}' for tag in self.image.tags]) - await self.exec('buildah', 'tag', self.sha, self.image.repository, tags, buildah=False) + tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags] + else: + tags = [self.image.repository] - if self.push: - user = os.getenv('DOCKER_USER') - passwd = os.getenv('DOCKER_PASS') - if user and passwd and os.getenv('CI') and self.registry: - await self.exec( - 'podman', - 'login', - '-u', - user, - '-p', - passwd, - self.registry, - buildah=False, - ) + for tag in tags: + await self.exec('buildah', 'tag', self.sha, tag, buildah=False) - for tag in self.image.tags: - await self.exec('podman', 'push', f'{self.image.repository}:{tag}', buildah=False) + async def push(self): + user = os.getenv('DOCKER_USER') + passwd = os.getenv('DOCKER_PASS') + if user and passwd and os.getenv('CI') and self.image.registry: + await self.exec('buildah', 'login', '-u', user, '-p', passwd, self.image.registry) + for tag in self.image.tags: + await self.exec('buildah', 'push', f'{self.image.repository}:{tag}') async def clean(self, *args, **kwargs): if self.is_runnable(): @@ -177,22 +170,12 @@ class Buildah(Localhost): await self.exec('umount', self.mnt / str(dst)[1:], buildah=False) if self.status == 'success': - if 'test' in self.kwargs and 'test' in args: - self.output.test(self) - await self.action('Docker', - *self.kwargs['test'].actions, - image=self.image, - mount={'.': '/app'}, - workdir='/app', - )(*args, **kwargs) await self.commit() + if 'push' in args: + await self.push() if self.mnt is not None: await self.exec('buildah', 'umount', self.ctr, buildah=False) if self.ctr is not None: await self.exec('buildah', 'rm', self.ctr, buildah=False) - - if 'push' in args: - for tag in self.image.tags: - await self.exec('podman', 'push', f'{self.image.repository}:{tag}', buildah=False) From fe70c69b0c0a0c50328bd1b1c0c027565dedf666 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 18:39:50 +0100 Subject: [PATCH 19/27] Update shlax in shlax --- shlax/actions/__init__.py | 1 - shlax/actions/commit.py | 87 --------------------------------------- shlaxfile.py | 4 +- 3 files changed, 2 insertions(+), 90 deletions(-) delete mode 100644 shlax/actions/commit.py diff --git a/shlax/actions/__init__.py b/shlax/actions/__init__.py index 7519565..44accfa 100644 --- a/shlax/actions/__init__.py +++ b/shlax/actions/__init__.py @@ -1,4 +1,3 @@ -from .commit import Commit from .copy import Copy from .packages import Packages # noqa from .base import Action # noqa diff --git a/shlax/actions/commit.py b/shlax/actions/commit.py deleted file mode 100644 index ee6dd55..0000000 --- a/shlax/actions/commit.py +++ /dev/null @@ -1,87 +0,0 @@ -import os -import subprocess - -from .base import Action - -from ..exceptions import WrongResult - -CI_VARS = ( - # gitlab - 'CI_COMMIT_SHORT_SHA', - 'CI_COMMIT_REF_NAME', - 'CI_COMMIT_TAG', - # CircleCI - 'CIRCLE_SHA1', - 'CIRCLE_TAG', - 'CIRCLE_BRANCH', -) - - -class Commit(Action): - def __init__(self, repo, tags=None, format=None, push=None, registry=None): - self.repo = repo - self.registry = registry or 'localhost' - self.push = push or os.getenv('CI') - - # figure out registry host - if '/' in self.repo and not registry: - first = self.repo.split('/')[0] - if '.' in first or ':' in first: - self.registry = self.repo.split('/')[0] - - # docker.io currently has issues with oci format - self.format = format or 'oci' - if self.registry == 'docker.io': - self.format = 'docker' - - self.tags = tags or [] - - # figure tags from CI vars - if not self.tags: - for name in CI_VARS: - value = os.getenv(name) - if value: - self.tags.append(value) - - # filter out tags which resolved to None - self.tags = [t for t in self.tags if t is not None] - - # default tag by default ... - if not self.tags: - self.tags = ['latest'] - - async def call(self, *args, ctr=None, **kwargs): - self.sha = (await self.parent.parent.exec( - 'buildah', - 'commit', - '--format=' + self.format, - ctr, - )).out - - if 'master' in self.tags: - self.tags.append('latest') - - if self.tags: - tags = ' '.join([f'{self.repo}:{tag}' for tag in self.tags]) - await script.exec('buildah', 'tag', self.sha, self.repo, tags) - - if self.push: - user = os.getenv('DOCKER_USER') - passwd = os.getenv('DOCKER_PASS') - if user and passwd and os.getenv('CI') and self.registry: - await script.exec( - 'podman', - 'login', - '-u', - user, - '-p', - passwd, - self.registry, - ) - - for tag in self.tags: - await script.exec('podman', 'push', f'{self.repo}:{tag}') - await script.umount() - - def __repr__(self): - return f'Commit({self.registry}/{self.repo}:{self.tags})' diff --git a/shlaxfile.py b/shlaxfile.py index 26f7a36..004530d 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -38,12 +38,12 @@ gitlabci = GitLabCI( build=dict( stage='build', image='yourlabs/shlax', - script='CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push', + script='pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push', cache=dict(paths=['.cache'], key='cache'), ), test=dict( stage='test', - script='./shlaxfile.py -d test', + script='pip install -U --user -e . && ./shlaxfile.py -d test', image=build, ), pypi=dict( From a374ac58b3a57808985698c153fefbdea6b8f091 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 18:45:34 +0100 Subject: [PATCH 20/27] Gitlab update --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 46054b2..10123f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,12 +3,13 @@ build: key: cache paths: [.cache] image: yourlabs/shlax - script: CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push + script: pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d + shlax build push stage: build pypi: image: yourlabs/python only: [tags] script: pypi-release stage: deploy -test: {image: 'yourlabs/shlax:$CI_COMMIT_SHORT_SHA', script: ./shlaxfile.py -d test, - stage: test} +test: {image: 'yourlabs/shlax:$CI_COMMIT_SHORT_SHA', script: pip install -U --user + -e . && ./shlaxfile.py -d test, stage: test} From 92b62799f478ba19e66b19f9b4fa5e880e2fbf4b Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 18:59:39 +0100 Subject: [PATCH 21/27] Fix pre-commit hook --- .pre-commit-hooks.yaml | 2 +- shlax/targets/buildah.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index b021449..cbfc7b9 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,5 +1,5 @@ - id: shlaxfile-gitlabci name: Regenerate .gitlab-ci.yml description: Regenerate gitlabci - entry: pip install -e . && ./shlaxfile.py gitlabci + entry: ./shlaxfile.py gitlabci language: python diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index f5f3571..95a993a 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -159,8 +159,12 @@ class Buildah(Localhost): async def push(self): user = os.getenv('DOCKER_USER') passwd = os.getenv('DOCKER_PASS') - if user and passwd and os.getenv('CI') and self.image.registry: - await self.exec('buildah', 'login', '-u', user, '-p', passwd, self.image.registry) + if user and passwd and os.getenv('CI'): + self.output.cmd('buildah', 'login', '-u', '...', '-p', '...', self.image.registry) + old = self.output.debug + self.output.debug = False + await self.exec('buildah', 'login', '-u', user, '-p', passwd, self.image.registry or 'docker.io', ) + self.output.debug = old for tag in self.image.tags: await self.exec('buildah', 'push', f'{self.image.repository}:{tag}') From 3ea029efec059f9adb5908ee1488d70cc0f6888d Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 19:09:27 +0100 Subject: [PATCH 22/27] Fix buildah login --- shlax/targets/buildah.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 95a993a..9181c6f 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -119,19 +119,6 @@ class Buildah(Localhost): ] await self.exec(*argv) - ''' - if debug is True or 'cmd' in str(debug): - self.output.cmd(' '.join(argv)) - - proc = await asyncio.create_subprocess_shell( - shlex.join(argv), - stderr=sys.stderr, - stdin=sys.stdin, - stdout=sys.stdout, - ) - await proc.communicate() - cli.exit_code = await proc.wait() - ''' async def commit(self): if not self.image: @@ -160,13 +147,13 @@ class Buildah(Localhost): user = os.getenv('DOCKER_USER') passwd = os.getenv('DOCKER_PASS') if user and passwd and os.getenv('CI'): - self.output.cmd('buildah', 'login', '-u', '...', '-p', '...', self.image.registry) + self.output.cmd('buildah login -u ... -p ...' + self.image.registry) old = self.output.debug self.output.debug = False await self.exec('buildah', 'login', '-u', user, '-p', passwd, self.image.registry or 'docker.io', ) self.output.debug = old for tag in self.image.tags: - await self.exec('buildah', 'push', f'{self.image.repository}:{tag}') + await self.exec('buildah', 'push', self.image.registry or 'docker.io', f'{self.image.repository}:{tag}') async def clean(self, *args, **kwargs): if self.is_runnable(): From f4844b0e730bb53b4132c3e43ccbc5c3a64ece00 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 19:28:38 +0100 Subject: [PATCH 23/27] Fix login --- shlax/actions/base.py | 3 ++- shlax/output.py | 3 ++- shlax/strategies/pod.py | 17 +---------------- shlax/targets/buildah.py | 14 ++++++-------- shlax/targets/localhost.py | 3 ++- shlaxfile.py | 12 ++++++------ 6 files changed, 19 insertions(+), 33 deletions(-) diff --git a/shlax/actions/base.py b/shlax/actions/base.py index d2e0e76..3872926 100644 --- a/shlax/actions/base.py +++ b/shlax/actions/base.py @@ -111,8 +111,9 @@ class Action: self.output = self.output_factory(*args, **kwargs) self.output_start() self.status = 'running' + call = getattr(self, kwargs.pop('method', 'call')) try: - result = await self.call(*args, **kwargs) + result = await call(*args, **kwargs) except Exception as e: self.output_fail(e) self.status = 'fail' diff --git a/shlax/output.py b/shlax/output.py index 21dda39..18e58b2 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -25,13 +25,14 @@ class Output: def colorize(self, code, content): return self.color(code) + content + self.color() - def __init__(self, prefix=None, regexps=None, debug=False, write=None, flush=None): + def __init__(self, prefix=None, regexps=None, debug=False, write=None, flush=None, **kwargs): self.prefix = prefix self.debug = debug self.prefix_length = 0 self.regexps = regexps or dict() self.write = write or sys.stdout.buffer.write self.flush = flush or sys.stdout.flush + self.kwargs = kwargs def prefix_line(self): if self.prefix not in self.prefixes: diff --git a/shlax/strategies/pod.py b/shlax/strategies/pod.py index ce7ca39..5cd5e61 100644 --- a/shlax/strategies/pod.py +++ b/shlax/strategies/pod.py @@ -18,22 +18,7 @@ class Container(Script): )(*args, **kwargs) if not args or 'push' in args: - user = os.getenv('DOCKER_USER') - passwd = os.getenv('DOCKER_PASS') - if user and passwd and os.getenv('CI') and self.registry: - await self.exec( - 'podman', - 'login', - '-u', - user, - '-p', - passwd, - self.registry, - buildah=False, - ) - - for tag in self.image.tags: - await self.exec('podman', 'push', f'{self.image.repository}:{tag}') + await self.kwargs['build'](method='push', **kwargs) #name = kwargs.get('name', os.getcwd()).split('/')[-1] diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 9181c6f..e2cfed4 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -143,17 +143,15 @@ class Buildah(Localhost): for tag in tags: await self.exec('buildah', 'tag', self.sha, tag, buildah=False) - async def push(self): + async def push(self, *args, **kwargs): user = os.getenv('DOCKER_USER') passwd = os.getenv('DOCKER_PASS') - if user and passwd and os.getenv('CI'): + if user and passwd: self.output.cmd('buildah login -u ... -p ...' + self.image.registry) - old = self.output.debug - self.output.debug = False - await self.exec('buildah', 'login', '-u', user, '-p', passwd, self.image.registry or 'docker.io', ) - self.output.debug = old + await self.exec('buildah', 'login', '-u', user, '-p', passwd, self.image.registry or 'docker.io', debug=False) + for tag in self.image.tags: - await self.exec('buildah', 'push', self.image.registry or 'docker.io', f'{self.image.repository}:{tag}') + await self.exec('buildah', 'push', f'{self.image.repository}:{tag}') async def clean(self, *args, **kwargs): if self.is_runnable(): @@ -162,7 +160,7 @@ class Buildah(Localhost): if self.status == 'success': await self.commit() - if 'push' in args: + if 'push' in args or os.getenv('CI'): await self.push() if self.mnt is not None: diff --git a/shlax/targets/localhost.py b/shlax/targets/localhost.py index 8bc0294..d424179 100644 --- a/shlax/targets/localhost.py +++ b/shlax/targets/localhost.py @@ -30,7 +30,8 @@ class Localhost(Script): return args, kwargs async def exec(self, *args, **kwargs): - kwargs.setdefault('debug', self.call_kwargs.get('debug', False)) + if 'debug' not in kwargs: + kwargs['debug'] = getattr(self, 'call_kwargs', {}).get('debug', False) kwargs.setdefault('output', self.output) args, kwargs = self.shargs(*args, **kwargs) proc = await Proc(*args, **kwargs)() diff --git a/shlaxfile.py b/shlaxfile.py index 004530d..685cd93 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -35,17 +35,17 @@ shlax = Container( ) gitlabci = GitLabCI( - build=dict( + test=dict( stage='build', + script='pip install -U --user -e .[test] && ' + PYTEST, + image='yourlabs/python', + ), + build=dict( + stage='test', image='yourlabs/shlax', script='pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push', cache=dict(paths=['.cache'], key='cache'), ), - test=dict( - stage='test', - script='pip install -U --user -e . && ./shlaxfile.py -d test', - image=build, - ), pypi=dict( stage='deploy', only=['tags'], From 2e3b9d8e5b93cbc57d409b7fef6046d3a23d2d13 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 19:37:18 +0100 Subject: [PATCH 24/27] Avoid double push --- shlax/targets/buildah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index e2cfed4..11860c5 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -160,7 +160,7 @@ class Buildah(Localhost): if self.status == 'success': await self.commit() - if 'push' in args or os.getenv('CI'): + if 'push' in args: await self.push() if self.mnt is not None: From ba87dea428a969c5516c29e4f76833257000f22d Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 19:41:20 +0100 Subject: [PATCH 25/27] Container strategies swallows args --- shlax/strategies/pod.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shlax/strategies/pod.py b/shlax/strategies/pod.py index 5cd5e61..6f496bf 100644 --- a/shlax/strategies/pod.py +++ b/shlax/strategies/pod.py @@ -5,7 +5,7 @@ from .script import Script class Container(Script): async def call(self, *args, **kwargs): if not args or 'build' in args: - await self.kwargs['build'](*args, **kwargs) + await self.kwargs['build'](**kwargs) self.image = self.kwargs['build'].image if not args or 'test' in args: @@ -15,7 +15,7 @@ class Container(Script): image=self.image, mount={'.': '/app'}, workdir='/app', - )(*args, **kwargs) + )(**kwargs) if not args or 'push' in args: await self.kwargs['build'](method='push', **kwargs) From ee252550280257a90ab643d848ff41a94e3bcdd0 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 19:57:44 +0100 Subject: [PATCH 26/27] Prevent double clean --- shlax/actions/base.py | 3 +-- shlax/image.py | 11 +++++++++++ shlax/output.py | 2 +- shlax/strategies/pod.py | 2 +- shlax/targets/buildah.py | 16 +--------------- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/shlax/actions/base.py b/shlax/actions/base.py index 3872926..d2e0e76 100644 --- a/shlax/actions/base.py +++ b/shlax/actions/base.py @@ -111,9 +111,8 @@ class Action: self.output = self.output_factory(*args, **kwargs) self.output_start() self.status = 'running' - call = getattr(self, kwargs.pop('method', 'call')) try: - result = await call(*args, **kwargs) + result = await self.call(*args, **kwargs) except Exception as e: self.output_fail(e) self.status = 'fail' diff --git a/shlax/image.py b/shlax/image.py index bee742f..80b4562 100644 --- a/shlax/image.py +++ b/shlax/image.py @@ -62,3 +62,14 @@ class Image: def __str__(self): return f'{self.repository}:{self.tags[-1]}' + + async def push(self, *args, **kwargs): + user = os.getenv('DOCKER_USER') + passwd = os.getenv('DOCKER_PASS') + action = kwargs.get('action', self) + if user and passwd: + action.output.cmd('buildah login -u ... -p ...' + self.registry) + await action.exec('buildah', 'login', '-u', user, '-p', passwd, self.registry or 'docker.io', debug=False) + + for tag in self.tags: + await action.exec('buildah', 'push', f'{self.repository}:{tag}') diff --git a/shlax/output.py b/shlax/output.py index 18e58b2..9a8955f 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -119,7 +119,7 @@ class Output: ])) def start(self, action): - if self.debug is True: + if self.debug is True or 'visit' in str(self.debug): self(''.join([ self.colors['orangebold'], '⚠ START ', diff --git a/shlax/strategies/pod.py b/shlax/strategies/pod.py index 6f496bf..97da07c 100644 --- a/shlax/strategies/pod.py +++ b/shlax/strategies/pod.py @@ -18,7 +18,7 @@ class Container(Script): )(**kwargs) if not args or 'push' in args: - await self.kwargs['build'](method='push', **kwargs) + await self.image.push(action=self) #name = kwargs.get('name', os.getcwd()).split('/')[-1] diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index 11860c5..bf5e28c 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -55,10 +55,6 @@ class Buildah(Localhost): def __repr__(self): return f'Base({self.base})' - async def __call__(self, *args, **kwargs): - result = await super().__call__(*args, **kwargs) - return result - async def config(self, line): """Run buildah config.""" return await self.exec(f'buildah config {line} {self.ctr}', buildah=False) @@ -143,16 +139,6 @@ class Buildah(Localhost): for tag in tags: await self.exec('buildah', 'tag', self.sha, tag, buildah=False) - async def push(self, *args, **kwargs): - user = os.getenv('DOCKER_USER') - passwd = os.getenv('DOCKER_PASS') - if user and passwd: - self.output.cmd('buildah login -u ... -p ...' + self.image.registry) - await self.exec('buildah', 'login', '-u', user, '-p', passwd, self.image.registry or 'docker.io', debug=False) - - for tag in self.image.tags: - await self.exec('buildah', 'push', f'{self.image.repository}:{tag}') - async def clean(self, *args, **kwargs): if self.is_runnable(): for src, dst in self.mounts.items(): @@ -161,7 +147,7 @@ class Buildah(Localhost): if self.status == 'success': await self.commit() if 'push' in args: - await self.push() + await self.image.push(action=self) if self.mnt is not None: await self.exec('buildah', 'umount', self.ctr, buildah=False) From ebcde95d5a8597256d1a97be1a6c5fcf00691ec3 Mon Sep 17 00:00:00 2001 From: jpic Date: Sun, 16 Feb 2020 20:02:24 +0100 Subject: [PATCH 27/27] Simplify gitlabci for now --- .gitlab-ci.yml | 4 ++-- shlaxfile.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10123f3..b8fb8b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,5 +11,5 @@ pypi: only: [tags] script: pypi-release stage: deploy -test: {image: 'yourlabs/shlax:$CI_COMMIT_SHORT_SHA', script: pip install -U --user - -e . && ./shlaxfile.py -d test, stage: test} +test: {image: yourlabs/python, script: 'pip install -U --user -e .[test] && py.test + -svv tests', stage: build} diff --git a/shlaxfile.py b/shlaxfile.py index 685cd93..3921ed5 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -41,7 +41,7 @@ gitlabci = GitLabCI( image='yourlabs/python', ), build=dict( - stage='test', + stage='build', image='yourlabs/shlax', script='pip install -U --user -e . && CACHE_DIR=$(pwd)/.cache ./shlaxfile.py -d shlax build push', cache=dict(paths=['.cache'], key='cache'),