From 638f0fa690adb33f952a4a784d9d4dc1576dca93 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 15 Feb 2020 13:51:32 +0100 Subject: [PATCH] Fix cli that would play all scripts --- shlax/__init__.py | 1 + shlax/actions/__init__.py | 2 ++ shlax/actions/base.py | 11 ++++++- shlax/actions/copy.py | 10 ++++++ shlax/actions/pip.py | 51 +++++++++++++++++++++++++++++ shlax/cli.py | 13 ++------ shlax/contrib/gitlab.py | 4 +-- shlax/output.py | 69 +++++++++++++++++++++++++-------------- shlax/proc.py | 14 +++++--- shlax/targets/buildah.py | 2 +- shlaxfile.py | 6 ++++ tests/test_output.py | 24 ++++++++++++++ 12 files changed, 163 insertions(+), 44 deletions(-) create mode 100644 shlax/actions/copy.py create mode 100644 shlax/actions/pip.py create mode 100644 tests/test_output.py diff --git a/shlax/__init__.py b/shlax/__init__.py index 3acb890..0021973 100644 --- a/shlax/__init__.py +++ b/shlax/__init__.py @@ -1,6 +1,7 @@ from .actions import * from .image import Image from .strategies import * +from .output import Output from .proc import Proc from .targets import * from .shlaxfile import Shlaxfile diff --git a/shlax/actions/__init__.py b/shlax/actions/__init__.py index 4a21bed..7519565 100644 --- a/shlax/actions/__init__.py +++ b/shlax/actions/__init__.py @@ -1,5 +1,7 @@ from .commit import Commit +from .copy import Copy from .packages import Packages # noqa from .base import Action # noqa from .run import Run # noqa +from .pip import Pip from .service import Service diff --git a/shlax/actions/base.py b/shlax/actions/base.py index 928c93b..5494389 100644 --- a/shlax/actions/base.py +++ b/shlax/actions/base.py @@ -8,6 +8,9 @@ from ..exceptions import WrongResult class Action: parent = None contextualize = [] + colorize = { + '[^ ]*([^:]*):': {1: 0}, + } def __init__(self, *args, **kwargs): self.args = args @@ -84,7 +87,8 @@ class Action: sys.exit(1) def output_factory(self, *args, **kwargs): - return Output(*args, kwargs) + kwargs.setdefault('regexps', self.colorize) + return Output(*args, **kwargs) async def __call__(self, *args, **kwargs): self.status = 'running' @@ -98,3 +102,8 @@ class Action: if self.status == 'running': self.status = 'success' return result + + def callable(self): + async def cb(*a, **k): + return await self(*a, **k) + return cb diff --git a/shlax/actions/copy.py b/shlax/actions/copy.py new file mode 100644 index 0000000..a96881a --- /dev/null +++ b/shlax/actions/copy.py @@ -0,0 +1,10 @@ +from .base import Action + + +class Copy(Action): + def __init__(self, *args): + self.src = args[:-1] + self.dst = args[-1] + + def call(self, *args, **kwargs): + self.copy(self.src, self.dst) diff --git a/shlax/actions/pip.py b/shlax/actions/pip.py new file mode 100644 index 0000000..e9d763e --- /dev/null +++ b/shlax/actions/pip.py @@ -0,0 +1,51 @@ +from glob import glob +import os + +from .base import Action + + +class Pip(Action): + packages = dict( + apt=['python3-pip'], + ) + + def __init__(self, *pip_packages, pip=None, requirements=None): + self.pip_packages = pip_packages + self.requirements = requirements + + async def call(self, *args, **kwargs): + breakpoint() + self.pip = await self.which(('pip3', 'pip', 'pip2')) + if not self.pip: + raise Exception('Could not find pip command') + + if 'CACHE_DIR' in os.environ: + cache = os.path.join(os.getenv('CACHE_DIR'), 'pip') + else: + cache = os.path.join(os.getenv('HOME'), '.cache', 'pip') + + await script.mount(cache, '/root/.cache/pip') + await script.crexec(f'{self.pip} install --upgrade pip') + + # https://github.com/pypa/pip/issues/5599 + self.pip = 'python3 -m pip' + + pip_packages = [] + for visitor in script.container.visitors: + pp = getattr(visitor, 'pip_packages', None) + if not pp: + continue + pip_packages += pip_packages + + source = [p for p in pip_packages if p.startswith('/')] + if source: + await script.crexec( + f'{self.pip} install --upgrade --editable {" ".join(source)}' + ) + + nonsource = [p for p in pip_packages if not p.startswith('/')] + if nonsource: + await script.crexec(f'{self.pip} install --upgrade {" ".join(nonsource)}') + + if self.requirements: + await script.crexec(f'{self.pip} install --upgrade -r {self.requirements}') diff --git a/shlax/cli.py b/shlax/cli.py index 40419c6..66708bf 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -24,7 +24,6 @@ async def runall(*args, **kwargs): @cli2.option('debug', alias='d', help='Display debug output.') async def test(*args, **kwargs): - breakpoint() """Run podctl test over a bunch of paths.""" report = [] @@ -112,28 +111,20 @@ async def test(*args, **kwargs): class ConsoleScript(cli2.ConsoleScript): - def call(self, *args, **kwargs): + def __call__(self, *args, **kwargs): self.shlaxfile = None shlaxfile = sys.argv.pop(1) if len(sys.argv) > 1 else '' if os.path.exists(shlaxfile.split('::')[0]): self.shlaxfile = Shlaxfile() self.shlaxfile.parse(shlaxfile) for name, action in self.shlaxfile.actions.items(): - async def cb(*args, **kwargs): - return await Localhost(action)(*args, **kwargs) self[name] = cli2.Callable( name, - cb, + action.callable(), color=getattr(action, 'color', cli2.YELLOW), ) return super().__call__(*args, **kwargs) - def __call__(self, command): - args = self.parser.funcargs - kwargs = self.parser.funckwargs - breakpoint() - return command(*args, **kwargs) - def call(self, command): try: return super().call(command) diff --git a/shlax/contrib/gitlab.py b/shlax/contrib/gitlab.py index c44c69a..39337a2 100644 --- a/shlax/contrib/gitlab.py +++ b/shlax/contrib/gitlab.py @@ -5,12 +5,12 @@ from shlax import * class GitLabCIConfig(Script): async def call(self, *args, write=True, **kwargs): - await super().__call__(*args, **kwargs) + await super().call(*args, **kwargs) self.kwargs = kwargs for name, definition in self.context.items(): self.kwargs[name] = definition output = yaml.dump(self.kwargs) - print(output) + self.output(output) if write: with open('.gitlab-ci.yml', 'w+') as f: f.write(output) diff --git a/shlax/output.py b/shlax/output.py index 36026f0..7c3ddb4 100644 --- a/shlax/output.py +++ b/shlax/output.py @@ -1,6 +1,9 @@ +import re +import sys class Output: + prefixes = dict() colors = ( '\x1b[1;36;45m', '\x1b[1;36;41m', @@ -8,53 +11,59 @@ class Output: '\x1b[1;37;45m', '\x1b[1;32m', '\x1b[1;37;44m', + '\u001b[30;1m', ) - def __init__(self, prefix=None): - self.prefix = prefix - self.prefixes = dict() - self.prefix_length = 0 + def color(self, code=None): + if not code: + return '\u001b[0m' + code = str(code) + return u"\u001b[38;5;" + code + "m" - def call(self, line, prefix, highlight=True, flush=True): - if prefix and prefix not in self.prefixes: - self.prefixes[prefix] = ( + def colorize(self, code, content): + return self.color(code) + content + self.color() + + def __init__(self, prefix=None, regexps=None, debug=True, write=None, flush=None): + self.prefix = prefix + self.debug = debug + self.prefix_length = 0 + self.regexps = regexps or dict() + self.write = write or sys.stdout.buffer.write + self.flush = flush or sys.stdout.flush + + def __call__(self, line, highlight=True, flush=True): + if self.prefix and self.prefix not in self.prefixes: + self.prefixes[self.prefix] = ( self.colors[len([*self.prefixes.keys()]) - 1] ) - if len(prefix) > self.prefix_length: - self.prefix_length = len(prefix) + if len(self.prefix) > self.prefix_length: + self.prefix_length = len(self.prefix) - prefix_color = self.prefixes[prefix] if prefix else '' - prefix_padding = '.' * (self.prefix_length - len(prefix) - 2) if prefix else '' + prefix_color = self.prefixes[self.prefix] if self.prefix else '' + prefix_padding = '.' * (self.prefix_length - len(self.prefix) - 2) if self.prefix else '' if prefix_padding: prefix_padding = ' ' + prefix_padding + ' ' - sys.stdout.buffer.write(( + self.write(( ( prefix_color + prefix_padding - + prefix + + self.prefix + ' ' - + Back.RESET - + Style.RESET_ALL - + Fore.LIGHTBLACK_EX + '| ' - + Style.RESET_ALL - if prefix + if self.prefix else '' ) + self.highlight(line, highlight) ).encode('utf8')) if flush: - sys.stdout.flush() + self.flush() - def cmd(self, line, prefix): + def cmd(self, line): self( - Fore.LIGHTBLACK_EX - + '+ ' - + Style.RESET_ALL + self.colorize(251, '+ ') + self.highlight(line, 'bash'), - prefix, highlight=False ) @@ -73,4 +82,16 @@ class Output: or '\\e[' in line ): return line + + for regexp, colors in self.regexps.items(): + match = re.match(regexp, line) + if not match: + continue + + for group, color in colors.items(): + res = match.group(group) + if not res: + continue + line = line.replace(res, self.colorize(color, res)) + return line diff --git a/shlax/proc.py b/shlax/proc.py index 528677c..f701031 100644 --- a/shlax/proc.py +++ b/shlax/proc.py @@ -22,16 +22,16 @@ class PrefixStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): super().__init__(*args, **kwargs) def pipe_data_received(self, fd, data): - if (self.debug is True or 'out' in str(self.debug)) and fd in (1, 2): + if (self.output.debug is True or 'out' in str(self.output.debug)) and fd in (1, 2): self.output(data, flush=False) sys.stdout.flush() super().pipe_data_received(fd, data) -def protocol_factory(prefix): +def protocol_factory(output): def _p(): return PrefixStreamProtocol( - prefix, + output, limit=asyncio.streams._DEFAULT_LIMIT, loop=asyncio.events.get_event_loop() ) @@ -79,12 +79,16 @@ class Proc: args = ['sh', '-euc', ' '.join(args)] return args + def output_factory(self, *args, **kwargs): + args = tuple(self.prefix) + args + return Output(*args, kwargs) + async def __call__(self, wait=True): if self.called: raise Exception('Already called: ' + self.cmd) if self.debug is True or 'cmd' in str(self.debug): - output.cmd(self.cmd, self.prefix) + self.output.cmd(self.cmd) if self.test: if self.test is True: @@ -94,7 +98,7 @@ class Proc: loop = asyncio.events.get_event_loop() transport, protocol = await loop.subprocess_exec( - protocol_factory(self.prefix), *self.args) + protocol_factory(self.output), *self.args) self.proc = asyncio.subprocess.Process(transport, protocol, loop) self.called = True diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index e7eb67d..9a88560 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -53,7 +53,7 @@ class Buildah(Localhost): async def copy(self, src, dst): """Run buildah copy to copy a file from host into container.""" - return await self.exec(f'buildah copy {self.ctr} {src} {dst}') + return await self.exec(f'buildah copy {self.ctr} {src} {self.mnt}{dst}') async def mount(self, src, dst): """Mount a host directory into the container.""" diff --git a/shlaxfile.py b/shlaxfile.py index 931c706..b6fbef5 100755 --- a/shlaxfile.py +++ b/shlaxfile.py @@ -3,6 +3,12 @@ from shlax.contrib.gitlab import * PYTEST = 'py.test -svv tests' +build = Buildah('alpine', + Copy('shlax', 'setup.py', '/app'), + Pip('/app'), + commit='yourlabs/shlax', +) + gitlabci = GitLabCIConfig( Job('test', stage='test', diff --git a/tests/test_output.py b/tests/test_output.py new file mode 100644 index 0000000..8a6be51 --- /dev/null +++ b/tests/test_output.py @@ -0,0 +1,24 @@ +import pytest +from shlax import Output + + +class Write: + def __init__(self): + self.output = '' + def __call__(self, out): + self.output += out.decode('utf8') + + +@pytest.fixture +def write(): + return Write() + + +def test_output_regexps(write): + output = Output( + regexps={'.*': {0: 0}}, + write=write, + flush=lambda: None, + ) + output('foo') + assert write.output == output.colorize(0, 'foo')