From 984f09d704d9017b27e88357c412a0f7e42dbac0 Mon Sep 17 00:00:00 2001 From: jpic Date: Sat, 15 Feb 2020 01:04:41 +0100 Subject: [PATCH] Refactor output and call Each action has its own output from now on. --- .pre-commit-config.yaml | 4 ++ .pre-commit-hooks.yaml | 4 +- shlax/__init__.py | 2 +- shlax/actions/base.py | 23 ++++++++- shlax/actions/commit.py | 2 +- shlax/actions/packages.py | 2 +- shlax/actions/run.py | 2 +- shlax/actions/service.py | 2 +- shlax/cli.py | 2 +- shlax/contrib/gitlab.py | 4 +- shlax/output.py | 76 +++++++++++++++++++++++++++++ shlax/proc.py | 99 +++----------------------------------- shlax/strategies/asyn.py | 2 +- shlax/strategies/script.py | 12 +---- shlax/targets/buildah.py | 8 +-- 15 files changed, 124 insertions(+), 120 deletions(-) create mode 100644 shlax/output.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a396c5..e42fb18 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,3 +8,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files +- repo: https://yourlabs.io/oss/shlax + rev: master + hooks: + - id: shlaxfile-gitlabci diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 40ac5ab..b021449 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,5 +1,5 @@ - id: shlaxfile-gitlabci - name: Check for added large files + name: Regenerate .gitlab-ci.yml description: Regenerate gitlabci - entry: ./shlaxfile.py gitlabci + entry: pip install -e . && ./shlaxfile.py gitlabci language: python diff --git a/shlax/__init__.py b/shlax/__init__.py index 137d01e..3acb890 100644 --- a/shlax/__init__.py +++ b/shlax/__init__.py @@ -1,6 +1,6 @@ from .actions import * from .image import Image from .strategies import * -from .proc import output, Proc +from .proc import Proc from .targets import * from .shlaxfile import Shlaxfile diff --git a/shlax/actions/base.py b/shlax/actions/base.py index f9b3448..928c93b 100644 --- a/shlax/actions/base.py +++ b/shlax/actions/base.py @@ -1,6 +1,9 @@ import inspect import sys +from ..output import Output +from ..exceptions import WrongResult + class Action: parent = None @@ -76,6 +79,22 @@ class Action: return getattr(a, name) raise AttributeError(name) - async def __call__(self, *args, **kwargs): - print(f'{self}.__call__(*args, **kwargs) not implemented') + async def call(self, *args, **kwargs): + print(f'{self}.call(*args, **kwargs) not implemented') sys.exit(1) + + def output_factory(self, *args, **kwargs): + return Output(*args, kwargs) + + async def __call__(self, *args, **kwargs): + self.status = 'running' + self.output = self.output_factory(*args, **kwargs) + try: + result = await self.call(*args, **kwargs) + except WrongResult as e: + print(e) + self.status = 'fail' + else: + if self.status == 'running': + self.status = 'success' + return result diff --git a/shlax/actions/commit.py b/shlax/actions/commit.py index 03523cc..ee6dd55 100644 --- a/shlax/actions/commit.py +++ b/shlax/actions/commit.py @@ -50,7 +50,7 @@ class Commit(Action): if not self.tags: self.tags = ['latest'] - async def __call__(self, *args, ctr=None, **kwargs): + async def call(self, *args, ctr=None, **kwargs): self.sha = (await self.parent.parent.exec( 'buildah', 'commit', diff --git a/shlax/actions/packages.py b/shlax/actions/packages.py index 7c14de5..6d37117 100644 --- a/shlax/actions/packages.py +++ b/shlax/actions/packages.py @@ -104,7 +104,7 @@ class Packages(Action): print(f'{self.container.name} | Waiting for update ...') await asyncio.sleep(1) - async def __call__(self, *args, **kwargs): + async def call(self, *args, **kwargs): cached = getattr(self, '_pagkages_mgr', None) if cached: self.mgr = cached diff --git a/shlax/actions/run.py b/shlax/actions/run.py index 959460a..21199ff 100644 --- a/shlax/actions/run.py +++ b/shlax/actions/run.py @@ -2,5 +2,5 @@ from .base import Action class Run(Action): - async def __call__(self, *args, **kwargs): + async def call(self, *args, **kwargs): return (await self.exec(*self.args, **self.kwargs)) diff --git a/shlax/actions/service.py b/shlax/actions/service.py index 972531f..3561c64 100644 --- a/shlax/actions/service.py +++ b/shlax/actions/service.py @@ -9,7 +9,7 @@ class Service(Action): self.names = names super().__init__() - async def __call__(self, *args, **kwargs): + async def call(self, *args, **kwargs): return asyncio.gather(*[ self.exec('systemctl', 'start', name, user='root') for name in self.names diff --git a/shlax/cli.py b/shlax/cli.py index 09e84f8..4a3e42f 100644 --- a/shlax/cli.py +++ b/shlax/cli.py @@ -112,7 +112,7 @@ async def test(*args, **kwargs): class ConsoleScript(cli2.ConsoleScript): - def __call__(self, *args, **kwargs): + def call(self, *args, **kwargs): self.shlaxfile = None shlaxfile = sys.argv.pop(1) if len(sys.argv) > 1 else '' if os.path.exists(shlaxfile.split('::')[0]): diff --git a/shlax/contrib/gitlab.py b/shlax/contrib/gitlab.py index bb9b3db..c44c69a 100644 --- a/shlax/contrib/gitlab.py +++ b/shlax/contrib/gitlab.py @@ -4,7 +4,7 @@ from shlax import * class GitLabCIConfig(Script): - async def __call__(self, *args, write=True, **kwargs): + async def call(self, *args, write=True, **kwargs): await super().__call__(*args, **kwargs) self.kwargs = kwargs for name, definition in self.context.items(): @@ -17,5 +17,5 @@ class GitLabCIConfig(Script): class Job(Action): - async def __call__(self, *args, **kwargs): + async def call(self, *args, **kwargs): self.context[self.args[0]] = self.kwargs diff --git a/shlax/output.py b/shlax/output.py new file mode 100644 index 0000000..36026f0 --- /dev/null +++ b/shlax/output.py @@ -0,0 +1,76 @@ + + +class Output: + colors = ( + '\x1b[1;36;45m', + '\x1b[1;36;41m', + '\x1b[1;36;40m', + '\x1b[1;37;45m', + '\x1b[1;32m', + '\x1b[1;37;44m', + ) + + def __init__(self, prefix=None): + self.prefix = prefix + self.prefixes = dict() + self.prefix_length = 0 + + def call(self, line, prefix, highlight=True, flush=True): + if prefix and prefix not in self.prefixes: + self.prefixes[prefix] = ( + self.colors[len([*self.prefixes.keys()]) - 1] + ) + if len(prefix) > self.prefix_length: + self.prefix_length = len(prefix) + + prefix_color = self.prefixes[prefix] if prefix else '' + prefix_padding = '.' * (self.prefix_length - len(prefix) - 2) if prefix else '' + if prefix_padding: + prefix_padding = ' ' + prefix_padding + ' ' + + sys.stdout.buffer.write(( + ( + prefix_color + + prefix_padding + + prefix + + ' ' + + Back.RESET + + Style.RESET_ALL + + Fore.LIGHTBLACK_EX + + '| ' + + Style.RESET_ALL + if prefix + else '' + ) + + self.highlight(line, highlight) + ).encode('utf8')) + + if flush: + sys.stdout.flush() + + def cmd(self, line, prefix): + self( + Fore.LIGHTBLACK_EX + + '+ ' + + Style.RESET_ALL + + self.highlight(line, 'bash'), + prefix, + highlight=False + ) + + def print(self, content): + self( + content, + prefix=None, + highlight=False + ) + + def highlight(self, line, highlight=True): + line = line.decode('utf8') if isinstance(line, bytes) else line + if not highlight or ( + '\x1b[' in line + or '\033[' in line + or '\\e[' in line + ): + return line + return line diff --git a/shlax/proc.py b/shlax/proc.py index e30eb0b..528677c 100644 --- a/shlax/proc.py +++ b/shlax/proc.py @@ -3,99 +3,12 @@ Asynchronous process execution wrapper. """ import asyncio -from colorama import Fore, Back, Style import os import shlex import sys from .exceptions import WrongResult -import pygments -from pygments import lexers -from pygments.formatters import TerminalFormatter -from pygments.formatters import Terminal256Formatter - - -class Output: - colors = ( - '\x1b[1;36;45m', - '\x1b[1;36;41m', - '\x1b[1;36;40m', - '\x1b[1;37;45m', - '\x1b[1;32m', - '\x1b[1;37;44m', - ) - def __init__(self): - self.prefixes = dict() - self.prefix_length = 0 - - def __call__(self, line, prefix, highlight=True, flush=True): - if prefix and prefix not in self.prefixes: - self.prefixes[prefix] = ( - self.colors[len([*self.prefixes.keys()]) - 1] - ) - if len(prefix) > self.prefix_length: - self.prefix_length = len(prefix) - - prefix_color = self.prefixes[prefix] if prefix else '' - prefix_padding = '.' * (self.prefix_length - len(prefix) - 2) if prefix else '' - if prefix_padding: - prefix_padding = ' ' + prefix_padding + ' ' - - sys.stdout.buffer.write(( - ( - prefix_color - + prefix_padding - + prefix - + ' ' - + Back.RESET - + Style.RESET_ALL - + Fore.LIGHTBLACK_EX - + '| ' - + Style.RESET_ALL - if prefix - else '' - ) - + self.highlight(line, highlight) - ).encode('utf8')) - - if flush: - sys.stdout.flush() - - def cmd(self, line, prefix): - self( - Fore.LIGHTBLACK_EX - + '+ ' - + Style.RESET_ALL - + self.highlight(line, 'bash'), - prefix, - highlight=False - ) - - def print(self, content): - self( - content, - prefix=None, - highlight=False - ) - - def highlight(self, line, highlight=True): - line = line.decode('utf8') if isinstance(line, bytes) else line - if not highlight or ( - '\x1b[' in line - or '\033[' in line - or '\\e[' in line - ): - return line - elif isinstance(highlight, str): - lexer = lexers.get_lexer_by_name(highlight) - else: - lexer = lexers.get_lexer_by_name('python') - formatter = Terminal256Formatter( - style=os.getenv('PODCTL_STYLE', 'fruity')) - return pygments.highlight(line, lexer, formatter) - - -output = Output() +from .output import Output class PrefixStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): @@ -104,14 +17,13 @@ class PrefixStreamProtocol(asyncio.subprocess.SubprocessStreamProtocol): make asynchronous output readable. """ - def __init__(self, prefix, *args, **kwargs): - self.debug = kwargs.get('debug', True) - self.prefix = prefix + def __init__(self, output, *args, **kwargs): + self.output = output super().__init__(*args, **kwargs) def pipe_data_received(self, fd, data): if (self.debug is True or 'out' in str(self.debug)) and fd in (1, 2): - output(data, self.prefix, flush=False) + self.output(data, flush=False) sys.stdout.flush() super().pipe_data_received(fd, data) @@ -142,8 +54,9 @@ class Proc: """ test = False - def __init__(self, *args, prefix=None, raises=True, debug=True): + def __init__(self, *args, prefix=None, raises=True, debug=True, output=None): self.debug = debug if not self.test else False + self.output = output or Output() self.cmd = ' '.join(args) self.args = args self.prefix = prefix diff --git a/shlax/strategies/asyn.py b/shlax/strategies/asyn.py index 8c6ff0b..ee45d87 100644 --- a/shlax/strategies/asyn.py +++ b/shlax/strategies/asyn.py @@ -4,7 +4,7 @@ from .script import Script class Async(Script): - async def __call__(self, *args, **kwargs): + async def call(self, *args, **kwargs): return asyncio.gather(*[ procs.append(action(*args, **kwargs)) for action in self.actions diff --git a/shlax/strategies/script.py b/shlax/strategies/script.py index f6aa1f0..fe7d621 100644 --- a/shlax/strategies/script.py +++ b/shlax/strategies/script.py @@ -27,17 +27,9 @@ class Script(Action): super().__init__(**kwargs) self.actions = Actions(self, actions) - async def __call__(self, *args, **kwargs): + async def call(self, *args, **kwargs): for action in self.actions: - try: - await action(*args, **kwargs) - except WrongResult as e: - print(e) - action.status = 'fail' - break - else: - if action.status == 'running': - action.status = 'success' + await action(*args, **kwargs) def shargs(self, *args, **kwargs): user = kwargs.pop('user', None) diff --git a/shlax/targets/buildah.py b/shlax/targets/buildah.py index de7e70d..e7eb67d 100644 --- a/shlax/targets/buildah.py +++ b/shlax/targets/buildah.py @@ -8,7 +8,7 @@ import subprocess import sys import textwrap -from ..proc import Proc, output +from ..proc import Proc from ..image import Image from .localhost import Localhost @@ -88,12 +88,12 @@ class Buildah(Localhost): def __repr__(self): return f'Build' - async def __call__(self, *args, debug=False, **kwargs): + async def call(self, *args, debug=False, **kwargs): if Proc.test or os.getuid() == 0 or self.parent.parent: self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out self.mnt = Path((await self.exec('buildah', 'mount', self.ctr, buildah=False)).out) - result = await super().__call__(*args, **kwargs) + result = await super().call(*args, **kwargs) #await self.umounts() #await self.umount() await self.exec('buildah', 'rm', self.ctr, raises=False, buildah=False) @@ -113,7 +113,7 @@ class Buildah(Localhost): cli.shlaxfile.path, cli.parser.command.name, # script name ? ] - output(' '.join(argv), 'EXECUTION', flush=True) + self.output(' '.join(argv), 'EXECUTION', flush=True) proc = await asyncio.create_subprocess_shell( shlex.join(argv),