sam/shlax
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Docker target, recursive calls

Browse Source
This commit is contained in:
jpic 2020-02-16 13:47:33 +01:00
parent baf295f145
commit 6924d39590
14 changed files with 268 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.gitlab-ci.yml
View File

@ -1,14 +1,14 @@
build:
image: yourlabs/shlax
script: pip install -U .[test] && py.test -svv tests
stage: test
script: ./shlaxfile.py build
stage: build
pypi:
image: yourlabs/python
only:
- tags
script: pypi-release
script: ./shlaxfile.py pypi
stage: deploy
test:
image: yourlabs/python
script: pip install -U .[test] && py.test -svv tests
image: yourlabs/shlax
script: ./shlaxfile.py test
stage: test

34
shlax/actions/base.py
View File

@ -1,4 +1,6 @@
import functools
import inspect
import importlib
import sys
from ..output import Output
@ -164,16 +166,46 @@ class Action:
def callable(self):
from ..targets import Localhost
async def cb(*a, **k):
return await Localhost(self, quiet=True)(*a, **k)
from shlax.cli import cli
script = Localhost(self, quiet=True)
result = await script(*a, **k)
success = functools.reduce(
lambda a, b: a + b,
[1 for c in script.children() if c.status == 'success'] or [0])
if success:
script.output.success(f'{success} PASS')
failures = functools.reduce(
lambda a, b: a + b,
[1 for c in script.children() if c.status == 'fail'] or [0])
if failures:
script.output.fail(f'{failures} FAIL')
cli.exit_code = failures
return result
return cb
def kwargs_output(self):
return self.kwargs
def action(self, action, *args, **kwargs):
if isinstance(action, str):
import cli2
a = cli2.Callable.factory(action).target
if not a:
a = cli2.Callable.factory(
'.'.join(['shlax', action])
).target
if a:
action = a
p = action(*args, **kwargs)
for parent in self.parents():
if hasattr(parent, 'actions'):
break
p.parent = parent
if 'actions' not in self.__dict__:
# "mutate" to Strategy
from ..strategies.script import Actions
self.actions = Actions(self, [p])
return p

2
shlax/actions/packages.py
View File

@ -113,7 +113,7 @@ class Packages(Action):
else:
mgr = await self.which(*self.mgrs.keys())
if mgr:
self.mgr = mgr.split('/')[-1]
self.mgr = mgr[0].split('/')[-1]
if not self.mgr:
raise Exception('Packages does not yet support this distro')

38
shlax/actions/pip.py
View File

@ -5,25 +5,27 @@ from .base import Action
class Pip(Action):
packages = dict(
apt=['python3-pip'],
)
def __init__(self, *pip_packages, pip=None, requirements=None):
self.pip_packages = pip_packages
self.requirements = requirements
super().__init__(*pip_packages, pip=pip, requirements=requirements)
async def call(self, *args, **kwargs):
self.pip = await self.which('pip3', 'pip', 'pip2')
if not self.pip:
pip = await self.which('pip3', 'pip', 'pip2')
if pip:
pip = pip[0]
else:
from .packages import Packages
action = self.action(Packages, 'python3,apk', 'python3-pip,apt', args=args, kwargs=kwargs)
action = self.action(
Packages,
'python3,apk', 'python3-pip,apt',
args=args, kwargs=kwargs
)
await action(*args, **kwargs)
self.pip = await self.which('pip3', 'pip', 'pip2')
if not self.pip:
pip = await self.which('pip3', 'pip', 'pip2')
if not pip:
raise Exception('Could not install a pip command')
else:
pip = pip[0]
if 'CACHE_DIR' in os.environ:
cache = os.path.join(os.getenv('CACHE_DIR'), 'pip')
@ -33,20 +35,20 @@ class Pip(Action):
if getattr(self, 'mount', None):
# we are in a target which shares a mount command
await self.mount(cache, '/root/.cache/pip')
await self.exec(f'{self.pip} install --upgrade pip')
await self.exec(f'{pip} install --upgrade pip')
# https://github.com/pypa/pip/issues/5599
self.pip = 'python3 -m pip'
pip = 'python3 -m pip'
source = [p for p in self.pip_packages if p.startswith('/')]
source = [p for p in self.args if p.startswith('/') or p.startswith('.')]
if source:
await self.exec(
f'{self.pip} install --upgrade --editable {" ".join(source)}'
f'{pip} install --upgrade --editable {" ".join(source)}'
)
nonsource = [p for p in self.pip_packages if not p.startswith('/')]
nonsource = [p for p in self.args if not p.startswith('/')]
if nonsource:
await self.exec(f'{self.pip} install --upgrade {" ".join(nonsource)}')
await self.exec(f'{pip} install --upgrade {" ".join(nonsource)}')
if self.requirements:
await self.exec(f'{self.pip} install --upgrade -r {self.requirements}')
await self.exec(f'{pip} install --upgrade -r {self.requirements}')

12
shlax/actions/run.py
View File

@ -1,6 +1,18 @@
from ..targets.buildah import Buildah
from ..targets.docker import Docker
from .base import Action
class Run(Action):
async def call(self, *args, **kwargs):
image = self.kwargs.get('image', None)
if not image:
return await self.exec(*self.args, **self.kwargs)
if isinstance(image, Buildah):
breakpoint()
result = await self.action(image, *args, **kwargs)
return await Docker(
image=image,
).exec(*args, **kwargs)

5
shlax/cli.py
View File

@ -8,6 +8,7 @@ shlax is a micro-framework to orchestrate commands.
import asyncio
import cli2
import copy
import inspect
import os
import sys
@ -130,8 +131,10 @@ class ConsoleScript(cli2.ConsoleScript):
return super().__call__(*args, **kwargs)
def call(self, command):
kwargs = copy.copy(self.parser.funckwargs)
kwargs.update(self.parser.options)
try:
return super().call(command)
return command(*self.parser.funcargs, **kwargs)
except WrongResult as e:
print(e)
self.exit_code = e.proc.rc

11
shlax/contrib/gitlab.py
View File

@ -3,9 +3,16 @@ import yaml
from shlax import *
class GitLabCIConfig(Script):
class GitLabCI(Script):
async def call(self, *args, write=True, **kwargs):
output = yaml.dump(self.kwargs)
output = dict()
for key, value in self.kwargs.items():
if isinstance(value, dict):
output[key] = value
output[key]['script'] = './shlaxfile.py ' + key
else:
output[key] = value
output = yaml.dump(output)
if kwargs['debug'] is True:
self.output(output)
if write:

7
shlax/image.py
View File

@ -55,3 +55,10 @@ class Image:
# default tag by default ...
if not self.tags:
self.tags = ['latest']
async def __call__(self, action, *args, **kwargs):
args = list(args)
return await action.exec(*args, **self.kwargs)
def __str__(self):
return self.repository

52
shlax/output.py
View File

@ -33,8 +33,8 @@ class Output:
self.write = write or sys.stdout.buffer.write
self.flush = flush or sys.stdout.flush
def __call__(self, line, highlight=True, flush=True):
if self.prefix and self.prefix not in self.prefixes:
def prefix_line(self):
if self.prefix not in self.prefixes:
self.prefixes[self.prefix] = self.prefix_colors[len(self.prefixes)]
if len(self.prefix) > self.prefix_length:
self.prefix_length = len(self.prefix)
@ -44,24 +44,24 @@ class Output:
if prefix_padding:
prefix_padding = ' ' + prefix_padding + ' '
self.write((
(
prefix_color
+ prefix_padding
+ self.prefix
+ ' '
+ self.colors['reset']
+ '| '
if self.prefix
else ''
)
+ self.highlight(line, highlight)
+ self.colors['reset']
).encode('utf8'))
return [
prefix_color,
prefix_padding,
self.prefix,
' ',
self.colors['reset'],
'| '
]
def __call__(self, line, highlight=True, flush=True):
line = [self.highlight(line) if highlight else line]
if self.prefix:
line = self.prefix_line() + line
line = ''.join(line)
self.write(line.encode('utf8'))
if flush:
if not line.endswith('\n'):
self.write(b'\n')
self.flush()
def cmd(self, line):
@ -70,7 +70,8 @@ class Output:
+ '\x1b[1;38;5;15m'
+ ' '
+ self.highlight(line, 'bash')
+ self.colors['reset'],
+ self.colors['reset']
+ '\n',
highlight=False
)
@ -92,25 +93,28 @@ class Output:
for regexp, colors in self.regexps.items():
line = re.sub(regexp, colors.format(**self.colors), line)
line = line + self.colors['reset']
return line
def clean(self, action):
if self.debug is True or 'visit' in str(self.debug):
if self.debug is True:
self(''.join([
self.colors['bluebold'],
'+ CLEAN ',
self.colors['reset'],
action.colorized(),
'\n',
]))
def start(self, action):
if self.debug is True or 'visit' in str(self.debug):
if self.debug is True:
self(''.join([
self.colors['orangebold'],
'⚠ START ',
self.colors['reset'],
action.colorized(),
'\n',
]))
def success(self, action):
@ -119,7 +123,8 @@ class Output:
self.colors['greenbold'],
'✔ SUCCESS ',
self.colors['reset'],
action.colorized(),
action.colorized() if hasattr(action, 'colorized') else str(action),
'\n',
]))
def fail(self, action, exception=None):
@ -128,5 +133,6 @@ class Output:
self.colors['redbold'],
'✘ FAIL ',
self.colors['reset'],
action.colorized(),
action.colorized() if hasattr(action, 'colorized') else str(action),
'\n',
]))

1
shlax/targets/__init__.py
View File

@ -1,3 +1,4 @@
from .buildah import Buildah
from .docker import Docker
from .localhost import Localhost
from .ssh import Ssh

37
shlax/targets/buildah.py
View File

@ -8,6 +8,8 @@ import subprocess
import sys
import textwrap
from ..actions.base import Action
from ..exceptions import Mistake
from ..proc import Proc
from ..image import Image
from .localhost import Localhost
@ -20,7 +22,10 @@ class Buildah(Localhost):
"""
contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount']
def __init__(self, base, *args, commit=None, push=False, **kwargs):
def __init__(self, base, *args, commit=None, push=False, cmd=None, **kwargs):
if isinstance(base, Action):
args = [base] + list(args)
base = 'alpine' # default selection in case of mistake
super().__init__(*args, **kwargs)
self.base = base
self.mounts = dict()
@ -28,6 +33,9 @@ class Buildah(Localhost):
self.mnt = None
self.image = Image(commit) if commit else None
self.push = push or os.getenv('CI')
self.config= dict(
cmd=cmd or 'sh',
)
def shargs(self, *args, user=None, buildah=True, **kwargs):
if not buildah or args[0].startswith('buildah'):
@ -52,9 +60,6 @@ class Buildah(Localhost):
"""Run buildah config."""
return await self.exec(f'buildah config {line} {self.ctr}', buildah=False)
async def mkdir(self, *dirs):
return await self.exec(*['mkdir', '-p'] + list(dirs))
async def copy(self, *args):
"""Run buildah copy to copy a file from host into container."""
src = args[:-1]
@ -81,19 +86,6 @@ class Buildah(Localhost):
await self.exec(f'mount -o bind {src} {target}', buildah=False)
self.mounts[src] = dst
async def which(self, *cmd):
"""
Return the first path to the cmd in the container.
If cmd argument is a list then it will try all commands.
"""
paths = (await self.env('PATH')).split(':')
for path in paths:
for c in cmd:
p = os.path.join(self.mnt, path[1:], c)
if os.path.exists(p):
return p[len(str(self.mnt)):]
def is_wrapper(self):
return not (
Proc.test
@ -101,7 +93,6 @@ class Buildah(Localhost):
or getattr(self.parent, 'parent', None)
)
async def call(self, *args, **kwargs):
if not self.is_wrapper():
self.ctr = (await self.exec('buildah', 'from', self.base, buildah=False)).out
@ -124,7 +115,11 @@ class Buildah(Localhost):
argv += [
cli.parser.command.name, # script name ?
]
self.output(' '.join(argv), 'EXECUTION', flush=True)
await self.exec(*argv)
'''
if debug is True or 'cmd' in str(debug):
self.output.cmd(' '.join(argv))
proc = await asyncio.create_subprocess_shell(
shlex.join(argv),
@ -134,11 +129,15 @@ class Buildah(Localhost):
)
await proc.communicate()
cli.exit_code = await proc.wait()
'''
async def commit(self):
if not self.image:
return
for key, value in self.config.items():
await self.exec(f'buildah config --{key} "{value}" {self.ctr}')
self.sha = (await self.exec(
'buildah',
'commit',

78
shlax/targets/docker.py Normal file
View File

@ -0,0 +1,78 @@
import asyncio
from pathlib import Path
import os
from ..image import Image
from .localhost import Localhost
class Docker(Localhost):
contextualize = Localhost.contextualize + ['mnt', 'ctr', 'mount']
def __init__(self, *args, **kwargs):
self.image = Image(kwargs.get('image', 'alpine'))
super().__init__(*args, **kwargs)
self.context['ctr'] = None
def shargs(self, *args, daemon=False, **kwargs):
if args[0] == 'docker':
return args, kwargs
extra = []
if 'user' in kwargs:
extra += ['--user', kwargs.pop('user')]
args, kwargs = super().shargs(*args, **kwargs)
if self.context['ctr']:
executor = 'exec'
extra = [self.context['ctr']]
return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + list(args), kwargs
executor = 'run'
cwd = os.getcwd()
if daemon:
extra += ['-d']
extra = extra + ['-v', f'{cwd}:{cwd}', '-w', f'{cwd}']
return [self.kwargs.get('docker', 'docker'), executor, '-t'] + extra + [str(self.image)] + list(args), kwargs
async def call(self, *args, **kwargs):
name = kwargs.get('name', os.getcwd()).split('/')[-1]
self.context['ctr'] = (
await self.exec(
'docker', 'ps', '-aq', '--filter',
'name=' + name,
raises=False
)
).out.split('\n')[0]
if 'recreate' in args and self.context['ctr']:
await self.exec('docker', 'rm', '-f', self.context['ctr'])
self.context['ctr'] = None
if self.context['ctr']:
self.context['ctr'] = (await self.exec('docker', 'start', name)).out
else:
self.context['ctr'] = (
await self.exec('sleep', '120', daemon=True)
).out
return await super().call(*args, **kwargs)
async def copy(self, *args):
src = args[:-1]
dst = args[-1]
await self.mkdir(dst)
procs = []
for s in src:
'''
if Path(s).is_dir():
await self.mkdir(s)
args = ['docker', 'copy', self.ctr, s, Path(dst) / s]
else:
args = ['docker', 'copy', self.ctr, s, dst]
'''
args = ['docker', 'cp', s, self.context['ctr'] + ':' + dst]
procs.append(self.exec(*args))
return await asyncio.gather(*procs)

19
shlax/targets/localhost.py
View File

@ -1,4 +1,5 @@
import os
import re
from shlax.proc import Proc
@ -50,12 +51,20 @@ class Localhost(Script):
If cmd argument is a list then it will try all commands.
"""
for path in (await self.env('PATH')).split(':'):
for c in cmd:
p = os.path.join(self.root, path[1:], c)
if os.path.exists(p):
return p[len(str(self.root)):]
proc = await self.exec('type ' + ' '.join(cmd), raises=False)
result = []
for res in proc.out.split('\n'):
match = re.match('([^ ]+) is ([^ ]+)$', res.strip())
if match:
result.append(match.group(1))
return result
async def copy(self, *args):
args = ['cp', '-ra'] + list(args)
return await self.exec(*args)
async def mount(self, *dirs):
pass
async def mkdir(self, *dirs):
return await self.exec(*['mkdir', '-p'] + list(dirs))

54
shlaxfile.py
View File

@ -3,32 +3,42 @@ from shlax.contrib.gitlab import *
PYTEST = 'py.test -svv tests'
build = Buildah('alpine',
build = Buildah(
Copy('shlax/', 'setup.py', '/app'),
Pip('/app'),
commit='yourlabs/shlax',
)
gitlabci = GitLabCIConfig(
build=dict(
stage='test',
image='yourlabs/shlax',
script='pip install -U .[test] && ' + PYTEST,
),
test=dict(
stage='test',
image='yourlabs/python',
script='pip install -U .[test] && ' + PYTEST,
),
pypi=dict(
stage='deploy',
image='yourlabs/python',
script='pypi-release',
only=['tags']
),
workdir='/app',
)
test = Script(
gitlabci,
Run('gitlab-runner exec docker test'),
Pip('.[test]'),
Run(PYTEST),
)
buildtest = Docker(
*test.actions,
mount={'.': '/app'},
workdir='/app',
)
pypi = Run(
'pypi-release',
stage='deploy',
image='yourlabs/python',
)
gitlabci = GitLabCI(
build=dict(
stage='build',
image='yourlabs/shlax',
),
test=dict(
stage='test',
image='yourlabs/shlax',
),
pypi=dict(
stage='deploy',
only=['tags'],
image='yourlabs/python',
),
)