sam/shlax
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'mostadvanced' into 'master'

Browse Source 
Mostadvanced

See merge request oss/shlax!4
This commit is contained in:
∞ 2020-06-11 17:43:06 +02:00
commit 150736f4f4
7 changed files with 170 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.gitlab-ci.yml
View File

@ -2,15 +2,20 @@ build:
cache:
key: cache
paths: [.cache]
image: quay.io/buildah/stable
image: yourlabs/buildah
script:
- dnf install -y curl python38
- curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
- python3 get-pip.py
- pip3 install -U --user -e .[cli]
- CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build
stage: build
#build-itself:
# cache:
# key: cache
# paths: [.cache]
# image: shlax:$CI_COMMIT_SHORT_SHA
# script: python3 ./shlaxfile.py build
# stage: test
test:
image: yourlabs/python
stage: build

73
shlax/actions/packages.py
View File

@ -37,11 +37,13 @@ class Packages:
update='pacman -Sy',
upgrade='pacman -Su --noconfirm',
install='pacman -S --noconfirm',
lastupdate='stat -c %Y /var/lib/pacman/sync/core.db',
),
dnf=dict(
update='dnf makecache --assumeyes',
upgrade='dnf upgrade --best --assumeyes --skip-broken', # noqa
install='dnf install --setopt=install_weak_deps=False --best --assumeyes', # noqa
lastupdate='stat -c %Y /var/cache/dnf/* | head -n1',
),
yum=dict(
update='yum update',
@ -52,54 +54,52 @@ class Packages:
installed = []
def __init__(self, *packages, upgrade=True):
def __init__(self, *packages, upgrade=False):
self.packages = []
self.upgrade = upgrade
for package in packages:
line = dedent(package).strip().replace('\n', ' ')
self.packages += line.split(' ')
@property
def cache_root(self):
async def cache_setup(self, target):
if 'CACHE_DIR' in os.environ:
return os.path.join(os.getenv('CACHE_DIR'))
self.cache_root = os.path.join(os.getenv('CACHE_DIR'))
else:
return os.path.join(os.getenv('HOME'), '.cache')
self.cache_root = os.path.join(await target.parent.getenv('HOME'), '.cache')
# run pkgmgr_setup functions ie. apk_setup
await getattr(self, self.mgr + '_setup')(target)
async def update(self, target):
# run pkgmgr_setup functions ie. apk_setup
cachedir = await getattr(self, self.mgr + '_setup')(target)
# lastupdate = await target.exec(self.cmds['lastupdate'], raises=False)
# lastupdate = int(lastupdate.out) if lastupdate.rc == 0 else None
lastupdate = None
if os.path.exists(cachedir + '/lastupdate'):
with open(cachedir + '/lastupdate', 'r') as f:
try:
lastupdate = int(f.read().strip())
except:
pass
if not os.path.exists(cachedir):
os.makedirs(cachedir)
now = int(datetime.now().strftime('%s'))
# cache for a week
if not lastupdate or now - lastupdate > 604800:
await target.rexec(self.cmds['update'])
return
# disabling with the above return call until needed again
# might have to rewrite this to not have our own lockfile
# or find a better place on the filesystem
# also make sure the lockfile is actually needed when running on
# targets that don't have isguest=True
if not lastupdate or now - lastupdate > 604800:
# crude lockfile implementation, should work against *most*
# race-conditions ...
lockfile = cachedir + '/update.lock'
if not os.path.exists(lockfile):
with open(lockfile, 'w+') as f:
f.write(str(os.getpid()))
if not await target.parent.exists(lockfile):
await target.parent.write(lockfile, str(os.getpid()))
try:
await target.rexec(self.cmds['update'])
finally:
os.unlink(lockfile)
await target.parent.rm(lockfile)
with open(cachedir + '/lastupdate', 'w+') as f:
f.write(str(now))
await target.parent.write(cachedir + '/lastupdate', str(now))
else:
while os.path.exists(lockfile):
while await target.parent.exists(lockfile):
print(f'{self.target} | Waiting for {lockfile} ...')
await asyncio.sleep(1)
@ -116,7 +116,13 @@ class Packages:
raise Exception('Packages does not yet support this distro')
self.cmds = self.mgrs[self.mgr]
if target.isguest:
# we're going to mount
await self.cache_setup(target)
await self.update(target)
if self.upgrade:
await target.rexec(self.cmds['upgrade'])
@ -147,19 +153,24 @@ class Packages:
return cachedir
async def apt_setup(self, target):
codename = (await self.rexec(
f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME'
codename = (await target.rexec(
f'source /etc/os-release; echo $VERSION_CODENAME'
)).out
cachedir = os.path.join(self.cache_root, self.mgr, codename)
await self.rexec('rm /etc/apt/apt.conf.d/docker-clean')
cache_archives = os.path.join(cachedir, 'archives')
await self.mount(cache_archives, f'/var/cache/apt/archives')
await target.mount(cache_archives, f'/var/cache/apt/archives')
cache_lists = os.path.join(cachedir, 'lists')
await self.mount(cache_lists, f'/var/lib/apt/lists')
await target.mount(cache_lists, f'/var/lib/apt/lists')
return cachedir
async def pacman_setup(self, target):
return self.cache_root + '/pacman'
cachedir = os.path.join(self.cache_root, self.mgr)
await target.mkdir(cachedir + '/cache', cachedir + '/sync')
await target.mount(cachedir + '/sync', '/var/lib/pacman/sync')
await target.mount(cachedir + '/cache', '/var/cache/pacman')
if await target.host.exists('/etc/pacman.d/mirrorlist'):
await target.copy('/etc/pacman.d/mirrorlist', '/etc/pacman.d/mirrorlist')
def __str__(self):
return f'Packages({self.packages}, upgrade={self.upgrade})'

5
shlax/actions/pip.py
View File

@ -19,7 +19,10 @@ class Pip(Action):
raise Exception('Could not find pip nor python')
# ensure pip module presence
result = await target.exec(python, '-m', 'pip', raises=False)
result = await target.exec(
python, '-m', 'pip',
raises=False, quiet=True
)
if result.rc != 0:
if not os.path.exists('get-pip.py'):
req = request.urlopen(

41
shlax/cli.py
View File

@ -20,22 +20,51 @@ class Group(cli2.Group):
self.cmdclass = Command
class TargetArgument(cli2.Argument):
"""DSN of the target to execute on, localhost by default, TBI"""
def __init__(self, cmd, param, doc=None, color=None, default=None):
from shlax.targets.base import Target
super().__init__(cmd, param, doc=self.__doc__, default=Target())
self.alias = ['target', 't']
def cast(self, value):
from shlax.targets.ssh import Ssh
if '@' in value:
user, host = value.split('@')
return Ssh(host=host, user=user)
class Command(cli2.Command):
def setargs(self):
super().setargs()
self['target'] = TargetArgument(
self,
self.sig.parameters['target'],
)
if 'actions' in self:
del self['actions']
def call(self, *args, **kwargs):
self.shlax_target = self['target'].value
return self.shlax_target(self.target)
def __call__(self, *argv):
from shlax.targets.base import Target
self.shlax_target = Target()
result = super().__call__(*argv)
super().__call__(*argv)
self.shlax_target.output.results(self.shlax_target)
return result
class ActionCommand(Command):
class ActionCommand(cli2.Command):
def setargs(self):
super().setargs()
self['target'] = TargetArgument(
self,
inspect.Parameter('target', inspect.Parameter.KEYWORD_ONLY),
)
def call(self, *args, **kwargs):
self.target = self.target(*args, **kwargs)
return super().call(*args, **kwargs)
return super().call(self['target'].value)
class ConsoleScript(Group):

49
shlax/targets/base.py
View File

@ -11,12 +11,17 @@ from ..result import Result, Results
class Target:
isguest = False
def __init__(self, *actions, root=None):
self.actions = actions
self.results = []
self.output = Output()
self.parent = None
self.root = root or os.getcwd()
self.root = root or ''
def __str__(self):
return 'localhost'
@property
def parent(self):
@ -127,20 +132,48 @@ class Target:
@root.setter
def root(self, value):
self._root = Path(value or os.getcwd())
self._root = Path(value) if value else ''
@property
def host(self):
current = self
while current.isguest:
current = self.parent
return current
def path(self, path):
if not self.root:
return path
if str(path).startswith('/'):
path = str(path)[1:]
return self.root / path
return str(self.root / path)
async def mkdir(self, path):
async def mkdir(self, *paths):
if '_mkdir' not in self.__dict__:
self._mkdir = []
path = str(path)
if path not in self._mkdir:
await self.exec('mkdir', '-p', path)
self._mkdir.append(path)
make = [str(path) for path in paths if str(path) not in self._mkdir]
if make:
await self.exec('mkdir', '-p', *make)
self._mkdir += make
async def copy(self, *args):
return await self.exec('cp', '-a', *args)
async def exists(self, path):
return (await self.exec('ls ' + self.path(path), raises=False)).rc == 0
async def read(self, path):
return (await self.exec('cat', self.path(path))).out
async def write(self, path, content):
return await self.exec('echo ' + content + ' > ' + self.path(path))
async def rm(self, path):
return await self.exec('rm', self.path(path))
async def getenv(self, key):
return (await self.exec('echo $' + key)).out
async def getcwd(self):
return (await self.exec('pwd')).out

52
shlax/targets/buildah.py
View File

@ -14,6 +14,7 @@ from ..proc import Proc
class Buildah(Target):
"""Build container image with buildah"""
isguest = True
def __init__(self,
*actions,
@ -63,22 +64,17 @@ class Buildah(Target):
if actions:
actions = actions[len(keep):]
if not actions:
return self.uptodate()
return self.output.success('Image up to date')
else:
self.actions = self.actions[len(keep):]
if not self.actions:
return self.uptodate()
return self.output.success('Image up to date')
self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out
self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out)
return await super().__call__(*actions)
def uptodate(self):
self.clean = None
self.output.success('Image up to date')
return
async def layers(self):
ret = set()
results = await self.parent.exec(
@ -144,21 +140,19 @@ class Buildah(Target):
return stop
async def clean(self, target, result):
if self.ctr is not None:
for src, dst in self.mounts.items():
await self.parent.exec('umount', self.root / str(dst)[1:])
if self.root is not None:
await self.parent.exec('buildah', 'umount', self.ctr)
if self.ctr is not None:
if result.status == 'success':
await self.commit()
await self.parent.exec('buildah', 'rm', self.ctr)
if result.status == 'success' and os.getenv('BUILDAH_PUSH'):
if os.getenv('BUILDAH_PUSH'):
await self.image.push(target)
if self.ctr is not None:
await self.parent.exec('buildah', 'rm', self.ctr)
async def mount(self, src, dst):
"""Mount a host directory into the container."""
target = self.root / str(dst)[1:]
@ -174,22 +168,13 @@ class Buildah(Target):
_args += [' '.join([str(a) for a in args])]
return await self.parent.exec(*_args, **kwargs)
async def commit(self, image=None):
image = image or self.image
if not image:
return
if not image:
# don't go through that if layer commit
async def commit(self):
for key, value in self.config.items():
await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}')
self.sha = (await self.parent.exec(
'buildah',
'commit',
'--format=' + image.format,
self.ctr,
)).out
await self.parent.exec(
f'buildah commit {self.ctr} {self.image.repository}:final'
)
ENV_TAGS = (
# gitlab
@ -209,16 +194,15 @@ class Buildah(Target):
if value:
self.image.tags.append(value)
if image.tags:
tags = [f'{image.repository}:{tag}' for tag in image.tags]
if self.image.tags:
tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags]
else:
tags = [image.repository]
tags = [self.image.repository]
for tag in tags:
await self.parent.exec('buildah', 'tag', self.sha, tag)
await self.parent.exec('buildah', 'tag', self.image.repository + ':final', *tags)
async def mkdir(self, path):
return await self.parent.mkdir(self.path(path))
async def mkdir(self, *paths):
return await self.parent.mkdir(*[self.path(path) for path in paths])
async def copy(self, *args):
return await self.parent.copy(*args[:-1], self.path(args[-1]))

15
shlax/targets/ssh.py Normal file
View File

@ -0,0 +1,15 @@
from .base import Target
class Ssh(Target):
def __init__(self, *actions, host, user=None):
self.host = host
self.user = user
super().__init__(*actions)
async def exec(self, *args, user=None, **kwargs):
_args = ['ssh', self.host]
if user == 'root':
_args += ['sudo']
_args += [' '.join([str(a) for a in args])]
return await self.parent.exec(*_args, **kwargs)