Merge branch 'mostadvanced' into 'master'
Mostadvanced See merge request oss/shlax!4
This commit is contained in:
∞
commit
150736f4f4
@ -2,15 +2,20 @@ build:
|
||||
cache:
|
||||
key: cache
|
||||
paths: [.cache]
|
||||
image: quay.io/buildah/stable
|
||||
image: yourlabs/buildah
|
||||
script:
|
||||
- dnf install -y curl python38
|
||||
- curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
|
||||
- python3 get-pip.py
|
||||
- pip3 install -U --user -e .[cli]
|
||||
- CACHE_DIR=$(pwd)/.cache python3 ./shlaxfile.py build
|
||||
stage: build
|
||||
|
||||
#build-itself:
|
||||
# cache:
|
||||
# key: cache
|
||||
# paths: [.cache]
|
||||
# image: shlax:$CI_COMMIT_SHORT_SHA
|
||||
# script: python3 ./shlaxfile.py build
|
||||
# stage: test
|
||||
|
||||
test:
|
||||
image: yourlabs/python
|
||||
stage: build
|
||||
|
||||
@ -37,11 +37,13 @@ class Packages:
|
||||
update='pacman -Sy',
|
||||
upgrade='pacman -Su --noconfirm',
|
||||
install='pacman -S --noconfirm',
|
||||
lastupdate='stat -c %Y /var/lib/pacman/sync/core.db',
|
||||
),
|
||||
dnf=dict(
|
||||
update='dnf makecache --assumeyes',
|
||||
upgrade='dnf upgrade --best --assumeyes --skip-broken', # noqa
|
||||
install='dnf install --setopt=install_weak_deps=False --best --assumeyes', # noqa
|
||||
lastupdate='stat -c %Y /var/cache/dnf/* | head -n1',
|
||||
),
|
||||
yum=dict(
|
||||
update='yum update',
|
||||
@ -52,54 +54,52 @@ class Packages:
|
||||
|
||||
installed = []
|
||||
|
||||
def __init__(self, *packages, upgrade=True):
|
||||
def __init__(self, *packages, upgrade=False):
|
||||
self.packages = []
|
||||
self.upgrade = upgrade
|
||||
for package in packages:
|
||||
line = dedent(package).strip().replace('\n', ' ')
|
||||
self.packages += line.split(' ')
|
||||
|
||||
@property
|
||||
def cache_root(self):
|
||||
async def cache_setup(self, target):
|
||||
if 'CACHE_DIR' in os.environ:
|
||||
return os.path.join(os.getenv('CACHE_DIR'))
|
||||
self.cache_root = os.path.join(os.getenv('CACHE_DIR'))
|
||||
else:
|
||||
return os.path.join(os.getenv('HOME'), '.cache')
|
||||
self.cache_root = os.path.join(await target.parent.getenv('HOME'), '.cache')
|
||||
|
||||
# run pkgmgr_setup functions ie. apk_setup
|
||||
await getattr(self, self.mgr + '_setup')(target)
|
||||
|
||||
async def update(self, target):
|
||||
# run pkgmgr_setup functions ie. apk_setup
|
||||
cachedir = await getattr(self, self.mgr + '_setup')(target)
|
||||
|
||||
# lastupdate = await target.exec(self.cmds['lastupdate'], raises=False)
|
||||
# lastupdate = int(lastupdate.out) if lastupdate.rc == 0 else None
|
||||
lastupdate = None
|
||||
if os.path.exists(cachedir + '/lastupdate'):
|
||||
with open(cachedir + '/lastupdate', 'r') as f:
|
||||
try:
|
||||
lastupdate = int(f.read().strip())
|
||||
except:
|
||||
pass
|
||||
|
||||
if not os.path.exists(cachedir):
|
||||
os.makedirs(cachedir)
|
||||
|
||||
now = int(datetime.now().strftime('%s'))
|
||||
# cache for a week
|
||||
if not lastupdate or now - lastupdate > 604800:
|
||||
await target.rexec(self.cmds['update'])
|
||||
|
||||
return
|
||||
|
||||
# disabling with the above return call until needed again
|
||||
# might have to rewrite this to not have our own lockfile
|
||||
# or find a better place on the filesystem
|
||||
# also make sure the lockfile is actually needed when running on
|
||||
# targets that don't have isguest=True
|
||||
if not lastupdate or now - lastupdate > 604800:
|
||||
# crude lockfile implementation, should work against *most*
|
||||
# race-conditions ...
|
||||
lockfile = cachedir + '/update.lock'
|
||||
if not os.path.exists(lockfile):
|
||||
with open(lockfile, 'w+') as f:
|
||||
f.write(str(os.getpid()))
|
||||
if not await target.parent.exists(lockfile):
|
||||
await target.parent.write(lockfile, str(os.getpid()))
|
||||
|
||||
try:
|
||||
await target.rexec(self.cmds['update'])
|
||||
finally:
|
||||
os.unlink(lockfile)
|
||||
await target.parent.rm(lockfile)
|
||||
|
||||
with open(cachedir + '/lastupdate', 'w+') as f:
|
||||
f.write(str(now))
|
||||
await target.parent.write(cachedir + '/lastupdate', str(now))
|
||||
else:
|
||||
while os.path.exists(lockfile):
|
||||
while await target.parent.exists(lockfile):
|
||||
print(f'{self.target} | Waiting for {lockfile} ...')
|
||||
await asyncio.sleep(1)
|
||||
|
||||
@ -116,7 +116,13 @@ class Packages:
|
||||
raise Exception('Packages does not yet support this distro')
|
||||
|
||||
self.cmds = self.mgrs[self.mgr]
|
||||
|
||||
if target.isguest:
|
||||
# we're going to mount
|
||||
await self.cache_setup(target)
|
||||
|
||||
await self.update(target)
|
||||
|
||||
if self.upgrade:
|
||||
await target.rexec(self.cmds['upgrade'])
|
||||
|
||||
@ -147,19 +153,24 @@ class Packages:
|
||||
return cachedir
|
||||
|
||||
async def apt_setup(self, target):
|
||||
codename = (await self.rexec(
|
||||
f'source {self.mnt}/etc/os-release; echo $VERSION_CODENAME'
|
||||
codename = (await target.rexec(
|
||||
f'source /etc/os-release; echo $VERSION_CODENAME'
|
||||
)).out
|
||||
cachedir = os.path.join(self.cache_root, self.mgr, codename)
|
||||
await self.rexec('rm /etc/apt/apt.conf.d/docker-clean')
|
||||
cache_archives = os.path.join(cachedir, 'archives')
|
||||
await self.mount(cache_archives, f'/var/cache/apt/archives')
|
||||
await target.mount(cache_archives, f'/var/cache/apt/archives')
|
||||
cache_lists = os.path.join(cachedir, 'lists')
|
||||
await self.mount(cache_lists, f'/var/lib/apt/lists')
|
||||
await target.mount(cache_lists, f'/var/lib/apt/lists')
|
||||
return cachedir
|
||||
|
||||
async def pacman_setup(self, target):
|
||||
return self.cache_root + '/pacman'
|
||||
cachedir = os.path.join(self.cache_root, self.mgr)
|
||||
await target.mkdir(cachedir + '/cache', cachedir + '/sync')
|
||||
await target.mount(cachedir + '/sync', '/var/lib/pacman/sync')
|
||||
await target.mount(cachedir + '/cache', '/var/cache/pacman')
|
||||
if await target.host.exists('/etc/pacman.d/mirrorlist'):
|
||||
await target.copy('/etc/pacman.d/mirrorlist', '/etc/pacman.d/mirrorlist')
|
||||
|
||||
def __str__(self):
|
||||
return f'Packages({self.packages}, upgrade={self.upgrade})'
|
||||
|
||||
@ -19,7 +19,10 @@ class Pip(Action):
|
||||
raise Exception('Could not find pip nor python')
|
||||
|
||||
# ensure pip module presence
|
||||
result = await target.exec(python, '-m', 'pip', raises=False)
|
||||
result = await target.exec(
|
||||
python, '-m', 'pip',
|
||||
raises=False, quiet=True
|
||||
)
|
||||
if result.rc != 0:
|
||||
if not os.path.exists('get-pip.py'):
|
||||
req = request.urlopen(
|
||||
|
||||
41
shlax/cli.py
41
shlax/cli.py
@ -20,22 +20,51 @@ class Group(cli2.Group):
|
||||
self.cmdclass = Command
|
||||
|
||||
|
||||
class TargetArgument(cli2.Argument):
|
||||
"""DSN of the target to execute on, localhost by default, TBI"""
|
||||
|
||||
def __init__(self, cmd, param, doc=None, color=None, default=None):
|
||||
from shlax.targets.base import Target
|
||||
super().__init__(cmd, param, doc=self.__doc__, default=Target())
|
||||
self.alias = ['target', 't']
|
||||
|
||||
def cast(self, value):
|
||||
from shlax.targets.ssh import Ssh
|
||||
if '@' in value:
|
||||
user, host = value.split('@')
|
||||
return Ssh(host=host, user=user)
|
||||
|
||||
|
||||
class Command(cli2.Command):
|
||||
def setargs(self):
|
||||
super().setargs()
|
||||
self['target'] = TargetArgument(
|
||||
self,
|
||||
self.sig.parameters['target'],
|
||||
)
|
||||
if 'actions' in self:
|
||||
del self['actions']
|
||||
|
||||
def call(self, *args, **kwargs):
|
||||
self.shlax_target = self['target'].value
|
||||
return self.shlax_target(self.target)
|
||||
|
||||
def __call__(self, *argv):
|
||||
from shlax.targets.base import Target
|
||||
self.shlax_target = Target()
|
||||
result = super().__call__(*argv)
|
||||
super().__call__(*argv)
|
||||
self.shlax_target.output.results(self.shlax_target)
|
||||
return result
|
||||
|
||||
|
||||
class ActionCommand(Command):
|
||||
class ActionCommand(cli2.Command):
|
||||
def setargs(self):
|
||||
super().setargs()
|
||||
self['target'] = TargetArgument(
|
||||
self,
|
||||
inspect.Parameter('target', inspect.Parameter.KEYWORD_ONLY),
|
||||
)
|
||||
|
||||
def call(self, *args, **kwargs):
|
||||
self.target = self.target(*args, **kwargs)
|
||||
return super().call(*args, **kwargs)
|
||||
return super().call(self['target'].value)
|
||||
|
||||
|
||||
class ConsoleScript(Group):
|
||||
|
||||
@ -11,12 +11,17 @@ from ..result import Result, Results
|
||||
|
||||
|
||||
class Target:
|
||||
isguest = False
|
||||
|
||||
def __init__(self, *actions, root=None):
|
||||
self.actions = actions
|
||||
self.results = []
|
||||
self.output = Output()
|
||||
self.parent = None
|
||||
self.root = root or os.getcwd()
|
||||
self.root = root or ''
|
||||
|
||||
def __str__(self):
|
||||
return 'localhost'
|
||||
|
||||
@property
|
||||
def parent(self):
|
||||
@ -127,20 +132,48 @@ class Target:
|
||||
|
||||
@root.setter
|
||||
def root(self, value):
|
||||
self._root = Path(value or os.getcwd())
|
||||
self._root = Path(value) if value else ''
|
||||
|
||||
@property
|
||||
def host(self):
|
||||
current = self
|
||||
while current.isguest:
|
||||
current = self.parent
|
||||
return current
|
||||
|
||||
def path(self, path):
|
||||
if not self.root:
|
||||
return path
|
||||
if str(path).startswith('/'):
|
||||
path = str(path)[1:]
|
||||
return self.root / path
|
||||
return str(self.root / path)
|
||||
|
||||
async def mkdir(self, path):
|
||||
async def mkdir(self, *paths):
|
||||
if '_mkdir' not in self.__dict__:
|
||||
self._mkdir = []
|
||||
path = str(path)
|
||||
if path not in self._mkdir:
|
||||
await self.exec('mkdir', '-p', path)
|
||||
self._mkdir.append(path)
|
||||
|
||||
make = [str(path) for path in paths if str(path) not in self._mkdir]
|
||||
if make:
|
||||
await self.exec('mkdir', '-p', *make)
|
||||
self._mkdir += make
|
||||
|
||||
async def copy(self, *args):
|
||||
return await self.exec('cp', '-a', *args)
|
||||
|
||||
async def exists(self, path):
|
||||
return (await self.exec('ls ' + self.path(path), raises=False)).rc == 0
|
||||
|
||||
async def read(self, path):
|
||||
return (await self.exec('cat', self.path(path))).out
|
||||
|
||||
async def write(self, path, content):
|
||||
return await self.exec('echo ' + content + ' > ' + self.path(path))
|
||||
|
||||
async def rm(self, path):
|
||||
return await self.exec('rm', self.path(path))
|
||||
|
||||
async def getenv(self, key):
|
||||
return (await self.exec('echo $' + key)).out
|
||||
|
||||
async def getcwd(self):
|
||||
return (await self.exec('pwd')).out
|
||||
|
||||
@ -14,6 +14,7 @@ from ..proc import Proc
|
||||
|
||||
class Buildah(Target):
|
||||
"""Build container image with buildah"""
|
||||
isguest = True
|
||||
|
||||
def __init__(self,
|
||||
*actions,
|
||||
@ -63,22 +64,17 @@ class Buildah(Target):
|
||||
if actions:
|
||||
actions = actions[len(keep):]
|
||||
if not actions:
|
||||
return self.uptodate()
|
||||
return self.output.success('Image up to date')
|
||||
else:
|
||||
self.actions = self.actions[len(keep):]
|
||||
if not self.actions:
|
||||
return self.uptodate()
|
||||
return self.output.success('Image up to date')
|
||||
|
||||
self.ctr = (await self.parent.exec('buildah', 'from', self.base)).out
|
||||
self.root = Path((await self.parent.exec('buildah', 'mount', self.ctr)).out)
|
||||
|
||||
return await super().__call__(*actions)
|
||||
|
||||
def uptodate(self):
|
||||
self.clean = None
|
||||
self.output.success('Image up to date')
|
||||
return
|
||||
|
||||
async def layers(self):
|
||||
ret = set()
|
||||
results = await self.parent.exec(
|
||||
@ -144,21 +140,19 @@ class Buildah(Target):
|
||||
return stop
|
||||
|
||||
async def clean(self, target, result):
|
||||
for src, dst in self.mounts.items():
|
||||
await self.parent.exec('umount', self.root / str(dst)[1:])
|
||||
|
||||
if self.root is not None:
|
||||
if self.ctr is not None:
|
||||
for src, dst in self.mounts.items():
|
||||
await self.parent.exec('umount', self.root / str(dst)[1:])
|
||||
await self.parent.exec('buildah', 'umount', self.ctr)
|
||||
|
||||
if self.ctr is not None:
|
||||
if result.status == 'success':
|
||||
await self.commit()
|
||||
|
||||
await self.parent.exec('buildah', 'rm', self.ctr)
|
||||
|
||||
if result.status == 'success' and os.getenv('BUILDAH_PUSH'):
|
||||
if result.status == 'success':
|
||||
await self.commit()
|
||||
if os.getenv('BUILDAH_PUSH'):
|
||||
await self.image.push(target)
|
||||
|
||||
if self.ctr is not None:
|
||||
await self.parent.exec('buildah', 'rm', self.ctr)
|
||||
|
||||
async def mount(self, src, dst):
|
||||
"""Mount a host directory into the container."""
|
||||
target = self.root / str(dst)[1:]
|
||||
@ -174,22 +168,13 @@ class Buildah(Target):
|
||||
_args += [' '.join([str(a) for a in args])]
|
||||
return await self.parent.exec(*_args, **kwargs)
|
||||
|
||||
async def commit(self, image=None):
|
||||
image = image or self.image
|
||||
if not image:
|
||||
return
|
||||
async def commit(self):
|
||||
for key, value in self.config.items():
|
||||
await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}')
|
||||
|
||||
if not image:
|
||||
# don't go through that if layer commit
|
||||
for key, value in self.config.items():
|
||||
await self.parent.exec(f'buildah config --{key} "{value}" {self.ctr}')
|
||||
|
||||
self.sha = (await self.parent.exec(
|
||||
'buildah',
|
||||
'commit',
|
||||
'--format=' + image.format,
|
||||
self.ctr,
|
||||
)).out
|
||||
await self.parent.exec(
|
||||
f'buildah commit {self.ctr} {self.image.repository}:final'
|
||||
)
|
||||
|
||||
ENV_TAGS = (
|
||||
# gitlab
|
||||
@ -209,16 +194,15 @@ class Buildah(Target):
|
||||
if value:
|
||||
self.image.tags.append(value)
|
||||
|
||||
if image.tags:
|
||||
tags = [f'{image.repository}:{tag}' for tag in image.tags]
|
||||
if self.image.tags:
|
||||
tags = [f'{self.image.repository}:{tag}' for tag in self.image.tags]
|
||||
else:
|
||||
tags = [image.repository]
|
||||
tags = [self.image.repository]
|
||||
|
||||
for tag in tags:
|
||||
await self.parent.exec('buildah', 'tag', self.sha, tag)
|
||||
await self.parent.exec('buildah', 'tag', self.image.repository + ':final', *tags)
|
||||
|
||||
async def mkdir(self, path):
|
||||
return await self.parent.mkdir(self.path(path))
|
||||
async def mkdir(self, *paths):
|
||||
return await self.parent.mkdir(*[self.path(path) for path in paths])
|
||||
|
||||
async def copy(self, *args):
|
||||
return await self.parent.copy(*args[:-1], self.path(args[-1]))
|
||||
|
||||
15
shlax/targets/ssh.py
Normal file
15
shlax/targets/ssh.py
Normal file
@ -0,0 +1,15 @@
|
||||
from .base import Target
|
||||
|
||||
|
||||
class Ssh(Target):
|
||||
def __init__(self, *actions, host, user=None):
|
||||
self.host = host
|
||||
self.user = user
|
||||
super().__init__(*actions)
|
||||
|
||||
async def exec(self, *args, user=None, **kwargs):
|
||||
_args = ['ssh', self.host]
|
||||
if user == 'root':
|
||||
_args += ['sudo']
|
||||
_args += [' '.join([str(a) for a in args])]
|
||||
return await self.parent.exec(*_args, **kwargs)
|
||||
Loading…
x
Reference in New Issue
Block a user