X-Git-Url: https://main.carlh.net/gitweb/?a=blobdiff_plain;f=cdist;h=7480499c4e8fa9aed3af6dd24780aa11732278c9;hb=1f3f05c6df033eaa155952a2da9ad107888a25f1;hp=fcd20c67276df732ba7f333e0d2b18e8b068653f;hpb=a427425204d25f1eb815e0b6dbc8decc225039f4;p=cdist.git diff --git a/cdist b/cdist index fcd20c6..7480499 100755 --- a/cdist +++ b/cdist @@ -1,6 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/python3 -# Copyright (C) 2012 Carl Hetherington +# Copyright (C) 2012-2020 Carl Hetherington # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,46 +16,128 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -import os -import sys -import shutil -import glob -import tempfile +from __future__ import print_function + import argparse +import copy import datetime -import subprocess +import getpass +import glob +import inspect +import multiprocessing +import os +import platform import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import time class Error(Exception): def __init__(self, value): self.value = value def __str__(self): - return '\x1b[31m%s\x1b[0m' % repr(self.value) + return self.value def __repr__(self): return str(self) +class Trees: + """ + Store for Tree objects which re-uses already-created objects + and checks for requests for different versions of the same thing. + """ + + def __init__(self): + self.trees = [] + + def get(self, name, specifier, target, required_by=None): + for t in self.trees: + if t.name == name and t.specifier == specifier and t.target == target: + return t + elif t.name == name and t.specifier != specifier: + a = specifier if specifier is not None else "[Any]" + if required_by is not None: + a += ' by %s' % required_by + b = t.specifier if t.specifier is not None else "[Any]" + if t.required_by is not None: + b += ' by %s' % t.required_by + raise Error('conflicting versions of %s required (%s versus %s)' % (name, a, b)) + + nt = Tree(name, specifier, target, required_by) + self.trees.append(nt) + return nt + + def add_built(self, name, specifier, target): + self.trees.append(Tree(name, specifier, target, None, built=True)) + + +class Globals: + quiet = False + command = None + dry_run = False + trees = Trees() + +globals = Globals() + + # # Configuration # +class Option(object): + def __init__(self, key, default=None): + self.key = key + self.value = default + + def offer(self, key, value): + if key == self.key: + self.value = value + +class BoolOption(object): + def __init__(self, key): + self.key = key + self.value = False + + def offer(self, key, value): + if key == self.key: + self.value = (value == 'yes' or value == '1' or value == 'true') + class Config: def __init__(self): - self.keys = ['linux_dir_in_chroot', - 'linux_chroot_prefix', - 'windows_environment_prefix', - 'mingw_prefix', - 'git_prefix', - 'osx_build_host', - 'osx_dir_in_host', - 'osx_environment_prefix', - 'osx_sdk_prefix', - 'osx_sdk'] - - self.dict = dict() + self.options = [ Option('mxe_prefix'), + Option('git_prefix'), + Option('git_reference'), + Option('osx_environment_prefix'), + Option('osx_sdk_prefix'), + Option('osx_sdk'), + Option('osx_keychain_file'), + Option('osx_keychain_password'), + Option('apple_id'), + Option('apple_password'), + BoolOption('docker_sudo'), + BoolOption('docker_no_user'), + Option('docker_hub_repository'), + Option('flatpak_state_dir'), + Option('parallel', multiprocessing.cpu_count()), + Option('temp', '/var/tmp')] + + config_dir = '%s/.config' % os.path.expanduser('~') + if not os.path.exists(config_dir): + os.mkdir(config_dir) + config_file = '%s/cdist' % config_dir + if not os.path.exists(config_file): + f = open(config_file, 'w') + for o in self.options: + print('# %s ' % o.key, file=f) + f.close() + print('Template config file written to %s; please edit and try again.' % config_file, file=sys.stderr) + sys.exit(1) try: f = open('%s/.config/cdist' % os.path.expanduser('~'), 'r') - while 1: + while True: l = f.readline() if l == '': break @@ -65,63 +147,136 @@ class Config: s = l.strip().split() if len(s) == 2: - for k in self.keys: - if k == s[0]: - self.dict[k] = s[1] + for k in self.options: + k.offer(s[0], s[1]) except: raise - def get(self, k): - if k in self.dict: - return self.dict[k] + def has(self, k): + for o in self.options: + if o.key == k and o.value is not None: + return True + return False - raise Error('Required setting %s not found' % k) + def get(self, k): + for o in self.options: + if o.key == k: + if o.value is None: + raise Error('Required setting %s not found' % k) + return o.value + + def set(self, k, v): + for o in self.options: + o.offer(k, v) + + def docker(self): + if self.get('docker_sudo'): + return 'sudo docker' + else: + return 'docker' config = Config() # # Utility bits -# +# + +def log_normal(m): + if not globals.quiet: + print('\x1b[33m* %s\x1b[0m' % m) + +def log_verbose(m): + if globals.verbose: + print('\x1b[35m* %s\x1b[0m' % m) + +def escape_spaces(s): + return s.replace(' ', '\\ ') + +def scp_escape(n): + """Escape a host:filename string for use with an scp command""" + s = n.split(':') + assert(len(s) == 1 or len(s) == 2) + if len(s) == 2: + return '%s:"\'%s\'"' % (s[0], s[1]) + else: + return '\"%s\"' % s[0] -def log(m): - if not args.quiet: - print '\x1b[33m* %s\x1b[0m' % m +def mv_escape(n): + return '\"%s\"' % n.substr(' ', '\\ ') def copytree(a, b): - log('copy %s -> %s' % (a, b)) - shutil.copytree(a, b) + log_normal('copy %s -> %s' % (scp_escape(a), scp_escape(b))) + if b.startswith('s3://'): + command('s3cmd -P -r put "%s" "%s"' % (a, b)) + else: + command('scp -r %s %s' % (scp_escape(a), scp_escape(b))) def copyfile(a, b): - log('copy %s -> %s' % (a, b)) - shutil.copyfile(a, b) + log_normal('copy %s -> %s with cwd %s' % (scp_escape(a), scp_escape(b), os.getcwd())) + if b.startswith('s3://'): + command('s3cmd -P put "%s" "%s"' % (a, b)) + else: + bc = b.find(":") + if bc != -1: + host = b[:bc] + path = b[bc+1:] + temp_path = os.path.join(os.path.dirname(path), ".tmp." + os.path.basename(path)) + command('scp %s %s' % (scp_escape(a), scp_escape(host + ":" + temp_path))) + command('ssh %s -- mv "%s" "%s"' % (host, escape_spaces(temp_path), escape_spaces(path))) + else: + command('scp %s %s' % (scp_escape(a), scp_escape(b))) + +def makedirs(d): + """ + Make directories either locally or on a remote host; remotely if + d includes a colon, otherwise locally. + """ + if d.startswith('s3://'): + # No need to create folders on S3 + return + + if d.find(':') == -1: + try: + os.makedirs(d) + except OSError as e: + if e.errno != 17: + raise e + else: + s = d.split(':') + command('ssh %s -- mkdir -p %s' % (s[0], s[1])) def rmdir(a): - log('remove %s' % a) + log_normal('remove %s' % a) os.rmdir(a) def rmtree(a): - log('remove %s' % a) + log_normal('remove %s' % a) shutil.rmtree(a, ignore_errors=True) -def command(c, can_fail=False): - log(c) - r = os.system(c) - if (r >> 8) and not can_fail: - raise Error('command %s failed' % c) +def command(c): + log_normal(c) + try: + r = subprocess.run(c, shell=True) + if r.returncode != 0: + raise Error('command %s failed (%d)' % (c, r.returncode)) + except Exception as e: + raise Error('command %s failed (%s)' % (c, e)) def command_and_read(c): - log(c) - p = subprocess.Popen(c.split(), stdout=subprocess.PIPE) - f = os.fdopen(os.dup(p.stdout.fileno())) - return f + log_normal(c) + p = subprocess.Popen(c.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (out, err) = p.communicate() + if p.returncode != 0: + raise Error('command %s failed (%s)' % (c, err)) + return str(out, 'utf-8').splitlines() def read_wscript_variable(directory, variable): f = open('%s/wscript' % directory, 'r') - while 1: + while True: l = f.readline() if l == '': break - + s = l.split() if len(s) == 3 and s[0] == variable: f.close() @@ -130,66 +285,92 @@ def read_wscript_variable(directory, variable): f.close() return None + +def devel_to_git(git_commit, filename): + if git_commit is not None: + filename = filename.replace('devel', '-%s' % git_commit) + return filename + + +def get_command_line_options(args): + """Get the options specified by --option on the command line""" + options = dict() + if args.option is not None: + for o in args.option: + b = o.split(':') + if len(b) != 2: + raise Error("Bad option `%s'" % o) + if b[1] == 'False': + options[b[0]] = False + elif b[1] == 'True': + options[b[0]] = True + else: + options[b[0]] = b[1] + return options + + +class TreeDirectory: + def __init__(self, tree): + self.tree = tree + def __enter__(self): + self.cwd = os.getcwd() + os.chdir('%s/src/%s' % (self.tree.target.directory, self.tree.name)) + def __exit__(self, type, value, traceback): + os.chdir(self.cwd) + # # Version # class Version: def __init__(self, s): - self.pre = False - self.beta = None + self.devel = False if s.startswith("'"): s = s[1:] if s.endswith("'"): s = s[0:-1] - + + if s.endswith('devel'): + s = s[0:-5] + self.devel = True + if s.endswith('pre'): s = s[0:-3] - self.pre = True - - b = s.find("beta") - if b != -1: - self.beta = int(s[b+4:]) - s = s[0:b] p = s.split('.') self.major = int(p[0]) self.minor = int(p[1]) + if len(p) == 3: + self.micro = int(p[2]) + else: + self.micro = 0 + + @classmethod + def from_git_tag(cls, tag): + bits = tag.split('-') + c = cls(bits[0]) + if len(bits) > 1 and int(bits[1]) > 0: + c.devel = True + return c - def bump(self): + def bump_minor(self): self.minor += 1 - self.pre = False - self.beta = None + self.micro = 0 - def to_pre(self): - self.pre = True - self.beta = None + def bump_micro(self): + self.micro += 1 - def bump_and_to_pre(self): - self.bump() - self.pre = True - self.beta = None + def to_devel(self): + self.devel = True def to_release(self): - self.pre = False - self.beta = None - - def bump_beta(self): - if self.pre: - self.pre = False - self.beta = 1 - elif self.beta is not None: - self.beta += 1 - elif self.beta is None: - self.beta = 1 + self.devel = False def __str__(self): - s = '%d.%02d' % (self.major, self.minor) - if self.beta is not None: - s += 'beta%d' % self.beta - elif self.pre: - s += 'pre' + s = '%d.%d.%d' % (self.major, self.minor, self.micro) + if self.devel: + s += 'devel' return s @@ -198,58 +379,93 @@ class Version: # class Target(object): - def __init__(self, platform, parallel): + """ + Class representing the target that we are building for. This is exposed to cscripts, + though not all of it is guaranteed 'API'. cscripts may expect: + + platform: platform string (e.g. 'windows', 'linux', 'osx') + parallel: number of parallel jobs to run + directory: directory to work in + variables: dict of environment variables + debug: True to build a debug version, otherwise False + ccache: True to use ccache, False to not + set(a, b): set the value of variable 'a' to 'b' + unset(a): unset the value of variable 'a' + command(c): run the command 'c' in the build environment + + """ + + def __init__(self, platform, directory=None): + """ + platform -- platform string (e.g. 'windows', 'linux', 'osx') + directory -- directory to work in; if None we will use a temporary directory + Temporary directories will be removed after use; specified directories will not. + """ self.platform = platform - self.parallel = parallel + self.parallel = int(config.get('parallel')) + # Environment variables that we will use when we call cscripts self.variables = {} self.debug = False + self._ccache = False + # True to build our dependencies ourselves; False if this is taken care + # of in some other way + self.build_dependencies = True - def build_dependencies(self, project): - cwd = os.getcwd() - if 'dependencies' in project.cscript: - print project.cscript['dependencies'](self) - for d in project.cscript['dependencies'](self): - log('Building dependency %s %s of %s' % (d[0], d[1], project.name)) - dep = Project(d[0], '.', d[1]) - dep.checkout(self) - self.build_dependencies(dep) - - # Make the options to pass in from the option_defaults of the thing - # we are building and any options specified by the parent. - # The presence of option_defaults() is taken to mean that this - # cscript understands and expects options - options = {} - if 'option_defaults' in dep.cscript: - options = dep.cscript['option_defaults']() - if len(d) > 2: - for k, v in d[2].iteritems(): - options[k] = v - - self.build(dep, options) - else: - # Backwards compatibility - self.build(dep) + if directory is None: + self.directory = tempfile.mkdtemp('', 'tmp', config.get('temp')) + self.rmdir = True + self.set('CCACHE_BASEDIR', os.path.realpath(self.directory)) + self.set('CCACHE_NOHASHDIR', '') + else: + self.directory = os.path.realpath(directory) + self.rmdir = False - os.chdir(cwd) - def build(self, project, options=None): - if options is not None: - project.cscript['build'](self, options) + def setup(self): + pass + + def _cscript_package(self, tree, options): + """ + Call package() in the cscript and return what it returns, except that + anything not in a list will be put into one. + """ + if len(inspect.getfullargspec(tree.cscript['package']).args) == 3: + packages = tree.call('package', tree.version, options) else: - project.cscript['build'](self) + log_normal("Deprecated cscript package() method with no options parameter") + packages = tree.call('package', tree.version) - def package(self, project): - project.checkout(self) - self.build_dependencies(project) - project.cscript['build'](self) - return project.cscript['package'](self, project.version) + return packages if isinstance(packages, list) else [packages] - def test(self, project): - project.checkout(self) - self.build_dependencies(project) - project.cscript['build'](self) - project.cscript['test'](self) + def _copy_packages(self, tree, packages, output_dir): + for p in packages: + copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) + + def package(self, project, checkout, output_dir, options, notarize): + tree = self.build(project, checkout, options) + tree.add_defaults(options) + p = self._cscript_package(tree, options) + self._copy_packages(tree, p, output_dir) + + def build(self, project, checkout, options): + tree = globals.trees.get(project, checkout, self) + if self.build_dependencies: + tree.build_dependencies(options) + tree.build(options) + return tree + + def test(self, project, checkout, target, test, options): + """test is the test case to run, or None""" + tree = globals.trees.get(project, checkout, target) + + tree.add_defaults(options) + with TreeDirectory(tree): + if len(inspect.getfullargspec(tree.cscript['test']).args) == 3: + return tree.call('test', options, test) + else: + log_normal('Deprecated cscript test() method with no options parameter') + return tree.call('test', test) def set(self, a, b): self.variables[a] = b @@ -260,543 +476,916 @@ class Target(object): def get(self, a): return self.variables[a] + def append(self, k, v, s): + if (not k in self.variables) or len(self.variables[k]) == 0: + self.variables[k] = '"%s"' % v + else: + e = self.variables[k] + if e[0] == '"' and e[-1] == '"': + self.variables[k] = '"%s%s%s"' % (e[1:-1], s, v) + else: + self.variables[k] = '"%s%s%s"' % (e, s, v) + + def append_with_space(self, k, v): + return self.append(k, v, ' ') + + def append_with_colon(self, k, v): + return self.append(k, v, ':') + def variables_string(self, escaped_quotes=False): e = '' - for k, v in self.variables.iteritems(): + for k, v in self.variables.items(): if escaped_quotes: v = v.replace('"', '\\"') e += '%s=%s ' % (k, v) return e def cleanup(self): + if self.rmdir: + rmtree(self.directory) + + def mount(self, m): pass -# -# Windows -# + @property + def ccache(self): + return self._ccache -class WindowsTarget(Target): - # @param directory directory to work in; if None, we will use a temporary directory - def __init__(self, bits, directory=None): - super(WindowsTarget, self).__init__('windows', 2) - self.bits = bits - if directory is None: - self.directory = tempfile.mkdtemp() - self.rmdir = True - else: - self.directory = directory - self.rmdir = False - - self.windows_prefix = '%s/%d' % (config.get('windows_environment_prefix'), self.bits) - if not os.path.exists(self.windows_prefix): - raise Error('windows prefix %s does not exist' % self.windows_prefix) - - if self.bits == 32: - self.mingw_name = 'i686' - else: - self.mingw_name = 'x86_64' - - mingw_path = '%s/%d/bin' % (config.get('mingw_prefix'), self.bits) - self.mingw_prefixes = ['/%s/%d' % (config.get('mingw_prefix'), self.bits), '%s/%d/%s-w64-mingw32' % (config.get('mingw_prefix'), bits, self.mingw_name)] - - self.set('PKG_CONFIG_LIBDIR', '%s/lib/pkgconfig' % self.windows_prefix) - self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.work_dir_cscript(), self.work_dir_cscript())) - self.set('PATH', '%s/bin:%s:%s' % (self.windows_prefix, mingw_path, os.environ['PATH'])) - self.set('CC', '%s-w64-mingw32-gcc' % self.mingw_name) - self.set('CXX', '%s-w64-mingw32-g++' % self.mingw_name) - self.set('LD', '%s-w64-mingw32-ld' % self.mingw_name) - self.set('RANLIB', '%s-w64-mingw32-ranlib' % self.mingw_name) - self.set('WINRC', '%s-w64-mingw32-windres' % self.mingw_name) - cxx = '-I%s/include -I%s/include' % (self.windows_prefix, self.work_dir_cscript()) - link = '-L%s/lib -L%s/lib' % (self.windows_prefix, self.work_dir_cscript()) - for p in self.mingw_prefixes: - cxx += ' -I%s/include' % p - link += ' -L%s/lib' % p - self.set('CXXFLAGS', '"%s"' % cxx) - self.set('LINKFLAGS', '"%s"' % link) + @ccache.setter + def ccache(self, v): + self._ccache = v - def work_dir_cdist(self): - return '%s/%d' % (self.directory, self.bits) - def work_dir_cscript(self): - return '%s/%d' % (self.directory, self.bits) +class DockerTarget(Target): + def __init__(self, platform, directory): + super(DockerTarget, self).__init__(platform, directory) + self.mounts = [] + self.privileged = False - def command(self, c): - log('host -> %s' % c) - command('%s %s' % (self.variables_string(), c)) + def _user_tag(self): + if config.get('docker_no_user'): + return '' + return '-u %s' % getpass.getuser() + + def _mount_option(self, d): + return '-v %s:%s ' % (os.path.realpath(d), os.path.realpath(d)) + + def setup(self): + opts = self._mount_option(self.directory) + for m in self.mounts: + opts += self._mount_option(m) + if config.has('git_reference'): + opts += self._mount_option(config.get('git_reference')) + if self.privileged: + opts += '--privileged=true ' + if self.ccache: + opts += "-e CCACHE_DIR=/ccache/%s-%d --mount source=ccache,target=/ccache" % (self.image, os.getuid()) + + tag = self.image + if config.has('docker_hub_repository'): + tag = '%s:%s' % (config.get('docker_hub_repository'), tag) + + self.container = command_and_read('%s run %s %s -itd %s /bin/bash' % (config.docker(), self._user_tag(), opts, tag))[0].strip() + + def command(self, cmd): + dir = os.path.join(self.directory, os.path.relpath(os.getcwd(), self.directory)) + interactive_flag = '-i ' if sys.stdin.isatty() else '' + command('%s exec %s %s -t %s /bin/bash -c \'export %s; cd %s; %s\'' % (config.docker(), self._user_tag(), interactive_flag, self.container, self.variables_string(), dir, cmd)) def cleanup(self): - if self.rmdir: - rmtree(self.directory) + super(DockerTarget, self).cleanup() + command('%s kill %s' % (config.docker(), self.container)) -# -# Linux -# + def mount(self, m): + self.mounts.append(m) -class LinuxTarget(Target): - def __init__(self, distro, version, bits, directory=None): - "directory -- directory to work in; if None, we will use the configured linux_dir_in_chroot" - super(LinuxTarget, self).__init__('linux', 2) - self.distro = distro - self.version = version - self.bits = bits - self.chroot = '%s-%s-%d' % (self.distro, self.version, self.bits) - if directory is None: - self.dir_in_chroot = config.get('linux_dir_in_chroot') - else: - self.dir_in_chroot = directory - for g in glob.glob('%s/*' % self.work_dir_cdist()): - rmtree(g) +class FlatpakTarget(Target): + def __init__(self, project, checkout): + super(FlatpakTarget, self).__init__('flatpak') + self.build_dependencies = False + self.project = project + self.checkout = checkout - self.set('CXXFLAGS', '-I%s/include' % self.work_dir_cscript()) - self.set('LINKFLAGS', '-L%s/lib' % self.work_dir_cscript()) - self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig' % self.work_dir_cscript()) - self.set('PATH', '%s:/usr/local/bin' % (os.environ['PATH'])) + def setup(self): + pass - def work_dir_cdist(self): - return '%s/%s%s' % (config.get('linux_chroot_prefix'), self.chroot, self.dir_in_chroot) + def command(self, cmd): + command(cmd) - def work_dir_cscript(self): - return self.dir_in_chroot + def checkout_dependencies(self): + tree = globals.trees.get(self.project, self.checkout, self) + return tree.checkout_dependencies() - def command(self, c): - # Work out the cwd for the chrooted command - cwd = os.getcwd() - prefix = '%s/%s' % (config.get('linux_chroot_prefix'), self.chroot) - assert(cwd.startswith(prefix)) - cwd = cwd[len(prefix):] + def flatpak(self): + return 'flatpak' - log('schroot [%s] -> %s' % (cwd, c)) - command('%s schroot -c %s -d %s -p -- %s' % (self.variables_string(), self.chroot, cwd, c)) + def flatpak_builder(self): + b = 'flatpak-builder' + if config.has('flatpak_state_dir'): + b += ' --state-dir=%s' % config.get('flatpak_state_dir') + return b - def cleanup(self): - for g in glob.glob('%s/*' % self.work_dir_cdist()): - rmtree(g) -# -# OS X -# +class WindowsDockerTarget(DockerTarget): + """ + This target exposes the following additional API: -class OSXTarget(Target): - def __init__(self, directory=None): - "directory -- directory to work in; if None, we will use the configured osx_dir_in_host" - super(OSXTarget, self).__init__('osx', 4) + version: Windows version ('xp' or None) + bits: bitness of Windows (32 or 64) + name: name of our target e.g. x86_64-w64-mingw32.shared + environment_prefix: path to Windows environment for the appropriate target (libraries and some tools) + tool_path: path to 32- and 64-bit tools + """ + def __init__(self, windows_version, bits, directory, environment_version): + super(WindowsDockerTarget, self).__init__('windows', directory) + self.version = windows_version + self.bits = bits - if directory is None: - self.dir_in_host = config.get('osx_dir_in_host') + self.tool_path = '%s/usr/bin' % config.get('mxe_prefix') + if self.bits == 32: + self.name = 'i686-w64-mingw32.shared' else: - self.dir_in_host = directory + self.name = 'x86_64-w64-mingw32.shared' + self.environment_prefix = '%s/usr/%s' % (config.get('mxe_prefix'), self.name) + + self.set('PKG_CONFIG_LIBDIR', '%s/lib/pkgconfig' % self.environment_prefix) + self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.directory, self.directory)) + self.set('PATH', '%s/bin:%s:%s' % (self.environment_prefix, self.tool_path, os.environ['PATH'])) + self.set('LD', '%s-ld' % self.name) + self.set('RANLIB', '%s-ranlib' % self.name) + self.set('WINRC', '%s-windres' % self.name) + cxx = '-I%s/include -I%s/include' % (self.environment_prefix, self.directory) + link = '-L%s/lib -L%s/lib' % (self.environment_prefix, self.directory) + self.set('CXXFLAGS', '"%s"' % cxx) + self.set('CPPFLAGS', '') + self.set('LINKFLAGS', '"%s"' % link) + self.set('LDFLAGS', '"%s"' % link) - for g in glob.glob('%s/*' % self.dir_in_host): - rmtree(g) + self.image = 'windows' + if environment_version is not None: + self.image += '_%s' % environment_version - def command(self, c): - command('%s %s' % (self.variables_string(False), c)) + def setup(self): + super().setup() + if self.ccache: + self.set('CC', '"ccache %s-gcc"' % self.name) + self.set('CXX', '"ccache %s-g++"' % self.name) + else: + self.set('CC', '%s-gcc' % self.name) + self.set('CXX', '%s-g++' % self.name) + + @property + def library_prefix(self): + log_normal('Deprecated property library_prefix: use environment_prefix') + return self.environment_prefix + + @property + def windows_prefix(self): + log_normal('Deprecated property windows_prefix: use environment_prefix') + return self.environment_prefix + + @property + def mingw_prefixes(self): + log_normal('Deprecated property mingw_prefixes: use environment_prefix') + return [self.environment_prefix] + + @property + def mingw_path(self): + log_normal('Deprecated property mingw_path: use tool_path') + return self.tool_path + + @property + def mingw_name(self): + log_normal('Deprecated property mingw_name: use name') + return self.name + + +class WindowsNativeTarget(Target): + """ + This target exposes the following additional API: + + version: Windows version ('xp' or None) + bits: bitness of Windows (32 or 64) + name: name of our target e.g. x86_64-w64-mingw32.shared + environment_prefix: path to Windows environment for the appropriate target (libraries and some tools) + """ + def __init__(self, directory): + super().__init__('windows', directory) + self.version = None + self.bits = 64 + self.environment_prefix = config.get('windows_native_environmnet_prefix') -class OSXSingleTarget(OSXTarget): - def __init__(self, bits, directory=None): - super(OSXSingleTarget, self).__init__(directory) + self.set('PATH', '%s/bin:%s' % (self.environment_prefix, os.environ['PATH'])) + + def command(self, cmd): + command(cmd) + + +class LinuxTarget(DockerTarget): + """ + Build for Linux in a docker container. + This target exposes the following additional API: + + distro: distribution ('debian', 'ubuntu', 'centos' or 'fedora') + version: distribution version (e.g. '12.04', '8', '6.5') + bits: bitness of the distribution (32 or 64) + detail: None or 'appimage' if we are building for appimage + """ + + def __init__(self, distro, version, bits, directory=None): + super(LinuxTarget, self).__init__('linux', directory) + self.distro = distro + self.version = version self.bits = bits + self.detail = None - if bits == 32: - arch = 'i386' + self.set('CXXFLAGS', '-I%s/include' % self.directory) + self.set('CPPFLAGS', '') + self.set('LINKFLAGS', '-L%s/lib' % self.directory) + self.set('PKG_CONFIG_PATH', + '%s/lib/pkgconfig:%s/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig' % (self.directory, self.directory)) + self.set('PATH', '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin') + + if self.version is None: + self.image = '%s-%s' % (self.distro, self.bits) else: - arch = 'x86_64' + self.image = '%s-%s-%s' % (self.distro, self.version, self.bits) + + def setup(self): + super(LinuxTarget, self).setup() + if self.ccache: + self.set('CC', '"ccache gcc"') + self.set('CXX', '"ccache g++"') + + def test(self, project, checkout, target, test, options): + self.append_with_colon('PATH', '%s/bin' % self.directory) + self.append_with_colon('LD_LIBRARY_PATH', '%s/lib' % self.directory) + super(LinuxTarget, self).test(project, checkout, target, test, options) + + +class AppImageTarget(LinuxTarget): + def __init__(self, work): + super(AppImageTarget, self).__init__('ubuntu', '18.04', 64, work) + self.detail = 'appimage' + self.privileged = True + + +def notarize_dmg(dmg, bundle_id): + p = subprocess.run( + ['xcrun', 'altool', '--notarize-app', '-t', 'osx', '-f', dmg, '--primary-bundle-id', bundle_id, '-u', config.get('apple_id'), '-p', config.get('apple_password'), '--output-format', 'xml'], + capture_output=True + ) + + def string_after(process, key): + lines = p.stdout.decode('utf-8').splitlines() + for i in range(0, len(lines)): + if lines[i].find(key) != -1: + return lines[i+1].strip().replace('', '').replace('', '') + + request_uuid = string_after(p, "RequestUUID") + if request_uuid is None: + raise Error('No RequestUUID found in response from Apple') + + for i in range(0, 30): + print('Checking up on %s' % request_uuid) + p = subprocess.run(['xcrun', 'altool', '--notarization-info', request_uuid, '-u', config.get('apple_id'), '-p', config.get('apple_password'), '--output-format', 'xml'], capture_output=True) + status = string_after(p, 'Status') + print('Got %s' % status) + if status == 'invalid': + raise Error("Notarization failed") + elif status == 'success': + subprocess.run(['xcrun', 'stapler', 'staple', dmg]) + return + elif status != "in progress": + print("Could not understand xcrun response") + print(p) + time.sleep(30) + + raise Error("Notarization timed out") - flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (config.get('osx_sdk_prefix'), config.get('osx_sdk'), arch) - enviro = '%s/%d' % (config.get('osx_environment_prefix'), bits) - # Environment variables - self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.work_dir_cscript(), enviro, flags)) - self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.work_dir_cscript(), enviro, flags)) - self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.work_dir_cscript(), enviro, flags)) - self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.work_dir_cscript(), enviro, flags)) - self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.work_dir_cscript(), enviro)) - self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % enviro) - self.set('MACOSX_DEPLOYMENT_TARGET', config.get('osx_sdk')) +class OSXTarget(Target): + def __init__(self, directory=None): + super(OSXTarget, self).__init__('osx', directory) + self.sdk_prefix = config.get('osx_sdk_prefix') + self.environment_prefix = config.get('osx_environment_prefix') + self.apple_id = config.get('apple_id') + self.apple_password = config.get('apple_password') + self.osx_keychain_file = config.get('osx_keychain_file') + self.osx_keychain_password = config.get('osx_keychain_password') - def work_dir_cdist(self): - return self.work_dir_cscript() + def command(self, c): + command('%s %s' % (self.variables_string(False), c)) - def work_dir_cscript(self): - return '%s/%d' % (self.dir_in_host, self.bits) + def unlock_keychain(self): + self.command('security unlock-keychain -p %s %s' % (self.osx_keychain_password, self.osx_keychain_file)) - def package(self, project): - raise Error('cannot package non-universal OS X versions') + def _cscript_package_and_notarize(self, tree, options, notarize): + """ + Call package() in the cscript and notarize the .dmgs that are returned, if notarize = True + """ + p = self._cscript_package(tree, options) + for x in p: + if not isinstance(x, tuple): + raise Error('macOS packages must be returned from cscript as tuples of (dmg-filename, bundle-id)') + if notarize: + notarize_dmg(x[0], x[1]) + return [x[0] for x in p] -class OSXUniversalTarget(OSXTarget): - def __init__(self, directory=None): - super(OSXUniversalTarget, self).__init__(directory) - self.parts = [] - self.parts.append(OSXSingleTarget(32, directory)) - self.parts.append(OSXSingleTarget(64, directory)) +class OSXSingleTarget(OSXTarget): + def __init__(self, arch, sdk, directory=None): + super(OSXSingleTarget, self).__init__(directory) + self.arch = arch + self.sdk = sdk - def work_dir_cscript(self): - return self.dir_in_host + flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (self.sdk_prefix, sdk, arch) + host_enviro = '%s/x86_64/10.9' % config.get('osx_environment_prefix') + target_enviro = '%s/%s/%s' % (config.get('osx_environment_prefix'), arch, sdk) - def package(self, project): - for p in self.parts: - project.checkout(p) - p.build_dependencies(project) - p.build(project) + self.bin = '%s/bin' % target_enviro - return project.cscript['package'](self, project.version) - + # Environment variables + self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, target_enviro, flags)) + self.set('CPPFLAGS', '') + self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, target_enviro, flags)) + self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, target_enviro, flags)) + self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, target_enviro, flags)) + self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.directory, target_enviro)) + self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % host_enviro) + self.set('MACOSX_DEPLOYMENT_TARGET', sdk) + self.set('CCACHE_BASEDIR', self.directory) + + @Target.ccache.setter + def ccache(self, v): + Target.ccache.fset(self, v) + if v: + self.set('CC', '"ccache gcc"') + self.set('CXX', '"ccache g++"') + + def package(self, project, checkout, output_dir, options, notarize): + tree = self.build(project, checkout, options) + tree.add_defaults(options) + self.unlock_keychain() + p = self._cscript_package_and_notarize(tree, options, notarize) + self._copy_packages(tree, p, output_dir) -# -# Source -# + +class OSXUniversalTarget(OSXTarget): + def __init__(self, archs, directory=None): + super(OSXUniversalTarget, self).__init__(directory) + self.archs = archs + self.sdk = config.get('osx_sdk') + for a in self.archs: + if a.find('arm') != -1: + self.sdk = '11.0' + + def package(self, project, checkout, output_dir, options, notarize): + for a in self.archs: + target = OSXSingleTarget(a, self.sdk, os.path.join(self.directory, a)) + target.ccache = self.ccache + tree = globals.trees.get(project, checkout, target) + tree.build_dependencies(options) + tree.build(options) + + self.unlock_keychain() + tree = globals.trees.get(project, checkout, self) + with TreeDirectory(tree): + for p in self._cscript_package_and_notarize(tree, options, notarize): + copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) class SourceTarget(Target): + """Build a source .tar.bz2""" def __init__(self): - super(SourceTarget, self).__init__('source', 2) - self.directory = tempfile.mkdtemp() - - def work_dir_cdist(self): - return self.directory - - def work_dir_cscript(self): - return self.directory + super(SourceTarget, self).__init__('source') def command(self, c): - log('host -> %s' % c) + log_normal('host -> %s' % c) command('%s %s' % (self.variables_string(), c)) def cleanup(self): rmtree(self.directory) - def package(self, project): - project.checkout(self) - name = read_wscript_variable(os.getcwd(), 'APPNAME') - command('./waf dist') - return os.path.abspath('%s-%s.tar.bz2' % (name, project.version)) - + def package(self, project, checkout, output_dir, options, notarize): + tree = globals.trees.get(project, checkout, self) + with TreeDirectory(tree): + name = read_wscript_variable(os.getcwd(), 'APPNAME') + command('./waf dist') + p = os.path.abspath('%s-%s.tar.bz2' % (name, tree.version)) + copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) # @param s Target string: # windows-{32,64} # or ubuntu-version-{32,64} # or debian-version-{32,64} -# or osx-{32,64} -# or source +# or centos-version-{32,64} +# or fedora-version-{32,64} +# or mageia-version-{32,64} +# or osx-{intel,arm} +# or source +# or flatpak +# or appimage # @param debug True to build with debugging symbols (where possible) -def target_factory(s, debug, work): +def target_factory(args): + s = args.target target = None if s.startswith('windows-'): - target = WindowsTarget(int(s.split('-')[1]), work) - elif s.startswith('ubuntu-') or s.startswith('debian-'): + x = s.split('-') + if platform.system() == "Windows": + target = WindowsNativeTarget(args.work) + else: + if len(x) == 2: + target = WindowsDockerTarget(None, int(x[1]), args.work, args.environment_version) + elif len(x) == 3: + target = WindowsDockerTarget(x[1], int(x[2]), args.work, args.environment_version) + else: + raise Error("Bad Windows target name `%s'") + elif s.startswith('ubuntu-') or s.startswith('debian-') or s.startswith('centos-') or s.startswith('fedora-') or s.startswith('mageia-'): p = s.split('-') if len(p) != 3: - print >>sys.stderr,"Bad Linux target name `%s'; must be something like ubuntu-12.04-32 (i.e. distro-version-bits)" % s - sys.exit(1) - target = LinuxTarget(p[0], p[1], int(p[2]), work) - elif s.startswith('osx-'): - target = OSXSingleTarget(int(s.split('-')[1]), work) - elif s == 'osx': - if args.command == 'build': - target = OSXSingleTarget(64, work) - else: - target = OSXUniversalTarget(work) + raise Error("Bad Linux target name `%s'; must be something like ubuntu-16.04-32 (i.e. distro-version-bits)" % s) + target = LinuxTarget(p[0], p[1], int(p[2]), args.work) + elif s.startswith('arch-'): + p = s.split('-') + if len(p) != 2: + raise Error("Bad Arch target name `%s'; must be arch-32 or arch-64") + target = LinuxTarget(p[0], None, int(p[1]), args.work) + elif s == 'raspbian': + target = LinuxTarget(s, None, None, args.work) + elif s == 'osx-intel': + # Intel 64-bit built for config's os_sdk + target = OSXSingleTarget('x86_64', config.get('osx_sdk'), args.work) + elif s == 'osx-arm-intel': + # Universal arm64 and Intel 64-bit built for SDK 11.0 + target = OSXUniversalTarget(('arm64', 'x86_64'), args.work) + elif s == 'osx-arm64': + target = OSXSingleTarget('arm64', '11.0', args.work) elif s == 'source': target = SourceTarget() + elif s == 'flatpak': + target = FlatpakTarget(args.project, args.checkout) + elif s == 'appimage': + target = AppImageTarget(args.work) + + if target is None: + raise Error("Bad target `%s'" % s) - if target is not None: - target.debug = debug + target.debug = args.debug + target.ccache = args.ccache + if args.environment is not None: + for e in args.environment: + target.set(e, os.environ[e]) + + if args.mount is not None: + for m in args.mount: + target.mount(m) + + target.setup() return target # -# Project +# Tree # - -class Project(object): - def __init__(self, name, directory, specifier=None): + +class Tree(object): + """Description of a tree, which is a checkout of a project, + possibly built. This class is never exposed to cscripts. + Attributes: + name -- name of git repository (without the .git) + specifier -- git tag or revision to use + target -- target object that we are using + version -- version from the wscript (if one is present) + git_commit -- git revision that is actually being used + built -- true if the tree has been built yet in this run + required_by -- name of the tree that requires this one + """ + + def __init__(self, name, specifier, target, required_by, built=False): self.name = name - self.directory = directory - self.version = None self.specifier = specifier - if self.specifier is None: - self.specifier = 'master' - - def checkout(self, target): - flags = '' - redirect = '' - if args.quiet: - flags = '-q' - redirect = '>/dev/null' - command('git clone %s %s/%s.git %s/src/%s' % (flags, config.get('git_prefix'), self.name, target.work_dir_cdist(), self.name)) - os.chdir('%s/src/%s' % (target.work_dir_cdist(), self.name)) - command('git checkout %s %s %s' % (flags, self.specifier, redirect)) - command('git submodule init') - command('git submodule update') - os.chdir(self.directory) - - proj = '%s/src/%s/%s' % (target.work_dir_cdist(), self.name, self.directory) - - self.read_cscript('%s/cscript' % proj) - + self.target = target + self.version = None + self.git_commit = None + self.built = built + self.required_by = required_by + + cwd = os.getcwd() + proj = '%s/src/%s' % (target.directory, self.name) + + if not built: + flags = '' + redirect = '' + if globals.quiet: + flags = '-q' + redirect = '>/dev/null' + if config.has('git_reference'): + ref = '--reference-if-able %s/%s.git' % (config.get('git_reference'), self.name) + else: + ref = '' + command('git clone %s %s %s/%s.git %s/src/%s' % (flags, ref, config.get('git_prefix'), self.name, target.directory, self.name)) + os.chdir('%s/src/%s' % (target.directory, self.name)) + + spec = self.specifier + if spec is None: + spec = 'master' + + command('git checkout %s %s %s' % (flags, spec, redirect)) + self.git_commit = command_and_read('git rev-parse --short=7 HEAD')[0].strip() + + self.cscript = {} + exec(open('%s/cscript' % proj).read(), self.cscript) + + if not built: + # cscript can include submodules = False to stop submodules being fetched + if (not 'submodules' in self.cscript or self.cscript['submodules'] == True) and os.path.exists('.gitmodules'): + command('git submodule --quiet init') + paths = command_and_read('git config --file .gitmodules --get-regexp path') + urls = command_and_read('git config --file .gitmodules --get-regexp url') + for path, url in zip(paths, urls): + ref = '' + if config.has('git_reference'): + url = url.split(' ')[1] + ref_path = os.path.join(config.get('git_reference'), os.path.basename(url)) + if os.path.exists(ref_path): + ref = '--reference %s' % ref_path + path = path.split(' ')[1] + command('git submodule --quiet update %s %s' % (ref, path)) + if os.path.exists('%s/wscript' % proj): v = read_wscript_variable(proj, "VERSION"); if v is not None: - self.version = Version(v) + try: + self.version = Version(v) + except: + try: + tag = command_and_read('git -C %s describe --tags' % proj)[0][1:] + self.version = Version.from_git_tag(tag) + except: + # We'll leave version as None if we can't read it; maybe this is a bad idea + # Should probably just install git on the Windows VM + pass - def read_cscript(self, s): - self.cscript = {} - execfile(s, self.cscript) + os.chdir(cwd) -def set_version_in_wscript(version): - f = open('wscript', 'rw') - o = open('wscript.tmp', 'w') - while 1: - l = f.readline() - if l == '': - break + def call(self, function, *args): + with TreeDirectory(self): + return self.cscript[function](self.target, *args) - s = l.split() - if len(s) == 3 and s[0] == "VERSION": - print "Writing %s" % version - print >>o,"VERSION = '%s'" % version + def add_defaults(self, options): + """Add the defaults from self into a dict options""" + if 'option_defaults' in self.cscript: + from_cscript = self.cscript['option_defaults'] + if isinstance(from_cscript, dict): + defaults_dict = from_cscript + else: + log_normal("Deprecated cscript option_defaults method; replace with a dict") + defaults_dict = from_cscript() + for k, v in defaults_dict.items(): + if not k in options: + options[k] = v + + def dependencies(self, options): + """ + yield details of the dependencies of this tree. Each dependency is returned + as a tuple of (tree, options, parent_tree). The 'options' parameter are the options that + we want to force for 'self'. + """ + if not 'dependencies' in self.cscript: + return + + if len(inspect.getfullargspec(self.cscript['dependencies']).args) == 2: + self_options = copy.copy(options) + self.add_defaults(self_options) + deps = self.call('dependencies', self_options) else: - print >>o,l, - f.close() - o.close() + log_normal("Deprecated cscript dependencies() method with no options parameter") + deps = self.call('dependencies') - os.rename('wscript.tmp', 'wscript') + # Loop over our immediate dependencies + for d in deps: + dep = globals.trees.get(d[0], d[1], self.target, self.name) -def append_version_to_changelog(version): - try: - f = open('ChangeLog', 'r') - except: - log('Could not open ChangeLog') - return + # deps only get their options from the parent's cscript + dep_options = d[2] if len(d) > 2 else {} + for i in dep.dependencies(dep_options): + yield i + yield (dep, dep_options, self) - c = f.read() - f.close() + def checkout_dependencies(self, options={}): + for i in self.dependencies(options): + pass - f = open('ChangeLog', 'w') - now = datetime.datetime.now() - f.write('%d-%02d-%02d Carl Hetherington \n\n\t* Version %s released.\n\n' % (now.year, now.month, now.day, version)) - f.write(c) + def build_dependencies(self, options): + """ + Called on the 'main' project tree (-p on the command line) to build all dependencies. + 'options' will be the ones from the command line. + """ + for i in self.dependencies(options): + i[0].build(i[1]) -def append_version_to_debian_changelog(version): - if not os.path.exists('debian'): - log('Could not find debian directory') - return + def build(self, options): + if self.built: + return + + log_verbose("Building %s %s %s with %s" % (self.name, self.specifier, self.version, options)) + + variables = copy.copy(self.target.variables) + + options = copy.copy(options) + self.add_defaults(options) + + if not globals.dry_run: + if len(inspect.getfullargspec(self.cscript['build']).args) == 2: + self.call('build', options) + else: + self.call('build') + + self.target.variables = variables + self.built = True - command('dch -b -v %s-1 "New upstream release."' % version) # # Command-line parser # -parser = argparse.ArgumentParser() -parser.add_argument('command') -parser.add_argument('-p', '--project', help='project name') -parser.add_argument('-d', '--directory', help='directory within project repo', default='.') -parser.add_argument('--beta', help='beta release', action='store_true') -parser.add_argument('--full', help='full release', action='store_true') -parser.add_argument('-c', '--checkout', help='string to pass to git for checkout') -parser.add_argument('-o', '--output', help='output directory', default='.') -parser.add_argument('-q', '--quiet', help='be quiet', action='store_true') -parser.add_argument('-t', '--target', help='target') -parser.add_argument('-k', '--keep', help='keep working tree', action='store_true') -parser.add_argument('--debug', help='build with debugging symbols where possible', action='store_true') -parser.add_argument('-w', '--work', help='override default work directory') -args = parser.parse_args() - -args.output = os.path.abspath(args.output) -if args.work is not None: - args.work = os.path.abspath(args.work) - -if args.project is None and args.command != 'shell': - raise Error('you must specify -p or --project') - -project = Project(args.project, args.directory, args.checkout) - -if args.command == 'build': - if args.target is None: - raise Error('you must specify -t or --target') - - target = target_factory(args.target, args.debug, args.work) - project.checkout(target) - target.build_dependencies(project) - target.build(project) - if not args.keep: - target.cleanup() +def main(): + + commands = { + "build": "build project", + "package": "build and package the project", + "release": "release a project using its next version number (adding a tag)", + "pot": "build the project's .pot files", + "manual": "build the project's manual", + "doxygen": "build the project's Doxygen documentation", + "latest": "print out the latest version", + "test": "build the project and run its unit tests", + "shell": "start a shell in the project''s work directory", + "checkout": "check out the project", + "revision": "print the head git revision number", + "dependencies" : "print details of the project's dependencies as a .dot file" + } + + one_of = "" + summary = "" + for k, v in commands.items(): + one_of += "\t%s%s\n" % (k.ljust(20), v) + summary += k + " " + + parser = argparse.ArgumentParser() + parser.add_argument('-p', '--project', help='project name') + parser.add_argument('-c', '--checkout', help='string to pass to git for checkout') + parser.add_argument('-o', '--output', help='output directory', default='.') + parser.add_argument('-q', '--quiet', help='be quiet', action='store_true') + parser.add_argument('-t', '--target', help='target', action='append') + parser.add_argument('--environment-version', help='version of environment to use') + parser.add_argument('-k', '--keep', help='keep working tree', action='store_true') + parser.add_argument('--debug', help='build with debugging symbols where possible', action='store_true') + parser.add_argument('-w', '--work', help='override default work directory') + parser.add_argument('-g', '--git-prefix', help='override configured git prefix') + parser.add_argument('-n', '--dry-run', help='run the process without building anything', action='store_true') + parser.add_argument('-e', '--environment', help='pass the value of the named environment variable into the build', action='append') + parser.add_argument('-m', '--mount', help='mount a given directory in the build environment', action='append') + parser.add_argument('--option', help='set an option for the build (use --option key:value)', action='append') + parser.add_argument('--ccache', help='use ccache', action='store_true') + parser.add_argument('--verbose', help='be verbose', action='store_true') + + subparsers = parser.add_subparsers(help='command to run', dest='command') + parser_build = subparsers.add_parser("build", help="build project") + parser_package = subparsers.add_parser("package", help="build and package project") + parser_package.add_argument('--no-notarize', help='do not notarize .dmg packages', action='store_true') + parser_release = subparsers.add_parser("release", help="release a project using its next version number (adding a tag)") + parser_release.add_argument('--minor', help='minor version number bump', action='store_true') + parser_release.add_argument('--micro', help='micro version number bump', action='store_true') + parser_pot = subparsers.add_parser("pot", help="build the project's .pot files") + parser_manual = subparsers.add_parser("manual", help="build the project's manual") + parser_doxygen = subparsers.add_parser("doxygen", help="build the project's Doxygen documentation") + parser_latest = subparsers.add_parser("latest", help="print out the latest version") + parser_latest.add_argument('--major', help='major version to return', type=int) + parser_latest.add_argument('--minor', help='minor version to return', type=int) + parser_test = subparsers.add_parser("test", help="build the project and run its unit tests") + parser_test.add_argument('--no-implicit-build', help='do not build first', action='store_true') + parser_test.add_argument('--test', help="name of test to run, defaults to all") + parser_shell = subparsers.add_parser("shell", help="build the project then start a shell") + parser_checkout = subparsers.add_parser("checkout", help="check out the project") + parser_revision = subparsers.add_parser("revision", help="print the head git revision number") + parser_dependencies = subparsers.add_parser("dependencies", help="print details of the project's dependencies as a .dot file") + + global args + args = parser.parse_args() + + # Check for incorrect multiple parameters + if args.target is not None: + if len(args.target) > 1: + parser.error('multiple -t options specified') + sys.exit(1) + else: + args.target = args.target[0] + + # Override configured stuff + if args.git_prefix is not None: + config.set('git_prefix', args.git_prefix) + + if args.output.find(':') == -1: + # This isn't of the form host:path so make it absolute + args.output = os.path.abspath(args.output) + '/' + else: + if args.output[-1] != ':' and args.output[-1] != '/': + args.output += '/' + + # Now, args.output is 'host:', 'host:path/' or 'path/' + + if args.work is not None: + args.work = os.path.abspath(args.work) + if not os.path.exists(args.work): + os.makedirs(args.work) + + if args.project is None and args.command != 'shell': + raise Error('you must specify -p or --project') -elif args.command == 'package': - if args.target is None: - raise Error('you must specify -t or --target') - - target = target_factory(args.target, args.debug, args.work) + globals.quiet = args.quiet + globals.verbose = args.verbose + globals.dry_run = args.dry_run - packages = target.package(project) - if hasattr(packages, 'strip') or (not hasattr(packages, '__getitem__') and not hasattr(packages, '__iter__')): - packages = [packages] + if args.command == 'build': + if args.target is None: + raise Error('you must specify -t or --target') - if target.platform == 'linux': - out = '%s/%s-%d' % (args.output, target.version, target.bits) + target = target_factory(args) + target.build(args.project, args.checkout, get_command_line_options(args)) + if not args.keep: + target.cleanup() + + elif args.command == 'package': + if args.target is None: + raise Error('you must specify -t or --target') + + target = None try: - os.makedirs(out) - except: - pass - for p in packages: - copyfile(p, '%s/%s' % (out, os.path.basename(p))) - else: - for p in packages: - copyfile(p, '%s/%s' % (args.output, os.path.basename(p))) + target = target_factory(args) - target.cleanup() + if target.platform == 'linux' and target.detail != "appimage": + if target.distro != 'arch': + output_dir = os.path.join(args.output, '%s-%s-%d' % (target.distro, target.version, target.bits)) + else: + output_dir = os.path.join(args.output, '%s-%d' % (target.distro, target.bits)) + else: + output_dir = args.output + + makedirs(output_dir) + target.package(args.project, args.checkout, output_dir, get_command_line_options(args), not args.no_notarize) + except Error as e: + if target is not None and not args.keep: + target.cleanup() + raise -elif args.command == 'release': - if args.full is False and args.beta is False: - raise Error('you must specify --full or --beta') + if target is not None and not args.keep: + target.cleanup() - target = SourceTarget() - project.checkout(target) + elif args.command == 'release': + if args.minor is False and args.micro is False: + raise Error('you must specify --minor or --micro') - version = project.version - if args.full: + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + + version = tree.version version.to_release() - else: - version.bump_beta() + if args.minor: + version.bump_minor() + else: + version.bump_micro() - set_version_in_wscript(version) - append_version_to_changelog(version) - append_version_to_debian_changelog(version) + with TreeDirectory(tree): + command('git tag -m "v%s" v%s' % (version, version)) + command('git push --tags') - command('git commit -a -m "Bump version"') - command('git tag -m "v%s" v%s' % (version, version)) + target.cleanup() - if args.full: - version.bump_and_to_pre() - set_version_in_wscript(version) - command('git commit -a -m "Bump version"') + elif args.command == 'pot': + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) - command('git push') - command('git push --tags') + pots = tree.call('make_pot') + for p in pots: + copyfile(p, '%s%s' % (args.output, os.path.basename(p))) - target.cleanup() + target.cleanup() -elif args.command == 'pot': - target = SourceTarget() - project.checkout(target) + elif args.command == 'manual': + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) - pots = project.cscript['make_pot'](target) - for p in pots: - copyfile(p, '%s/%s' % (args.output, os.path.basename(p))) + outs = tree.call('make_manual') + for o in outs: + if os.path.isfile(o): + copyfile(o, '%s%s' % (args.output, os.path.basename(o))) + else: + copytree(o, '%s%s' % (args.output, os.path.basename(o))) - target.cleanup() + target.cleanup() -elif args.command == 'changelog': - target = SourceTarget() - project.checkout(target) + elif args.command == 'doxygen': + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) - text = open('ChangeLog', 'r') - html = open('%s/changelog.html' % args.output, 'w') - versions = 8 - - last = None - changes = [] - - while 1: - l = text.readline() - if l == '': - break - - if len(l) > 0 and l[0] == "\t": - s = l.split() - if len(s) == 4 and s[1] == "Version" and s[3] == "released.": - if not "beta" in s[2]: - if last is not None and len(changes) > 0: - print >>html,"

Changes between version %s and %s

" % (s[2], last) - print >>html,"
    " - for c in changes: - print >>html,"
  • %s" % c - print >>html,"
" - last = s[2] - changes = [] - versions -= 1 - if versions < 0: - break - else: - c = l.strip() - if len(c) > 0: - if c[0] == '*': - changes.append(c[2:]) - else: - changes[-1] += " " + c - - target.cleanup() - -elif args.command == 'manual': - target = SourceTarget() - project.checkout(target) - - outs = project.cscript['make_manual'](target) - for o in outs: - if os.path.isfile(o): - copyfile(o, '%s/%s' % (args.output, os.path.basename(o))) - else: - copytree(o, '%s/%s' % (args.output, os.path.basename(o))) + dirs = tree.call('make_doxygen') + if hasattr(dirs, 'strip') or (not hasattr(dirs, '__getitem__') and not hasattr(dirs, '__iter__')): + dirs = [dirs] - target.cleanup() + for d in dirs: + copytree(d, args.output) -elif args.command == 'doxygen': - target = SourceTarget() - project.checkout(target) + target.cleanup() + + elif args.command == 'latest': + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + + with TreeDirectory(tree): + f = command_and_read('git log --tags --simplify-by-decoration --pretty="%d"') + latest = None + line = 0 + while latest is None: + t = f[line] + line += 1 + m = re.compile(".*\((.*)\).*").match(t) + if m: + tags = m.group(1).split(', ') + for t in tags: + s = t.split() + if len(s) > 1: + t = s[1] + if len(t) > 0 and t[0] == 'v': + v = Version(t[1:]) + if (args.major is None or v.major == args.major) and (args.minor is None or v.minor == args.minor): + latest = v + + print(latest) + target.cleanup() + + elif args.command == 'test': + if args.target is None: + raise Error('you must specify -t or --target') + + target = None + try: + target = target_factory(args) + options = get_command_line_options(args) + if args.no_implicit_build: + globals.trees.add_built(args.project, args.checkout, target) + else: + target.build(args.project, args.checkout, options) + target.test(args.project, args.checkout, target, args.test, options) + finally: + if target is not None and not args.keep: + target.cleanup() - dirs = project.cscript['make_doxygen'](target) - if hasattr(dirs, 'strip') or (not hasattr(dirs, '__getitem__') and not hasattr(dirs, '__iter__')): - dirs = [dirs] + elif args.command == 'shell': + if args.target is None: + raise Error('you must specify -t or --target') - for d in dirs: - copytree(d, '%s/%s' % (args.output, 'doc')) + target = target_factory(args) + target.command('bash') - target.cleanup() + elif args.command == 'revision': -elif args.command == 'latest': - target = SourceTarget() - project.checkout(target) + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + with TreeDirectory(tree): + print(command_and_read('git rev-parse HEAD')[0].strip()[:7]) + target.cleanup() - f = command_and_read('git log --tags --simplify-by-decoration --pretty="%d"') - t = f.readline() - m = re.compile(".*\((.*)\).*").match(t) - latest = None - if m: - tags = m.group(1).split(', ') - for t in tags: - if len(t) > 0 and t[0] == 'v': - latest = t[1:] + elif args.command == 'checkout': - print latest - target.cleanup() + if args.output is None: + raise Error('you must specify -o or --output') -elif args.command == 'test': - if args.target is None: - raise Error('you must specify -t or --target') + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + with TreeDirectory(tree): + shutil.copytree('.', args.output) + target.cleanup() - target = None - try: - target = target_factory(args.target, args.debug, args.work) - target.test(project) - except Error as e: - if target is not None: - target.cleanup(project) - raise - - if target is not None: - target.cleanup(project) - -elif args.command == 'shell': - if args.target is None: - raise Error('you must specify -t or --target') - - target = target_factory(args.target, args.debug, args.work) - target.command('bash') - -else: - raise Error('invalid command %s' % args.command) + elif args.command == 'dependencies': + if args.target is None: + raise Error('you must specify -t or --target') + if args.checkout is None: + raise Error('you must specify -c or --checkout') + + target = target_factory(args) + tree = globals.trees.get(args.project, args.checkout, target) + print("strict digraph {") + for d in list(tree.dependencies({})): + print("%s -> %s;" % (d[2].name.replace("-", "-"), d[0].name.replace("-", "_"))) + print("}") + + +try: + main() +except Error as e: + print('cdist: %s' % str(e), file=sys.stderr) + sys.exit(1)