#!/usr/bin/python3 # Copyright (C) 2012-2020 Carl Hetherington # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. from __future__ import print_function import argparse import copy import datetime import getpass import glob import inspect import multiprocessing import os import platform import re import shlex import shutil import subprocess import sys import tempfile import time TEMPORARY_DIRECTORY = '/var/tmp' class Error(Exception): def __init__(self, value): self.value = value def __str__(self): return self.value def __repr__(self): return str(self) class Trees: """ Store for Tree objects which re-uses already-created objects and checks for requests for different versions of the same thing. """ def __init__(self): self.trees = [] def get(self, name, specifier, target, required_by=None): for t in self.trees: if t.name == name and t.specifier == specifier and t.target == target: return t elif t.name == name and t.specifier != specifier: a = specifier if specifier is not None else "[Any]" if required_by is not None: a += ' by %s' % required_by b = t.specifier if t.specifier is not None else "[Any]" if t.required_by is not None: b += ' by %s' % t.required_by raise Error('conflicting versions of %s required (%s versus %s)' % (name, a, b)) nt = Tree(name, specifier, target, required_by) self.trees.append(nt) return nt def add_built(self, name, specifier, target): self.trees.append(Tree(name, specifier, target, None, built=True)) class Globals: quiet = False command = None dry_run = False trees = Trees() globals = Globals() # # Configuration # class Option(object): def __init__(self, key, default=None): self.key = key self.value = default def offer(self, key, value): if key == self.key: self.value = value class BoolOption(object): def __init__(self, key): self.key = key self.value = False def offer(self, key, value): if key == self.key: self.value = (value == 'yes' or value == '1' or value == 'true') class Config: def __init__(self): self.options = [ Option('mxe_prefix'), Option('git_prefix'), Option('git_reference'), Option('osx_environment_prefix'), Option('osx_sdk_prefix'), Option('osx_sdk'), Option('osx_keychain_file'), Option('osx_keychain_password'), Option('apple_id'), Option('apple_password'), BoolOption('docker_sudo'), BoolOption('docker_no_user'), Option('docker_hub_repository'), Option('flatpak_state_dir'), Option('parallel', multiprocessing.cpu_count()) ] config_dir = '%s/.config' % os.path.expanduser('~') if not os.path.exists(config_dir): os.mkdir(config_dir) config_file = '%s/cdist' % config_dir if not os.path.exists(config_file): f = open(config_file, 'w') for o in self.options: print('# %s ' % o.key, file=f) f.close() print('Template config file written to %s; please edit and try again.' % config_file, file=sys.stderr) sys.exit(1) try: f = open('%s/.config/cdist' % os.path.expanduser('~'), 'r') while True: l = f.readline() if l == '': break if len(l) > 0 and l[0] == '#': continue s = l.strip().split() if len(s) == 2: for k in self.options: k.offer(s[0], s[1]) except: raise def has(self, k): for o in self.options: if o.key == k and o.value is not None: return True return False def get(self, k): for o in self.options: if o.key == k: if o.value is None: raise Error('Required setting %s not found' % k) return o.value def set(self, k, v): for o in self.options: o.offer(k, v) def docker(self): if self.get('docker_sudo'): return 'sudo docker' else: return 'docker' config = Config() # # Utility bits # def log_normal(m): if not globals.quiet: print('\x1b[33m* %s\x1b[0m' % m) def log_verbose(m): if globals.verbose: print('\x1b[35m* %s\x1b[0m' % m) def escape_spaces(s): return s.replace(' ', '\\ ') def scp_escape(n): """Escape a host:filename string for use with an scp command""" s = n.split(':') assert(len(s) == 1 or len(s) == 2) if len(s) == 2: return '%s:"\'%s\'"' % (s[0], s[1]) else: return '\"%s\"' % s[0] def mv_escape(n): return '\"%s\"' % n.substr(' ', '\\ ') def copytree(a, b): log_normal('copy %s -> %s' % (scp_escape(a), scp_escape(b))) if b.startswith('s3://'): command('s3cmd -P -r put "%s" "%s"' % (a, b)) else: command('scp -r %s %s' % (scp_escape(a), scp_escape(b))) def copyfile(a, b): log_normal('copy %s -> %s with cwd %s' % (scp_escape(a), scp_escape(b), os.getcwd())) if b.startswith('s3://'): command('s3cmd -P put "%s" "%s"' % (a, b)) else: bc = b.find(":") if bc != -1: host = b[:bc] path = b[bc+1:] temp_path = os.path.join(os.path.dirname(path), ".tmp." + os.path.basename(path)) command('scp %s %s' % (scp_escape(a), scp_escape(host + ":" + temp_path))) command('ssh %s -- mv "%s" "%s"' % (host, escape_spaces(temp_path), escape_spaces(path))) else: command('scp %s %s' % (scp_escape(a), scp_escape(b))) def makedirs(d): """ Make directories either locally or on a remote host; remotely if d includes a colon, otherwise locally. """ if d.startswith('s3://'): # No need to create folders on S3 return if d.find(':') == -1: try: os.makedirs(d) except OSError as e: if e.errno != 17: raise e else: s = d.split(':') command('ssh %s -- mkdir -p %s' % (s[0], s[1])) def rmdir(a): log_normal('remove %s' % a) os.rmdir(a) def rmtree(a): log_normal('remove %s' % a) shutil.rmtree(a, ignore_errors=True) def command(c): log_normal(c) r = os.system(c) if (r >> 8): raise Error('command %s failed' % c) def command_and_read(c): log_normal(c) p = subprocess.Popen(c.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = p.communicate() if p.returncode != 0: raise Error('command %s failed (%s)' % (c, err)) return str(out, 'utf-8').splitlines() def read_wscript_variable(directory, variable): f = open('%s/wscript' % directory, 'r') while True: l = f.readline() if l == '': break s = l.split() if len(s) == 3 and s[0] == variable: f.close() return s[2][1:-1] f.close() return None def devel_to_git(git_commit, filename): if git_commit is not None: filename = filename.replace('devel', '-%s' % git_commit) return filename def get_command_line_options(args): """Get the options specified by --option on the command line""" options = dict() if args.option is not None: for o in args.option: b = o.split(':') if len(b) != 2: raise Error("Bad option `%s'" % o) if b[1] == 'False': options[b[0]] = False elif b[1] == 'True': options[b[0]] = True else: options[b[0]] = b[1] return options class TreeDirectory: def __init__(self, tree): self.tree = tree def __enter__(self): self.cwd = os.getcwd() os.chdir('%s/src/%s' % (self.tree.target.directory, self.tree.name)) def __exit__(self, type, value, traceback): os.chdir(self.cwd) # # Version # class Version: def __init__(self, s): self.devel = False if s.startswith("'"): s = s[1:] if s.endswith("'"): s = s[0:-1] if s.endswith('devel'): s = s[0:-5] self.devel = True if s.endswith('pre'): s = s[0:-3] p = s.split('.') self.major = int(p[0]) self.minor = int(p[1]) if len(p) == 3: self.micro = int(p[2]) else: self.micro = 0 @classmethod def from_git_tag(cls, tag): bits = tag.split('-') c = cls(bits[0]) if len(bits) > 1 and int(bits[1]) > 0: c.devel = True return c def bump_minor(self): self.minor += 1 self.micro = 0 def bump_micro(self): self.micro += 1 def to_devel(self): self.devel = True def to_release(self): self.devel = False def __str__(self): s = '%d.%d.%d' % (self.major, self.minor, self.micro) if self.devel: s += 'devel' return s # # Targets # class Target(object): """ Class representing the target that we are building for. This is exposed to cscripts, though not all of it is guaranteed 'API'. cscripts may expect: platform: platform string (e.g. 'windows', 'linux', 'osx') parallel: number of parallel jobs to run directory: directory to work in variables: dict of environment variables debug: True to build a debug version, otherwise False ccache: True to use ccache, False to not set(a, b): set the value of variable 'a' to 'b' unset(a): unset the value of variable 'a' command(c): run the command 'c' in the build environment """ def __init__(self, platform, directory=None): """ platform -- platform string (e.g. 'windows', 'linux', 'osx') directory -- directory to work in; if None we will use a temporary directory Temporary directories will be removed after use; specified directories will not. """ self.platform = platform self.parallel = int(config.get('parallel')) # Environment variables that we will use when we call cscripts self.variables = {} self.debug = False self._ccache = False # True to build our dependencies ourselves; False if this is taken care # of in some other way self.build_dependencies = True if directory is None: self.directory = tempfile.mkdtemp('', 'tmp', TEMPORARY_DIRECTORY) self.rmdir = True self.set('CCACHE_BASEDIR', os.path.realpath(self.directory)) self.set('CCACHE_NOHASHDIR', '') else: self.directory = os.path.realpath(directory) self.rmdir = False def setup(self): pass def _build_packages(self, tree, options): if len(inspect.getfullargspec(tree.cscript['package']).args) == 3: packages = tree.call('package', tree.version, options) else: log_normal("Deprecated cscript package() method with no options parameter") packages = tree.call('package', tree.version) return packages if isinstance(packages, list) else [packages] def _copy_packages(self, tree, packages, output_dir): for p in packages: copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) def package(self, project, checkout, output_dir, options, no_notarize): tree = self.build(project, checkout, options) tree.add_defaults(options) p = self._build_packages(tree, options) self._copy_packages(tree, p, output_dir) def build(self, project, checkout, options): tree = globals.trees.get(project, checkout, self) if self.build_dependencies: tree.build_dependencies(options) tree.build(options) return tree def test(self, project, checkout, target, test, options): """test is the test case to run, or None""" tree = globals.trees.get(project, checkout, target) tree.add_defaults(options) with TreeDirectory(tree): if len(inspect.getfullargspec(tree.cscript['test']).args) == 3: return tree.call('test', options, test) else: log_normal('Deprecated cscript test() method with no options parameter') return tree.call('test', test) def set(self, a, b): self.variables[a] = b def unset(self, a): del(self.variables[a]) def get(self, a): return self.variables[a] def append(self, k, v, s): if (not k in self.variables) or len(self.variables[k]) == 0: self.variables[k] = '"%s"' % v else: e = self.variables[k] if e[0] == '"' and e[-1] == '"': self.variables[k] = '"%s%s%s"' % (e[1:-1], s, v) else: self.variables[k] = '"%s%s%s"' % (e, s, v) def append_with_space(self, k, v): return self.append(k, v, ' ') def append_with_colon(self, k, v): return self.append(k, v, ':') def variables_string(self, escaped_quotes=False): e = '' for k, v in self.variables.items(): if escaped_quotes: v = v.replace('"', '\\"') e += '%s=%s ' % (k, v) return e def cleanup(self): if self.rmdir: rmtree(self.directory) def mount(self, m): pass @property def ccache(self): return self._ccache @ccache.setter def ccache(self, v): self._ccache = v class DockerTarget(Target): def __init__(self, platform, directory): super(DockerTarget, self).__init__(platform, directory) self.mounts = [] self.privileged = False def _user_tag(self): if config.get('docker_no_user'): return '' return '-u %s' % getpass.getuser() def _mount_option(self, d): return '-v %s:%s ' % (os.path.realpath(d), os.path.realpath(d)) def setup(self): opts = self._mount_option(self.directory) for m in self.mounts: opts += self._mount_option(m) if config.has('git_reference'): opts += self._mount_option(config.get('git_reference')) if self.privileged: opts += '--privileged=true ' if self.ccache: opts += "-e CCACHE_DIR=/ccache/%s-%d --mount source=ccache,target=/ccache" % (self.image, os.getuid()) tag = self.image if config.has('docker_hub_repository'): tag = '%s:%s' % (config.get('docker_hub_repository'), tag) self.container = command_and_read('%s run %s %s -itd %s /bin/bash' % (config.docker(), self._user_tag(), opts, tag))[0].strip() def command(self, cmd): dir = os.path.join(self.directory, os.path.relpath(os.getcwd(), self.directory)) interactive_flag = '-i ' if sys.stdin.isatty() else '' command('%s exec %s %s -t %s /bin/bash -c \'export %s; cd %s; %s\'' % (config.docker(), self._user_tag(), interactive_flag, self.container, self.variables_string(), dir, cmd)) def cleanup(self): super(DockerTarget, self).cleanup() command('%s kill %s' % (config.docker(), self.container)) def mount(self, m): self.mounts.append(m) class FlatpakTarget(Target): def __init__(self, project, checkout): super(FlatpakTarget, self).__init__('flatpak') self.build_dependencies = False self.project = project self.checkout = checkout def setup(self): pass def command(self, cmd): command(cmd) def checkout_dependencies(self): tree = globals.trees.get(self.project, self.checkout, self) return tree.checkout_dependencies() def flatpak(self): return 'flatpak' def flatpak_builder(self): b = 'flatpak-builder' if config.has('flatpak_state_dir'): b += ' --state-dir=%s' % config.get('flatpak_state_dir') return b class WindowsDockerTarget(DockerTarget): """ This target exposes the following additional API: version: Windows version ('xp' or None) bits: bitness of Windows (32 or 64) name: name of our target e.g. x86_64-w64-mingw32.shared environment_prefix: path to Windows environment for the appropriate target (libraries and some tools) tool_path: path to 32- and 64-bit tools """ def __init__(self, windows_version, bits, directory, environment_version): super(WindowsDockerTarget, self).__init__('windows', directory) self.version = windows_version self.bits = bits self.tool_path = '%s/usr/bin' % config.get('mxe_prefix') if self.bits == 32: self.name = 'i686-w64-mingw32.shared' else: self.name = 'x86_64-w64-mingw32.shared' self.environment_prefix = '%s/usr/%s' % (config.get('mxe_prefix'), self.name) self.set('PKG_CONFIG_LIBDIR', '%s/lib/pkgconfig' % self.environment_prefix) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.directory, self.directory)) self.set('PATH', '%s/bin:%s:%s' % (self.environment_prefix, self.tool_path, os.environ['PATH'])) self.set('LD', '%s-ld' % self.name) self.set('RANLIB', '%s-ranlib' % self.name) self.set('WINRC', '%s-windres' % self.name) cxx = '-I%s/include -I%s/include' % (self.environment_prefix, self.directory) link = '-L%s/lib -L%s/lib' % (self.environment_prefix, self.directory) self.set('CXXFLAGS', '"%s"' % cxx) self.set('CPPFLAGS', '') self.set('LINKFLAGS', '"%s"' % link) self.set('LDFLAGS', '"%s"' % link) self.image = 'windows' if environment_version is not None: self.image += '_%s' % environment_version def setup(self): super().setup() if self.ccache: self.set('CC', '"ccache %s-gcc"' % self.name) self.set('CXX', '"ccache %s-g++"' % self.name) else: self.set('CC', '%s-gcc' % self.name) self.set('CXX', '%s-g++' % self.name) @property def library_prefix(self): log_normal('Deprecated property library_prefix: use environment_prefix') return self.environment_prefix @property def windows_prefix(self): log_normal('Deprecated property windows_prefix: use environment_prefix') return self.environment_prefix @property def mingw_prefixes(self): log_normal('Deprecated property mingw_prefixes: use environment_prefix') return [self.environment_prefix] @property def mingw_path(self): log_normal('Deprecated property mingw_path: use tool_path') return self.tool_path @property def mingw_name(self): log_normal('Deprecated property mingw_name: use name') return self.name class WindowsNativeTarget(Target): """ This target exposes the following additional API: version: Windows version ('xp' or None) bits: bitness of Windows (32 or 64) name: name of our target e.g. x86_64-w64-mingw32.shared environment_prefix: path to Windows environment for the appropriate target (libraries and some tools) """ def __init__(self, directory): super().__init__('windows', directory) self.version = None self.bits = 64 self.environment_prefix = config.get('windows_native_environmnet_prefix') self.set('PATH', '%s/bin:%s' % (self.environment_prefix, os.environ['PATH'])) def command(self, cmd): command(cmd) class LinuxTarget(DockerTarget): """ Build for Linux in a docker container. This target exposes the following additional API: distro: distribution ('debian', 'ubuntu', 'centos' or 'fedora') version: distribution version (e.g. '12.04', '8', '6.5') bits: bitness of the distribution (32 or 64) detail: None or 'appimage' if we are building for appimage """ def __init__(self, distro, version, bits, directory=None): super(LinuxTarget, self).__init__('linux', directory) self.distro = distro self.version = version self.bits = bits self.detail = None self.set('CXXFLAGS', '-I%s/include' % self.directory) self.set('CPPFLAGS', '') self.set('LINKFLAGS', '-L%s/lib' % self.directory) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig' % (self.directory, self.directory)) self.set('PATH', '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin') if self.version is None: self.image = '%s-%s' % (self.distro, self.bits) else: self.image = '%s-%s-%s' % (self.distro, self.version, self.bits) def setup(self): super(LinuxTarget, self).setup() if self.ccache: self.set('CC', '"ccache gcc"') self.set('CXX', '"ccache g++"') def test(self, project, checkout, target, test, options): self.append_with_colon('PATH', '%s/bin' % self.directory) self.append_with_colon('LD_LIBRARY_PATH', '%s/lib' % self.directory) super(LinuxTarget, self).test(project, checkout, target, test, options) class AppImageTarget(LinuxTarget): def __init__(self, work): super(AppImageTarget, self).__init__('ubuntu', '18.04', 64, work) self.detail = 'appimage' self.privileged = True def notarize(dmg, bundle_id): p = subprocess.run( ['xcrun', 'altool', '--notarize-app', '-t', 'osx', '-f', dmg, '--primary-bundle-id', bundle_id, '-u', config.get('apple_id'), '-p', config.get('apple_password'), '--output-format', 'xml'], capture_output=True ) def string_after(process, key): lines = p.stdout.decode('utf-8').splitlines() for i in range(0, len(lines)): if lines[i].find(key) != -1: return lines[i+1].strip().replace('', '').replace('', '') request_uuid = string_after(p, "RequestUUID") if request_uuid is None: raise Error('No RequestUUID found in response from Apple') for i in range(0, 30): print('Checking up on %s' % request_uuid) p = subprocess.run(['xcrun', 'altool', '--notarization-info', request_uuid, '-u', config.get('apple_id'), '-p', config.get('apple_password'), '--output-format', 'xml'], capture_output=True) status = string_after(p, 'Status') print('Got %s' % status) if status == 'invalid': raise Error("Notarization failed") elif status == 'success': subprocess.run(['xcrun', 'stapler', 'staple', dmg]) return time.sleep(30) raise Error("Notarization timed out") class OSXTarget(Target): def __init__(self, directory=None): super(OSXTarget, self).__init__('osx', directory) self.sdk = config.get('osx_sdk') self.sdk_prefix = config.get('osx_sdk_prefix') self.environment_prefix = config.get('osx_environment_prefix') self.apple_id = config.get('apple_id') self.apple_password = config.get('apple_password') self.osx_keychain_file = config.get('osx_keychain_file') self.osx_keychain_password = config.get('osx_keychain_password') def command(self, c): command('%s %s' % (self.variables_string(False), c)) def build(self, *a, **k): self.command('security unlock-keychain -p %s %s' % (self.osx_keychain_password, self.osx_keychain_file)) return super().build(*a, **k) class OSXSingleTarget(OSXTarget): def __init__(self, bits, directory=None): super(OSXSingleTarget, self).__init__(directory) self.bits = bits if bits == 32: arch = 'i386' else: arch = 'x86_64' flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (self.sdk_prefix, self.sdk, arch) enviro = '%s/%d' % (config.get('osx_environment_prefix'), bits) # Environment variables self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, enviro, flags)) self.set('CPPFLAGS', '') self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, enviro, flags)) self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, enviro, flags)) self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, enviro, flags)) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.directory, enviro)) self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % enviro) self.set('MACOSX_DEPLOYMENT_TARGET', config.get('osx_sdk')) self.set('CCACHE_BASEDIR', self.directory) @Target.ccache.setter def ccache(self, v): Target.ccache.fset(self, v) if v: self.set('CC', '"ccache gcc"') self.set('CXX', '"ccache g++"') def package(self, project, checkout, output_dir, options, no_notarize): tree = self.build(project, checkout, options) tree.add_defaults(options) p = self._build_packages(tree, options) for x in p: if not isinstance(x, tuple): raise Error('macOS packages must be returned from cscript as tuples of (dmg-filename, bundle-id)') if not no_notarize: notarize(x[0], x[1]) self._copy_packages(tree, [x[0] for x in p], output_dir) class OSXUniversalTarget(OSXTarget): def __init__(self, directory=None): super(OSXUniversalTarget, self).__init__(directory) self.bits = None def package(self, project, checkout, output_dir, options, no_notarize): for b in [32, 64]: target = OSXSingleTarget(b, os.path.join(self.directory, '%d' % b)) target.ccache = self.ccache tree = globals.trees.get(project, checkout, target) tree.build_dependencies(options) tree.build(options) tree = globals.trees.get(project, checkout, self) with TreeDirectory(tree): if len(inspect.getfullargspec(tree.cscript['package']).args) == 3: packages = tree.call('package', tree.version, options) else: log_normal("Deprecated cscript package() method with no options parameter") packages = tree.call('package', tree.version) for p in packages: copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) class SourceTarget(Target): """Build a source .tar.bz2""" def __init__(self): super(SourceTarget, self).__init__('source') def command(self, c): log_normal('host -> %s' % c) command('%s %s' % (self.variables_string(), c)) def cleanup(self): rmtree(self.directory) def package(self, project, checkout, output_dir, options): tree = globals.trees.get(project, checkout, self) with TreeDirectory(tree): name = read_wscript_variable(os.getcwd(), 'APPNAME') command('./waf dist') p = os.path.abspath('%s-%s.tar.bz2' % (name, tree.version)) copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) # @param s Target string: # windows-{32,64} # or ubuntu-version-{32,64} # or debian-version-{32,64} # or centos-version-{32,64} # or fedora-version-{32,64} # or mageia-version-{32,64} # or osx-{32,64} # or source # or flatpak # or appimage # @param debug True to build with debugging symbols (where possible) def target_factory(args): s = args.target target = None if s.startswith('windows-'): x = s.split('-') if platform.system() == "Windows": target = WindowsNativeTarget(args.work) else: if len(x) == 2: target = WindowsDockerTarget(None, int(x[1]), args.work, args.environment_version) elif len(x) == 3: target = WindowsDockerTarget(x[1], int(x[2]), args.work, args.environment_version) else: raise Error("Bad Windows target name `%s'") elif s.startswith('ubuntu-') or s.startswith('debian-') or s.startswith('centos-') or s.startswith('fedora-') or s.startswith('mageia-'): p = s.split('-') if len(p) != 3: raise Error("Bad Linux target name `%s'; must be something like ubuntu-16.04-32 (i.e. distro-version-bits)" % s) target = LinuxTarget(p[0], p[1], int(p[2]), args.work) elif s.startswith('arch-'): p = s.split('-') if len(p) != 2: raise Error("Bad Arch target name `%s'; must be arch-32 or arch-64") target = LinuxTarget(p[0], None, int(p[1]), args.work) elif s == 'raspbian': target = LinuxTarget(s, None, None, args.work) elif s.startswith('osx-'): target = OSXSingleTarget(int(s.split('-')[1]), args.work) elif s == 'osx': if args.command == 'build': target = OSXSingleTarget(64, args.work) else: target = OSXUniversalTarget(args.work) elif s == 'source': target = SourceTarget() elif s == 'flatpak': target = FlatpakTarget(args.project, args.checkout) elif s == 'appimage': target = AppImageTarget(args.work) if target is None: raise Error("Bad target `%s'" % s) target.debug = args.debug target.ccache = args.ccache if args.environment is not None: for e in args.environment: target.set(e, os.environ[e]) if args.mount is not None: for m in args.mount: target.mount(m) target.setup() return target # # Tree # class Tree(object): """Description of a tree, which is a checkout of a project, possibly built. This class is never exposed to cscripts. Attributes: name -- name of git repository (without the .git) specifier -- git tag or revision to use target -- target object that we are using version -- version from the wscript (if one is present) git_commit -- git revision that is actually being used built -- true if the tree has been built yet in this run required_by -- name of the tree that requires this one """ def __init__(self, name, specifier, target, required_by, built=False): self.name = name self.specifier = specifier self.target = target self.version = None self.git_commit = None self.built = built self.required_by = required_by cwd = os.getcwd() proj = '%s/src/%s' % (target.directory, self.name) if not built: flags = '' redirect = '' if globals.quiet: flags = '-q' redirect = '>/dev/null' if config.has('git_reference'): ref = '--reference-if-able %s/%s.git' % (config.get('git_reference'), self.name) else: ref = '' command('git clone %s %s %s/%s.git %s/src/%s' % (flags, ref, config.get('git_prefix'), self.name, target.directory, self.name)) os.chdir('%s/src/%s' % (target.directory, self.name)) spec = self.specifier if spec is None: spec = 'master' command('git checkout %s %s %s' % (flags, spec, redirect)) self.git_commit = command_and_read('git rev-parse --short=7 HEAD')[0].strip() self.cscript = {} exec(open('%s/cscript' % proj).read(), self.cscript) if not built: # cscript can include submodules = False to stop submodules being fetched if (not 'submodules' in self.cscript or self.cscript['submodules'] == True) and os.path.exists('.gitmodules'): command('git submodule --quiet init') paths = command_and_read('git config --file .gitmodules --get-regexp path') urls = command_and_read('git config --file .gitmodules --get-regexp url') for path, url in zip(paths, urls): ref = '' if config.has('git_reference'): url = url.split(' ')[1] ref_path = os.path.join(config.get('git_reference'), os.path.basename(url)) if os.path.exists(ref_path): ref = '--reference %s' % ref_path path = path.split(' ')[1] command('git submodule --quiet update %s %s' % (ref, path)) if os.path.exists('%s/wscript' % proj): v = read_wscript_variable(proj, "VERSION"); if v is not None: try: self.version = Version(v) except: try: tag = command_and_read('git -C %s describe --tags' % proj)[0][1:] self.version = Version.from_git_tag(tag) except: # We'll leave version as None if we can't read it; maybe this is a bad idea # Should probably just install git on the Windows VM pass os.chdir(cwd) def call(self, function, *args): with TreeDirectory(self): return self.cscript[function](self.target, *args) def add_defaults(self, options): """Add the defaults from self into a dict options""" if 'option_defaults' in self.cscript: from_cscript = self.cscript['option_defaults'] if isinstance(from_cscript, dict): defaults_dict = from_cscript else: log_normal("Deprecated cscript option_defaults method; replace with a dict") defaults_dict = from_cscript() for k, v in defaults_dict.items(): if not k in options: options[k] = v def dependencies(self, options): """ yield details of the dependencies of this tree. Each dependency is returned as a tuple of (tree, options, parent_tree). The 'options' parameter are the options that we want to force for 'self'. """ if not 'dependencies' in self.cscript: return if len(inspect.getfullargspec(self.cscript['dependencies']).args) == 2: self_options = copy.copy(options) self.add_defaults(self_options) deps = self.call('dependencies', self_options) else: log_normal("Deprecated cscript dependencies() method with no options parameter") deps = self.call('dependencies') # Loop over our immediate dependencies for d in deps: dep = globals.trees.get(d[0], d[1], self.target, self.name) # deps only get their options from the parent's cscript dep_options = d[2] if len(d) > 2 else {} for i in dep.dependencies(dep_options): yield i yield (dep, dep_options, self) def checkout_dependencies(self, options={}): for i in self.dependencies(options): pass def build_dependencies(self, options): """ Called on the 'main' project tree (-p on the command line) to build all dependencies. 'options' will be the ones from the command line. """ for i in self.dependencies(options): i[0].build(i[1]) def build(self, options): if self.built: return log_verbose("Building %s %s %s with %s" % (self.name, self.specifier, self.version, options)) variables = copy.copy(self.target.variables) options = copy.copy(options) self.add_defaults(options) if not globals.dry_run: if len(inspect.getfullargspec(self.cscript['build']).args) == 2: self.call('build', options) else: self.call('build') self.target.variables = variables self.built = True # # Command-line parser # def main(): commands = { "build": "build project", "package": "build and package the project", "release": "release a project using its next version number (adding a tag)", "pot": "build the project's .pot files", "manual": "build the project's manual", "doxygen": "build the project's Doxygen documentation", "latest": "print out the latest version", "test": "build the project and run its unit tests", "shell": "start a shell in the project''s work directory", "checkout": "check out the project", "revision": "print the head git revision number", "dependencies" : "print details of the project's dependencies as a .dot file" } one_of = "" summary = "" for k, v in commands.items(): one_of += "\t%s%s\n" % (k.ljust(20), v) summary += k + " " parser = argparse.ArgumentParser() parser.add_argument('-p', '--project', help='project name') parser.add_argument('-c', '--checkout', help='string to pass to git for checkout') parser.add_argument('-o', '--output', help='output directory', default='.') parser.add_argument('-q', '--quiet', help='be quiet', action='store_true') parser.add_argument('-t', '--target', help='target', action='append') parser.add_argument('--environment-version', help='version of environment to use') parser.add_argument('-k', '--keep', help='keep working tree', action='store_true') parser.add_argument('--debug', help='build with debugging symbols where possible', action='store_true') parser.add_argument('-w', '--work', help='override default work directory') parser.add_argument('-g', '--git-prefix', help='override configured git prefix') parser.add_argument('-n', '--dry-run', help='run the process without building anything', action='store_true') parser.add_argument('-e', '--environment', help='pass the value of the named environment variable into the build', action='append') parser.add_argument('-m', '--mount', help='mount a given directory in the build environment', action='append') parser.add_argument('--option', help='set an option for the build (use --option key:value)', action='append') parser.add_argument('--ccache', help='use ccache', action='store_true') parser.add_argument('--verbose', help='be verbose', action='store_true') subparsers = parser.add_subparsers(help='command to run', dest='command') parser_build = subparsers.add_parser("build", help="build project") parser_package = subparsers.add_parser("package", help="build and package project") parser_package.add_argument('--no-notarize', help='do not notarize .dmg packages', action='store_true') parser_release = subparsers.add_parser("release", help="release a project using its next version number (adding a tag)") parser_release.add_argument('--minor', help='minor version number bump', action='store_true') parser_release.add_argument('--micro', help='micro version number bump', action='store_true') parser_pot = subparsers.add_parser("pot", help="build the project's .pot files") parser_manual = subparsers.add_parser("manual", help="build the project's manual") parser_doxygen = subparsers.add_parser("doxygen", help="build the project's Doxygen documentation") parser_latest = subparsers.add_parser("latest", help="print out the latest version") parser_latest.add_argument('--major', help='major version to return', type=int) parser_latest.add_argument('--minor', help='minor version to return', type=int) parser_test = subparsers.add_parser("test", help="build the project and run its unit tests") parser_test.add_argument('--no-implicit-build', help='do not build first', action='store_true') parser_test.add_argument('--test', help="name of test to run, defaults to all") parser_shell = subparsers.add_parser("shell", help="build the project then start a shell") parser_checkout = subparsers.add_parser("checkout", help="check out the project") parser_revision = subparsers.add_parser("revision", help="print the head git revision number") parser_dependencies = subparsers.add_parser("dependencies", help="print details of the project's dependencies as a .dot file") global args args = parser.parse_args() # Check for incorrect multiple parameters if args.target is not None: if len(args.target) > 1: parser.error('multiple -t options specified') sys.exit(1) else: args.target = args.target[0] # Override configured stuff if args.git_prefix is not None: config.set('git_prefix', args.git_prefix) if args.output.find(':') == -1: # This isn't of the form host:path so make it absolute args.output = os.path.abspath(args.output) + '/' else: if args.output[-1] != ':' and args.output[-1] != '/': args.output += '/' # Now, args.output is 'host:', 'host:path/' or 'path/' if args.work is not None: args.work = os.path.abspath(args.work) if not os.path.exists(args.work): os.makedirs(args.work) if args.project is None and args.command != 'shell': raise Error('you must specify -p or --project') globals.quiet = args.quiet globals.verbose = args.verbose globals.dry_run = args.dry_run if args.command == 'build': if args.target is None: raise Error('you must specify -t or --target') target = target_factory(args) target.build(args.project, args.checkout, get_command_line_options(args)) if not args.keep: target.cleanup() elif args.command == 'package': if args.target is None: raise Error('you must specify -t or --target') target = None try: target = target_factory(args) if target.platform == 'linux' and target.detail != "appimage": if target.distro != 'arch': output_dir = os.path.join(args.output, '%s-%s-%d' % (target.distro, target.version, target.bits)) else: output_dir = os.path.join(args.output, '%s-%d' % (target.distro, target.bits)) else: output_dir = args.output makedirs(output_dir) target.package(args.project, args.checkout, output_dir, get_command_line_options(args), args.no_notarize) except Error as e: if target is not None and not args.keep: target.cleanup() raise if target is not None and not args.keep: target.cleanup() elif args.command == 'release': if args.minor is False and args.micro is False: raise Error('you must specify --minor or --micro') target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) version = tree.version version.to_release() if args.minor: version.bump_minor() else: version.bump_micro() with TreeDirectory(tree): command('git tag -m "v%s" v%s' % (version, version)) command('git push --tags') target.cleanup() elif args.command == 'pot': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) pots = tree.call('make_pot') for p in pots: copyfile(p, '%s%s' % (args.output, os.path.basename(p))) target.cleanup() elif args.command == 'manual': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) outs = tree.call('make_manual') for o in outs: if os.path.isfile(o): copyfile(o, '%s%s' % (args.output, os.path.basename(o))) else: copytree(o, '%s%s' % (args.output, os.path.basename(o))) target.cleanup() elif args.command == 'doxygen': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) dirs = tree.call('make_doxygen') if hasattr(dirs, 'strip') or (not hasattr(dirs, '__getitem__') and not hasattr(dirs, '__iter__')): dirs = [dirs] for d in dirs: copytree(d, args.output) target.cleanup() elif args.command == 'latest': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): f = command_and_read('git log --tags --simplify-by-decoration --pretty="%d"') latest = None line = 0 while latest is None: t = f[line] line += 1 m = re.compile(".*\((.*)\).*").match(t) if m: tags = m.group(1).split(', ') for t in tags: s = t.split() if len(s) > 1: t = s[1] if len(t) > 0 and t[0] == 'v': v = Version(t[1:]) if (args.major is None or v.major == args.major) and (args.minor is None or v.minor == args.minor): latest = v print(latest) target.cleanup() elif args.command == 'test': if args.target is None: raise Error('you must specify -t or --target') target = None try: target = target_factory(args) options = get_command_line_options(args) if args.no_implicit_build: globals.trees.add_built(args.project, args.checkout, target) else: target.build(args.project, args.checkout, options) target.test(args.project, args.checkout, target, args.test, options) finally: if target is not None and not args.keep: target.cleanup() elif args.command == 'shell': if args.target is None: raise Error('you must specify -t or --target') target = target_factory(args) target.command('bash') elif args.command == 'revision': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): print(command_and_read('git rev-parse HEAD')[0].strip()[:7]) target.cleanup() elif args.command == 'checkout': if args.output is None: raise Error('you must specify -o or --output') target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): shutil.copytree('.', args.output) target.cleanup() elif args.command == 'dependencies': if args.target is None: raise Error('you must specify -t or --target') if args.checkout is None: raise Error('you must specify -c or --checkout') target = target_factory(args) tree = globals.trees.get(args.project, args.checkout, target) print("strict digraph {") for d in list(tree.dependencies({})): print("%s -> %s;" % (d[2].name.replace("-", "-"), d[0].name.replace("-", "_"))) print("}") try: main() except Error as e: print('cdist: %s' % str(e), file=sys.stderr) sys.exit(1)