#!/usr/bin/python3 # Copyright (C) 2012-2022 Carl Hetherington # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. from __future__ import print_function import argparse import copy import datetime import getpass import glob import inspect import multiprocessing import os from pathlib import Path import platform import re import signal import shlex import shutil import subprocess import sys import tempfile import time class Error(Exception): def __init__(self, value): self.value = value def __str__(self): return self.value def __repr__(self): return str(self) class Trees: """ Store for Tree objects which re-uses already-created objects and checks for requests for different versions of the same thing. """ def __init__(self): self.trees = [] def get(self, name, commit_ish, target, required_by=None): for t in self.trees: if t.name == name and t.commit_ish == commit_ish and t.target == target: return t elif t.name == name and t.commit_ish != commit_ish: a = commit_ish if commit_ish is not None else "[Any]" if required_by is not None: a += ' by %s' % required_by b = t.commit_ish if t.commit_ish is not None else "[Any]" if t.required_by is not None: b += ' by %s' % t.required_by raise Error('conflicting versions of %s required (%s versus %s)' % (name, a, b)) nt = Tree(name, commit_ish, target, required_by) self.trees.append(nt) return nt def add_built(self, name, commit_ish, target): self.trees.append(Tree(name, commit_ish, target, None, built=True)) class Globals: quiet = False command = None dry_run = False use_git_reference = True trees = Trees() globals = Globals() # # Configuration # class Option: def __init__(self, key, default=None): self.key = key self.value = default def offer(self, key, value): if key == self.key: self.value = value class BoolOption: def __init__(self, key): self.key = key self.value = False def offer(self, key, value): if key == self.key: self.value = value in ['yes', '1', 'true'] class Config: def __init__(self): self.options = [ Option('mxe_prefix'), Option('git_prefix'), Option('git_reference'), Option('osx_environment_prefix'), Option('osx_sdk_prefix'), Option('osx_sdk'), Option('osx_intel_deployment'), Option('osx_arm_deployment'), Option('osx_old_deployment'), Option('osx_keychain_file'), Option('osx_keychain_password'), Option('apple_id'), Option('apple_password'), Option('apple_team_id'), BoolOption('docker_sudo'), BoolOption('docker_no_user'), Option('docker_hub_repository'), Option('flatpak_state_dir'), Option('parallel', multiprocessing.cpu_count()), Option('temp', '/var/tmp'), Option('osx_notarytool', ['xcrun', 'notarytool'])] config_dir = '%s/.config' % os.path.expanduser('~') if not os.path.exists(config_dir): os.mkdir(config_dir) config_file = '%s/cdist' % config_dir if not os.path.exists(config_file): f = open(config_file, 'w') for o in self.options: print('# %s ' % o.key, file=f) f.close() print('Template config file written to %s; please edit and try again.' % config_file, file=sys.stderr) sys.exit(1) f = open('%s/.config/cdist' % os.path.expanduser('~'), 'r') while True: l = f.readline() if l == '': break if len(l) > 0 and l[0] == '#': continue s = l.strip().split() if len(s) == 2: for k in self.options: k.offer(s[0], s[1]) if not isinstance(self.get('osx_notarytool'), list): self.set('osx_notarytool', [self.get('osx_notarytool')]) def has(self, k): for o in self.options: if o.key == k and o.value is not None: return True return False def get(self, k): for o in self.options: if o.key == k: if o.value is None: raise Error('Required setting %s not found' % k) return o.value def set(self, k, v): for o in self.options: o.offer(k, v) def docker(self): if self.get('docker_sudo'): return 'sudo docker' else: return 'docker' config = Config() # # Utility bits # def log_normal(m): if not globals.quiet: print('\x1b[33m* %s\x1b[0m' % m) def log_verbose(m): if globals.verbose: print('\x1b[35m* %s\x1b[0m' % m) def escape_spaces(s): return s.replace(' ', '\\ ') def scp_escape(n): """Escape a host:filename string for use with an scp command""" s = n.split(':') assert(len(s) == 1 or len(s) == 2) if len(s) == 2: return '%s:"\'%s\'"' % (s[0], s[1]) else: return '\"%s\"' % s[0] def mv_escape(n): return '\"%s\"' % n.substr(' ', '\\ ') def copytree(a, b): log_normal('copy %s -> %s' % (scp_escape(a), scp_escape(b))) if b.startswith('s3://'): command('s3cmd -P -r put "%s" "%s"' % (a, b)) else: command('scp -r %s %s' % (scp_escape(a), scp_escape(b))) def copyfile(a, b): log_normal('copy %s -> %s with cwd %s' % (scp_escape(a), scp_escape(b), os.getcwd())) if b.startswith('s3://'): command('s3cmd -P put "%s" "%s"' % (a, b)) else: bc = b.find(":") if bc != -1: host = b[:bc] path = b[bc+1:] temp_path = os.path.join(os.path.dirname(path), ".tmp." + os.path.basename(path)) command('scp %s %s' % (scp_escape(a), scp_escape(host + ":" + temp_path))) command('ssh %s -- mv "%s" "%s"' % (host, escape_spaces(temp_path), escape_spaces(path))) else: command('scp %s %s' % (scp_escape(a), scp_escape(b))) def makedirs(d): """ Make directories either locally or on a remote host; remotely if d includes a colon, otherwise locally. """ if d.startswith('s3://'): # No need to create folders on S3 return if d.find(':') == -1: try: os.makedirs(d) except OSError as e: if e.errno != 17: raise e else: s = d.split(':') command('ssh %s -- mkdir -p %s' % (s[0], s[1])) def rmdir(a): log_normal('remove %s' % a) os.rmdir(a) def rmtree(a): log_normal('remove %s' % a) shutil.rmtree(a, ignore_errors=True) def command(c): log_normal(c) try: r = subprocess.run(c, shell=True) if r.returncode != 0: raise Error('command %s failed (%d)' % (c, r.returncode)) except Exception as e: raise Error('command %s failed (%s)' % (c, e)) def command_and_read(c): log_normal(c) p = subprocess.Popen(c.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = p.communicate() if p.returncode != 0: raise Error('command %s failed (%s)' % (c, err)) return str(out, 'utf-8').splitlines() def read_wscript_variable(directory, variable): f = open('%s/wscript' % directory, 'r') while True: l = f.readline() if l == '': break s = l.split() if len(s) == 3 and s[0] == variable: f.close() return s[2][1:-1] f.close() return None def devel_to_git(git_commit, filename): if git_commit is not None: filename = filename.replace('devel', '-%s' % git_commit) return filename def get_command_line_options(args): """Get the options specified by --option on the command line""" options = dict() if args.option is not None: for o in args.option: b = o.split(':') if len(b) != 2: raise Error("Bad option `%s'" % o) if b[1] == 'False': options[b[0]] = False elif b[1] == 'True': options[b[0]] = True else: options[b[0]] = b[1] return options class TreeDirectory: def __init__(self, tree): self.tree = tree def __enter__(self): self.cwd = os.getcwd() os.chdir('%s/src/%s' % (self.tree.target.directory, self.tree.name)) def __exit__(self, type, value, traceback): os.chdir(self.cwd) # # Version # class Version: def __init__(self, s): self.devel = False if s.startswith("'"): s = s[1:] if s.endswith("'"): s = s[0:-1] if s.endswith('devel'): s = s[0:-5] self.devel = True if s.endswith('pre'): s = s[0:-3] p = s.split('.') self.major = int(p[0]) self.minor = int(p[1]) if len(p) == 3: self.micro = int(p[2]) else: self.micro = 0 @classmethod def from_git_tag(cls, tag): bits = tag.split('-') c = cls(bits[0]) if len(bits) > 1 and int(bits[1]) > 0: c.devel = True return c def bump_minor(self): self.minor += 1 self.micro = 0 def bump_micro(self): self.micro += 1 def to_devel(self): self.devel = True def to_release(self): self.devel = False def __str__(self): s = '%d.%d.%d' % (self.major, self.minor, self.micro) if self.devel: s += 'devel' return s # # Targets # class Target: """ Class representing the target that we are building for. This is exposed to cscripts, though not all of it is guaranteed 'API'. cscripts may expect: platform: platform string (e.g. 'windows', 'linux', 'osx') parallel: number of parallel jobs to run directory: directory to work in variables: dict of environment variables debug: True to build a debug version, otherwise False ccache: True to use ccache, False to not set(a, b): set the value of variable 'a' to 'b' unset(a): unset the value of variable 'a' command(c): run the command 'c' in the build environment """ def __init__(self, platform, directory=None, dependencies_only=False): """ platform -- platform string (e.g. 'windows', 'linux', 'osx') directory -- directory to work in; if None we will use a temporary directory Temporary directories will be removed after use; specified directories will not. """ self.platform = platform self.parallel = int(config.get('parallel')) # Environment variables that we will use when we call cscripts self.variables = {} self.debug = False self._ccache = False # True to build our dependencies ourselves; False if this is taken care # of in some other way self.build_dependencies = True self.dependencies_only = dependencies_only if directory is None: try: os.makedirs(config.get('temp')) except OSError as e: if e.errno != 17: raise e self.directory = tempfile.mkdtemp('', 'tmp', config.get('temp')) self.rmdir = True self.set('CCACHE_BASEDIR', os.path.realpath(self.directory)) self.set('CCACHE_NOHASHDIR', '') else: self.directory = os.path.realpath(directory) self.rmdir = False def setup(self): pass def _cscript_package(self, tree, options): """ Call package() in the cscript and return what it returns, except that anything not in a list will be put into one. options: from command line """ if len(inspect.getfullargspec(tree.cscript['package']).args) == 3: packages = tree.call('package', tree.version, tree.add_defaults(options)) else: log_normal("Deprecated cscript package() method with no options parameter") packages = tree.call('package', tree.version) return packages if isinstance(packages, list) else [packages] def _copy_packages(self, tree, packages, output_dir): for p in packages: copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.commit, p)))) def package(self, project, checkout, output_dir, options, notarize): """ options: from command line """ tree = self.build(project, checkout, options, for_package=True) p = self._cscript_package(tree, options) self._copy_packages(tree, p, output_dir) def build(self, project, checkout, options, for_package=False): tree = globals.trees.get(project, checkout, self) if self.build_dependencies: tree.build_dependencies(options) if not self.dependencies_only: tree.build(options, for_package=for_package) return tree def test(self, project, checkout, target, test, options): """test is the test case to run, or None""" tree = globals.trees.get(project, checkout, target) with TreeDirectory(tree): if len(inspect.getfullargspec(tree.cscript['test']).args) == 3: return tree.call('test', tree.add_defaults(options), test) else: log_normal('Deprecated cscript test() method with no options parameter') return tree.call('test', test) def set(self, a, b): self.variables[a] = b def unset(self, a): del(self.variables[a]) def get(self, a): return self.variables[a] def append(self, k, v, s): if (not k in self.variables) or len(self.variables[k]) == 0: self.variables[k] = '"%s"' % v else: e = self.variables[k] if e[0] == '"' and e[-1] == '"': self.variables[k] = '"%s%s%s"' % (e[1:-1], s, v) else: self.variables[k] = '"%s%s%s"' % (e, s, v) def append_with_space(self, k, v): return self.append(k, v, ' ') def append_with_colon(self, k, v): return self.append(k, v, ':') def variables_string(self, escaped_quotes=False): e = '' for k, v in self.variables.items(): if escaped_quotes: v = v.replace('"', '\\"') e += '%s=%s ' % (k, v) return e def cleanup(self): if self.rmdir: rmtree(self.directory) def mount(self, m): pass @property def ccache(self): return self._ccache @ccache.setter def ccache(self, v): self._ccache = v class DockerTarget(Target): def __init__(self, platform, directory): super(DockerTarget, self).__init__(platform, directory) self.mounts = [] self.privileged = False def _user_tag(self): if config.get('docker_no_user'): return '' return '-u %s' % getpass.getuser() def _mount_option(self, d): return '-v %s:%s ' % (os.path.realpath(d), os.path.realpath(d)) def setup(self): opts = self._mount_option(self.directory) for m in self.mounts: opts += self._mount_option(m) if config.has('git_reference'): opts += self._mount_option(config.get('git_reference')) if self.privileged: opts += '--privileged=true ' if self.ccache: opts += "-e CCACHE_DIR=/ccache/%s-%d --mount source=ccache,target=/ccache " % (self.image, os.getuid()) opts += "--rm " tag = self.image if config.has('docker_hub_repository'): tag = '%s:%s' % (config.get('docker_hub_repository'), tag) def signal_handler(signum, frame): raise Error('Killed') signal.signal(signal.SIGTERM, signal_handler) self.container = command_and_read('%s run %s %s -itd %s /bin/bash' % (config.docker(), self._user_tag(), opts, tag))[0].strip() def command(self, cmd): dir = os.path.join(self.directory, os.path.relpath(os.getcwd(), self.directory)) interactive_flag = '-i ' if sys.stdin.isatty() else '' command('%s exec %s %s -t %s /bin/bash -c \'export %s; cd %s; %s\'' % (config.docker(), self._user_tag(), interactive_flag, self.container, self.variables_string(), dir, cmd)) def cleanup(self): super(DockerTarget, self).cleanup() command('%s kill %s' % (config.docker(), self.container)) def mount(self, m): self.mounts.append(m) class WindowsDockerTarget(DockerTarget): """ This target exposes the following additional API: bits: bitness of Windows (32 or 64) name: name of our target e.g. x86_64-w64-mingw32.shared environment_prefix: path to Windows environment for the appropriate target (libraries and some tools) tool_path: path to 32- and 64-bit tools """ def __init__(self, bits, directory, environment_version): super(WindowsDockerTarget, self).__init__('windows', directory) self.bits = bits # This was used to differentiate "normal" Windows from XP, and is no longer important, # but old cscripts still look for it self.version = None self.tool_path = '%s/usr/bin' % config.get('mxe_prefix') if self.bits == 32: self.name = 'i686-w64-mingw32.shared' else: self.name = 'x86_64-w64-mingw32.shared' self.environment_prefix = '%s/usr/%s' % (config.get('mxe_prefix'), self.name) self.set('PKG_CONFIG_LIBDIR', '%s/lib/pkgconfig' % self.environment_prefix) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.directory, self.directory)) self.set('PATH', '%s/bin:%s:%s' % (self.environment_prefix, self.tool_path, os.environ['PATH'])) self.set('LD', '%s-ld' % self.name) self.set('RANLIB', '%s-ranlib' % self.name) self.set('WINRC', '%s-windres' % self.name) cxx = '-I%s/include -I%s/include' % (self.environment_prefix, self.directory) link = '-L%s/lib -L%s/lib' % (self.environment_prefix, self.directory) self.set('CXXFLAGS', '"%s"' % cxx) self.set('CPPFLAGS', '') self.set('LINKFLAGS', '"%s"' % link) self.set('LDFLAGS', '"%s"' % link) self.image = 'windows' if environment_version is not None: self.image += '_%s' % environment_version def setup(self): super().setup() if self.ccache: self.set('CC', '"ccache %s-gcc"' % self.name) self.set('CXX', '"ccache %s-g++"' % self.name) else: self.set('CC', '%s-gcc' % self.name) self.set('CXX', '%s-g++' % self.name) @property def library_prefix(self): log_normal('Deprecated property library_prefix: use environment_prefix') return self.environment_prefix @property def windows_prefix(self): log_normal('Deprecated property windows_prefix: use environment_prefix') return self.environment_prefix @property def mingw_prefixes(self): log_normal('Deprecated property mingw_prefixes: use environment_prefix') return [self.environment_prefix] @property def mingw_path(self): log_normal('Deprecated property mingw_path: use tool_path') return self.tool_path @property def mingw_name(self): log_normal('Deprecated property mingw_name: use name') return self.name class WindowsNativeTarget(Target): """ This target exposes the following additional API: bits: bitness of Windows (32 or 64) name: name of our target e.g. x86_64-w64-mingw32.shared environment_prefix: path to Windows environment for the appropriate target (libraries and some tools) """ def __init__(self, directory): super().__init__('windows', directory) self.bits = 64 self.environment_prefix = config.get('windows_native_environmnet_prefix') self.set('PATH', '%s/bin:%s' % (self.environment_prefix, os.environ['PATH'])) def command(self, cmd): command(cmd) class LinuxTarget(DockerTarget): """ Build for Linux in a docker container. This target exposes the following additional API: distro: distribution ('debian', 'ubuntu', 'centos' or 'fedora') version: distribution version (e.g. '12.04', '8', '6.5') bits: bitness of the distribution (32 or 64) detail: None or 'appimage' if we are building for appimage """ def __init__(self, distro, version, bits, directory=None): super(LinuxTarget, self).__init__('linux', directory) self.distro = distro self.version = version self.bits = bits self.detail = None self.set('CXXFLAGS', '-I%s/include' % self.directory) self.set('CPPFLAGS', '') self.set('LINKFLAGS', '-L%s/lib' % self.directory) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig' % (self.directory, self.directory)) self.set('PATH', '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin') if self.version is None: self.image = '%s-%s' % (self.distro, self.bits) else: self.image = '%s-%s-%s' % (self.distro, self.version, self.bits) def setup(self): super(LinuxTarget, self).setup() if self.ccache: self.set('CC', '"ccache gcc"') self.set('CXX', '"ccache g++"') def test(self, project, checkout, target, test, options): self.append_with_colon('PATH', '%s/bin' % self.directory) self.append_with_colon('LD_LIBRARY_PATH', '%s/lib' % self.directory) super(LinuxTarget, self).test(project, checkout, target, test, options) class AppImageTarget(LinuxTarget): def __init__(self, work): super(AppImageTarget, self).__init__('ubuntu', '18.04', 64, work) self.detail = 'appimage' self.privileged = True class FlatpakTarget(Target): def __init__(self, project, checkout, work): super(FlatpakTarget, self).__init__('flatpak') self.build_dependencies = False self.project = project self.checkout = checkout # If we use git references we end up with a checkout in one mount trying # to link to the git reference repo in other, which doesn't work. globals.use_git_reference = False if config.has('flatpak_state_dir'): self.mount(config.get('flatpak_state_dir')) def command(self, c): log_normal('host -> %s' % c) command('%s %s' % (self.variables_string(), c)) def setup(self): super().setup() globals.trees.get(self.project, self.checkout, self).checkout_dependencies() def flatpak(self): return 'flatpak' def flatpak_builder(self): b = 'flatpak-builder' if config.has('flatpak_state_dir'): b += ' --state-dir=%s' % config.get('flatpak_state_dir') return b def notarize_dmg(dmg): p = subprocess.run( config.get('osx_notarytool') + [ 'submit', '--apple-id', config.get('apple_id'), '--password', config.get('apple_password'), '--team-id', config.get('apple_team_id'), '--wait', dmg ], capture_output=True) last_line = [x.strip() for x in p.stdout.decode('utf-8').splitlines() if x.strip()][-1] if last_line != 'status: Accepted': print("Could not understand notarytool response") print(p) print(f"Last line: {last_line}") raise Error('Notarization failed') subprocess.run(['xcrun', 'stapler', 'staple', dmg]) class OSXTarget(Target): def __init__(self, directory=None): super(OSXTarget, self).__init__('osx', directory) self.sdk_prefix = config.get('osx_sdk_prefix') self.environment_prefix = config.get('osx_environment_prefix') self.apple_id = config.get('apple_id') self.apple_password = config.get('apple_password') self.osx_keychain_file = config.get('osx_keychain_file') self.osx_keychain_password = config.get('osx_keychain_password') def command(self, c): command('%s %s' % (self.variables_string(False), c)) def unlock_keychain(self): self.command('security unlock-keychain -p %s %s' % (self.osx_keychain_password, self.osx_keychain_file)) def package(self, project, checkout, output_dir, options, notarize): self.unlock_keychain() tree = globals.trees.get(project, checkout, self) with TreeDirectory(tree): p = self._cscript_package_and_notarize(tree, options, self.can_notarize and notarize) self._copy_packages(tree, p, output_dir) def _copy_packages(self, tree, packages, output_dir): for p in packages: dest = os.path.join(output_dir, os.path.basename(devel_to_git(tree.commit, p))) copyfile(p, dest) def _cscript_package_and_notarize(self, tree, options, notarize): """ Call package() in the cscript and notarize the .dmgs that are returned, if notarize == True """ output = [] for x in self._cscript_package(tree, options): # Some older cscripts give us the DMG filename and the bundle ID, even though # (since using notarytool instead of altool for notarization) the bundle ID # is no longer necessary. Cope with either type of cscript. dmg = x[0] if isinstance(x, tuple) else x if notarize: notarize_dmg(dmg) output.append(dmg) return output class OSXSingleTarget(OSXTarget): def __init__(self, arch, sdk, deployment, directory=None, can_notarize=True): super(OSXSingleTarget, self).__init__(directory) self.arch = arch self.sdk = sdk self.deployment = deployment self.can_notarize = can_notarize self.sub_targets = [self] flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (self.sdk_prefix, sdk, arch) if arch == 'x86_64': host_enviro = '%s/x86_64/%s' % (config.get('osx_environment_prefix'), deployment) else: host_enviro = '%s/x86_64/10.10' % config.get('osx_environment_prefix') target_enviro = '%s/%s/%s' % (config.get('osx_environment_prefix'), arch, deployment) self.bin = '%s/bin' % target_enviro # Environment variables self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, target_enviro, flags)) self.set('CPPFLAGS', '') self.set('CXXFLAGS', '"-I%s/include -I%s/include -stdlib=libc++ %s"' % (self.directory, target_enviro, flags)) self.set('LDFLAGS', '"-L%s/lib -L%s/lib -stdlib=libc++ %s"' % (self.directory, target_enviro, flags)) self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, target_enviro, flags)) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.directory, target_enviro)) self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % host_enviro) self.set('MACOSX_DEPLOYMENT_TARGET', self.deployment) self.set('CCACHE_BASEDIR', self.directory) @Target.ccache.setter def ccache(self, v): Target.ccache.fset(self, v) if v: self.set('CC', '"ccache gcc"') self.set('CXX', '"ccache g++"') def package(self, project, checkout, output_dir, options, notarize): tree = self.build(project, checkout, options, for_package=True) super().package(project, checkout, output_dir, options, notarize) class OSXUniversalTarget(OSXTarget): def __init__(self, directory=None): super(OSXUniversalTarget, self).__init__(directory) self.sdk = config.get('osx_sdk') self.sub_targets = [] for arch, deployment in (('x86_64', config.get('osx_intel_deployment')), ('arm64', config.get('osx_arm_deployment'))): target = OSXSingleTarget(arch, self.sdk, deployment, os.path.join(self.directory, arch, deployment)) target.ccache = self.ccache self.sub_targets.append(target) self.can_notarize = True def package(self, project, checkout, output_dir, options, notarize): for target in self.sub_targets: tree = globals.trees.get(project, checkout, target) tree.build_dependencies(options) tree.build(options, for_package=True) super().package(project, checkout, output_dir, options, notarize) @Target.ccache.setter def ccache(self, v): for target in self.sub_targets: target.ccache = v class SourceTarget(Target): """Build a source .tar.bz2 and .zst""" def __init__(self): super(SourceTarget, self).__init__('source') def command(self, c): log_normal('host -> %s' % c) command('%s %s' % (self.variables_string(), c)) def cleanup(self): rmtree(self.directory) def package(self, project, checkout, output_dir, options, notarize): tree = globals.trees.get(project, checkout, self) with TreeDirectory(tree): name = read_wscript_variable(os.getcwd(), 'APPNAME') command('./waf dist') bz2 = os.path.abspath('%s-%s.tar.bz2' % (name, tree.version)) copyfile(bz2, os.path.join(output_dir, os.path.basename(devel_to_git(tree.commit, bz2)))) command('tar xjf %s' % bz2) command('tar --zstd -cf %s-%s.tar.zst %s-%s' % (name, tree.version, name, tree.version)) zstd = os.path.abspath('%s-%s.tar.zst' % (name, tree.version)) copyfile(zstd, os.path.join(output_dir, os.path.basename(devel_to_git(tree.commit, zstd)))) class LocalTarget(Target): """Build on the local machine with its environment""" def __init__(self, work, dependencies_only=False): super(LocalTarget, self).__init__('linux', work, dependencies_only=dependencies_only) # Hack around ffmpeg.git which sees that the target isn't windows/osx and then assumes # distro will be there. self.distro = None self.detail = None self.bits = 64 self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.directory, self.directory)) self.append_with_colon('LD_LIBRARY_PATH', '%s/lib' % self.directory) self.set('CXXFLAGS', '-I%s/include' % self.directory) self.set('LINKFLAGS', '-L%s/lib' % self.directory) def command(self, c): log_normal('host -> %s' % c) command('%s %s' % (self.variables_string(), c)) def cleanup(self): rmtree(self.directory) # @param s Target string: # windows-{32,64} # or ubuntu-version-{32,64} # or debian-version-{32,64} # or centos-version-{32,64} # or fedora-version-{32,64} # or mageia-version-{32,64} # or osx # or source # or flatpak # or appimage def target_factory(args): s = args.target target = None if s.startswith('windows-'): x = s.split('-') if platform.system() == "Windows": target = WindowsNativeTarget(args.work) elif len(x) == 2: target = WindowsDockerTarget(int(x[1]), args.work, args.environment_version) else: raise Error("Bad Windows target name `%s'") elif s.startswith('ubuntu-') or s.startswith('debian-') or s.startswith('centos-') or s.startswith('fedora-') or s.startswith('mageia-'): p = s.split('-') if len(p) != 3: raise Error("Bad Linux target name `%s'; must be something like ubuntu-16.04-32 (i.e. distro-version-bits)" % s) target = LinuxTarget(p[0], p[1], int(p[2]), args.work) elif s.startswith('arch-'): p = s.split('-') if len(p) != 2: raise Error("Bad Arch target name `%s'; must be arch-32 or arch-64") target = LinuxTarget(p[0], None, int(p[1]), args.work) elif s == 'raspbian': target = LinuxTarget(s, None, None, args.work) elif s == 'osx': target = OSXUniversalTarget(args.work) elif s == 'osx-intel': target = OSXSingleTarget('x86_64', config.get('osx_sdk'), config.get('osx_intel_deployment'), args.work) elif s == 'osx-old': target = OSXSingleTarget('x86_64', config.get('osx_sdk'), config.get('osx_old_deployment'), args.work, False) elif s == 'source': target = SourceTarget() elif s == 'flatpak': target = FlatpakTarget(args.project, args.checkout, args.work) elif s == 'appimage': target = AppImageTarget(args.work) elif s == 'local': target = LocalTarget(args.work, args.dependencies_only) if target is None: raise Error("Bad target `%s'" % s) target.debug = args.debug target.ccache = args.ccache if args.environment is not None: for e in args.environment: target.set(e, os.environ[e]) if args.mount is not None: for m in args.mount: target.mount(m) target.setup() return target # # Tree # class Tree: """Description of a tree, which is a checkout of a project, possibly built. This class is never exposed to cscripts. Attributes: name -- name of git repository (without the .git) commit_ish -- git tag or revision to use target -- target object that we are using version -- version from the wscript (if one is present) commit -- git revision that is actually being used built -- true if the tree has been built yet in this run required_by -- name of the tree that requires this one """ def __init__(self, name, commit_ish, target, required_by, built=False): self.name = name self.commit_ish = commit_ish self.target = target self.version = None self.commit = None self.built = built self.required_by = required_by cwd = os.getcwd() proj = '%s/src/%s' % (target.directory, self.name) if not built: flags = '' redirect = '' if globals.quiet: flags = '-q' redirect = '>/dev/null' if config.has('git_reference') and globals.use_git_reference: ref = '--reference-if-able %s/%s.git' % (config.get('git_reference'), self.name) else: ref = '' command('git -c protocol.file.allow=always clone %s %s %s/%s.git %s/src/%s' % (flags, ref, config.get('git_prefix'), self.name, target.directory, self.name)) os.chdir('%s/src/%s' % (target.directory, self.name)) if self.commit_ish is not None: command('git checkout %s %s %s' % (flags, self.commit_ish, redirect)) self.commit = command_and_read('git rev-parse --short=7 HEAD')[0].strip() self.cscript = {} exec(open('%s/cscript' % proj).read(), self.cscript) if not built: # cscript can include submodules = False to stop submodules being fetched if (not 'submodules' in self.cscript or self.cscript['submodules'] == True) and os.path.exists('.gitmodules'): command('git submodule --quiet init') paths = command_and_read('git config --file .gitmodules --get-regexp path') urls = command_and_read('git config --file .gitmodules --get-regexp url') for path, url in zip(paths, urls): ref = '' if config.has('git_reference') and globals.use_git_reference: url = url.split(' ')[1] ref_path = os.path.join(config.get('git_reference'), os.path.basename(url)) if os.path.exists(ref_path): ref = '--reference %s' % ref_path path = path.split(' ')[1] command('git -c protocol.file.allow=always submodule --quiet update %s %s' % (ref, path)) if os.path.exists('%s/wscript' % proj): v = read_wscript_variable(proj, "VERSION"); if v is not None: try: self.version = Version(v) except: try: tag = command_and_read('git -C %s describe --match v* --tags' % proj)[0][1:] self.version = Version.from_git_tag(tag) except: # We'll leave version as None if we can't read it; maybe this is a bad idea # Should probably just install git on the Windows VM pass os.chdir(cwd) def call(self, function, *args): with TreeDirectory(self): return self.cscript[function](self.target, *args) def add_defaults(self, options): """Add the defaults from self into a dict options and returns a new dict""" new_options = copy.copy(options) if 'option_defaults' in self.cscript: from_cscript = self.cscript['option_defaults'] if isinstance(from_cscript, dict): defaults_dict = from_cscript else: log_normal("Deprecated cscript option_defaults method; replace with a dict") defaults_dict = from_cscript() for k, v in defaults_dict.items(): if not k in new_options: new_options[k] = v return new_options def dependencies(self, options): """ yield details of the dependencies of this tree. Each dependency is returned as a tuple of (tree, options). options: either from command line (for top-level tree) or from parent's dependencies() (for other trees) """ if not 'dependencies' in self.cscript: return if len(inspect.getfullargspec(self.cscript['dependencies']).args) == 2: deps = self.call('dependencies', self.add_defaults(options)) else: log_normal("Deprecated cscript dependencies() method with no options parameter") deps = self.call('dependencies') # Loop over our immediate dependencies for d in deps: dep = globals.trees.get(d[0], d[1], self.target, self.name) # deps only get their options from the parent's cscript dep_options = d[2] if len(d) > 2 else {} for i in dep.dependencies(dep_options): yield i yield (dep, dep_options) def checkout_dependencies(self, options={}): for i in self.dependencies(options): pass def build_dependencies(self, options): """ Called on the 'main' project tree (-p on the command line) to build all dependencies. options: either from command line (for top-level tree) or from parent's dependencies() (for other trees) """ for dependency, dependency_options in self.dependencies(options): dependency.build(dependency_options) def build(self, options, for_package=False): """ options: either from command line (for top-level tree) or from parent's dependencies() (for other trees) """ if self.built: return log_verbose("Building %s %s %s with %s" % (self.name, self.commit_ish, self.version, self.add_defaults(options))) variables = copy.copy(self.target.variables) if not globals.dry_run: num_args = len(inspect.getfullargspec(self.cscript['build']).args) if num_args == 3: self.call('build', self.add_defaults(options), for_package) elif num_args == 2: self.call('build', self.add_defaults(options)) else: self.call('build') self.target.variables = variables self.built = True # # Command-line parser # def main(): parser = argparse.ArgumentParser() parser.add_argument('-p', '--project', help='project name') parser.add_argument('-c', '--checkout', help='string to pass to git for checkout') parser.add_argument('-o', '--output', help='output directory', default='.') parser.add_argument('-q', '--quiet', help='be quiet', action='store_true') parser.add_argument('-t', '--target', help='target', action='append') parser.add_argument('--environment-version', help='version of environment to use') parser.add_argument('-k', '--keep', help='keep working tree', action='store_true') parser.add_argument('--debug', help='build with debugging symbols where possible', action='store_true') parser.add_argument('-w', '--work', help='override default work directory') parser.add_argument('-g', '--git-prefix', help='override configured git prefix') parser.add_argument('-n', '--dry-run', help='run the process without building anything', action='store_true') parser.add_argument('-e', '--environment', help='pass the value of the named environment variable into the build', action='append') parser.add_argument('-m', '--mount', help='mount a given directory in the build environment', action='append') parser.add_argument('--option', help='set an option for the build (use --option key:value)', action='append') parser.add_argument('--ccache', help='use ccache', action='store_true') parser.add_argument('--verbose', help='be verbose', action='store_true') subparsers = parser.add_subparsers(help='command to run', dest='command') parser_build = subparsers.add_parser("build", help="build project") parser_build.add_argument('--dependencies-only', help='only build dependencies', action='store_true') parser_package = subparsers.add_parser("package", help="build and package project") parser_package.add_argument('--no-notarize', help='do not notarize .dmg packages', action='store_true') parser_release = subparsers.add_parser("release", help="release a project using its next version number (adding a tag)") parser_release.add_argument('--minor', help='minor version number bump', action='store_true') parser_release.add_argument('--micro', help='micro version number bump', action='store_true') parser_pot = subparsers.add_parser("pot", help="build the project's .pot files") parser_manual = subparsers.add_parser("manual", help="build the project's manual") parser_doxygen = subparsers.add_parser("doxygen", help="build the project's Doxygen documentation") parser_latest = subparsers.add_parser("latest", help="print out the latest version") parser_latest.add_argument('--major', help='major version to return', type=int) parser_latest.add_argument('--minor', help='minor version to return', type=int) parser_test = subparsers.add_parser("test", help="build the project and run its unit tests") parser_test.add_argument('--no-implicit-build', help='do not build first', action='store_true') parser_test.add_argument('--test', help="name of test to run, defaults to all") parser_shell = subparsers.add_parser("shell", help="start a shell in the project's work directory") parser_checkout = subparsers.add_parser("checkout", help="check out the project") parser_revision = subparsers.add_parser("revision", help="print the head git revision number") parser_dependencies = subparsers.add_parser("dependencies", help="print details of the project's dependencies as a .dot file") parser_notarize = subparsers.add_parser("notarize", help="notarize .dmgs in a directory") parser_notarize.add_argument('--dmgs', help='directory containing *.dmg') global args args = parser.parse_args() # Check for incorrect multiple parameters if args.target is not None: if len(args.target) > 1: parser.error('multiple -t options specified') sys.exit(1) else: args.target = args.target[0] # Override configured stuff if args.git_prefix is not None: config.set('git_prefix', args.git_prefix) if args.output.find(':') == -1: # This isn't of the form host:path so make it absolute args.output = os.path.abspath(args.output) + '/' else: if args.output[-1] != ':' and args.output[-1] != '/': args.output += '/' # Now, args.output is 'host:', 'host:path/' or 'path/' if args.work is not None: args.work = os.path.abspath(args.work) if not os.path.exists(args.work): os.makedirs(args.work) if args.project is None and not args.command in ['shell', 'notarize']: raise Error('you must specify -p or --project') globals.quiet = args.quiet globals.verbose = args.verbose globals.dry_run = args.dry_run if args.command == 'build': if args.target is None: raise Error('you must specify -t or --target') target = target_factory(args) try: target.build(args.project, args.checkout, get_command_line_options(args)) finally: if not args.keep: target.cleanup() elif args.command == 'package': if args.target is None: raise Error('you must specify -t or --target') target = None try: target = target_factory(args) if target.platform == 'linux' and target.detail != "appimage": if target.distro != 'arch': output_dir = os.path.join(args.output, '%s-%s-%d' % (target.distro, target.version, target.bits)) else: output_dir = os.path.join(args.output, '%s-%d' % (target.distro, target.bits)) else: output_dir = args.output makedirs(output_dir) target.package(args.project, args.checkout, output_dir, get_command_line_options(args), not args.no_notarize) finally: if target is not None and not args.keep: target.cleanup() elif args.command == 'release': if args.minor is False and args.micro is False: raise Error('you must specify --minor or --micro') target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) version = tree.version version.to_release() if args.minor: version.bump_minor() else: version.bump_micro() with TreeDirectory(tree): command('git tag -m "v%s" v%s' % (version, version)) command('git push --tags') target.cleanup() elif args.command == 'pot': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) pots = tree.call('make_pot') for p in pots: copyfile(p, '%s%s' % (args.output, os.path.basename(p))) target.cleanup() elif args.command == 'manual': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) tree.checkout_dependencies() outs = tree.call('make_manual') for o in outs: if os.path.isfile(o): copyfile(o, '%s%s' % (args.output, os.path.basename(o))) else: copytree(o, '%s%s' % (args.output, os.path.basename(o))) target.cleanup() elif args.command == 'doxygen': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) dirs = tree.call('make_doxygen') if hasattr(dirs, 'strip') or (not hasattr(dirs, '__getitem__') and not hasattr(dirs, '__iter__')): dirs = [dirs] for d in dirs: copytree(d, args.output) target.cleanup() elif args.command == 'latest': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): f = command_and_read('git log --tags --simplify-by-decoration --pretty="%d"') latest = None line = 0 while latest is None: t = f[line] line += 1 m = re.compile(".*\((.*)\).*").match(t) if m: tags = m.group(1).split(', ') for t in tags: s = t.split() if len(s) > 1: t = s[1] if len(t) > 0 and t[0] == 'v': v = Version(t[1:]) if (args.major is None or v.major == args.major) and (args.minor is None or v.minor == args.minor): latest = v print(latest) target.cleanup() elif args.command == 'test': if args.target is None: raise Error('you must specify -t or --target') target = None try: target = target_factory(args) options = get_command_line_options(args) if args.no_implicit_build: globals.trees.add_built(args.project, args.checkout, target) else: target.build(args.project, args.checkout, options) target.test(args.project, args.checkout, target, args.test, options) finally: if target is not None and not args.keep: target.cleanup() elif args.command == 'shell': if args.target is None: raise Error('you must specify -t or --target') target = target_factory(args) target.command('bash') elif args.command == 'revision': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): print(command_and_read('git rev-parse HEAD')[0].strip()[:7]) target.cleanup() elif args.command == 'checkout': if args.output is None: raise Error('you must specify -o or --output') target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): shutil.copytree('.', args.output) target.cleanup() elif args.command == 'notarize': if args.dmgs is None: raise Error('you must specify ---dmgs') for dmg in Path(args.dmgs).glob('*.dmg'): notarize_dmg(dmg) try: main() except Error as e: print('cdist: %s' % str(e), file=sys.stderr) sys.exit(1)