#!/usr/bin/python # Copyright (C) 2012-2020 Carl Hetherington # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. from __future__ import print_function import os import sys import shutil import glob import tempfile import argparse import datetime import subprocess import re import copy import inspect import getpass import shlex import multiprocessing TEMPORARY_DIRECTORY = '/var/tmp' class Error(Exception): def __init__(self, value): self.value = value def __str__(self): return self.value def __repr__(self): return str(self) class Trees: """ Store for Tree objects which re-uses already-created objects and checks for requests for different versions of the same thing. """ def __init__(self): self.trees = [] def get(self, name, specifier, target, required_by=None): for t in self.trees: if t.name == name and t.specifier == specifier and t.target == target: return t elif t.name == name and t.specifier != specifier: a = specifier if specifier is not None else "[Any]" if required_by is not None: a += ' by %s' % required_by b = t.specifier if t.specifier is not None else "[Any]" if t.required_by is not None: b += ' by %s' % t.required_by raise Error('conflicting versions of %s required (%s versus %s)' % (name, a, b)) nt = Tree(name, specifier, target, required_by) self.trees.append(nt) return nt class Globals: quiet = False command = None dry_run = False trees = Trees() globals = Globals() # # Configuration # class Option(object): def __init__(self, key, default=None): self.key = key self.value = default def offer(self, key, value): if key == self.key: self.value = value class BoolOption(object): def __init__(self, key): self.key = key self.value = False def offer(self, key, value): if key == self.key: self.value = (value == 'yes' or value == '1' or value == 'true') class Config: def __init__(self): self.options = [ Option('mxe_prefix'), Option('git_prefix'), Option('osx_environment_prefix'), Option('osx_sdk_prefix'), Option('osx_sdk'), Option('apple_id'), Option('apple_password'), BoolOption('docker_sudo'), BoolOption('docker_no_user'), Option('docker_hub_repository'), Option('flatpak_state_dir'), Option('parallel', multiprocessing.cpu_count()) ] config_dir = '%s/.config' % os.path.expanduser('~') if not os.path.exists(config_dir): os.mkdir(config_dir) config_file = '%s/cdist' % config_dir if not os.path.exists(config_file): f = open(config_file, 'w') for o in self.options: print('# %s ' % o.key, file=f) f.close() print('Template config file written to %s; please edit and try again.' % config_file, file=sys.stderr) sys.exit(1) try: f = open('%s/.config/cdist' % os.path.expanduser('~'), 'r') while True: l = f.readline() if l == '': break if len(l) > 0 and l[0] == '#': continue s = l.strip().split() if len(s) == 2: for k in self.options: k.offer(s[0], s[1]) except: raise def has(self, k): for o in self.options: if o.key == k and o.value is not None: return True return False def get(self, k): for o in self.options: if o.key == k: if o.value is None: raise Error('Required setting %s not found' % k) return o.value def set(self, k, v): for o in self.options: o.offer(k, v) def docker(self): if self.get('docker_sudo'): return 'sudo docker' else: return 'docker' config = Config() # # Utility bits # def log_normal(m): if not globals.quiet: print('\x1b[33m* %s\x1b[0m' % m) def log_verbose(m): if globals.verbose: print('\x1b[35m* %s\x1b[0m' % m) def escape_spaces(s): return s.replace(' ', '\\ ') def scp_escape(n): """Escape a host:filename string for use with an scp command""" s = n.split(':') assert(len(s) == 1 or len(s) == 2) if len(s) == 2: return '%s:"\'%s\'"' % (s[0], s[1]) else: return '\"%s\"' % s[0] def mv_escape(n): return '\"%s\"' % n.substr(' ', '\\ ') def copytree(a, b): log_normal('copy %s -> %s' % (scp_escape(a), scp_escape(b))) if b.startswith('s3://'): command('s3cmd -P -r put "%s" "%s"' % (a, b)) else: command('scp -r %s %s' % (scp_escape(a), scp_escape(b))) def copyfile(a, b): log_normal('copy %s -> %s' % (scp_escape(a), scp_escape(b))) if b.startswith('s3://'): command('s3cmd -P put "%s" "%s"' % (a, b)) else: bc = b.find(":") if bc != -1: host = b[:bc] path = b[bc+1:] temp_path = os.path.join(os.path.dirname(path), ".tmp." + os.path.basename(path)) command('scp %s %s' % (scp_escape(a), scp_escape(host + ":" + temp_path))) command('ssh %s -- mv "%s" "%s"' % (host, escape_spaces(temp_path), escape_spaces(path))) else: command('scp %s %s' % (scp_escape(a), scp_escape(b))) def makedirs(d): """ Make directories either locally or on a remote host; remotely if d includes a colon, otherwise locally. """ if d.startswith('s3://'): # No need to create folders on S3 return if d.find(':') == -1: try: os.makedirs(d) except OSError as e: if e.errno != 17: raise e else: s = d.split(':') command('ssh %s -- mkdir -p %s' % (s[0], s[1])) def rmdir(a): log_normal('remove %s' % a) os.rmdir(a) def rmtree(a): log_normal('remove %s' % a) shutil.rmtree(a, ignore_errors=True) def command(c): log_normal(c) r = os.system(c) if (r >> 8): raise Error('command %s failed' % c) def command_and_read(c): log_normal(c) p = subprocess.Popen(c.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = p.communicate() if p.returncode != 0: raise Error('command %s failed (%s)' % (c, err)) return out.splitlines() def read_wscript_variable(directory, variable): f = open('%s/wscript' % directory, 'r') while True: l = f.readline() if l == '': break s = l.split() if len(s) == 3 and s[0] == variable: f.close() return s[2][1:-1] f.close() return None def set_version_in_wscript(version): f = open('wscript', 'rw') o = open('wscript.tmp', 'w') while True: l = f.readline() if l == '': break s = l.split() if len(s) == 3 and s[0] == "VERSION": print("VERSION = '%s'" % version, file=o) else: print(l, file=o, end="") f.close() o.close() os.rename('wscript.tmp', 'wscript') def append_version_to_changelog(version): try: f = open('ChangeLog', 'r') except: log_normal('Could not open ChangeLog') return c = f.read() f.close() f = open('ChangeLog', 'w') now = datetime.datetime.now() f.write('%d-%02d-%02d Carl Hetherington \n\n\t* Version %s released.\n\n' % (now.year, now.month, now.day, version)) f.write(c) def append_version_to_debian_changelog(version): if not os.path.exists('debian'): log_normal('Could not find debian directory') return command('dch -b -v %s-1 "New upstream release."' % version) def devel_to_git(git_commit, filename): if git_commit is not None: filename = filename.replace('devel', '-%s' % git_commit) return filename def get_command_line_options(args): """Get the options specified by --option on the command line""" options = dict() if args.option is not None: for o in args.option: b = o.split(':') if len(b) != 2: raise Error("Bad option `%s'" % o) if b[1] == 'False': options[b[0]] = False elif b[1] == 'True': options[b[0]] = True else: options[b[0]] = b[1] return options class TreeDirectory: def __init__(self, tree): self.tree = tree def __enter__(self): self.cwd = os.getcwd() os.chdir('%s/src/%s' % (self.tree.target.directory, self.tree.name)) def __exit__(self, type, value, traceback): os.chdir(self.cwd) # # Version # class Version: def __init__(self, s): self.devel = False if s.startswith("'"): s = s[1:] if s.endswith("'"): s = s[0:-1] if s.endswith('devel'): s = s[0:-5] self.devel = True if s.endswith('pre'): s = s[0:-3] p = s.split('.') self.major = int(p[0]) self.minor = int(p[1]) if len(p) == 3: self.micro = int(p[2]) else: self.micro = 0 @classmethod def from_git_tag(cls, tag): bits = tag.split('-') c = cls(bits[0]) if len(bits) > 1 and int(bits[1]) > 0: c.devel = True return c def bump_minor(self): self.minor += 1 self.micro = 0 def bump_micro(self): self.micro += 1 def to_devel(self): self.devel = True def to_release(self): self.devel = False def __str__(self): s = '%d.%d.%d' % (self.major, self.minor, self.micro) if self.devel: s += 'devel' return s # # Targets # class Target(object): """ Class representing the target that we are building for. This is exposed to cscripts, though not all of it is guaranteed 'API'. cscripts may expect: platform: platform string (e.g. 'windows', 'linux', 'osx') parallel: number of parallel jobs to run directory: directory to work in variables: dict of environment variables debug: True to build a debug version, otherwise False ccache: True to use ccache, False to not set(a, b): set the value of variable 'a' to 'b' unset(a): unset the value of variable 'a' command(c): run the command 'c' in the build environment """ def __init__(self, platform, directory=None): """ platform -- platform string (e.g. 'windows', 'linux', 'osx') directory -- directory to work in; if None we will use a temporary directory Temporary directories will be removed after use; specified directories will not. """ self.platform = platform self.parallel = int(config.get('parallel')) # Environment variables that we will use when we call cscripts self.variables = {} self.debug = False self._ccache = False # True to build our dependencies ourselves; False if this is taken care # of in some other way self.build_dependencies = True if directory is None: self.directory = tempfile.mkdtemp('', 'tmp', TEMPORARY_DIRECTORY) self.rmdir = True self.set('CCACHE_BASEDIR', os.path.realpath(self.directory)) self.set('CCACHE_NOHASHDIR', '') else: self.directory = directory self.rmdir = False def setup(self): pass def package(self, project, checkout, output_dir, options): tree = self.build(project, checkout, options) if len(inspect.getargspec(tree.cscript['package']).args) == 3: packages = tree.call('package', tree.version, options) else: log_normal("Deprecated cscript package() method with no options parameter") packages = tree.call('package', tree.version) if isinstance(packages, (str, unicode)): copyfile(packages, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, packages)))) else: for p in packages: copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) def build(self, project, checkout, options): tree = globals.trees.get(project, checkout, self) if self.build_dependencies: tree.build_dependencies(options) tree.build(options) return tree def test(self, tree, test, options): """test is the test case to run, or None""" if self.build_dependencies: tree.build_dependencies(options) tree.build(options) return tree.call('test', test) def set(self, a, b): self.variables[a] = b def unset(self, a): del(self.variables[a]) def get(self, a): return self.variables[a] def append(self, k, v, s): if (not k in self.variables) or len(self.variables[k]) == 0: self.variables[k] = '"%s"' % v else: e = self.variables[k] if e[0] == '"' and e[-1] == '"': self.variables[k] = '"%s%s%s"' % (e[1:-1], s, v) else: self.variables[k] = '"%s%s%s"' % (e, s, v) def append_with_space(self, k, v): return self.append(k, v, ' ') def append_with_colon(self, k, v): return self.append(k, v, ':') def variables_string(self, escaped_quotes=False): e = '' for k, v in self.variables.items(): if escaped_quotes: v = v.replace('"', '\\"') e += '%s=%s ' % (k, v) return e def cleanup(self): if self.rmdir: rmtree(self.directory) def mount(self, m): pass @property def ccache(self): return self._ccache @ccache.setter def ccache(self, v): self._ccache = v class DockerTarget(Target): def __init__(self, platform, directory): super(DockerTarget, self).__init__(platform, directory) self.mounts = [] self.privileged = False def _user_tag(self): if config.get('docker_no_user'): return '' return '-u %s' % getpass.getuser() def setup(self): opts = '-v %s:%s ' % (self.directory, self.directory) for m in self.mounts: opts += '-v %s:%s ' % (m, m) if self.privileged: opts += '--privileged=true ' if self.ccache: opts += "-e CCACHE_DIR=/ccache --volumes-from ccache-%s" % self.image tag = self.image if config.has('docker_hub_repository'): tag = '%s:%s' % (config.get('docker_hub_repository'), tag) self.container = command_and_read('%s run %s %s -itd %s /bin/bash' % (config.docker(), self._user_tag(), opts, tag))[0].strip() def command(self, cmd): dir = os.path.join(self.directory, os.path.relpath(os.getcwd(), self.directory)) interactive_flag = '-i ' if sys.stdin.isatty() else '' command('%s exec %s %s -t %s /bin/bash -c \'export %s; cd %s; %s\'' % (config.docker(), self._user_tag(), interactive_flag, self.container, self.variables_string(), dir, cmd)) def cleanup(self): super(DockerTarget, self).cleanup() command('%s kill %s' % (config.docker(), self.container)) def mount(self, m): self.mounts.append(m) class FlatpakTarget(Target): def __init__(self, project, checkout): super(FlatpakTarget, self).__init__('flatpak') self.build_dependencies = False self.project = project self.checkout = checkout def setup(self): pass def command(self, cmd): command(cmd) def checkout_dependencies(self): tree = globals.trees.get(self.project, self.checkout, self) return tree.checkout_dependencies() def flatpak(self): return 'flatpak' def flatpak_builder(self): b = 'flatpak-builder' if config.has('flatpak_state_dir'): b += ' --state-dir=%s' % config.get('flatpak_state_dir') return b class WindowsTarget(DockerTarget): """ This target exposes the following additional API: version: Windows version ('xp' or None) bits: bitness of Windows (32 or 64) name: name of our target e.g. x86_64-w64-mingw32.shared environment_prefix: path to Windows environment for the appropriate target (libraries and some tools) tool_path: path to 32- and 64-bit tools """ def __init__(self, windows_version, bits, directory, environment_version): super(WindowsTarget, self).__init__('windows', directory) self.version = windows_version self.bits = bits self.tool_path = '%s/usr/bin' % config.get('mxe_prefix') if self.bits == 32: self.name = 'i686-w64-mingw32.shared' else: self.name = 'x86_64-w64-mingw32.shared' self.environment_prefix = '%s/usr/%s' % (config.get('mxe_prefix'), self.name) self.set('PKG_CONFIG_LIBDIR', '%s/lib/pkgconfig' % self.environment_prefix) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.directory, self.directory)) self.set('PATH', '%s/bin:%s:%s' % (self.environment_prefix, self.tool_path, os.environ['PATH'])) self.set('CC', '%s-gcc' % self.name) self.set('CXX', '%s-g++' % self.name) self.set('LD', '%s-ld' % self.name) self.set('RANLIB', '%s-ranlib' % self.name) self.set('WINRC', '%s-windres' % self.name) cxx = '-I%s/include -I%s/include' % (self.environment_prefix, self.directory) link = '-L%s/lib -L%s/lib' % (self.environment_prefix, self.directory) self.set('CXXFLAGS', '"%s"' % cxx) self.set('CPPFLAGS', '') self.set('LINKFLAGS', '"%s"' % link) self.set('LDFLAGS', '"%s"' % link) self.image = 'windows' if environment_version is not None: self.image += '_%s' % environment_version @property def library_prefix(self): log_normal('Deprecated property library_prefix: use environment_prefix') return self.environment_prefix @property def windows_prefix(self): log_normal('Deprecated property windows_prefix: use environment_prefix') return self.environment_prefix @property def mingw_prefixes(self): log_normal('Deprecated property mingw_prefixes: use environment_prefix') return [self.environment_prefix] @property def mingw_path(self): log_normal('Deprecated property mingw_path: use tool_path') return self.tool_path @property def mingw_name(self): log_normal('Deprecated property mingw_name: use name') return self.name class LinuxTarget(DockerTarget): """ Build for Linux in a docker container. This target exposes the following additional API: distro: distribution ('debian', 'ubuntu', 'centos' or 'fedora') version: distribution version (e.g. '12.04', '8', '6.5') bits: bitness of the distribution (32 or 64) detail: None or 'appimage' if we are building for appimage """ def __init__(self, distro, version, bits, directory=None): super(LinuxTarget, self).__init__('linux', directory) self.distro = distro self.version = version self.bits = bits self.detail = None self.set('CXXFLAGS', '-I%s/include' % self.directory) self.set('CPPFLAGS', '') self.set('LINKFLAGS', '-L%s/lib' % self.directory) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig' % (self.directory, self.directory)) self.set('PATH', '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin') if self.version is None: self.image = '%s-%s' % (self.distro, self.bits) else: self.image = '%s-%s-%s' % (self.distro, self.version, self.bits) def setup(self): super(LinuxTarget, self).setup() if self.ccache: self.set('CC', '"ccache gcc"') self.set('CXX', '"ccache g++"') def test(self, tree, test, options): self.append_with_colon('PATH', '%s/bin' % self.directory) self.append_with_colon('LD_LIBRARY_PATH', '%s/lib' % self.directory) super(LinuxTarget, self).test(tree, test, options) class AppImageTarget(LinuxTarget): def __init__(self, work): super(AppImageTarget, self).__init__('ubuntu', '16.04', 64, work) self.detail = 'appimage' self.privileged = True class OSXTarget(Target): def __init__(self, directory=None): super(OSXTarget, self).__init__('osx', directory) self.sdk = config.get('osx_sdk') self.sdk_prefix = config.get('osx_sdk_prefix') self.environment_prefix = config.get('osx_environment_prefix') self.apple_id = config.get('apple_id') self.apple_password = config.get('apple_password') def command(self, c): command('%s %s' % (self.variables_string(False), c)) class OSXSingleTarget(OSXTarget): def __init__(self, bits, directory=None): super(OSXSingleTarget, self).__init__(directory) self.bits = bits if bits == 32: arch = 'i386' else: arch = 'x86_64' flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (self.sdk_prefix, self.sdk, arch) enviro = '%s/%d' % (config.get('osx_environment_prefix'), bits) # Environment variables self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, enviro, flags)) self.set('CPPFLAGS', '') self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, enviro, flags)) self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, enviro, flags)) self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, enviro, flags)) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.directory, enviro)) self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % enviro) self.set('MACOSX_DEPLOYMENT_TARGET', config.get('osx_sdk')) self.set('CCACHE_BASEDIR', self.directory) @Target.ccache.setter def ccache(self, v): Target.ccache.fset(self, v) if v: self.set('CC', '"ccache gcc"') self.set('CXX', '"ccache g++"') class OSXUniversalTarget(OSXTarget): def __init__(self, directory=None): super(OSXUniversalTarget, self).__init__(directory) self.bits = None def package(self, project, checkout, output_dir, options): for b in [32, 64]: target = OSXSingleTarget(b, os.path.join(self.directory, '%d' % b)) target.ccache = self.ccache tree = globals.trees.get(project, checkout, target) tree.build_dependencies(options) tree.build(options) tree = globals.trees.get(project, checkout, self) with TreeDirectory(tree): if len(inspect.getargspec(tree.cscript['package']).args) == 3: packages = tree.call('package', tree.version, options) else: log_normal("Deprecated cscript package() method with no options parameter") packages = tree.call('package', tree.version) for p in packages: copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) class SourceTarget(Target): """Build a source .tar.bz2""" def __init__(self): super(SourceTarget, self).__init__('source') def command(self, c): log_normal('host -> %s' % c) command('%s %s' % (self.variables_string(), c)) def cleanup(self): rmtree(self.directory) def package(self, project, checkout, output_dir, options): tree = globals.trees.get(project, checkout, self) with TreeDirectory(tree): name = read_wscript_variable(os.getcwd(), 'APPNAME') command('./waf dist') p = os.path.abspath('%s-%s.tar.bz2' % (name, tree.version)) copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) # @param s Target string: # windows-{32,64} # or ubuntu-version-{32,64} # or debian-version-{32,64} # or centos-version-{32,64} # or fedora-version-{32,64} # or mageia-version-{32,64} # or osx-{32,64} # or source # or flatpak # or appimage # @param debug True to build with debugging symbols (where possible) def target_factory(args): s = args.target target = None if s.startswith('windows-'): x = s.split('-') if len(x) == 2: target = WindowsTarget(None, int(x[1]), args.work, args.environment_version) elif len(x) == 3: target = WindowsTarget(x[1], int(x[2]), args.work, args.environment_version) else: raise Error("Bad Windows target name `%s'") elif s.startswith('ubuntu-') or s.startswith('debian-') or s.startswith('centos-') or s.startswith('fedora-') or s.startswith('mageia-'): p = s.split('-') if len(p) != 3: raise Error("Bad Linux target name `%s'; must be something like ubuntu-16.04-32 (i.e. distro-version-bits)" % s) target = LinuxTarget(p[0], p[1], int(p[2]), args.work) elif s.startswith('arch-'): p = s.split('-') if len(p) != 2: raise Error("Bad Arch target name `%s'; must be arch-32 or arch-64") target = LinuxTarget(p[0], None, int(p[1]), args.work) elif s == 'raspbian': target = LinuxTarget(s, None, None, args.work) elif s.startswith('osx-'): target = OSXSingleTarget(int(s.split('-')[1]), args.work) elif s == 'osx': if globals.command == 'build': target = OSXSingleTarget(64, args.work) else: target = OSXUniversalTarget(args.work) elif s == 'source': target = SourceTarget() elif s == 'flatpak': target = FlatpakTarget(args.project, args.checkout) elif s == 'appimage': target = AppImageTarget(args.work) if target is None: raise Error("Bad target `%s'" % s) target.debug = args.debug target.ccache = args.ccache if args.environment is not None: for e in args.environment: target.set(e, os.environ[e]) if args.mount is not None: for m in args.mount: target.mount(m) target.setup() return target # # Tree # class Tree(object): """Description of a tree, which is a checkout of a project, possibly built. This class is never exposed to cscripts. Attributes: name -- name of git repository (without the .git) specifier -- git tag or revision to use target -- target object that we are using version -- version from the wscript (if one is present) git_commit -- git revision that is actually being used built -- true if the tree has been built yet in this run required_by -- name of the tree that requires this one """ def __init__(self, name, specifier, target, required_by): self.name = name self.specifier = specifier self.target = target self.version = None self.git_commit = None self.built = False self.required_by = required_by cwd = os.getcwd() flags = '' redirect = '' if globals.quiet: flags = '-q' redirect = '>/dev/null' command('git clone %s %s/%s.git %s/src/%s' % (flags, config.get('git_prefix'), self.name, target.directory, self.name)) os.chdir('%s/src/%s' % (target.directory, self.name)) spec = self.specifier if spec is None: spec = 'master' command('git checkout %s %s %s' % (flags, spec, redirect)) self.git_commit = command_and_read('git rev-parse --short=7 HEAD')[0].strip() proj = '%s/src/%s' % (target.directory, self.name) self.cscript = {} exec(open('%s/cscript' % proj).read(), self.cscript) # cscript can include submodules = False to stop submodules being fetched if not 'submodules' in self.cscript or self.cscript['submodules'] == True: command('git submodule init --quiet') command('git submodule update --quiet') if os.path.exists('%s/wscript' % proj): v = read_wscript_variable(proj, "VERSION"); if v is not None: try: self.version = Version(v) except: tag = subprocess.Popen(shlex.split('git -C %s describe --tags' % proj), stdout=subprocess.PIPE).communicate()[0][1:] self.version = Version.from_git_tag(tag) os.chdir(cwd) def call(self, function, *args): with TreeDirectory(self): return self.cscript[function](self.target, *args) def add_defaults(self, options): """Add the defaults from self into a dict options""" if 'option_defaults' in self.cscript: from_cscript = self.cscript['option_defaults'] if isinstance(from_cscript, dict): defaults_dict = from_cscript else: log_normal("Deprecated cscript option_defaults method; replace with a dict") defaults_dict = from_cscript() for k, v in defaults_dict.items(): if not k in options: options[k] = v def dependencies(self, options): """ yield details of the dependencies of this tree. Each dependency is returned as a tuple of (tree, options). The 'options' parameter are the options that we want to force for 'self'. """ if not 'dependencies' in self.cscript: return if len(inspect.getargspec(self.cscript['dependencies']).args) == 2: self_options = copy.copy(options) self.add_defaults(self_options) deps = self.call('dependencies', self_options) else: log_normal("Deprecated cscript dependencies() method with no options parameter") deps = self.call('dependencies') # Loop over our immediate dependencies for d in deps: dep = globals.trees.get(d[0], d[1], self.target, self.name) # deps only get their options from the parent's cscript dep_options = d[2] if len(d) > 2 else {} for i in dep.dependencies(dep_options): yield i yield (dep, dep_options) def checkout_dependencies(self, options={}): for i in self.dependencies(options): pass def build_dependencies(self, options): """ Called on the 'main' project tree (-p on the command line) to build all dependencies. 'options' will be the ones from the command line. """ for i in self.dependencies(options): i[0].build(i[1]) def build(self, options): if self.built: return log_verbose("Building %s %s %s with %s" % (self.name, self.specifier, self.version, options)) variables = copy.copy(self.target.variables) options = copy.copy(options) self.add_defaults(options) if not globals.dry_run: if len(inspect.getargspec(self.cscript['build']).args) == 2: self.call('build', options) else: self.call('build') self.target.variables = variables self.built = True # # Command-line parser # def main(): commands = { "build": "build project", "package": "package and build project", "release": "release a project using its next version number (changing wscript and tagging)", "pot": "build the project's .pot files", "manual": "build the project's manual", "doxygen": "build the project's Doxygen documentation", "latest": "print out the latest version", "test": "run the project's unit tests", "shell": "build the project then start a shell", "checkout": "check out the project", "revision": "print the head git revision number" } one_of = "Command is one of:\n" summary = "" for k, v in commands.items(): one_of += "\t%s\t%s\n" % (k, v) summary += k + " " parser = argparse.ArgumentParser() parser.add_argument('command', help=summary) parser.add_argument('-p', '--project', help='project name') parser.add_argument('--minor', help='minor version number bump', action='store_true') parser.add_argument('--micro', help='micro version number bump', action='store_true') parser.add_argument('--latest-major', help='major version to return with latest', type=int) parser.add_argument('--latest-minor', help='minor version to return with latest', type=int) parser.add_argument('-c', '--checkout', help='string to pass to git for checkout') parser.add_argument('-o', '--output', help='output directory', default='.') parser.add_argument('-q', '--quiet', help='be quiet', action='store_true') parser.add_argument('-t', '--target', help='target', action='append') parser.add_argument('--environment-version', help='version of environment to use') parser.add_argument('-k', '--keep', help='keep working tree', action='store_true') parser.add_argument('--debug', help='build with debugging symbols where possible', action='store_true') parser.add_argument('-w', '--work', help='override default work directory') parser.add_argument('-g', '--git-prefix', help='override configured git prefix') parser.add_argument('--test', help="name of test to run (with `test'), defaults to all") parser.add_argument('-n', '--dry-run', help='run the process without building anything', action='store_true') parser.add_argument('-e', '--environment', help='pass the value of the named environment variable into the build', action='append') parser.add_argument('-m', '--mount', help='mount a given directory in the build environment', action='append') parser.add_argument('--no-version-commit', help="use just tags for versioning, don't modify wscript, ChangeLog etc.", action='store_true') parser.add_argument('--option', help='set an option for the build (use --option key:value)', action='append') parser.add_argument('--ccache', help='use ccache', action='store_true') parser.add_argument('--verbose', help='be verbose', action='store_true') global args args = parser.parse_args() # Check for incorrect multiple parameters if args.target is not None: if len(args.target) > 1: parser.error('multiple -t options specified') sys.exit(1) else: args.target = args.target[0] # Override configured stuff if args.git_prefix is not None: config.set('git_prefix', args.git_prefix) if args.output.find(':') == -1: # This isn't of the form host:path so make it absolute args.output = os.path.abspath(args.output) + '/' else: if args.output[-1] != ':' and args.output[-1] != '/': args.output += '/' # Now, args.output is 'host:', 'host:path/' or 'path/' if args.work is not None: args.work = os.path.abspath(args.work) if not os.path.exists(args.work): os.makedirs(args.work) if args.project is None and args.command != 'shell': raise Error('you must specify -p or --project') globals.quiet = args.quiet globals.verbose = args.verbose globals.command = args.command globals.dry_run = args.dry_run if not globals.command in commands: e = 'command must be one of:\n' + one_of raise Error('command must be one of:\n%s' % one_of) if globals.command == 'build': if args.target is None: raise Error('you must specify -t or --target') target = target_factory(args) target.build(args.project, args.checkout, get_command_line_options(args)) if not args.keep: target.cleanup() elif globals.command == 'package': if args.target is None: raise Error('you must specify -t or --target') target = None try: target = target_factory(args) if target.platform == 'linux' and target.detail != "appimage": if target.distro != 'arch': output_dir = os.path.join(args.output, '%s-%s-%d' % (target.distro, target.version, target.bits)) else: output_dir = os.path.join(args.output, '%s-%d' % (target.distro, target.bits)) else: output_dir = args.output makedirs(output_dir) target.package(args.project, args.checkout, output_dir, get_command_line_options(args)) except Error as e: if target is not None and not args.keep: target.cleanup() raise if target is not None and not args.keep: target.cleanup() elif globals.command == 'release': if args.minor is False and args.micro is False: raise Error('you must specify --minor or --micro') target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) version = tree.version version.to_release() if args.minor: version.bump_minor() else: version.bump_micro() with TreeDirectory(tree): if not args.no_version_commit: set_version_in_wscript(version) append_version_to_changelog(version) append_version_to_debian_changelog(version) command('git commit -a -m "Bump version"') command('git tag -m "v%s" v%s' % (version, version)) if not args.no_version_commit: version.to_devel() set_version_in_wscript(version) command('git commit -a -m "Bump version"') command('git push') command('git push --tags') target.cleanup() elif globals.command == 'pot': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) pots = tree.call('make_pot') for p in pots: copyfile(p, '%s%s' % (args.output, os.path.basename(p))) target.cleanup() elif globals.command == 'manual': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) outs = tree.call('make_manual') for o in outs: if os.path.isfile(o): copyfile(o, '%s%s' % (args.output, os.path.basename(o))) else: copytree(o, '%s%s' % (args.output, os.path.basename(o))) target.cleanup() elif globals.command == 'doxygen': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) dirs = tree.call('make_doxygen') if hasattr(dirs, 'strip') or (not hasattr(dirs, '__getitem__') and not hasattr(dirs, '__iter__')): dirs = [dirs] for d in dirs: copytree(d, args.output) target.cleanup() elif globals.command == 'latest': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): f = command_and_read('git log --tags --simplify-by-decoration --pretty="%d"') latest = None line = 0 while latest is None: t = f[line] line += 1 m = re.compile(".*\((.*)\).*").match(t) if m: tags = m.group(1).split(', ') for t in tags: s = t.split() if len(s) > 1: t = s[1] if len(t) > 0 and t[0] == 'v': v = Version(t[1:]) if (args.latest_major is None or v.major == args.latest_major) and (args.latest_minor is None or v.minor == args.latest_minor): latest = v print(latest) target.cleanup() elif globals.command == 'test': if args.target is None: raise Error('you must specify -t or --target') target = None try: target = target_factory(args) tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): target.test(tree, args.test, get_command_line_options(args)) except Error as e: if target is not None and not args.keep: target.cleanup() raise if target is not None and not args.keep: target.cleanup() elif globals.command == 'shell': if args.target is None: raise Error('you must specify -t or --target') target = target_factory(args) target.command('bash') elif globals.command == 'revision': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): print(command_and_read('git rev-parse HEAD')[0].strip()[:7]) target.cleanup() elif globals.command == 'checkout': if args.output is None: raise Error('you must specify -o or --output') target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) with TreeDirectory(tree): shutil.copytree('.', args.output) target.cleanup() else: raise Error('invalid command %s' % globals.command) try: main() except Error as e: print('cdist: %s' % str(e), file=sys.stderr) sys.exit(1)