#!/usr/bin/python # Copyright (C) 2012-2014 Carl Hetherington # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. import os import sys import shutil import glob import tempfile import argparse import datetime import subprocess import re import copy import inspect class Error(Exception): def __init__(self, value): self.value = value def __str__(self): return '\x1b[31m%s\x1b[0m' % repr(self.value) def __repr__(self): return str(self) # # Configuration # class Config: def __init__(self): self.keys = ['linux_dir_in_chroot', 'linux_chroot_prefix', 'windows_environment_prefix', 'mingw_prefix', 'git_prefix', 'osx_build_host', 'osx_dir_in_host', 'osx_environment_prefix', 'osx_sdk_prefix', 'osx_sdk'] self.dict = dict() try: f = open('%s/.config/cdist' % os.path.expanduser('~'), 'r') while 1: l = f.readline() if l == '': break if len(l) > 0 and l[0] == '#': continue s = l.strip().split() if len(s) == 2: for k in self.keys: if k == s[0]: self.dict[k] = s[1] except: raise def get(self, k): if k in self.dict: return self.dict[k] raise Error('Required setting %s not found' % k) config = Config() # # Utility bits # def log(m): if not args.quiet: print '\x1b[33m* %s\x1b[0m' % m def copytree(a, b): log('copy %s -> %s' % (a, b)) shutil.copytree(a, b) def copyfile(a, b): log('copy %s -> %s' % (a, b)) shutil.copyfile(a, b) def rmdir(a): log('remove %s' % a) os.rmdir(a) def rmtree(a): log('remove %s' % a) shutil.rmtree(a, ignore_errors=True) def command(c, can_fail=False): log(c) r = os.system(c) if (r >> 8) and not can_fail: raise Error('command %s failed' % c) def command_and_read(c): log(c) p = subprocess.Popen(c.split(), stdout=subprocess.PIPE) f = os.fdopen(os.dup(p.stdout.fileno())) return f def read_wscript_variable(directory, variable): f = open('%s/wscript' % directory, 'r') while 1: l = f.readline() if l == '': break s = l.split() if len(s) == 3 and s[0] == variable: f.close() return s[2][1:-1] f.close() return None # # Version # class Version: def __init__(self, s): self.devel = False if s.startswith("'"): s = s[1:] if s.endswith("'"): s = s[0:-1] if s.endswith('devel'): s = s[0:-5] self.devel = True if s.endswith('pre'): s = s[0:-3] p = s.split('.') self.major = int(p[0]) self.minor = int(p[1]) if len(p) == 3: self.micro = int(p[2]) else: self.micro = 0 def bump_minor(self): self.minor += 1 self.micro = 0 def bump_micro(self): self.micro += 1 def to_devel(self): self.devel = True def to_release(self): self.devel = False def __str__(self): s = '%d.%02d.%d' % (self.major, self.minor, self.micro) if self.devel: s += 'devel' return s # # Targets # class Target(object): def __init__(self, platform, parallel): self.platform = platform self.parallel = parallel # Environment variables that we will use when we call cscripts self.variables = {} self.debug = False def build_dependencies(self, project): cwd = os.getcwd() if 'dependencies' in project.cscript: print project.cscript['dependencies'](self) for d in project.cscript['dependencies'](self): log('Building dependency %s %s of %s' % (d[0], d[1], project.name)) dep = Project(d[0], '.', d[1]) dep.checkout(self) self.build_dependencies(dep) # Make the options to pass in from the option_defaults of the thing # we are building and any options specified by the parent. options = {} if 'option_defaults' in dep.cscript: options = dep.cscript['option_defaults']() if len(d) > 2: for k, v in d[2].iteritems(): options[k] = v self.build(dep, options) os.chdir(cwd) def build(self, project, options=None): variables = copy.copy(self.variables) print 'Target %s builds %s with %s' % (self.platform, project.name, self.variables) if len(inspect.getargspec(project.cscript['build']).args) == 2: project.cscript['build'](self, options) else: project.cscript['build'](self) self.variables = variables def package(self, project): project.checkout(self) self.build_dependencies(project) self.build(project) return project.cscript['package'](self, project.version) def test(self, project): project.checkout(self) self.build_dependencies(project) self.build(project) project.cscript['test'](self) def set(self, a, b): print "Target set %s=%s" % (a, b) self.variables[a] = b def unset(self, a): del(self.variables[a]) def get(self, a): return self.variables[a] def append_with_space(self, k, v): if not k in self.variables: self.variables[k] = v else: self.variables[k] = '%s %s' % (self.variables[k], v) def variables_string(self, escaped_quotes=False): e = '' for k, v in self.variables.iteritems(): if escaped_quotes: v = v.replace('"', '\\"') e += '%s=%s ' % (k, v) return e def cleanup(self): pass # # Windows # class WindowsTarget(Target): # @param directory directory to work in; if None, we will use a temporary directory def __init__(self, bits, directory=None): super(WindowsTarget, self).__init__('windows', 2) self.bits = bits if directory is None: self.directory = tempfile.mkdtemp() self.rmdir = True else: self.directory = directory self.rmdir = False self.windows_prefix = '%s/%d' % (config.get('windows_environment_prefix'), self.bits) if not os.path.exists(self.windows_prefix): raise Error('windows prefix %s does not exist' % self.windows_prefix) if self.bits == 32: self.mingw_name = 'i686' else: self.mingw_name = 'x86_64' mingw_path = '%s/%d/bin' % (config.get('mingw_prefix'), self.bits) self.mingw_prefixes = ['/%s/%d' % (config.get('mingw_prefix'), self.bits), '%s/%d/%s-w64-mingw32' % (config.get('mingw_prefix'), bits, self.mingw_name)] self.set('PKG_CONFIG_LIBDIR', '%s/lib/pkgconfig' % self.windows_prefix) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.work_dir_cscript(), self.work_dir_cscript())) self.set('PATH', '%s/bin:%s:%s' % (self.windows_prefix, mingw_path, os.environ['PATH'])) self.set('CC', '%s-w64-mingw32-gcc' % self.mingw_name) self.set('CXX', '%s-w64-mingw32-g++' % self.mingw_name) self.set('LD', '%s-w64-mingw32-ld' % self.mingw_name) self.set('RANLIB', '%s-w64-mingw32-ranlib' % self.mingw_name) self.set('WINRC', '%s-w64-mingw32-windres' % self.mingw_name) cxx = '-I%s/include -I%s/include' % (self.windows_prefix, self.work_dir_cscript()) link = '-L%s/lib -L%s/lib' % (self.windows_prefix, self.work_dir_cscript()) for p in self.mingw_prefixes: cxx += ' -I%s/include' % p link += ' -L%s/lib' % p self.set('CXXFLAGS', '"%s"' % cxx) self.set('LINKFLAGS', '"%s"' % link) def work_dir_cdist(self): return '%s/%d' % (self.directory, self.bits) def work_dir_cscript(self): return '%s/%d' % (self.directory, self.bits) def command(self, c): log('host -> %s' % c) command('%s %s' % (self.variables_string(), c)) def cleanup(self): if self.rmdir: rmtree(self.directory) # # Linux # class LinuxTarget(Target): def __init__(self, distro, version, bits, directory=None): "directory -- directory to work in; if None, we will use the configured linux_dir_in_chroot" super(LinuxTarget, self).__init__('linux', 2) self.distro = distro self.version = version self.bits = bits self.chroot = '%s-%s-%d' % (self.distro, self.version, self.bits) if directory is None: self.dir_in_chroot = config.get('linux_dir_in_chroot') else: self.dir_in_chroot = directory for g in glob.glob('%s/*' % self.work_dir_cdist()): rmtree(g) self.set('CXXFLAGS', '-I%s/include' % self.work_dir_cscript()) self.set('LINKFLAGS', '-L%s/lib' % self.work_dir_cscript()) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:/usr/local/lib/pkgconfig' % self.work_dir_cscript()) self.set('PATH', '%s:/usr/local/bin' % (os.environ['PATH'])) def work_dir_cdist(self): return '%s/%s%s' % (config.get('linux_chroot_prefix'), self.chroot, self.dir_in_chroot) def work_dir_cscript(self): return self.dir_in_chroot def command(self, c): # Work out the cwd for the chrooted command cwd = os.getcwd() prefix = '%s/%s' % (config.get('linux_chroot_prefix'), self.chroot) assert(cwd.startswith(prefix)) cwd = cwd[len(prefix):] log('schroot [%s] -> %s' % (cwd, c)) command('%s schroot -c %s -d %s -p -- %s' % (self.variables_string(), self.chroot, cwd, c)) def cleanup(self): for g in glob.glob('%s/*' % self.work_dir_cdist()): rmtree(g) # # OS X # class OSXTarget(Target): def __init__(self, directory=None): "directory -- directory to work in; if None, we will use the configured osx_dir_in_host" super(OSXTarget, self).__init__('osx', 4) if directory is None: self.dir_in_host = config.get('osx_dir_in_host') else: self.dir_in_host = directory for g in glob.glob('%s/*' % self.dir_in_host): rmtree(g) def command(self, c): command('%s %s' % (self.variables_string(False), c)) class OSXSingleTarget(OSXTarget): def __init__(self, bits, directory=None): super(OSXSingleTarget, self).__init__(directory) self.bits = bits if bits == 32: arch = 'i386' else: arch = 'x86_64' flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (config.get('osx_sdk_prefix'), config.get('osx_sdk'), arch) enviro = '%s/%d' % (config.get('osx_environment_prefix'), bits) # Environment variables self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.work_dir_cscript(), enviro, flags)) self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.work_dir_cscript(), enviro, flags)) self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.work_dir_cscript(), enviro, flags)) self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.work_dir_cscript(), enviro, flags)) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.work_dir_cscript(), enviro)) self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % enviro) self.set('MACOSX_DEPLOYMENT_TARGET', config.get('osx_sdk')) def work_dir_cdist(self): return self.work_dir_cscript() def work_dir_cscript(self): return '%s/%d' % (self.dir_in_host, self.bits) def package(self, project): raise Error('cannot package non-universal OS X versions') class OSXUniversalTarget(OSXTarget): def __init__(self, directory=None): super(OSXUniversalTarget, self).__init__(directory) self.parts = [] self.parts.append(OSXSingleTarget(32, directory)) self.parts.append(OSXSingleTarget(64, directory)) def work_dir_cscript(self): return self.dir_in_host def package(self, project): for p in self.parts: project.checkout(p) p.build_dependencies(project) p.build(project) return project.cscript['package'](self, project.version) # # Source # class SourceTarget(Target): def __init__(self): super(SourceTarget, self).__init__('source', 2) self.directory = tempfile.mkdtemp() def work_dir_cdist(self): return self.directory def work_dir_cscript(self): return self.directory def command(self, c): log('host -> %s' % c) command('%s %s' % (self.variables_string(), c)) def cleanup(self): rmtree(self.directory) def package(self, project): project.checkout(self) name = read_wscript_variable(os.getcwd(), 'APPNAME') command('./waf dist') return os.path.abspath('%s-%s.tar.bz2' % (name, project.version)) # @param s Target string: # windows-{32,64} # or ubuntu-version-{32,64} # or debian-version-{32,64} # or centos-version-{32,64} # or osx-{32,64} # or source # @param debug True to build with debugging symbols (where possible) def target_factory(s, debug, work): target = None if s.startswith('windows-'): target = WindowsTarget(int(s.split('-')[1]), work) elif s.startswith('ubuntu-') or s.startswith('debian-') or s.startswith('centos-'): p = s.split('-') if len(p) != 3: print >>sys.stderr,"Bad Linux target name `%s'; must be something like ubuntu-12.04-32 (i.e. distro-version-bits)" % s sys.exit(1) target = LinuxTarget(p[0], p[1], int(p[2]), work) elif s.startswith('osx-'): target = OSXSingleTarget(int(s.split('-')[1]), work) elif s == 'osx': if args.command == 'build': target = OSXSingleTarget(64, work) else: target = OSXUniversalTarget(work) elif s == 'source': target = SourceTarget() if target is not None: target.debug = debug return target # # Project # class Project(object): def __init__(self, name, directory, specifier=None): self.name = name self.directory = directory self.version = None self.specifier = specifier if self.specifier is None: self.specifier = 'master' def checkout(self, target): flags = '' redirect = '' if args.quiet: flags = '-q' redirect = '>/dev/null' command('git clone %s %s/%s.git %s/src/%s' % (flags, config.get('git_prefix'), self.name, target.work_dir_cdist(), self.name)) os.chdir('%s/src/%s' % (target.work_dir_cdist(), self.name)) command('git checkout %s %s %s' % (flags, self.specifier, redirect)) command('git submodule init --quiet') command('git submodule update --quiet') os.chdir(self.directory) proj = '%s/src/%s/%s' % (target.work_dir_cdist(), self.name, self.directory) self.read_cscript('%s/cscript' % proj) if os.path.exists('%s/wscript' % proj): v = read_wscript_variable(proj, "VERSION"); if v is not None: self.version = Version(v) def read_cscript(self, s): self.cscript = {} execfile(s, self.cscript) def set_version_in_wscript(version): f = open('wscript', 'rw') o = open('wscript.tmp', 'w') while 1: l = f.readline() if l == '': break s = l.split() if len(s) == 3 and s[0] == "VERSION": print "Writing %s" % version print >>o,"VERSION = '%s'" % version else: print >>o,l, f.close() o.close() os.rename('wscript.tmp', 'wscript') def append_version_to_changelog(version): try: f = open('ChangeLog', 'r') except: log('Could not open ChangeLog') return c = f.read() f.close() f = open('ChangeLog', 'w') now = datetime.datetime.now() f.write('%d-%02d-%02d Carl Hetherington \n\n\t* Version %s released.\n\n' % (now.year, now.month, now.day, version)) f.write(c) def append_version_to_debian_changelog(version): if not os.path.exists('debian'): log('Could not find debian directory') return command('dch -b -v %s-1 "New upstream release."' % version) # # Command-line parser # parser = argparse.ArgumentParser() parser.add_argument('command') parser.add_argument('-p', '--project', help='project name') parser.add_argument('-d', '--directory', help='directory within project repo', default='.') parser.add_argument('--minor', help='minor version number bump', action='store_true') parser.add_argument('--micro', help='micro version number bump', action='store_true') parser.add_argument('-c', '--checkout', help='string to pass to git for checkout') parser.add_argument('-o', '--output', help='output directory', default='.') parser.add_argument('-q', '--quiet', help='be quiet', action='store_true') parser.add_argument('-t', '--target', help='target') parser.add_argument('-k', '--keep', help='keep working tree', action='store_true') parser.add_argument('--debug', help='build with debugging symbols where possible', action='store_true') parser.add_argument('-w', '--work', help='override default work directory') args = parser.parse_args() args.output = os.path.abspath(args.output) if args.work is not None: args.work = os.path.abspath(args.work) if args.project is None and args.command != 'shell': raise Error('you must specify -p or --project') project = Project(args.project, args.directory, args.checkout) if args.command == 'build': if args.target is None: raise Error('you must specify -t or --target') target = target_factory(args.target, args.debug, args.work) project.checkout(target) target.build_dependencies(project) target.build(project) if not args.keep: target.cleanup() elif args.command == 'package': if args.target is None: raise Error('you must specify -t or --target') target = target_factory(args.target, args.debug, args.work) packages = target.package(project) if hasattr(packages, 'strip') or (not hasattr(packages, '__getitem__') and not hasattr(packages, '__iter__')): packages = [packages] if target.platform == 'linux': out = '%s/%s-%s-%d' % (args.output, target.distro, target.version, target.bits) try: os.makedirs(out) except: pass for p in packages: copyfile(p, '%s/%s' % (out, os.path.basename(p))) else: for p in packages: copyfile(p, '%s/%s' % (args.output, os.path.basename(p))) if not args.keep: target.cleanup() elif args.command == 'release': if args.minor is False and args.micro is False: raise Error('you must specify --minor or --micro') target = SourceTarget() project.checkout(target) version = project.version version.to_release() if args.minor: version.bump_minor() else: version.bump_micro() set_version_in_wscript(version) append_version_to_changelog(version) append_version_to_debian_changelog(version) command('git commit -a -m "Bump version"') command('git tag -m "v%s" v%s' % (version, version)) version.to_devel() set_version_in_wscript(version) command('git commit -a -m "Bump version"') command('git push') command('git push --tags') target.cleanup() elif args.command == 'pot': target = SourceTarget() project.checkout(target) pots = project.cscript['make_pot'](target) for p in pots: copyfile(p, '%s/%s' % (args.output, os.path.basename(p))) target.cleanup() elif args.command == 'changelog': target = SourceTarget() project.checkout(target) text = open('ChangeLog', 'r') html = open('%s/changelog.html' % args.output, 'w') versions = 8 last = None changes = [] while 1: l = text.readline() if l == '': break if len(l) > 0 and l[0] == "\t": s = l.split() if len(s) == 4 and s[1] == "Version" and s[3] == "released.": v = Version(s[2]) if v.micro == 0: if last is not None and len(changes) > 0: print >>html,"

Changes between version %s and %s

" % (s[2], last) print >>html,"" last = s[2] changes = [] versions -= 1 if versions < 0: break else: c = l.strip() if len(c) > 0: if c[0] == '*': changes.append(c[2:]) else: changes[-1] += " " + c target.cleanup() elif args.command == 'manual': target = SourceTarget() project.checkout(target) outs = project.cscript['make_manual'](target) for o in outs: if os.path.isfile(o): copyfile(o, '%s/%s' % (args.output, os.path.basename(o))) else: copytree(o, '%s/%s' % (args.output, os.path.basename(o))) target.cleanup() elif args.command == 'doxygen': target = SourceTarget() project.checkout(target) dirs = project.cscript['make_doxygen'](target) if hasattr(dirs, 'strip') or (not hasattr(dirs, '__getitem__') and not hasattr(dirs, '__iter__')): dirs = [dirs] for d in dirs: copytree(d, '%s/%s' % (args.output, 'doc')) target.cleanup() elif args.command == 'latest': target = SourceTarget() project.checkout(target) f = command_and_read('git log --tags --simplify-by-decoration --pretty="%d"') t = f.readline() m = re.compile(".*\((.*)\).*").match(t) latest = None if m: tags = m.group(1).split(', ') for t in tags: s = t.split() if len(s) > 1: t = s[1] if len(t) > 0 and t[0] == 'v': latest = t[1:] print latest target.cleanup() elif args.command == 'test': if args.target is None: raise Error('you must specify -t or --target') target = None try: target = target_factory(args.target, args.debug, args.work) target.test(project) except Error as e: if target is not None: target.cleanup() raise if target is not None: target.cleanup() elif args.command == 'shell': if args.target is None: raise Error('you must specify -t or --target') target = target_factory(args.target, args.debug, args.work) target.command('bash') else: raise Error('invalid command %s' % args.command)