Display Apple server response in case of failure.
[cdist.git] / cdist
diff --git a/cdist b/cdist
index 516dcadaf34fe742fbc1291c4c76ce43b27884bb..bc11f4a0ae1c98d9150b168615ec7da5011a2a6b 100755 (executable)
--- a/cdist
+++ b/cdist
@@ -35,8 +35,6 @@ import sys
 import tempfile
 import time
 
-TEMPORARY_DIRECTORY = '/var/tmp'
-
 class Error(Exception):
     def __init__(self, value):
         self.value = value
@@ -113,7 +111,8 @@ class Config:
                          Option('git_reference'),
                          Option('osx_environment_prefix'),
                          Option('osx_sdk_prefix'),
-                         Option('osx_sdk'),
+                         Option('osx_intel_sdk'),
+                         Option('osx_arm_sdk'),
                          Option('osx_keychain_file'),
                          Option('osx_keychain_password'),
                          Option('apple_id'),
@@ -122,7 +121,8 @@ class Config:
                          BoolOption('docker_no_user'),
                          Option('docker_hub_repository'),
                          Option('flatpak_state_dir'),
-                         Option('parallel', multiprocessing.cpu_count()) ]
+                         Option('parallel', multiprocessing.cpu_count()),
+                         Option('temp', '/var/tmp')]
 
         config_dir = '%s/.config' % os.path.expanduser('~')
         if not os.path.exists(config_dir):
@@ -257,11 +257,11 @@ def rmtree(a):
 def command(c):
     log_normal(c)
     try:
-        r = subprocess.run(shlex.split(c), shell=True)
+        r = subprocess.run(c, shell=True)
         if r.returncode != 0:
-            raise Error(f'command {c} failed ({r.returncode})')
+            raise Error('command %s failed (%d)' % (c, r.returncode))
     except Exception as e:
-        raise Error(f'command {c} failed ({e})')
+        raise Error('command %s failed (%s)' % (c, e))
 
 def command_and_read(c):
     log_normal(c)
@@ -414,7 +414,7 @@ class Target(object):
         self.build_dependencies = True
 
         if directory is None:
-            self.directory = tempfile.mkdtemp('', 'tmp', TEMPORARY_DIRECTORY)
+            self.directory = tempfile.mkdtemp('', 'tmp', config.get('temp'))
             self.rmdir = True
             self.set('CCACHE_BASEDIR', os.path.realpath(self.directory))
             self.set('CCACHE_NOHASHDIR', '')
@@ -426,7 +426,11 @@ class Target(object):
     def setup(self):
         pass
 
-    def _build_packages(self, tree, options):
+    def _cscript_package(self, tree, options):
+        """
+        Call package() in the cscript and return what it returns, except that
+        anything not in a list will be put into one.
+        """
         if len(inspect.getfullargspec(tree.cscript['package']).args) == 3:
             packages = tree.call('package', tree.version, options)
         else:
@@ -439,10 +443,10 @@ class Target(object):
         for p in packages:
             copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p))))
 
-    def package(self, project, checkout, output_dir, options, no_notarize):
+    def package(self, project, checkout, output_dir, options, notarize):
         tree = self.build(project, checkout, options)
         tree.add_defaults(options)
-        p = self._build_packages(tree, options)
+        p = self._cscript_package(tree, options)
         self._copy_packages(tree, p, output_dir)
 
     def build(self, project, checkout, options):
@@ -729,7 +733,7 @@ class AppImageTarget(LinuxTarget):
         self.privileged = True
 
 
-def notarize(dmg, bundle_id):
+def notarize_dmg(dmg, bundle_id):
     p = subprocess.run(
         ['xcrun', 'altool', '--notarize-app', '-t', 'osx', '-f', dmg, '--primary-bundle-id', bundle_id, '-u', config.get('apple_id'), '-p', config.get('apple_password'), '--output-format', 'xml'],
         capture_output=True
@@ -743,6 +747,7 @@ def notarize(dmg, bundle_id):
 
     request_uuid = string_after(p, "RequestUUID")
     if request_uuid is None:
+        print("Response: %s" % p)
         raise Error('No RequestUUID found in response from Apple')
 
     for i in range(0, 30):
@@ -755,6 +760,9 @@ def notarize(dmg, bundle_id):
         elif status == 'success':
             subprocess.run(['xcrun', 'stapler', 'staple', dmg])
             return
+        elif status != "in progress":
+            print("Could not understand xcrun response")
+            print(p)
         time.sleep(30)
 
     raise Error("Notarization timed out")
@@ -763,7 +771,6 @@ def notarize(dmg, bundle_id):
 class OSXTarget(Target):
     def __init__(self, directory=None):
         super(OSXTarget, self).__init__('osx', directory)
-        self.sdk = config.get('osx_sdk')
         self.sdk_prefix = config.get('osx_sdk_prefix')
         self.environment_prefix = config.get('osx_environment_prefix')
         self.apple_id = config.get('apple_id')
@@ -774,33 +781,43 @@ class OSXTarget(Target):
     def command(self, c):
         command('%s %s' % (self.variables_string(False), c))
 
-    def build(self, *a, **k):
+    def unlock_keychain(self):
         self.command('security unlock-keychain -p %s %s' % (self.osx_keychain_password, self.osx_keychain_file))
-        return super().build(*a, **k)
+
+    def _cscript_package_and_notarize(self, tree, options, notarize):
+        """
+        Call package() in the cscript and notarize the .dmgs that are returned, if notarize == True
+        """
+        p = self._cscript_package(tree, options)
+        for x in p:
+            if not isinstance(x, tuple):
+                raise Error('macOS packages must be returned from cscript as tuples of (dmg-filename, bundle-id)')
+            if notarize:
+                notarize_dmg(x[0], x[1])
+        return [x[0] for x in p]
 
 
 class OSXSingleTarget(OSXTarget):
-    def __init__(self, bits, directory=None):
+    def __init__(self, arch, sdk, directory=None):
         super(OSXSingleTarget, self).__init__(directory)
-        self.bits = bits
+        self.arch = arch
+        self.sdk = sdk
 
-        if bits == 32:
-            arch = 'i386'
-        else:
-            arch = 'x86_64'
+        flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (self.sdk_prefix, sdk, arch)
+        host_enviro = '%s/x86_64/10.9' % config.get('osx_environment_prefix')
+        target_enviro = '%s/%s/%s' % (config.get('osx_environment_prefix'), arch, sdk)
 
-        flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (self.sdk_prefix, self.sdk, arch)
-        enviro = '%s/%d' % (config.get('osx_environment_prefix'), bits)
+        self.bin = '%s/bin' % target_enviro
 
         # Environment variables
-        self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, enviro, flags))
+        self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, target_enviro, flags))
         self.set('CPPFLAGS', '')
-        self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, enviro, flags))
-        self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, enviro, flags))
-        self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, enviro, flags))
-        self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.directory, enviro))
-        self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % enviro)
-        self.set('MACOSX_DEPLOYMENT_TARGET', config.get('osx_sdk'))
+        self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, target_enviro, flags))
+        self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, target_enviro, flags))
+        self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, target_enviro, flags))
+        self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.directory, target_enviro))
+        self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % host_enviro)
+        self.set('MACOSX_DEPLOYMENT_TARGET', sdk)
         self.set('CCACHE_BASEDIR', self.directory)
 
     @Target.ccache.setter
@@ -810,40 +827,32 @@ class OSXSingleTarget(OSXTarget):
             self.set('CC', '"ccache gcc"')
             self.set('CXX', '"ccache g++"')
 
-    def package(self, project, checkout, output_dir, options, no_notarize):
+    def package(self, project, checkout, output_dir, options, notarize):
         tree = self.build(project, checkout, options)
         tree.add_defaults(options)
-        p = self._build_packages(tree, options)
-        for x in p:
-            if not isinstance(x, tuple):
-                raise Error('macOS packages must be returned from cscript as tuples of (dmg-filename, bundle-id)')
-            if not no_notarize:
-                notarize(x[0], x[1])
-        self._copy_packages(tree, [x[0] for x in p], output_dir)
+        self.unlock_keychain()
+        p = self._cscript_package_and_notarize(tree, options, notarize)
+        self._copy_packages(tree, p, output_dir)
 
 
 class OSXUniversalTarget(OSXTarget):
     def __init__(self, directory=None):
         super(OSXUniversalTarget, self).__init__(directory)
-        self.bits = None
-
-    def package(self, project, checkout, output_dir, options, no_notarize):
+        self.intel_sdk = config.get('osx_intel_sdk')
+        self.arm_sdk = config.get('osx_arm_sdk')
 
-        for b in [32, 64]:
-            target = OSXSingleTarget(b, os.path.join(self.directory, '%d' % b))
+    def package(self, project, checkout, output_dir, options, notarize):
+        for arch, sdk in (('x86_64', self.intel_sdk), ('arm64', self.arm_sdk)):
+            target = OSXSingleTarget(arch, sdk, os.path.join(self.directory, arch))
             target.ccache = self.ccache
             tree = globals.trees.get(project, checkout, target)
             tree.build_dependencies(options)
             tree.build(options)
 
+        self.unlock_keychain()
         tree = globals.trees.get(project, checkout, self)
         with TreeDirectory(tree):
-            if len(inspect.getfullargspec(tree.cscript['package']).args) == 3:
-                packages = tree.call('package', tree.version, options)
-            else:
-                log_normal("Deprecated cscript package() method with no options parameter")
-                packages = tree.call('package', tree.version)
-            for p in packages:
+            for p in self._cscript_package_and_notarize(tree, options, notarize):
                 copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p))))
 
 class SourceTarget(Target):
@@ -858,7 +867,7 @@ class SourceTarget(Target):
     def cleanup(self):
         rmtree(self.directory)
 
-    def package(self, project, checkout, output_dir, options, no_notarize):
+    def package(self, project, checkout, output_dir, options, notarize):
         tree = globals.trees.get(project, checkout, self)
         with TreeDirectory(tree):
             name = read_wscript_variable(os.getcwd(), 'APPNAME')
@@ -873,7 +882,7 @@ class SourceTarget(Target):
 #    or centos-version-{32,64}
 #    or fedora-version-{32,64}
 #    or mageia-version-{32,64}
-#    or osx-{32,64}
+#    or osx
 #    or source
 #    or flatpak
 #    or appimage
@@ -904,13 +913,8 @@ def target_factory(args):
         target = LinuxTarget(p[0], None, int(p[1]), args.work)
     elif s == 'raspbian':
         target = LinuxTarget(s, None, None, args.work)
-    elif s.startswith('osx-'):
-        target = OSXSingleTarget(int(s.split('-')[1]), args.work)
     elif s == 'osx':
-        if args.command == 'build':
-            target = OSXSingleTarget(64, args.work)
-        else:
-            target = OSXUniversalTarget(args.work)
+        target = OSXUniversalTarget(args.work)
     elif s == 'source':
         target = SourceTarget()
     elif s == 'flatpak':
@@ -1225,7 +1229,7 @@ def main():
                 output_dir = args.output
 
             makedirs(output_dir)
-            target.package(args.project, args.checkout, output_dir, get_command_line_options(args), args.no_notarize)
+            target.package(args.project, args.checkout, output_dir, get_command_line_options(args), not args.no_notarize)
         except Error as e:
             if target is not None and not args.keep:
                 target.cleanup()